From 7eba09c0888fc07de0922cbc880ab66e5e5efdae Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 8 Oct 2026 23:39:35 -0400 Subject: [PATCH 01/67] feat!: move runtime storage to PostgreSQL with verified offline upgrades Use native PostgreSQL SQL and restricted telemetry/account roles. Preserve legacy SQLite only in the offline importer, which snapshots, resumes and verifies data before enabling runtime access. Update native backups, deployment, tests and the source-pinned paired benchmark harness. BREAKING CHANGE: runtime storage requires PostgreSQL 18. Existing SQLite instances must use the documented planned-downtime import before cutover. Refs #2144 --- .env.example | 27 +- .github/workflows/deploy.yml | 341 +++- .github/workflows/postgres-benchmark.yml | 119 ++ .gitignore | 2 + AGENTS.md | 65 +- DEPLOY.md | 62 +- Dockerfile | 56 +- Dockerfile.go | 61 - Makefile | 48 +- README.md | 121 +- cmd/decrypt/go.mod | 19 +- cmd/decrypt/go.sum | 30 +- cmd/decrypt/main.go | 59 +- cmd/decrypt/postgres_test.go | 43 + cmd/ingestor/README.md | 51 +- .../advert_evidence_migration_race_test.go | 48 +- cmd/ingestor/advert_evidence_scale_test.go | 10 +- cmd/ingestor/advert_route_evidence.go | 24 +- cmd/ingestor/advert_route_evidence_test.go | 60 +- cmd/ingestor/approved_channels.go | 84 +- cmd/ingestor/approved_channels_test.go | 149 +- cmd/ingestor/async_migration.go | 71 +- cmd/ingestor/async_migration_test.go | 10 +- cmd/ingestor/client_reception.go | 12 +- cmd/ingestor/client_reception_test.go | 64 +- cmd/ingestor/client_regions_test.go | 12 +- cmd/ingestor/client_rf_sample.go | 2 +- cmd/ingestor/config.go | 65 +- cmd/ingestor/config_test.go | 8 +- cmd/ingestor/coverage_boost_test.go | 42 +- cmd/ingestor/coverage_gate_test.go | 2 +- cmd/ingestor/coverage_query_bench_test.go | 12 +- cmd/ingestor/db.go | 1389 +++-------------- cmd/ingestor/db_test.go | 545 ++----- cmd/ingestor/db_writer_perf_test.go | 12 +- cmd/ingestor/decoder.go | 50 +- cmd/ingestor/from_pubkey_test.go | 8 +- cmd/ingestor/go.mod | 22 +- cmd/ingestor/go.sum | 32 +- cmd/ingestor/ingest_buffer.go | 2 +- cmd/ingestor/issue1283_test.go | 53 +- cmd/ingestor/issue1690_last_seen_test.go | 6 +- cmd/ingestor/issue1865_test.go | 10 +- cmd/ingestor/main.go | 102 +- cmd/ingestor/main_test.go | 10 +- cmd/ingestor/maintenance.go | 37 +- cmd/ingestor/multibyte_persist.go | 29 +- .../multibyte_persist_helpers_test.go | 21 +- cmd/ingestor/multibyte_persist_test.go | 16 +- cmd/ingestor/neighbor_builder.go | 70 +- cmd/ingestor/neighbor_builder_delta_test.go | 74 +- cmd/ingestor/neighbor_builder_test.go | 84 +- cmd/ingestor/observer_lastseen_1465_test.go | 6 +- cmd/ingestor/observer_purge_test.go | 16 +- cmd/ingestor/observer_status_meta_test.go | 2 +- cmd/ingestor/postgres_config_test.go | 63 + cmd/ingestor/postgres_test.go | 179 +++ cmd/ingestor/pragma_parity_test.go | 59 +- cmd/ingestor/prune_chunked_test.go | 36 +- cmd/ingestor/prune_geofilter.go | 13 +- cmd/ingestor/prune_geofilter_test.go | 4 +- cmd/ingestor/refresh_planner_stats_test.go | 210 +-- cmd/ingestor/region_keys.go | 4 +- cmd/ingestor/region_keys_test.go | 17 +- cmd/ingestor/regions_gate_test.go | 2 +- cmd/ingestor/relay_touch_test.go | 2 +- cmd/ingestor/resolved_path_test.go | 14 +- cmd/ingestor/retained_status_test.go | 16 +- cmd/ingestor/rf_gate_test.go | 2 +- cmd/ingestor/scope_match_tally.go | 17 +- cmd/ingestor/scope_match_tally_test.go | 12 +- cmd/ingestor/sig_validate_ingest_test.go | 18 +- cmd/ingestor/stats_file.go | 18 +- cmd/ingestor/stats_file_test.go | 8 +- cmd/ingestor/stats_file_timestamp_test.go | 2 +- cmd/ingestor/status_liveness_test.go | 10 +- cmd/migrate/cli_test.go | 106 ++ cmd/migrate/generated_columns_test.go | 82 + cmd/migrate/go.mod | 22 +- cmd/migrate/go.sum | 30 + cmd/migrate/import.go | 782 ++++++++++ cmd/migrate/import_safety_test.go | 294 ++++ cmd/migrate/import_test.go | 256 +++ cmd/migrate/layout.go | 329 ++++ cmd/migrate/main.go | 252 ++- cmd/migrate/main_test.go | 153 +- cmd/migrate/source.go | 220 +++ cmd/migrate/source_test.go | 188 +++ cmd/server/account_test.go | 3 +- cmd/server/advert_evidence_scale_test.go | 2 +- cmd/server/advert_route_evidence.go | 6 +- cmd/server/advert_route_evidence_test.go | 48 +- cmd/server/advert_stats.go | 8 +- cmd/server/advert_stats_test.go | 15 +- .../analytics_recompute_after_load_test.go | 13 +- cmd/server/auth_fixture_test.go | 61 +- cmd/server/auth_flow_test.go | 3 +- cmd/server/auth_login_audit_test.go | 5 +- cmd/server/auth_mail_test.go | 15 +- cmd/server/auth_service.go | 16 +- cmd/server/backup.go | 35 +- cmd/server/backup_test.go | 50 +- cmd/server/bounded_load_test.go | 189 +-- cmd/server/bridge_handle_nodes_test.go | 2 +- cmd/server/channel_message_hash_size_test.go | 13 +- cmd/server/channel_message_scope_name_test.go | 26 +- .../channels_message_order_1366_test.go | 54 +- .../channels_no_unknown_bucket_1373_test.go | 4 +- cmd/server/chunked_load.go | 22 +- cmd/server/chunked_load_id_zero_test.go | 25 +- cmd/server/chunked_load_oldest_test.go | 25 +- cmd/server/chunked_load_test.go | 8 +- cmd/server/collision_details_test.go | 12 +- cmd/server/config.go | 35 +- cmd/server/config_mode_test.go | 15 +- cmd/server/config_test.go | 65 +- cmd/server/coverage_test.go | 231 ++- cmd/server/db.go | 489 +++--- cmd/server/db_channel_messages_perf_test.go | 38 +- cmd/server/db_test.go | 298 ++-- cmd/server/decoder.go | 51 +- cmd/server/encrypted_channels_test.go | 8 +- cmd/server/eviction_test.go | 4 +- cmd/server/first_seen_1166_test.go | 2 +- cmd/server/go.mod | 33 +- cmd/server/go.sum | 79 +- cmd/server/hash_migrate.go | 136 +- cmd/server/hash_migrate_test.go | 32 +- .../hidden_name_prefix_1181_endpoints_test.go | 8 +- cmd/server/hidden_name_prefix_1181_test.go | 6 +- cmd/server/hop_disambig_e2e_test.go | 15 +- cmd/server/hot_startup_consistency_test.go | 2 +- cmd/server/hot_startup_test.go | 70 +- cmd/server/issue1189_distinct_iatas_test.go | 20 +- cmd/server/issue1690_cold_load_test.go | 18 +- cmd/server/issue1809_bg_load_race_test.go | 56 +- cmd/server/issue673_test.go | 4 +- cmd/server/issue804_repeater_region_test.go | 24 +- cmd/server/issue810_repro_test.go | 12 +- cmd/server/issue871_test.go | 2 +- cmd/server/load_graph_invariant_test.go | 23 +- cmd/server/loadchunk_ambiguous_prefix_test.go | 31 +- .../loadchunk_pathjson_fallback_test.go | 29 +- .../loadchunk_resolved_path_1558_test.go | 25 +- cmd/server/main.go | 62 +- cmd/server/memlimit.go | 17 +- cmd/server/memory.go | 20 +- cmd/server/multibyte_capability_test.go | 47 +- cmd/server/multibyte_region_filter_test.go | 14 +- cmd/server/neighbor_api_test.go | 35 +- cmd/server/neighbor_persist_legacy_test.go | 18 +- cmd/server/neighbor_recomputer_test.go | 27 +- cmd/server/node_battery.go | 4 +- cmd/server/node_battery_test.go | 12 +- cmd/server/node_blacklist_test.go | 14 +- cmd/server/node_declared_regions_api_test.go | 2 +- .../node_health_can_relay_case_1290_test.go | 2 +- cmd/server/node_hop_analytics_test.go | 24 +- cmd/server/node_reach.go | 13 +- cmd/server/node_reach_bench_test.go | 17 +- cmd/server/node_reach_endpoint_test.go | 24 +- cmd/server/node_reach_test.go | 11 +- cmd/server/node_resolve.go | 2 +- cmd/server/nodes_pagination_has_more_test.go | 2 +- cmd/server/nodes_perf_limit2000_test.go | 2 +- cmd/server/nodes_perf_test.go | 2 +- cmd/server/notify_service_test.go | 19 +- cmd/server/notify_source_test.go | 2 +- cmd/server/obs_raw_hex_test.go | 13 +- cmd/server/observer_blacklist_test.go | 8 +- cmd/server/observer_count_1888_test.go | 6 +- cmd/server/observer_naive_clock_1478_test.go | 6 +- .../observers_canrelay_tristate_1290_test.go | 6 +- cmd/server/openapi.go | 15 +- cmd/server/packet_scope_name_test.go | 12 +- cmd/server/packets_order_test.go | 14 +- cmd/server/parity_test.go | 17 +- cmd/server/pathhop_dedupe_2108_test.go | 10 +- cmd/server/paths_anchor_bias_test.go | 10 +- cmd/server/paths_hop_name_test.go | 8 +- cmd/server/paths_sort_test.go | 22 +- .../paths_through_collision_1352_test.go | 30 +- cmd/server/paths_through_test.go | 12 +- cmd/server/perf_io.go | 69 +- cmd/server/perf_io_test.go | 54 +- cmd/server/postgres_fixture_test.go | 85 + cmd/server/postgres_stats.go | 80 + cmd/server/postgres_stats_test.go | 186 +++ cmd/server/postgres_test.go | 250 +++ cmd/server/postgres_value_parity_test.go | 199 +++ cmd/server/preparable_schema_test.go | 28 +- cmd/server/proposal_handlers_test.go | 5 +- cmd/server/readonly_invariant_test.go | 18 +- cmd/server/region_nodes_test.go | 2 +- cmd/server/repeater_liveness_test.go | 14 +- cmd/server/resolve_context_test.go | 14 +- cmd/server/resolved_index.go | 21 +- cmd/server/resolved_index_test.go | 39 +- cmd/server/rf_noise.go | 4 +- cmd/server/rf_noise_test.go | 6 +- cmd/server/routes.go | 25 +- cmd/server/routes_test.go | 110 +- cmd/server/runstartup_load_test.go | 31 +- cmd/server/rx_coverage.go | 19 +- cmd/server/rx_coverage_endpoint_test.go | 4 +- cmd/server/rx_dashboard.go | 17 +- cmd/server/scope_audit.go | 25 +- cmd/server/scope_audit_test.go | 52 +- cmd/server/scope_config_state_api_test.go | 15 +- cmd/server/scope_verify.go | 4 +- cmd/server/state_dir.go | 15 + cmd/server/store.go | 90 +- cmd/server/store_memory_diagnostics_test.go | 27 +- cmd/server/subpaths_window_test.go | 10 +- cmd/server/testdata/golden/shapes.json | 34 +- cmd/server/topology_dedup_test.go | 77 +- cmd/server/traffic_share_score_test.go | 4 +- cmd/server/types.go | 24 +- .../usefulness_axes_handle_nodes_test.go | 4 +- cmd/server/user_mgmt_config.go | 24 +- cmd/server/user_mgmt_config_test.go | 20 +- cmd/server/user_mgmt_off_test.go | 38 +- cmd/server/users_backup.go | 16 +- cmd/server/users_backup_test.go | 44 +- cmd/server/websocket.go | 6 +- cmd/server/websocket_test.go | 12 +- cmd/server/zerohop_hashsize_test.go | 8 +- config.example.json | 21 +- docker-compose.example.yml | 29 +- docker-compose.no-mosquitto.yml | 23 + docker-compose.staging.no-mosquitto.yml | 23 + docker-compose.staging.yml | 23 + docker-compose.yml | 23 + docker/postgres-bootstrap.sh | 26 + docker/postgres-grants.sql | 22 + docker/postgres-init.sh | 40 + docker/postgres.compose.yml | 34 + docker/supervisord-go-no-caddy.conf | 2 +- .../supervisord-go-no-mosquitto-no-caddy.conf | 2 +- docker/supervisord-go-no-mosquitto.conf | 2 +- docker/supervisord-go.conf | 2 +- docs/deployment.md | 216 +-- docs/postgresql-migration-plan.md | 55 + docs/postgresql-performance.md | 51 + docs/postgresql-upgrade.md | 116 ++ docs/user-guide/accounts.md | 69 +- internal/dbconfig/dbconfig.go | 11 +- internal/dbschema/collation_test.go | 31 + internal/dbschema/dbschema.go | 826 ++-------- internal/dbschema/go.mod | 19 +- internal/dbschema/go.sum | 28 + .../dbschema/{ => legacy}/advert_evidence.go | 2 +- internal/dbschema/legacy/base.go | 707 +++++++++ internal/dbschema/legacy/check.go | 50 + internal/dbschema/legacy/dbschema.go | 753 +++++++++ .../dbschema/{ => legacy}/dbschema_test.go | 2 +- internal/dbschema/{ => legacy}/dedup_index.go | 2 +- .../dbschema/{ => legacy}/dedup_index_test.go | 2 +- .../legacy/ingestor_migration_test.go | 67 + .../dbschema/legacy/public_key_test.go | 21 +- internal/dbschema/{ => legacy}/writer_dsn.go | 2 +- .../dbschema/{ => legacy}/writer_dsn_test.go | 2 +- internal/dbschema/manifest.go | 36 + .../dbschema/postgres_integration_test.go | 179 +++ internal/dbschema/postgres_test.go | 29 + internal/dbschema/schema.sql | 325 ++++ internal/packetpath/hash.go | 61 + internal/packetpath/hash_test.go | 45 + internal/pgutil/dump.go | 81 + internal/pgutil/go.mod | 14 + internal/pgutil/go.sum | 28 + internal/pgutil/pgtest/pgtest.go | 141 ++ internal/pgutil/pgutil.go | 150 ++ internal/pgutil/pgutil_test.go | 232 +++ internal/users/audit.go | 23 +- internal/users/audit_list_test.go | 31 +- internal/users/export_test.go | 2 +- internal/users/go.mod | 27 +- internal/users/go.sum | 75 +- internal/users/helpers_test.go | 87 +- internal/users/legacy_schema.go | 130 ++ internal/users/mail.go | 23 +- internal/users/notify.go | 68 +- internal/users/notify_test.go | 27 +- internal/users/onetime.go | 48 +- internal/users/postgres_test.go | 389 +++++ internal/users/proposals.go | 52 +- internal/users/proposals_test.go | 32 +- internal/users/schema.go | 278 ++-- internal/users/sessions.go | 25 +- internal/users/settings.go | 27 +- internal/users/settings_test.go | 29 +- internal/users/snapshot.go | 33 +- internal/users/snapshot_test.go | 86 +- internal/users/stats.go | 36 +- internal/users/store.go | 113 +- internal/users/store_test.go | 117 +- internal/users/users.go | 70 +- manage.sh | 295 ++-- public/admin-audit.js | 2 +- public/geofilter-docs.html | 24 +- public/perf.js | 48 +- scripts/bash-path.js | 15 + scripts/install-postgres-ci.sh | 27 + scripts/migrate-dedup.js | 12 +- scripts/migrate-fixture-hashes.go | 167 +- scripts/non-unit-tests.json | 1 + scripts/postgres-benchmark/.gitignore | 1 + scripts/postgres-benchmark/README.md | 181 +++ .../overlays/clock_linux_test.go | 18 + .../overlays/clock_other_test.go | 6 + .../overlays/common_test.go | 122 ++ .../overlays/ingestor_test.go | 529 +++++++ .../overlays/postgres_adapter_test.go | 48 + .../overlays/server_test.go | 355 +++++ .../overlays/sqlite_adapter_test.go | 306 ++++ scripts/postgres-benchmark/run.py | 1009 ++++++++++++ scripts/postgres-benchmark/test_harness.py | 168 ++ scripts/prepare-postgres-fixture.go | 142 ++ scripts/prune-nodes-outside-geo-filter.py | 8 +- scripts/test-postgres-compose.sh | 205 +++ test-all.sh | 2 + tests/e2e/test-postgres-backup-e2e.js | 124 ++ tests/e2e/test-user-management-e2e.js | 15 +- tests/unit/test-issue-1648-m2-emoji-scan.js | 2 +- tests/unit/test-issue-1956-release-routing.js | 56 +- tests/unit/test-perf-disk-io-1120.js | 102 +- tests/unit/test-postgres-ops.js | 126 ++ 328 files changed, 16220 insertions(+), 7075 deletions(-) create mode 100644 .github/workflows/postgres-benchmark.yml delete mode 100644 Dockerfile.go create mode 100644 cmd/decrypt/postgres_test.go create mode 100644 cmd/ingestor/postgres_config_test.go create mode 100644 cmd/ingestor/postgres_test.go create mode 100644 cmd/migrate/cli_test.go create mode 100644 cmd/migrate/generated_columns_test.go create mode 100644 cmd/migrate/import.go create mode 100644 cmd/migrate/import_safety_test.go create mode 100644 cmd/migrate/import_test.go create mode 100644 cmd/migrate/layout.go create mode 100644 cmd/migrate/source.go create mode 100644 cmd/migrate/source_test.go create mode 100644 cmd/server/postgres_fixture_test.go create mode 100644 cmd/server/postgres_stats.go create mode 100644 cmd/server/postgres_stats_test.go create mode 100644 cmd/server/postgres_test.go create mode 100644 cmd/server/postgres_value_parity_test.go create mode 100644 cmd/server/state_dir.go create mode 100755 docker/postgres-bootstrap.sh create mode 100644 docker/postgres-grants.sql create mode 100755 docker/postgres-init.sh create mode 100644 docker/postgres.compose.yml create mode 100644 docs/postgresql-migration-plan.md create mode 100644 docs/postgresql-performance.md create mode 100644 docs/postgresql-upgrade.md create mode 100644 internal/dbschema/collation_test.go rename internal/dbschema/{ => legacy}/advert_evidence.go (98%) create mode 100644 internal/dbschema/legacy/base.go create mode 100644 internal/dbschema/legacy/check.go create mode 100644 internal/dbschema/legacy/dbschema.go rename internal/dbschema/{ => legacy}/dbschema_test.go (99%) rename internal/dbschema/{ => legacy}/dedup_index.go (99%) rename internal/dbschema/{ => legacy}/dedup_index_test.go (99%) create mode 100644 internal/dbschema/legacy/ingestor_migration_test.go rename cmd/ingestor/public_key_lowercase_migration_test.go => internal/dbschema/legacy/public_key_test.go (84%) rename internal/dbschema/{ => legacy}/writer_dsn.go (98%) rename internal/dbschema/{ => legacy}/writer_dsn_test.go (98%) create mode 100644 internal/dbschema/manifest.go create mode 100644 internal/dbschema/postgres_integration_test.go create mode 100644 internal/dbschema/postgres_test.go create mode 100644 internal/dbschema/schema.sql create mode 100644 internal/packetpath/hash.go create mode 100644 internal/packetpath/hash_test.go create mode 100644 internal/pgutil/dump.go create mode 100644 internal/pgutil/go.mod create mode 100644 internal/pgutil/go.sum create mode 100644 internal/pgutil/pgtest/pgtest.go create mode 100644 internal/pgutil/pgutil.go create mode 100644 internal/pgutil/pgutil_test.go create mode 100644 internal/users/legacy_schema.go create mode 100644 internal/users/postgres_test.go create mode 100644 scripts/bash-path.js create mode 100755 scripts/install-postgres-ci.sh create mode 100644 scripts/postgres-benchmark/.gitignore create mode 100644 scripts/postgres-benchmark/README.md create mode 100644 scripts/postgres-benchmark/overlays/clock_linux_test.go create mode 100644 scripts/postgres-benchmark/overlays/clock_other_test.go create mode 100644 scripts/postgres-benchmark/overlays/common_test.go create mode 100644 scripts/postgres-benchmark/overlays/ingestor_test.go create mode 100644 scripts/postgres-benchmark/overlays/postgres_adapter_test.go create mode 100644 scripts/postgres-benchmark/overlays/server_test.go create mode 100644 scripts/postgres-benchmark/overlays/sqlite_adapter_test.go create mode 100644 scripts/postgres-benchmark/run.py create mode 100644 scripts/postgres-benchmark/test_harness.py create mode 100644 scripts/prepare-postgres-fixture.go create mode 100755 scripts/test-postgres-compose.sh create mode 100644 tests/e2e/test-postgres-backup-e2e.js create mode 100644 tests/unit/test-postgres-ops.js diff --git a/.env.example b/.env.example index 548e3c0ba..e7b4a327e 100644 --- a/.env.example +++ b/.env.example @@ -1,15 +1,34 @@ # MeshCore Analyzer — Environment Configuration -# Copy to .env and customize. All values have sensible defaults. +# Copy to .env and customize. Generate the required database passwords below. # # This file is read by BOTH docker compose AND manage.sh — one source of truth. # manage.sh setup negotiates and updates only these production managed keys: # PROD_DATA_DIR, PROD_HTTP_PORT, PROD_HTTPS_PORT, PROD_MQTT_PORT, DISABLE_MOSQUITTO # Each environment keeps config + data together in one directory: -# ~/meshcore-data/config.json, meshcore.db, Caddyfile, theme.json -# ~/meshcore-staging-data/config.json, meshcore.db, Caddyfile +# ~/meshcore-data/config.json, Caddyfile, theme.json, queues and backups +# ~/meshcore-staging-data/config.json, Caddyfile, queues and backups +# PostgreSQL has a separate persistent directory; do not copy it as a backup. + +# --- PostgreSQL --- +# Generate a DIFFERENT value for each blank field: openssl rand -hex 32 +# Keep .env private. Hexadecimal passwords can be safely embedded in the +# generated container URLs; the initializer enforces at least 32 hex digits. +# These roles are created only when PostgreSQL's data directory is empty. +POSTGRES_ADMIN_PASSWORD= +CORESCOPE_OWNER_PASSWORD= +CORESCOPE_READER_PASSWORD= +CORESCOPE_WRITER_PASSWORD= +CORESCOPE_ACCOUNTS_PASSWORD= +CORESCOPE_CHANNELS_PASSWORD= +PROD_POSTGRES_DATA_DIR=~/corescope-postgres +STAGING_POSTGRES_DATA_DIR=~/corescope-staging-postgres +# docker-compose.example.yml uses this local default instead: +POSTGRES_DATA_DIR=./postgres-data +# Existing instances: stop both processes and follow docs/postgresql-upgrade.md +# before starting the PostgreSQL image. Never mount two servers on one directory. # --- Production --- -# Data directory (database, theme, etc.) +# State directory (config, theme, queues and backups) # Default: ~/meshcore-data # Used by: docker compose, manage.sh PROD_DATA_DIR=~/meshcore-data diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c020c1000..8263e421b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -23,10 +23,17 @@ on: description: 'Release fast path already published the tag images' type: boolean default: false + postgres_benchmark: + description: Run only the isolated PostgreSQL benchmark (no publish or deploy) + type: boolean + default: false + candidate_sha: + description: Full candidate commit for postgres_benchmark + type: string + default: '' permissions: contents: read - packages: write concurrency: group: ci-${{ github.event.pull_request.number || github.ref }} @@ -47,12 +54,21 @@ env: # PRs stop after build-and-publish (no GHCR push). Master continues to deploy + badges. jobs: + postgres-benchmark: + name: Isolated PostgreSQL comparison + if: github.event_name == 'workflow_dispatch' && inputs.postgres_benchmark + permissions: + contents: read + uses: ./.github/workflows/postgres-benchmark.yml + with: + candidate_sha: ${{ inputs.candidate_sha }} + # ─────────────────────────────────────────────────────────────── # 0. Change scope — decides whether the expensive jobs below run. # ─────────────────────────────────────────────────────────────── changes: name: "🔎 Change scope" - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: code: ${{ steps.scope.outputs.code }} ingestor: ${{ steps.scope.outputs.ingestor }} @@ -66,6 +82,12 @@ jobs: id: scope run: | set -euo pipefail + if [ "${{ inputs.postgres_benchmark }}" = "true" ]; then + echo "code=false" >> "$GITHUB_OUTPUT" + echo "ingestor=false" >> "$GITHUB_OUTPUT" + echo "benchmark-only dispatch" + exit 0 + fi # Only pull requests are scoped. Pushes and dispatches always count as # code, so master keeps producing an :edge image and tag builds are # never skipped. @@ -117,17 +139,37 @@ jobs: # set) whose safety is an argument until something checks it. race-test: name: "🏁 Race detector (ingestor)" - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: [changes] if: needs.changes.outputs.ingestor == 'true' + services: &postgres_test_services + postgres: + image: postgres:18.6-alpine3.24 + env: + POSTGRES_PASSWORD: "ci-${{ github.run_id }}-${{ github.run_attempt }}" + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --locale=C" + ports: ["127.0.0.1:5432:5432"] + options: >- + --health-cmd "pg_isready -h 127.0.0.1 -U postgres" + --health-interval 5s --health-timeout 5s --health-retries 20 + env: + CORESCOPE_TEST_POSTGRES_URL: "postgresql://postgres:ci-${{ github.run_id }}-${{ github.run_attempt }}@127.0.0.1:5432/postgres?sslmode=disable" + PGHOST: 127.0.0.1 + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: "ci-${{ github.run_id }}-${{ github.run_attempt }}" + PGSSLMODE: disable steps: - name: Checkout code uses: actions/checkout@v5 - - name: Set up Go 1.27 + - name: Install pinned PostgreSQL 18.6 clients + run: &install_postgres_clients bash scripts/install-postgres-ci.sh client + + - name: Set up Go 1.27.2 uses: actions/setup-go@v6 with: - go-version: '1.27' + go-version: '1.27.2' # All modules: a -race build of a cgo package is the slowest thing in # this pipeline, and setup-go caches ~/.cache/go-build as well as the # module cache, so a key that misses on an unrelated module hurts most @@ -145,9 +187,17 @@ jobs: # ─────────────────────────────────────────────────────────────── go-test: name: "✅ Go Build & Test" - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: [changes] if: needs.changes.outputs.code == 'true' + services: *postgres_test_services + env: + CORESCOPE_TEST_POSTGRES_URL: "postgresql://postgres:ci-${{ github.run_id }}-${{ github.run_attempt }}@127.0.0.1:5432/postgres?sslmode=disable" + PGHOST: 127.0.0.1 + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: "ci-${{ github.run_id }}-${{ github.run_attempt }}" + PGSSLMODE: disable steps: - name: Checkout code uses: actions/checkout@v5 @@ -157,21 +207,27 @@ jobs: - name: Clean Go module cache run: rm -rf ~/go/pkg/mod 2>/dev/null || true - - name: Set up Go 1.27 + - name: Install pinned PostgreSQL 18.6 clients + run: *install_postgres_clients + + + - name: Set up Go 1.27.2 uses: actions/setup-go@v6 with: - go-version: '1.27' - # All modules, not just two: the cgo SQLite build is expensive to + go-version: '1.27.2' + # All modules: the offline SQLite importer is expensive to # redo, and setup-go caches ~/.cache/go-build as well as the module # cache, so a key that misses on an unrelated module hurts. cache-dependency-path: "**/go.sum" + - name: Test PostgreSQL benchmark controls + run: python3 -m unittest discover -s scripts/postgres-benchmark -p 'test_*.py' + - name: Enforce gofmt + go vet (issue #1859) run: | set -e - # gofmt: every tracked *.go file except the misnamed Dockerfile.go, - # which is a Dockerfile (FROM ...) that gofmt cannot parse. - unformatted=$(gofmt -l $(git ls-files '*.go' | grep -vi 'Dockerfile.go')) + # Every tracked Go source file must be formatted. + unformatted=$(gofmt -l $(git ls-files '*.go')) if [ -n "$unformatted" ]; then echo "::error::gofmt required on these files:" echo "$unformatted" @@ -188,7 +244,7 @@ jobs: run: | set -e -o pipefail cd cmd/server - go build . + CGO_ENABLED=0 go build . # -race gates PR #1208's atomic.Pointer migration: the race-detector # is what makes path_inspect_atomic_race_test.go actually assert. go test -timeout 20m -race -coverprofile=server-coverage.out ./... 2>&1 | tee server-test.log @@ -201,7 +257,7 @@ jobs: run: | set -e -o pipefail cd cmd/ingestor - go build . + CGO_ENABLED=0 go build . go test -timeout 20m -coverprofile=ingestor-coverage.out ./... 2>&1 | tee ingestor-test.log echo "--- Go Ingestor Coverage ---" go tool cover -func=ingestor-coverage.out | tail -1 @@ -213,7 +269,7 @@ jobs: go test ./... echo "--- Channel library tests passed ---" cd ../../cmd/decrypt - go build -ldflags="-s -w" -o corescope-decrypt . + CGO_ENABLED=0 go build -ldflags="-s -w" -o corescope-decrypt . go test ./... echo "--- Decrypt CLI tests passed ---" @@ -229,10 +285,12 @@ jobs: set -e -o pipefail # Neither module had CI test execution before the mattn/go-sqlite3 # migration; both open the database, and dbschema owns the migrations. - cd internal/dbschema + cd internal/pgutil + go test ./... + cd ../dbschema go test ./... cd ../../cmd/migrate - go build -o /dev/null . + CGO_ENABLED=1 go build -o /dev/null . go test ./... echo "--- migrate + dbschema tests passed ---" @@ -344,17 +402,24 @@ jobs: e2e-shard: name: "🎭 Playwright E2E shard ${{ matrix.shard }}/3" needs: [changes] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: matrix: shard: [1, 2, 3] env: + CORESCOPE_TEST_POSTGRES_URL: "postgresql://postgres:ci-${{ github.run_id }}-${{ github.run_attempt }}@127.0.0.1:5432/postgres?sslmode=disable" + PGHOST: 127.0.0.1 + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: "ci-${{ github.run_id }}-${{ github.run_attempt }}" + PGSSLMODE: disable SHARD: ${{ matrix.shard }} E2E_SHARDS: 3 defaults: run: shell: bash if: needs.changes.outputs.code == 'true' && !(startsWith(github.ref, 'refs/tags/v') && inputs.images_published) + services: *postgres_test_services steps: - name: Checkout code uses: actions/checkout@v5 @@ -369,16 +434,20 @@ jobs: - name: Clean Go module cache run: rm -rf ~/go/pkg/mod 2>/dev/null || true - - name: Set up Go 1.27 + - name: Install pinned PostgreSQL 18.6 clients + run: *install_postgres_clients + + + - name: Set up Go 1.27.2 uses: actions/setup-go@v6 with: - go-version: '1.27' + go-version: '1.27.2' cache-dependency-path: "**/go.sum" - name: Build Go server run: | cd cmd/server - go build -o ../../corescope-server . + CGO_ENABLED=0 go build -o ../../corescope-server . echo "Go server built successfully" # Only test-user-management-e2e.js uses the e2etest build and its server, @@ -388,12 +457,12 @@ jobs: if: matrix.shard == 3 run: | cd cmd/server - go build -tags e2etest -o ../../corescope-server-e2e . + CGO_ENABLED=0 go build -tags e2etest -o ../../corescope-server-e2e . - name: Build Go migrate tool run: | cd cmd/migrate - go build -o ../../corescope-migrate . + CGO_ENABLED=1 go build -o ../../corescope-migrate . echo "Go migrate tool built successfully" - name: Install npm dependencies @@ -407,8 +476,17 @@ jobs: - name: Instrument frontend JS for coverage run: sh scripts/instrument-frontend.sh + - name: Prepare disposable legacy fixture copy + run: | + mkdir -p "$RUNNER_TEMP/corescope-fixture" + cd cmd/migrate + go run ../../scripts/prepare-postgres-fixture.go \ + -source ../../test-fixtures/e2e-fixture.db \ + -destination "$RUNNER_TEMP/corescope-fixture/fixture.sqlite" + echo "E2E_SQLITE=$RUNNER_TEMP/corescope-fixture/fixture.sqlite" >> "$GITHUB_ENV" + - name: Freshen fixture timestamps - run: bash tools/freshen-fixture.sh test-fixtures/e2e-fixture.db + run: bash tools/freshen-fixture.sh "$E2E_SQLITE" - name: Seed grouped-packet row for #1486 collapse test # The committed fixture has 499 packets, each with exactly ONE @@ -417,11 +495,9 @@ jobs: # (toggle-select) row. Insert a NEW transmission with 3 # observations. # - # The server's async hash-migrate (cmd/server/hash_migrate.go) - # recomputes `transmissions.hash` from `raw_hex` via - # ComputeContentHash(), so the inserted hash MUST equal that - # function's output for the chosen raw_hex — otherwise the row - # gets relabelled and the E2E can't find it. + # Fixture hashes are normalized offline before PostgreSQL import. + # The read-only server verifies them and never rewrites telemetry. + # This seeded hash must equal the shared runtime ContentHash output. # # raw_hex 15000102030405060708090a0b0c0d0e0f # → header=0x15 (route_type=1, payload_type=5) @@ -432,7 +508,7 @@ jobs: # The #1486 and #2104 tests reach it via an explicit hash + # ?timeWindow=0 deep-link. run: | - sqlite3 test-fixtures/e2e-fixture.db <<'SQL' + sqlite3 "$E2E_SQLITE" <<'SQL' -- Sort the seeded row LAST in BOTH default packets views: -- • flat view sorts by transmissions.id DESC → id=0 puts it last -- • grouped view (#default for the packets page) sorts by @@ -470,20 +546,11 @@ jobs: (-1000000,1,'rx',7.0,-88,0,'[]',CAST(strftime('%s','now') AS INTEGER),'[]'); SQL - - name: Migrate fixture DB to current schema (#1287) - # Server now ASSERTs schema is migrated and refuses to start - # otherwise (cmd/server/main.go: dbschema.AssertReady). In prod - # the ingestor owns dbschema.Apply, but CI starts only the - # server against the committed e2e fixture — so we run the - # standalone migrate tool here to bring the fixture up to the - # required shape before the server boots. - run: ./corescope-migrate -db test-fixtures/e2e-fixture.db - - name: 'Seed distinct observation bytes for #2104' # raw_hex exists only after migration. Keep the #1486 payload/hash # while encoding each observation's existing one-hop path. run: | - sqlite3 test-fixtures/e2e-fixture.db <<'SQL' + sqlite3 "$E2E_SQLITE" <<'SQL' UPDATE observations SET raw_hex = CASE observer_idx WHEN 1 THEN '1501aa0102030405060708090a0b0c0d0e0f' WHEN 2 THEN '1501bb0102030405060708090a0b0c0d0e0f' @@ -492,13 +559,42 @@ jobs: SQL - name: Seed deterministic Path Inspector routes (#2060) - run: sqlite3 test-fixtures/e2e-fixture.db < test-fixtures/path-inspector.sql + run: sqlite3 "$E2E_SQLITE" < test-fixtures/path-inspector.sql + + - name: Normalize fixture hashes using the runtime parser + run: | + cd cmd/migrate + go run ../../scripts/migrate-fixture-hashes.go "$E2E_SQLITE" + + - name: Import fixture and grant restricted runtime roles + run: | + set -euo pipefail + for key in CORESCOPE_OWNER_PASSWORD CORESCOPE_READER_PASSWORD CORESCOPE_WRITER_PASSWORD CORESCOPE_ACCOUNTS_PASSWORD CORESCOPE_CHANNELS_PASSWORD; do + printf -v "$key" '%s' "$(openssl rand -hex 32)" + export "$key" + echo "::add-mask::$(printenv "$key")" + done + sh docker/postgres-init.sh + owner_url="postgresql://corescope_owner:$CORESCOPE_OWNER_PASSWORD@127.0.0.1:5432/corescope_telemetry?sslmode=disable" + owner_accounts="postgresql://corescope_owner:$CORESCOPE_OWNER_PASSWORD@127.0.0.1:5432/corescope_accounts?sslmode=disable" + CORESCOPE_DATABASE_URL="$owner_url" CORESCOPE_USERS_DATABASE_URL="$owner_accounts" \ + ./corescope-migrate -offline -from-sqlite "$E2E_SQLITE" -state-dir "$RUNNER_TEMP/corescope-import" + for database in corescope_telemetry corescope_accounts; do + PGUSER=corescope_owner PGPASSWORD="$CORESCOPE_OWNER_PASSWORD" PGDATABASE="$database" \ + psql -X -v ON_ERROR_STOP=1 -f docker/postgres-grants.sql + done + echo "CORESCOPE_READER_DATABASE_URL=postgresql://corescope_reader:$CORESCOPE_READER_PASSWORD@127.0.0.1:5432/corescope_telemetry?sslmode=disable" >> "$GITHUB_ENV" + echo "CORESCOPE_USERS_DATABASE_URL=postgresql://corescope_accounts:$CORESCOPE_ACCOUNTS_PASSWORD@127.0.0.1:5432/corescope_accounts?sslmode=disable" >> "$GITHUB_ENV" + mkdir -p "$RUNNER_TEMP/corescope-state" + # Retain the masked owner password for the later restore step only. + umask 077 + printf '%s' "$CORESCOPE_OWNER_PASSWORD" > "$RUNNER_TEMP/corescope-e2e-owner-password" - name: Start Go server with fixture DB run: | fuser -k 13581/tcp 2>/dev/null || true sleep 1 - ./corescope-server -port 13581 -db test-fixtures/e2e-fixture.db -public public-instrumented & + env -u PGPASSWORD -u PGUSER -u CORESCOPE_TEST_POSTGRES_URL ./corescope-server -port 13581 -state-dir "$RUNNER_TEMP/corescope-state" -public public-instrumented & echo $! > .server.pid for i in $(seq 1 30); do if curl -sf http://localhost:13581/api/healthz > /dev/null 2>&1; then @@ -518,14 +614,12 @@ jobs: fuser -k 13582/tcp 2>/dev/null || true rm -rf /tmp/cs-um mkdir -p /tmp/cs-um - # Own copy of the (already migrated and seeded) fixture; fresh users.db every run. - cp test-fixtures/e2e-fixture.db /tmp/cs-um/on.db + # Shared read-only telemetry; the account writer has its own logical database. cat > /tmp/cs-um/config.json <<'JSON' { "port": 13582, "userManagement": { "enabled": true, - "dbPath": "/tmp/cs-um/users.db", "adminEmails": ["admin@e2e.test"], "publicBaseUrl": "http://localhost:13582", "mail": { "provider": "fake", "fromEmail": "noreply@e2e.test" }, @@ -534,7 +628,7 @@ jobs: } } JSON - ./corescope-server-e2e -config-dir /tmp/cs-um -port 13582 -db /tmp/cs-um/on.db -public public-instrumented & + env -u PGPASSWORD -u PGUSER -u CORESCOPE_TEST_POSTGRES_URL ./corescope-server-e2e -config-dir /tmp/cs-um -port 13582 -state-dir /tmp/cs-um -public public-instrumented & echo $! > .server-um.pid for i in $(seq 1 30); do if curl -sf http://localhost:13582/api/healthz > /dev/null 2>&1; then echo "UM server ready after ${i}s"; break; fi @@ -695,6 +789,57 @@ jobs: suite 2 test-show-neighbors.js BASE_URL=$B suite 3 test-user-management-e2e.js BASE_URL=http://localhost:13582 BASE_URL_OFF=$B + - name: Restore native HTTP backups and verify browser sessions + if: matrix.shard == 3 + run: | + set -euo pipefail + export E2E_BACKUP_DIR="$RUNNER_TEMP/corescope-http-backup" + export BASE_URL=http://localhost:13582 + echo "=== E2E SUITE: test-postgres-backup-e2e.js capture ===" | tee -a e2e-output.txt + node tests/e2e/test-postgres-backup-e2e.js capture 2>&1 | tee -a e2e-output.txt + for pidfile in .server.pid .server-um.pid; do + pid=$(cat "$pidfile") + kill "$pid" + for attempt in $(seq 1 30); do + kill -0 "$pid" 2>/dev/null || break + sleep 1 + done + ! kill -0 "$pid" 2>/dev/null || { echo 'Fixture server did not stop' >&2; exit 1; } + done + # Fresh databases, same deployment roles, no overwrite or role widening. + psql -X -v ON_ERROR_STOP=1 -d postgres <<'SQL' + CREATE DATABASE corescope_restore_telemetry OWNER corescope_owner TEMPLATE template0 ENCODING 'UTF8' LOCALE 'C'; + CREATE DATABASE corescope_restore_accounts OWNER corescope_owner TEMPLATE template0 ENCODING 'UTF8' LOCALE 'C'; + REVOKE ALL ON DATABASE corescope_restore_telemetry FROM PUBLIC; + REVOKE ALL ON DATABASE corescope_restore_accounts FROM PUBLIC; + GRANT CONNECT ON DATABASE corescope_restore_telemetry TO corescope_reader, corescope_writer; + GRANT CONNECT ON DATABASE corescope_restore_accounts TO corescope_accounts, corescope_channels; + SQL + for kind in telemetry accounts; do + PGUSER=corescope_owner PGPASSWORD="$(cat "$RUNNER_TEMP/corescope-e2e-owner-password")" \ + pg_restore --no-owner --no-privileges --exit-on-error --single-transaction \ + --dbname="corescope_restore_$kind" "$E2E_BACKUP_DIR/$kind.dump" + PGUSER=corescope_owner PGPASSWORD="$(cat "$RUNNER_TEMP/corescope-e2e-owner-password")" \ + psql -X -v ON_ERROR_STOP=1 -d "corescope_restore_$kind" -f docker/postgres-grants.sql + PGUSER=corescope_owner PGPASSWORD="$(cat "$RUNNER_TEMP/corescope-e2e-owner-password")" \ + psql -X -v ON_ERROR_STOP=1 -d "corescope_restore_$kind" -c 'ANALYZE' + done + export CORESCOPE_READER_DATABASE_URL="${CORESCOPE_READER_DATABASE_URL/\/corescope_telemetry/\/corescope_restore_telemetry}" + export CORESCOPE_USERS_DATABASE_URL="${CORESCOPE_USERS_DATABASE_URL/\/corescope_accounts/\/corescope_restore_accounts}" + env -u PGPASSWORD -u PGUSER -u CORESCOPE_TEST_POSTGRES_URL ./corescope-server -port 13581 -state-dir "$RUNNER_TEMP/corescope-state" -public public-instrumented & + echo $! > .server.pid + env -u PGPASSWORD -u PGUSER -u CORESCOPE_TEST_POSTGRES_URL ./corescope-server-e2e -config-dir /tmp/cs-um -port 13582 -state-dir /tmp/cs-um -public public-instrumented & + echo $! > .server-um.pid + for port in 13581 13582; do + for attempt in $(seq 1 30); do + curl -sf "http://localhost:$port/api/healthz" >/dev/null && break + [ "$attempt" -lt 30 ] || { echo 'Restored server did not become ready' >&2; exit 1; } + sleep 1 + done + done + echo "=== E2E SUITE: test-postgres-backup-e2e.js verify ===" | tee -a e2e-output.txt + node tests/e2e/test-postgres-backup-e2e.js verify 2>&1 | tee -a e2e-output.txt + # #1616: slide-over focus-restore flake-gate. Runs the slide-over # E2E 3 more times against the SAME backend instance (it already # ran once in the list above) so the Chromium-headless focus race @@ -775,7 +920,7 @@ jobs: e2e-test: name: "🎭 Playwright E2E Tests" needs: [e2e-shard, changes] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: ${{ !cancelled() && needs.changes.outputs.code == 'true' && !(startsWith(github.ref, 'refs/tags/v') && inputs.images_published) }} steps: - name: Require every shard to have passed @@ -830,7 +975,7 @@ jobs: # ─────────────────────────────────────────────────────────────── # 3a. Two-arch image build — beside the tests, never publishes # ─────────────────────────────────────────────────────────────── - # Since the SQLite driver became cgo, the container build depends on a zig + # The offline SQLite importer uses cgo, so the container build needs a zig # cross-toolchain and static musl linking. Builds both architectures and # actually runs the arm64 binaries under QEMU, on every event: for a PR it is # the only image check (the GHCR push below is push/tag-only, so without it @@ -845,7 +990,7 @@ jobs: image-check: name: "🧱 Two-arch image build" needs: [changes] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: needs.changes.outputs.code == 'true' && !(startsWith(github.ref, 'refs/tags/v') && inputs.images_published) outputs: build_time: ${{ steps.meta.outputs.build_time }} @@ -875,7 +1020,11 @@ jobs: # GHCR on its own runner. It rebuilt the same Dockerfile as the step # below for two minutes. Validating the compose file is what is left. - name: Validate staging compose file - run: docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging config --quiet + run: | + for key in POSTGRES_ADMIN_PASSWORD CORESCOPE_OWNER_PASSWORD CORESCOPE_READER_PASSWORD CORESCOPE_WRITER_PASSWORD CORESCOPE_ACCOUNTS_PASSWORD CORESCOPE_CHANNELS_PASSWORD; do + export "$key=$(openssl rand -hex 32)" + done + docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging config --quiet - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -897,26 +1046,50 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max + - name: Load amd64 image for packaged Compose smoke (no publish) + uses: docker/build-push-action@v6 + with: + context: . + push: false + load: true + platforms: linux/amd64 + tags: corescope-pg-smoke:${{ github.sha }} + build-args: | + APP_VERSION=${{ steps.meta.outputs.app_version }} + GIT_COMMIT=${{ steps.meta.outputs.git_commit }} + BUILD_TIME=${{ steps.meta.outputs.build_time }} + cache-from: type=gha + - name: Smoke the arm64 binaries under QEMU + env: + APP_VERSION: ${{ steps.meta.outputs.app_version }} + GIT_COMMIT: ${{ steps.meta.outputs.git_commit }} + BUILD_TIME: ${{ steps.meta.outputs.build_time }} run: | set -e # Export the arm64 binaries so the runner's own `file` can confirm # they are static, then execute one under QEMU. Cheap: the build above # already populated the layer cache. docker buildx build . --platform linux/arm64 \ - --build-arg APP_VERSION=smoke --cache-from type=gha \ + --build-arg APP_VERSION="$APP_VERSION" \ + --build-arg GIT_COMMIT="$GIT_COMMIT" \ + --build-arg BUILD_TIME="$BUILD_TIME" --cache-from type=gha \ --output type=local,dest=./arm64-image - for b in corescope-server corescope-ingestor corescope-decrypt; do + for b in corescope-server corescope-ingestor corescope-decrypt corescope-migrate; do file "./arm64-image/app/$b" | tee /dev/stderr | grep -q 'statically linked' \ || { echo "::error::$b is not statically linked"; exit 1; } file "./arm64-image/app/$b" | grep -q 'ARM aarch64' \ || { echo "::error::$b is not an arm64 binary"; exit 1; } done - docker run --rm --platform linux/arm64 \ - -v "$PWD/arm64-image/app:/bin-under-test:ro" alpine:3.20 \ + docker run --rm --platform linux/arm64 --network none \ + -v "$PWD/arm64-image/app:/bin-under-test:ro" alpine:3.24 \ /bin-under-test/corescope-decrypt --version rm -rf ./arm64-image - echo "arm64 static binaries run under QEMU ✅" + echo "All four arm64 binaries are static; decrypt --version ran under QEMU." + + - name: Exercise packaged PostgreSQL Compose startup + timeout-minutes: 10 + run: timeout --kill-after=150s 7m bash scripts/test-postgres-compose.sh "corescope-pg-smoke:$GITHUB_SHA" # ─────────────────────────────────────────────────────────────── # 3b. Publish Docker Image — only after every test job passed @@ -945,10 +1118,13 @@ jobs: # a skipped required check as passing. race-test is skipped when no ingestor # Go file changed, so for it skipped is accepted; a master push always runs it. build-and-publish: + permissions: + contents: read + packages: write name: "🏗️ Build & Publish Docker Image" needs: [go-test, race-test, e2e-test, image-check, changes] - runs-on: ubuntu-latest - if: ${{ !cancelled() && needs.changes.outputs.code == 'true' && !(startsWith(github.ref, 'refs/tags/v') && inputs.images_published) }} + runs-on: ubuntu-24.04 + if: ${{ !cancelled() && needs.changes.outputs.code == 'true' && !(startsWith(github.ref, 'refs/tags/v') && inputs.images_published) && !inputs.postgres_benchmark }} steps: - name: Require every test job and the image check to have passed run: | @@ -1022,49 +1198,57 @@ jobs: release-artifacts: name: "📦 Release Artifacts" needs: [go-test, changes] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: write - if: startsWith(github.ref, 'refs/tags/v') && needs.changes.outputs.code == 'true' + if: (startsWith(github.ref, 'refs/tags/v') && needs.changes.outputs.code == 'true') && !inputs.postgres_benchmark steps: - name: Checkout code uses: actions/checkout@v5 - - name: Set up Go 1.27 + - name: Set up Go 1.27.2 uses: actions/setup-go@v6 with: - go-version: '1.27' + go-version: '1.27.2' cache-dependency-path: | cmd/decrypt/go.sum - internal/channel/go.sum + cmd/migrate/go.sum + internal/**/go.sum - # The SQLite driver is cgo now (github.com/mattn/go-sqlite3), so the Go - # toolchain alone cannot cross-compile these. zig cc is the C compiler - # that can target both architectures; musl makes the result static. + # The offline importer alone needs cgo and a cross C compiler. - name: Set up Zig uses: mlugg/setup-zig@v2 with: version: 0.16.0 - - name: Build corescope-decrypt (static, linux/amd64) + - name: Build static release tools for both architectures run: | - cd cmd/decrypt - CGO_ENABLED=1 GOOS=linux GOARCH=amd64 CC="zig cc -target x86_64-linux-musl" go build -trimpath -tags netgo,osusergo,sqlite_omit_load_extension -ldflags="-s -w -extldflags '-static -Wl,-s' -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-amd64 . - - - name: Build corescope-decrypt (static, linux/arm64) - run: | - cd cmd/decrypt - CGO_ENABLED=1 GOOS=linux GOARCH=arm64 CC="zig cc -target aarch64-linux-musl" go build -trimpath -tags netgo,osusergo,sqlite_omit_load_extension -ldflags="-s -w -extldflags '-static -Wl,-s' -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-arm64 . + set -euo pipefail + for arch in amd64 arm64; do + case "$arch" in amd64) target=x86_64-linux-musl ;; arm64) target=aarch64-linux-musl ;; esac + ( + cd cmd/decrypt + CGO_ENABLED=0 GOOS=linux GOARCH="$arch" go build -trimpath -tags netgo,osusergo \ + -ldflags="-s -w -X main.version=$GITHUB_REF_NAME" -o "../../corescope-decrypt-linux-$arch" . + ) + ( + cd cmd/migrate + CGO_ENABLED=1 GOOS=linux GOARCH="$arch" CC="zig cc -target $target" \ + go build -trimpath -tags netgo,osusergo,sqlite_omit_load_extension \ + -ldflags="-s -w -extldflags '-static -Wl,-s'" -o "../../corescope-migrate-linux-$arch" . + ) + done - name: Verify release binaries are static and runnable run: | set -e - for f in corescope-decrypt-linux-amd64 corescope-decrypt-linux-arm64; do + for f in corescope-{decrypt,migrate}-linux-{amd64,arm64}; do file "$f" | grep -q 'statically linked' || { echo "::error::$f is not statically linked"; exit 1; } done # amd64 runs natively on the runner; arm64 is only checked structurally # here (the container image job exercises arm64 under QEMU). ./corescope-decrypt-linux-amd64 --version + ./corescope-migrate-linux-amd64 -help - name: Locate the release notes for this tag id: notes @@ -1101,6 +1285,8 @@ jobs: files: | corescope-decrypt-linux-amd64 corescope-decrypt-linux-arm64 + corescope-migrate-linux-amd64 + corescope-migrate-linux-arm64 # ─────────────────────────────────────────────────────────────── # 4b. Deploy Staging (master only) @@ -1125,7 +1311,8 @@ jobs: # variable is set while the runner is still missing the job fails in ten # minutes rather than blocking the branch again. if: | - vars.ENABLE_STAGING_DEPLOY == 'true' + !inputs.postgres_benchmark + && vars.ENABLE_STAGING_DEPLOY == 'true' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/master' needs: [build-and-publish] @@ -1216,14 +1403,14 @@ jobs: # ─────────────────────────────────────────────────────────────── publish: name: "📝 Publish Badges & Summary" - if: github.event_name == 'push' + if: github.event_name == 'push' && !inputs.postgres_benchmark # Was needs: [deploy]. The badges are built from the go-test and e2e-test # artifacts and never needed the deploy step; depending on it meant the # coverage badges stopped updating whenever the self-hosted runner was # unavailable. build-and-publish already transitively requires both test # jobs, so ordering is unchanged. needs: [build-and-publish] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout code uses: actions/checkout@v5 diff --git a/.github/workflows/postgres-benchmark.yml b/.github/workflows/postgres-benchmark.yml new file mode 100644 index 000000000..12bc47b2b --- /dev/null +++ b/.github/workflows/postgres-benchmark.yml @@ -0,0 +1,119 @@ +name: PostgreSQL paired benchmark + +on: + workflow_dispatch: + inputs: + candidate_sha: + description: Exact 40-character candidate commit + type: string + required: true + workflow_call: + inputs: + candidate_sha: + type: string + required: true + +permissions: + contents: read + +concurrency: + group: postgres-benchmark-${{ github.run_id }} + cancel-in-progress: false + +jobs: + benchmark: + name: Five paired runs (128K transmissions / 2.048M observations) + runs-on: ubuntu-24.04 + timeout-minutes: 240 + env: + CANDIDATE_SHA: ${{ inputs.candidate_sha }} + CGO_ENABLED: '1' + GOMAXPROCS: '3' + steps: + - name: Validate exact candidate and runner capacity + run: | + set -euo pipefail + [[ "$CANDIDATE_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo 'candidate_sha must be a full lowercase commit SHA' >&2; exit 1; } + [[ $(nproc) -ge 4 ]] || { echo 'At least four CPUs are required' >&2; exit 1; } + awk '/MemTotal:/ {exit !($2 >= 8*1024*1024)}' /proc/meminfo || { echo 'At least 8 GiB RAM is required' >&2; exit 1; } + test -f /sys/fs/cgroup/cgroup.controllers + echo "BENCH_OUTPUT=$RUNNER_TEMP/corescope-bench-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" >> "$GITHUB_ENV" + echo "SMOKE_OUTPUT=$RUNNER_TEMP/corescope-bench-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-smoke" >> "$GITHUB_ENV" + + - name: Checkout the exact candidate + uses: actions/checkout@v5 + with: + ref: ${{ inputs.candidate_sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Verify candidate and workflow snapshot + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$CANDIDATE_SHA" + test "$GITHUB_SHA" = "$CANDIDATE_SHA" || { echo 'Dispatch this workflow from the candidate commit/branch so workflow and code match' >&2; exit 1; } + git cat-file -e 9dbc287579a237ffa744dd0c91fa7227d09763ac^{commit} + + - name: Test benchmark controls + run: python3 -m unittest discover -s scripts/postgres-benchmark -p 'test_*.py' + + - name: Use the same compiler for both revisions + uses: actions/setup-go@v6 + with: + go-version: '1.27.2' + cache-dependency-path: '**/go.sum' + + - name: Install exact PostgreSQL 18.6 binaries + run: bash scripts/install-postgres-ci.sh server + + - name: Provision owned cgroup and run primary comparison + run: | + set -euo pipefail + delegated="/sys/fs/cgroup/corescope-benchmark-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + test ! -e "$delegated" + sudo mkdir "$delegated" + printf '+cpu +memory\n' | sudo tee "$delegated/cgroup.subtree_control" >/dev/null + sudo mkdir "$delegated/coordinator" + sudo chown "$(id -u):$(id -g)" "$delegated" "$delegated/cgroup.procs" "$delegated/cgroup.subtree_control" "$delegated/coordinator" "$delegated/coordinator/cgroup.procs" + # The coordinator/generator live in a sibling of the measured app+DB + # group. The common ancestor is delegated so unprivileged child moves + # work; the runner validates the 3 CPU / 6 GiB measured envelope. + printf '%s\n' "$$" | sudo tee "$delegated/coordinator/cgroup.procs" >/dev/null + # Prove the controller and Linux prerequisites before preparing corpus B. + python3 scripts/postgres-benchmark/run.py --repo . \ + --candidate-sha "$CANDIDATE_SHA" --corpus S --pairs 1 --profile smoke \ + --warmup 2 --seconds 10 --postgres-version 18.6 --go-version 1.27.2 \ + --postgres-bin /usr/lib/postgresql/18/bin \ + --output "$SMOKE_OUTPUT" --cgroup "$delegated" + python3 scripts/postgres-benchmark/run.py --repo . \ + --candidate-sha "$CANDIDATE_SHA" --corpus B --pairs 5 --profile primary \ + --postgres-version 18.6 --go-version 1.27.2 \ + --postgres-bin /usr/lib/postgresql/18/bin \ + --output "$BENCH_OUTPUT" --cgroup "$delegated" + + - name: Upload only sanitized public evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: postgres-benchmark-${{ inputs.candidate_sha }} + path: | + ${{ env.SMOKE_OUTPUT }}/public/ + ${{ env.BENCH_OUTPUT }}/public/ + if-no-files-found: error + retention-days: 30 + + - name: Remove only this run's cgroup + if: always() + run: | + set -euo pipefail + delegated="/sys/fs/cgroup/corescope-benchmark-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + if [ -d "$delegated" ]; then + test "$(realpath "$delegated")" = "$delegated" + if [ -f "$delegated/cgroup.kill" ]; then echo 1 | sudo tee "$delegated/cgroup.kill" >/dev/null; fi + for attempt in $(seq 1 10); do + sudo find "$delegated" -depth -type d -exec rmdir {} \; 2>/dev/null || true + test ! -d "$delegated" && break + sleep 1 + done + test ! -d "$delegated" + fi diff --git a/.gitignore b/.gitignore index 33def5772..6bc2d49ed 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ node_modules/ data/ +postgres-data/ +*.dump *.bak *.db *.db-journal diff --git a/AGENTS.md b/AGENTS.md index 0ecf7a1fa..340f18b1b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,10 +13,12 @@ Go backend + static frontend. No build step. No framework. No bundler. cmd/server/ — Go API server (REST + WebSocket broadcast + static file serving) main.go — Entry point, flags, SPA handler routes.go — All /api/* endpoints - store.go — In-memory packet store + analytics + SQLite queries + store.go — In-memory packet store + analytics + PostgreSQL queries config.go — Configuration loading decoder.go — MeshCore packet decoder -cmd/ingestor/ — Go MQTT ingestor (separate binary, writes to shared SQLite DB) +cmd/ingestor/ — Go MQTT ingestor (separate binary, writes telemetry to PostgreSQL) +cmd/migrate/ — Offline schema bootstrap and verified SQLite upgrade importer +internal/pgutil/ — PostgreSQL connections, role checks, native backup helpers public/ — Frontend (vanilla JS, one file per page) — ACTIVE, NOT DEPRECATED app.js — SPA router, shared globals, theme loading roles.js — ROLE_COLORS, TYPE_COLORS, health thresholds, shared helpers @@ -34,33 +36,34 @@ public/ — Frontend (vanilla JS, one file per page) — ACTIVE, NOT live.css — Live page styles home.css — Home page styles index.html — SPA shell, script/style tags with __BUST__ placeholder (auto-replaced at server startup) -test-fixtures/ — Real data SQLite fixture from staging (used for E2E tests) +test-fixtures/ — Legacy SQLite fixture; prepare and import into PostgreSQL for E2E scripts/ — Tooling (coverage collector, fixture capture, frontend instrumentation) ``` ### Data Flow -1. MQTT brokers → Go ingestor (`cmd/ingestor/`) ingests packets → decodes → writes to SQLite -2. Go server (`cmd/server/`) polls SQLite for new packets, broadcasts via WebSocket +1. MQTT brokers → Go ingestor (`cmd/ingestor/`) ingests packets → decodes → writes to PostgreSQL +2. Go server (`cmd/server/`) polls PostgreSQL for new packets, broadcasts via WebSocket 3. Frontend fetches via REST API (`/api/*`), filters/sorts client-side ### Read/Write Separation Invariant (#1283) -- **All DB writes live in `cmd/ingestor/`.** INSERT / UPDATE / DELETE / VACUUM / - schema migrations / retention all run in the ingestor process. -- **`cmd/server/` never writes measurement data.** It opens the analyzer DB with - `mode=ro` and must not acquire a write lock on it. Adding a write-side helper - (e.g. a `cachedRW`-style RW connection) regresses this invariant and races the - ingestor → SQLITE_BUSY. -- **Single exception: `users.db` (optional user management).** When - `userManagement.enabled`, the server owns a *separate* SQLite file through - `internal/users` only. `users.Open` refuses the analyzer DB path, and - `TestUsersOpenIsTheOnlyServerWritePath` pins the one call site. Account data - never goes into the analyzer DB, and measurement writes never go through - `internal/users`. -- **The ingestor reads `users.db`, never writes it.** With - `userManagement.channelProposals.enabled` the ingestor opens `users.db` - read-only (`mode=ro`, raw SQL, no `internal/users` import) once a minute - for the approved hashtag channel names. `TestChannelKeySetIsReadOnly` pins - the read-only open. +- **Runtime telemetry writes live in `cmd/ingestor/`.** INSERT / UPDATE / + DELETE and retention use a restricted writer role. PostgreSQL handles + autovacuum; offline `cmd/migrate/` owns schema installation and upgrades. +- **`cmd/server/` never writes measurement data.** Its telemetry connection + uses a SELECT-only role and read-only transactions. Do not add a write-side + connection or grant the reader schema/owner privileges. +- **Single exception: the separate accounts database (optional user + management).** When `userManagement.enabled`, the server writes accounts + through `internal/users` only. `users.Open` rejects the actual telemetry + database, including URL aliases, and `TestUsersOpenIsTheOnlyServerWritePath` + pins the one call site. Account and measurement databases stay separate. +- **The ingestor reads approved channel names, never account rows.** With + `userManagement.channelProposals.enabled`, its dedicated role may SELECT + only `approved_channels` and the readiness marker. It uses a read-only + snapshot and retains the last good keys if readiness or refresh fails. +- **Runtime roles never run DDL or mark an import ready.** Bootstrap/import + uses separate owner credentials. Tests must cover denied writes and + cross-database access with the real runtime grants. - Enforcement: `cmd/server/readonly_invariant_test.go` reflect-asserts that `PruneOldPackets`, `PruneOldMetrics`, and `RemoveStaleObservers` are NOT methods on the server's `*DB`. If you need a new write, add it to @@ -102,9 +105,21 @@ Every change must consider performance impact BEFORE implementation. This codeba No proof = no merge. ### 1. No commit without tests -Every change that touches logic MUST have tests. For Go backend: `cd cmd/server && go test ./...` and `cd cmd/ingestor && go test ./...`, or `make test` for all 14 modules. For frontend: `node tests/unit/test-packet-filter.js && node tests/unit/test-aging.js && node tests/unit/test-frontend-helpers.js`. If you add new logic, add tests. No exceptions. - -The SQLite driver (`github.com/mattn/go-sqlite3`) is cgo. **`CGO_ENABLED=0` still *builds*** — that is the trap. It links a stub, and the binary dies on the first query with `go-sqlite3 requires cgo to work. This is a stub`, so a green build proves nothing. A plain `GOOS=linux go build` likewise cannot cross-compile it. Use `make build` / `make crossbuild` — the latter needs [`zig`](https://ziglang.org/download/) as the cross C compiler and produces static musl binaries. Never re-add `CGO_ENABLED=0`. +Every change that touches logic MUST have tests. For Go backend: `cd cmd/server && go test ./...` and `cd cmd/ingestor && go test ./...`, or `make test` for all Go modules. For frontend: `node tests/unit/test-packet-filter.js && node tests/unit/test-aging.js && node tests/unit/test-frontend-helpers.js`. If you add new logic, add tests. No exceptions. + +The PostgreSQL runtime uses native Go (`pgx`). SQLite is retained only for +the offline upgrade importer and legacy test fixtures. The importer still +requires cgo (`github.com/mattn/go-sqlite3`); `CGO_ENABLED=0` links a stub +that fails on the first SQLite query. Use `make build` / `make crossbuild` +for the complete release, including the importer. Cross-building that +importer requires [`zig`](https://ziglang.org/download/) or another suitable +C cross compiler. Validate a real import, not just a successful build. + +Database integration tests require a disposable PostgreSQL 18 instance and +`CORESCOPE_TEST_POSTGRES_URL` with test schema/database/role creation rights. +They fail when the database is unavailable rather than silently skipping. +Never point tests at a live instance. Runtime credentials stay in deployment +environment variables and must not appear in logs or committed fixtures. ### 2. No commit without browser validation After pushing, verify the change works in an actual browser. Use `browser profile=openclaw` against the running instance. Take a screenshot if the change is visual. If you can't validate it, say so — don't claim it works. diff --git a/DEPLOY.md b/DEPLOY.md index af011657a..e65ed6e42 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -4,33 +4,30 @@ Pre-built images are published to GHCR for `linux/amd64` and `linux/arm64` (Rasp ## Quick Start -### Docker run +### Complete Compose checkout -```bash -docker run -d --name corescope \ - -p 80:80 \ - -v corescope-data:/app/data \ - -e DISABLE_CADDY=true \ - ghcr.io/kpa-clawbot/corescope:latest -``` +PostgreSQL 18.6 runs as a separate service. Existing SQLite installations must complete the [offline upgrade](docs/postgresql-upgrade.md) before starting this version. -Open `http://localhost` — done. - -### Docker Compose +Set `CORESCOPE_REF` to the reviewed application revision, then retain the complete checkout: the Compose variants require `docker/postgres.compose.yml` and its initialization scripts. ```bash -curl -sL https://raw.githubusercontent.com/Kpa-clawbot/CoreScope/master/docker-compose.example.yml \ - -o docker-compose.yml -docker compose up -d +git clone https://github.com/Kpa-clawbot/CoreScope.git +cd CoreScope +git checkout --detach "$CORESCOPE_REF" +cp .env.example .env +chmod 600 .env +# Fill every PostgreSQL password field with a different openssl rand -hex 32 value. +# Set CORESCOPE_IMAGE in .env to the matching reviewed image tag or digest. +docker compose -f docker-compose.example.yml up -d ``` +The PostgreSQL initializer creates separate telemetry/account databases and roles. Bootstrap installs schemas and grants; the application starts only when bootstrap succeeds. Open `http://localhost` and verify `/api/healthz` plus real packet ingestion. The default HTTP port is 80; adjust `HTTP_PORT` when using the example variant. + ## Image Tags | Tag | Description | |-----|-------------| -| `v3.4.1` | Pinned release (recommended for production) | -| `v3.4` | Latest patch in v3.4.x | -| `v3` | Latest minor+patch in v3.x | +| Reviewed release tag or image digest | Pin the application and its matching repository files together | | `latest` | Latest release tag | | `edge` | Built from master — unstable, for testing | @@ -40,7 +37,7 @@ Settings can be overridden via environment variables: | Variable | Default | Description | |----------|---------|-------------| -| `DISABLE_CADDY` | `false` | Skip internal Caddy (set `true` behind a reverse proxy) | +| `DISABLE_CADDY` | Variant-specific; `true` in the example | Skip internal Caddy (set `true` behind a reverse proxy) | | `DISABLE_MOSQUITTO` | `true` in `docker-compose.staging.yml`; `false` elsewhere | Skip internal MQTT broker. Default flipped to `true` for the staging deploy in v3.7+ because a standalone `mqtt-broker` container owns MQTT on that host — see "Standalone MQTT broker (staging)" below. | | `HTTP_PORT` | `80` | Host port mapping | | `DATA_DIR` | `./data` | Host path for persistent data | @@ -50,18 +47,15 @@ For advanced configuration, mount a `config.json` into `/app/data/config.json`. ## Updating ```bash -docker compose pull -docker compose up -d +docker compose -f docker-compose.example.yml pull +docker compose -f docker-compose.example.yml up -d ``` ## Data -All persistent data lives in `/app/data`: -- `meshcore.db` — SQLite database (packets, nodes) -- `config.json` — custom config (optional) -- `theme.json` — custom theme (optional) +PostgreSQL persists telemetry and accounts in its own host directory (`POSTGRES_DATA_DIR`, or the production/staging equivalent). `/app/data` holds config, theme, queues, statistics and account backup files. Keep these locations separate and preserve the private `.env` with your recovery material. -**Backup:** `cp data/meshcore.db ~/backup/` +Use native `pg_dump` archives, the authenticated backup endpoints, or `./manage.sh backup `. Do not copy live PostgreSQL storage or a legacy SQLite file as a current backup. `./manage.sh restore ` accepts native archives only, refuses nonempty targets and leaves the application stopped for validation. See the [backup and account recovery boundaries](docs/postgresql-upgrade.md#native-backups-and-restores). ## TLS @@ -134,13 +128,13 @@ your fork or ensure the network exists. If you're currently deploying with `manage.sh` (git clone + local build), you have two options going forward: -### Option A: Keep using manage.sh (no changes needed) +### Option A: Keep using manage.sh -`manage.sh update` continues to work exactly as before — it fetches the latest tag, builds locally, and restarts. Nothing breaks. +Complete the PostgreSQL migration and configure the private database credentials before using `manage.sh start` or `update`. Its backup, restore, staging-copy and status commands use PostgreSQL clients from the selected database container. Restores require empty destinations; existing staging data is retained. ```bash ./manage.sh update # latest release -./manage.sh update v3.5.0 # specific version +./manage.sh update "$CORESCOPE_REF" # reviewed version ``` ### Option B: Switch to pre-built images (recommended) @@ -154,16 +148,18 @@ Pre-built images skip the build step entirely — faster updates, no Go toolchai ./manage.sh stop ``` -2. Your data is in `~/meshcore-data/` (or whatever `PROD_DATA_DIR` is set to). It's untouched — the database, config, and theme files persist. +2. Preserve `PROD_DATA_DIR`, the separate PostgreSQL directory, credentials and the native backup pair. For a SQLite instance, perform the offline upgrade first. -3. Copy `docker-compose.example.yml` to where you want to run from: +3. Keep the complete pinned checkout and choose the matching image in `.env`: ```bash - cp docker-compose.example.yml ~/docker-compose.yml + # CORESCOPE_IMAGE= + # DATA_DIR= + # POSTGRES_DATA_DIR= ``` 4. Start with the pre-built image: ```bash - cd ~ && docker compose up -d + docker compose -f docker-compose.example.yml up -d ``` 5. Verify it picked up your existing data: @@ -185,7 +181,7 @@ docker compose pull && docker compose up -d | `./manage.sh start` | `docker compose up -d` | | `./manage.sh logs` | `docker compose logs -f` | | `./manage.sh status` | `docker compose ps` | -| `./manage.sh setup` | Copy `docker-compose.example.yml`, edit env vars | +| `./manage.sh setup` | Retain the full checkout, configure private credentials and select the Compose variant | `manage.sh` remains available for advanced use cases (building from source, custom patches, development). Pre-built images are recommended for most production deployments. diff --git a/Dockerfile b/Dockerfile index 76514d2a9..90fc02e82 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,11 +2,9 @@ # Build stage always runs natively on the builder's arch ($BUILDPLATFORM) and # cross-compiles to $TARGETOS/$TARGETARCH. No QEMU for compilation. # -# The SQLite driver is github.com/mattn/go-sqlite3, which is cgo, so the Go -# toolchain alone can no longer cross-compile this: it needs a C compiler that -# can target the other architecture. `zig cc` is that compiler. Targeting musl -# makes the result fully static (see -extldflags below), so the runtime stage -# has no libc dependency on the base image at all. +# The PostgreSQL runtime binaries use pure Go. The offline SQLite importer +# still needs cgo and a cross C compiler. `zig cc` targets musl for that +# binary, keeping all four release binaries independent of the runtime libc. # # BUILDPLATFORM is auto-set by buildx; default to linux/amd64 so plain # `docker build` (without buildx) doesn't fail on an empty platform string. @@ -20,9 +18,8 @@ ARG BUILD_TIME=unknown ARG TARGETOS ARG TARGETARCH -# Keep these in step with the Makefile: netgo/osusergo preserve the pure-Go -# resolver and user lookup the binaries had under CGO_ENABLED=0, and -# sqlite_omit_load_extension drops the dlopen path so -static links cleanly. +# Keep these in step with the Makefile. The importer's netgo/osusergo tags +# retain Go DNS/user lookup; sqlite_omit_load_extension permits static linking. ENV GO_BUILD_TAGS=netgo,osusergo,sqlite_omit_load_extension \ ZIG_GLOBAL_CACHE_DIR=/tmp/zig-cache @@ -42,8 +39,7 @@ RUN apk add --no-cache curl xz && \ ln -s /opt/zig/zig /usr/local/bin/zig && rm /tmp/zig.tar.xz && \ zig version -# zigcc resolves TARGETARCH to a zig target triple once, so the three build -# steps below stay readable and cannot disagree with each other. +# zigcc selects the importer's C target from the requested image architecture. RUN printf '%s\n' '#!/bin/sh' \ 'case "$TARGETARCH" in' \ ' amd64) t=x86_64-linux-musl ;;' \ @@ -51,7 +47,7 @@ RUN printf '%s\n' '#!/bin/sh' \ ' *) echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 1 ;;' \ 'esac' \ 'exec zig cc -target "$t" "$@"' > /usr/local/bin/zigcc && chmod +x /usr/local/bin/zigcc -ENV CC=zigcc CGO_ENABLED=1 +ENV CC=zigcc CGO_ENABLED=0 # Build server WORKDIR /build/server @@ -68,6 +64,7 @@ COPY internal/lora/ ../../internal/lora/ COPY internal/users/ ../../internal/users/ COPY internal/channel/ ../../internal/channel/ COPY internal/mailer/ ../../internal/mailer/ +COPY internal/pgutil/ ../../internal/pgutil/ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/tmp/zig-cache \ @@ -78,7 +75,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/tmp/zig-cache \ GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ go build -trimpath -tags ${GO_BUILD_TAGS} \ - -ldflags "-s -w -extldflags '-static -Wl,-s' -X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" \ + -ldflags "-s -w -X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" \ -o /corescope-server . # Build ingestor @@ -93,6 +90,7 @@ COPY internal/dbschema/ ../../internal/dbschema/ COPY internal/prunequeue/ ../../internal/prunequeue/ COPY internal/perfio/ ../../internal/perfio/ COPY internal/mbcapqueue/ ../../internal/mbcapqueue/ +COPY internal/pgutil/ ../../internal/pgutil/ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/tmp/zig-cache \ @@ -103,13 +101,15 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/tmp/zig-cache \ GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ go build -trimpath -tags ${GO_BUILD_TAGS} \ - -ldflags "-s -w -extldflags '-static -Wl,-s'" \ + -ldflags "-s -w" \ -o /corescope-ingestor . # Build decrypt CLI WORKDIR /build/decrypt COPY cmd/decrypt/go.mod cmd/decrypt/go.sum ./ COPY internal/channel/ ../../internal/channel/ +COPY internal/dbschema/ ../../internal/dbschema/ +COPY internal/pgutil/ ../../internal/pgutil/ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/tmp/zig-cache \ @@ -120,18 +120,37 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/tmp/zig-cache \ GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ go build -trimpath -tags ${GO_BUILD_TAGS} \ - -ldflags "-s -w -extldflags '-static -Wl,-s' -X main.version=${APP_VERSION}" \ + -ldflags "-s -w -X main.version=${APP_VERSION}" \ -o /corescope-decrypt . -# Runtime image -FROM alpine:3.20 +# Offline bootstrap/import CLI retains SQLite support for existing instances. +ENV CGO_ENABLED=1 +WORKDIR /build/migrate +COPY cmd/migrate/go.mod cmd/migrate/go.sum ./ +COPY internal/dbschema/ ../../internal/dbschema/ +COPY internal/users/ ../../internal/users/ +COPY internal/pgutil/ ../../internal/pgutil/ +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + go mod download +COPY cmd/migrate/ ./ +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/tmp/zig-cache \ + GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -tags ${GO_BUILD_TAGS} \ + -ldflags "-s -w -extldflags '-static -Wl,-s'" \ + -o /corescope-migrate . + +# The client major matches PostgreSQL 18 for native backup/restore support. +FROM alpine:3.24 -RUN apk add --no-cache mosquitto mosquitto-clients supervisor caddy wget +RUN apk add --no-cache mosquitto mosquitto-clients supervisor caddy wget postgresql18-client WORKDIR /app # Go binaries (statically linked; they do not use this image's libc) -COPY --from=builder /corescope-server /corescope-ingestor /corescope-decrypt /app/ +COPY --from=builder /corescope-server /corescope-ingestor /corescope-decrypt /corescope-migrate /app/ # Frontend assets + config COPY public/ ./public/ @@ -155,6 +174,7 @@ RUN mkdir -p /app/data /var/lib/mosquitto /data/caddy && \ # Entrypoint COPY docker/entrypoint-go.sh /entrypoint.sh +COPY docker/postgres-bootstrap.sh docker/postgres-grants.sql /app/ RUN chmod +x /entrypoint.sh EXPOSE 80 443 1883 diff --git a/Dockerfile.go b/Dockerfile.go deleted file mode 100644 index ec2725f52..000000000 --- a/Dockerfile.go +++ /dev/null @@ -1,61 +0,0 @@ -FROM golang:1.27-alpine AS builder - -RUN apk add --no-cache build-base - -ARG APP_VERSION=unknown -ARG GIT_COMMIT=unknown -ARG BUILD_TIME=unknown - -# Build server -WORKDIR /build/server -COPY cmd/server/go.mod cmd/server/go.sum ./ -RUN go mod download -COPY cmd/server/ ./ -RUN go build -ldflags "-X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" -o /corescope-server . - -# Build ingestor -WORKDIR /build/ingestor -COPY cmd/ingestor/go.mod cmd/ingestor/go.sum ./ -RUN go mod download -COPY cmd/ingestor/ ./ -RUN go build -o /corescope-ingestor . - -# Runtime image -FROM alpine:3.20 - -RUN apk add --no-cache mosquitto mosquitto-clients supervisor caddy wget - -WORKDIR /app - -# Go binaries -COPY --from=builder /corescope-server /corescope-ingestor /app/ - -# Frontend assets + config -COPY public/ ./public/ -COPY config.example.json channel-rainbow.json ./ - -# Bake git commit SHA (CI writes .git-commit before build; fallback for non-ldflags usage) -COPY .git-commi[t] ./ -RUN if [ ! -f .git-commit ]; then echo "unknown" > .git-commit; fi - -# Supervisor + Mosquitto + Caddy config -COPY docker/supervisord-go.conf /etc/supervisor/conf.d/supervisord.conf -COPY docker/supervisord-go-no-mosquitto.conf /etc/supervisor/conf.d/supervisord-no-mosquitto.conf -COPY docker/supervisord-go-no-caddy.conf /etc/supervisor/conf.d/supervisord-no-caddy.conf -COPY docker/supervisord-go-no-mosquitto-no-caddy.conf /etc/supervisor/conf.d/supervisord-no-mosquitto-no-caddy.conf -COPY docker/mosquitto.conf /etc/mosquitto/mosquitto.conf -COPY docker/Caddyfile /etc/caddy/Caddyfile - -# Data directory -RUN mkdir -p /app/data /var/lib/mosquitto /data/caddy && \ - chown -R mosquitto:mosquitto /var/lib/mosquitto - -# Entrypoint -COPY docker/entrypoint-go.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh - -EXPOSE 80 443 1883 - -VOLUME ["/app/data", "/data/caddy"] - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/Makefile b/Makefile index 65c25df4d..878d1025b 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,8 @@ # corescope build entry point. # -# This repo is 14 Go modules wired with `replace ../../internal/*` and no -# go.work, so every recipe changes into its module. The build flags live here -# because the SQLite driver is cgo now (github.com/mattn/go-sqlite3) and the -# flags stopped being something you can safely retype at each call site. -# -# Cross-compilation uses `zig cc` as the C toolchain, targeting musl so the -# binaries are fully static and run on the alpine runtime image (or scratch) -# with no libc dependency at all. +# Each Go module resolves internal dependencies through local replace directives. +# Runtime binaries use pure Go. Only the offline SQLite importer needs cgo; +# its Linux cross-build uses zig cc with musl for a fully static binary. # # Quick reference: # make build # all four binaries for the host @@ -21,21 +16,17 @@ GIT_VERSION ?= $(shell git describe --tags --match "v*" 2>/dev/null || ech GIT_COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown) BUILD_TIME ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ) -GOENV_GOOS := $(shell $(GO) env GOOS) -GOENV_GOARCH := $(shell $(GO) env GOARCH) +GOENV_GOOS := $(shell "$(GO)" env GOOS) +GOENV_GOARCH := $(shell "$(GO)" env GOARCH) GOOS ?= $(GOENV_GOOS) GOARCH ?= $(GOENV_GOARCH) -# Exported: a bare make variable never reaches the go toolchain. -export CGO_ENABLED ?= 1 - -# netgo/osusergo keep the pure-Go resolver and user lookup the binaries had -# under CGO_ENABLED=0, so enabling cgo for SQLite does not quietly switch DNS -# resolution to the C resolver. sqlite_omit_load_extension drops the dlopen -# path, which is what lets -extldflags -static link cleanly. -Wl,-s on the -# cross builds strips the musl objects zig links in: Go's own -s -w does not -# reach them, and they account for well over half the binary. -GO_BUILD_TAGS ?= netgo,osusergo,sqlite_omit_load_extension +CGO_server := 0 +CGO_ingestor := 0 +CGO_decrypt := 0 +CGO_migrate := 1 +GO_BUILD_TAGS ?= netgo,osusergo +TAGS_migrate := ,sqlite_omit_load_extension GO_BUILD_FLAGS ?= -trimpath GO_LDFLAGS_OPTIMS ?= -s -w @@ -53,6 +44,7 @@ LDFLAGS_migrate := # zig target triples for the platforms we ship. musl, so the result is static. ZIG_TARGET_linux_amd64 := x86_64-linux-musl ZIG_TARGET_linux_arm64 := aarch64-linux-musl +comma := , CROSS_PLATFORMS := linux/amd64 linux/arm64 DOCKER_IMAGE ?= ghcr.io/kpa-clawbot/corescope @@ -73,8 +65,8 @@ build: $(addprefix build-,$(CMDS)) build-%: @mkdir -p $(DIST) - cd cmd/$* && GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO) build \ - -tags $(GO_BUILD_TAGS) $(GO_BUILD_FLAGS) \ + cd cmd/$* && CGO_ENABLED=$(CGO_$*) GOOS=$(GOOS) GOARCH=$(GOARCH) "$(GO)" build \ + -tags $(GO_BUILD_TAGS)$(TAGS_$*) $(GO_BUILD_FLAGS) \ -ldflags "$(GO_LDFLAGS_OPTIMS) $(LDFLAGS_$*)" \ -o ../../$(DIST)/corescope-$* . @@ -89,10 +81,10 @@ define CROSSBUILD_RULE .PHONY: crossbuild-$(2)-$(3)-$(4) crossbuild-$(2)-$(3)-$(4): @mkdir -p $$(DIST) - @command -v zig >/dev/null || { echo "zig not found: needed to cross-compile cgo. See https://ziglang.org/download/"; exit 1; } - cd cmd/$(2) && CGO_ENABLED=1 GOOS=$(3) GOARCH=$(4) CC="zig cc -target $(1)" \ - $$(GO) build -tags $$(GO_BUILD_TAGS) $$(GO_BUILD_FLAGS) \ - -ldflags '$$(GO_LDFLAGS_OPTIMS) -extldflags "-static -Wl,-s" $$(LDFLAGS_$(2))' \ + $(if $(filter migrate,$(2)),@command -v zig >/dev/null || { echo "zig not found: needed to cross-compile the offline importer. See https://ziglang.org/download/"; exit 1; },@:) + cd cmd/$(2) && CGO_ENABLED=$$(CGO_$(2)) GOOS=$(3) GOARCH=$(4) $(if $(filter migrate,$(2)),CC="zig cc -target $(1)",) \ + "$$(GO)" build -tags $$(GO_BUILD_TAGS)$$(TAGS_$(2)) $$(GO_BUILD_FLAGS) \ + -ldflags '$$(GO_LDFLAGS_OPTIMS) $(if $(filter migrate,$(2)),-extldflags "-static -Wl$(comma)-s",) $$(LDFLAGS_$(2))' \ -o ../../$$(DIST)/corescope-$(2)-$(3)-$(4) . endef @@ -103,13 +95,13 @@ $(foreach c,$(CMDS),\ # -- checks -------------------------------------------------------------------- test: - bash scripts/allmod.sh test ./... + CGO_ENABLED=1 bash scripts/allmod.sh test ./... vet: bash scripts/allmod.sh vet ./... fmt-check: - @unformatted=$$(gofmt -l $$(git ls-files '*.go' | grep -vi 'Dockerfile.go')); \ + @unformatted=$$(gofmt -l $$(git ls-files '*.go')); \ if [ -n "$$unformatted" ]; then echo "gofmt required on:"; echo "$$unformatted"; exit 1; fi; \ echo "gofmt: clean" diff --git a/README.md b/README.md index 7345d3d02..151684b81 100644 --- a/README.md +++ b/README.md @@ -6,24 +6,15 @@ [![Frontend Coverage](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/Kpa-clawbot/CoreScope/master/.badges/frontend-coverage.json)](https://github.com/Kpa-clawbot/CoreScope/actions/workflows/deploy.yml) [![Deploy](https://github.com/Kpa-clawbot/CoreScope/actions/workflows/deploy.yml/badge.svg)](https://github.com/Kpa-clawbot/CoreScope/actions/workflows/deploy.yml) -> High-performance mesh network analyzer powered by Go. Sub-millisecond packet queries, ~300 MB memory for 56K+ packets, real-time WebSocket broadcast, full channel decryption. +> MeshCore packet analysis with a Go backend, PostgreSQL persistence, real-time WebSocket updates and channel decryption. Self-hosted, open-source MeshCore packet analyzer. Collects MeshCore packets via MQTT, decodes them in real time, and presents a full web UI with live packet feed, interactive maps, channel chat, packet tracing, and per-node analytics. ## ⚡ Performance -The Go backend serves all 40+ API endpoints from an in-memory packet store with 5 indexes (hash, txID, obsID, observer, node). SQLite is for persistence only — reads never touch disk. +The Go backend combines a packet cache with PostgreSQL persistence and SQL-backed analytics. The ingestor owns telemetry writes; the server uses a restricted telemetry reader and a separate optional account writer. Existing SQLite instances require the [offline PostgreSQL upgrade](docs/postgresql-upgrade.md). -| Metric | Value | -|--------|-------| -| Packet queries | **< 1 ms** (in-memory) | -| All API endpoints | **< 100 ms** | -| Memory (56K packets) | **~300 MB** (vs 1.3 GB on Node.js) | -| WebSocket broadcast | **Real-time** to all connected browsers | -| Channel decryption | **AES-128-ECB** with rainbow table | -| GOMEMLIMIT (memory-constrained hosts) | **set to ≥1.5× working set** (e.g. 1536 MiB on a 2 GB Pi for a ~1 GB store). Lower values trigger a GC death-spiral. Configure via the `GOMEMLIMIT` env var or `runtime.maxMemoryMB` in `config.json`; env wins. Applies to both server and ingestor. See [#1010](https://github.com/Kpa-clawbot/CoreScope/issues/1010). | - -See [PERFORMANCE.md](PERFORMANCE.md) for full benchmarks. +Budget for the application and PostgreSQL together, including retained data, indexes, WAL and the packet cache. Historical SQLite measurements in [PERFORMANCE.md](PERFORMANCE.md) are baseline context, not PostgreSQL results. Use the [paired benchmark workflow](.github/workflows/postgres-benchmark.yml) for an exact candidate comparison. ## ✨ Features @@ -75,50 +66,32 @@ Full experience on your phone — proper touch controls, iOS safe area support, ## Quick Start -### Pre-built Image (Recommended) +### Pre-built image with PostgreSQL -No build step required — just run: - -```bash -docker run -d --name corescope \ - --restart=unless-stopped \ - -p 80:80 -p 1883:1883 \ - -v /your/data:/app/data \ - ghcr.io/kpa-clawbot/corescope:latest -``` - -Open `http://localhost` — done. No config file needed; CoreScope starts with sensible defaults. - -For HTTPS with a custom domain, add `-p 443:443` and mount your Caddyfile: -```bash -docker run -d --name corescope \ - --restart=unless-stopped \ - -p 80:80 -p 443:443 -p 1883:1883 \ - -v /your/data:/app/data \ - -v /your/Caddyfile:/etc/caddy/Caddyfile:ro \ - -v /your/caddy-data:/data/caddy \ - ghcr.io/kpa-clawbot/corescope:latest -``` - -Disable built-in services with `-e DISABLE_MOSQUITTO=true` or `-e DISABLE_CADDY=true`, or drop a `.env` file in your data volume. See [docs/deployment.md](docs/deployment.md) for the full reference. - -### Build from Source +Retain the complete repository at the same reviewed revision as the image. The Compose variants require the common PostgreSQL service and initialization scripts. ```bash git clone https://github.com/Kpa-clawbot/CoreScope.git cd CoreScope -./manage.sh setup +git checkout --detach "$CORESCOPE_REF" +cp .env.example .env +chmod 600 .env +# Fill each PostgreSQL password field with a different openssl rand -hex 32 value. +# Set CORESCOPE_IMAGE to the matching reviewed image tag or digest. +docker compose -f docker-compose.example.yml up -d ``` -The setup wizard walks you through config, domain, HTTPS, build, and run. +Open `http://localhost`. Bootstrap creates separate telemetry and account databases and grants restricted runtime roles before the application starts. Existing SQLite data requires the [offline upgrade](docs/postgresql-upgrade.md) first. See [DEPLOY.md](DEPLOY.md) for ports, MQTT, HTTPS and native backups. + +### Build from source + +Use the same complete checkout and private `.env`, then run `./manage.sh setup`. The wizard configures, builds and starts the source deployment. PostgreSQL credentials must already be filled in. ```bash -./manage.sh status # Health check + packet/node counts -./manage.sh logs # Follow logs -./manage.sh backup # Backup database -./manage.sh update # Pull latest + rebuild + restart -./manage.sh mqtt-test # Check if observer data is flowing -./manage.sh help # All commands +./manage.sh status +./manage.sh logs +./manage.sh backup ./backups/instance +./manage.sh update ``` ### Configure @@ -156,32 +129,33 @@ Copy `config.example.json` to `config.json` and edit: | `mqttSources` | External MQTT broker connections (optional) | | `channelKeys` | Channel decryption keys (hex). Hashtag channels auto-derived via SHA256 | | `defaultRegion` | Default IATA region code for the UI | -| `dbPath` | SQLite database path (default: `data/meshcore.db`) | +| `databaseURL` | PostgreSQL telemetry URL; prefer the per-process environment credentials below | +| `stateDir` | Filesystem directory for queues, statistics and account backups (default: `data`) | ### Environment Variables | Variable | Description | |----------|-------------| | `PORT` | Override config port | -| `DB_PATH` | Override SQLite database path | +| `CORESCOPE_READER_DATABASE_URL` | Server telemetry reader URL | +| `CORESCOPE_WRITER_DATABASE_URL` | Ingestor telemetry writer URL | +| `CORESCOPE_USERS_DATABASE_URL` | Optional account writer URL in a separate database | +| `CORESCOPE_APPROVED_CHANNELS_DATABASE_URL` | Ingestor's restricted approved-channel reader URL | +| `CORESCOPE_STATE_DIR` | Shared state-directory path; never a database URL | ## Architecture -``` - ┌─────────────────────────────────────────────┐ - │ Docker Container │ - │ │ -Observer → USB → │ Mosquitto ──→ Go Ingestor ──→ SQLite DB │ - meshcoretomqtt → MQTT ──→│ │ │ - │ Go HTTP Server ──→ WebSocket │ - │ │ │ │ - │ Caddy (HTTPS) ←───────┘ │ - └────────────────────┼────────────────────────┘ - │ - Browser +```mermaid +flowchart LR + MQTT[MQTT observers] --> Ingestor[Go ingestor] + Ingestor -->|telemetry writer| Telemetry[(PostgreSQL telemetry)] + Telemetry -->|read-only role| Server[Go API server] + Server <-->|account writer| Accounts[(Separate account database)] + Accounts -->|approved-channel view| Ingestor + Server <-->|REST and WebSocket| Browser ``` -**Two-process model:** The Go ingestor handles MQTT ingestion and packet decoding. The Go HTTP server loads all packets into an in-memory store on startup (5 indexes for fast lookups) and serves the REST API + WebSocket broadcast. Both are managed by supervisord inside a single container with Caddy for HTTPS and Mosquitto for local MQTT. +**Two-process model:** The ingestor handles MQTT, decoding, telemetry writes and retention. The HTTP server loads its packet cache and serves REST/WebSocket requests using a read-only telemetry role. Accounts use a separate PostgreSQL database and writer role. Supervisord runs the application processes with Caddy and optional Mosquitto; PostgreSQL runs as a separate service. Bootstrap/import credentials are not runtime credentials. ## MQTT Setup @@ -201,15 +175,15 @@ corescope/ │ │ ├── main.go # Entry point │ │ ├── routes.go # 40+ API endpoint handlers │ │ ├── store.go # In-memory packet store (5 indexes) -│ │ ├── db.go # SQLite persistence layer +│ │ ├── db.go # Read-only PostgreSQL access │ │ ├── decoder.go # MeshCore packet decoder │ │ ├── websocket.go # WebSocket broadcast -│ │ └── *_test.go # 327 test functions +│ │ └── *_test.go # Server tests │ └── ingestor/ # Go MQTT ingestor │ ├── main.go # MQTT subscription + packet processing │ ├── decoder.go # Packet decoder (shared logic) -│ ├── db.go # SQLite write path -│ └── *_test.go # 53 test functions +│ ├── db.go # PostgreSQL writer and retention path +│ └── *_test.go # Ingestor tests ├── proto/ # Protobuf API definitions ├── public/ # Vanilla JS frontend (no build step) │ ├── index.html # SPA shell @@ -243,9 +217,9 @@ make build-server # just one make crossbuild # static linux/amd64 + linux/arm64 binaries ``` -The SQLite driver is [`mattn/go-sqlite3`](https://github.com/mattn/go-sqlite3), which -is cgo, so a plain `GOOS=linux go build` from a Mac will not work: cross-compiling -needs a C compiler that can target the other platform. `make crossbuild` uses +Runtime database access uses pgx through `database/sql`. The offline importer retains +the cgo [`mattn/go-sqlite3`](https://github.com/mattn/go-sqlite3) driver for legacy +sources, so cross-compiling that binary needs a C compiler. `make crossbuild` uses [`zig`](https://ziglang.org/download/) as that compiler (install it and it just works) and links statically against musl, so the result is one self-contained file that runs on Alpine or scratch. The container build does the same thing — see @@ -253,14 +227,17 @@ that runs on Alpine or scratch. The container build does the same thing — see ### Test Suite -**380 Go tests** covering the backend, plus **150+ Node.js tests** for the frontend and legacy logic, plus **49 Playwright E2E tests** for browser validation. +Go suites exercise real PostgreSQL, role boundaries, concurrency, migration and native backup/restore. The standalone frontend list remains `test-all.sh`; Playwright covers the imported fixture and accounts on/off. CI pins Go 1.27.2 and PostgreSQL 18.6. ```bash +# Point this at an isolated PostgreSQL 18 administrator database. +# Keep the URL in private environment configuration; pg_dump/pg_restore 18 must be on PATH. +# CORESCOPE_TEST_POSTGRES_URL is required; database tests do not silently skip. # Go backend tests cd cmd/server && go test ./... -v cd cmd/ingestor && go test ./... -v -# Or across all 14 modules at once +# Or across all modules make test # Node.js frontend + integration tests @@ -278,7 +255,7 @@ node tools/generate-packets.js --api --count 200 ### Migrating from Node.js -If you're running an existing Node.js deployment, see [docs/go-migration.md](docs/go-migration.md) for a step-by-step guide. The Go engine reads the same SQLite database and `config.json` — no data migration needed. +Historical Node.js/SQLite deployments must first reach a supported SQLite layout using the last SQLite release, then follow the [offline PostgreSQL upgrade](docs/postgresql-upgrade.md). Preserve the original data, configuration and keys; a container/image update alone does not perform this conversion. ## Contributing diff --git a/cmd/decrypt/go.mod b/cmd/decrypt/go.mod index 226dbcf0e..954b4763f 100644 --- a/cmd/decrypt/go.mod +++ b/cmd/decrypt/go.mod @@ -1,10 +1,25 @@ module github.com/corescope/decrypt -go 1.22 +go 1.25.0 require ( - github.com/mattn/go-sqlite3 v1.14.52 github.com/meshcore-analyzer/channel v0.0.0 + github.com/meshcore-analyzer/dbschema v0.0.0 + github.com/meshcore-analyzer/pgutil v0.0.0 +) + +require ( + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.7.6 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + golang.org/x/crypto v0.37.0 // indirect + golang.org/x/sync v0.13.0 // indirect + golang.org/x/text v0.24.0 // indirect ) replace github.com/meshcore-analyzer/channel => ../../internal/channel + +replace github.com/meshcore-analyzer/dbschema => ../../internal/dbschema + +replace github.com/meshcore-analyzer/pgutil => ../../internal/pgutil diff --git a/cmd/decrypt/go.sum b/cmd/decrypt/go.sum index 75bf6eb53..7600ae8d6 100644 --- a/cmd/decrypt/go.sum +++ b/cmd/decrypt/go.sum @@ -1,2 +1,28 @@ -github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= -github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cmd/decrypt/main.go b/cmd/decrypt/main.go index 8c897d6ff..4b8395469 100644 --- a/cmd/decrypt/main.go +++ b/cmd/decrypt/main.go @@ -1,8 +1,8 @@ -// corescope-decrypt decrypts and exports hashtag channel messages from a CoreScope SQLite database. +// corescope-decrypt decrypts and exports hashtag channel messages from PostgreSQL. // // Usage: // -// corescope-decrypt --channel "#wardriving" --db meshcore.db [--format json|html] [--output file] +// corescope-decrypt --channel "#wardriving" [--format json|html] [--output file] package main import ( @@ -18,8 +18,9 @@ import ( "strings" "time" - _ "github.com/mattn/go-sqlite3" "github.com/meshcore-analyzer/channel" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" ) // Version info (set via ldflags). @@ -47,7 +48,12 @@ type Observer struct { func main() { channelName := flag.String("channel", "", "Channel name (e.g. \"#wardriving\")") - dbPath := flag.String("db", "", "Path to CoreScope SQLite database") + defaultURL := os.Getenv("CORESCOPE_READER_DATABASE_URL") + if defaultURL == "" { + defaultURL = os.Getenv("CORESCOPE_DATABASE_URL") + } + databaseURL := flag.String("database-url", defaultURL, "PostgreSQL reader URL (prefer CORESCOPE_READER_DATABASE_URL)") + legacyDB := flag.String("db", "", "Removed: import SQLite with corescope-migrate before exporting") format := flag.String("format", "json", "Output format: json, html, irc (or log)") output := flag.String("output", "", "Output file (default: stdout)") showVersion := flag.Bool("version", false, "Print version and exit") @@ -56,12 +62,13 @@ func main() { fmt.Fprintf(os.Stderr, `corescope-decrypt — Decrypt and export MeshCore hashtag channel messages USAGE - corescope-decrypt --channel NAME --db PATH [--format FORMAT] [--output FILE] + corescope-decrypt --channel NAME [--format FORMAT] [--output FILE] FLAGS --channel NAME Channel name to decrypt (e.g. "#wardriving", "wardriving") The "#" prefix is added automatically if missing. - --db PATH Path to a CoreScope SQLite database file (read-only access). + --database-url PostgreSQL reader URL. Prefer CORESCOPE_READER_DATABASE_URL + or CORESCOPE_DATABASE_URL to keep credentials out of argv. --format FORMAT Output format (default: json): json — Machine-readable JSON array with full metadata html — Self-contained HTML viewer with search and sorting @@ -72,17 +79,17 @@ FLAGS EXAMPLES # Export #wardriving messages as JSON - corescope-decrypt --channel "#wardriving" --db /app/data/meshcore.db + corescope-decrypt --channel "#wardriving" # Generate an interactive HTML viewer - corescope-decrypt --channel wardriving --db meshcore.db --format html --output wardriving.html + corescope-decrypt --channel wardriving --format html --output wardriving.html # Greppable IRC log - corescope-decrypt --channel "#MeshCore" --db meshcore.db --format irc --output meshcore.log + corescope-decrypt --channel "#MeshCore" --format irc --output meshcore.log grep "KE6QR" meshcore.log # From the Docker container - docker exec corescope-prod /app/corescope-decrypt --channel "#wardriving" --db /app/data/meshcore.db + docker exec corescope-prod /app/corescope-decrypt --channel "#wardriving" RETROACTIVE DECRYPTION MeshCore hashtag channels use symmetric encryption — the key is derived from the @@ -107,7 +114,10 @@ LIMITATIONS os.Exit(0) } - if *channelName == "" || *dbPath == "" { + if *legacyDB != "" { + log.Fatal("SQLite runtime access has been removed; use corescope-migrate -from-sqlite during the documented offline upgrade") + } + if *channelName == "" || *databaseURL == "" { flag.Usage() os.Exit(1) } @@ -121,7 +131,7 @@ LIMITATIONS key := channel.DeriveKey(ch) chHash := channel.ChannelHash(key) - db, err := sql.Open("sqlite3", "file:"+*dbPath+"?mode=ro") + db, err := openExportDB(*databaseURL) if err != nil { log.Fatalf("Failed to open database: %v", err) } @@ -276,9 +286,26 @@ func extractGRPPayload(rawHex string) ([]byte, error) { return buf[offset:], nil } +func openExportDB(databaseURL string) (*sql.DB, error) { + db, err := pgutil.Open(databaseURL, true) + if err != nil { + return nil, err + } + db.SetMaxOpenConns(4) + if err := pgutil.AssertReadOnly(db); err != nil { + db.Close() + return nil, err + } + if err := dbschema.AssertReady(db); err != nil { + db.Close() + return nil, err + } + return db, nil +} + func getPathFromDB(db *sql.DB, txID int) []string { var decodedJSON sql.NullString - err := db.QueryRow(`SELECT decoded_json FROM transmissions WHERE id = ?`, txID).Scan(&decodedJSON) + err := db.QueryRow(`SELECT decoded_json FROM transmissions WHERE id = $1`, txID).Scan(&decodedJSON) if err != nil || !decodedJSON.Valid { return nil } @@ -298,9 +325,9 @@ func getObservers(db *sql.DB, txID int) []Observer { rows, err := db.Query(` SELECT o.name, obs.snr, obs.rssi, obs.timestamp FROM observations obs - LEFT JOIN observers o ON o.id = CAST(obs.observer_idx AS TEXT) - WHERE obs.transmission_id = ? - ORDER BY obs.timestamp + LEFT JOIN observers o ON o.rowid = obs.observer_idx + WHERE obs.transmission_id = $1 + ORDER BY obs.timestamp, obs.id `, txID) if err != nil { return nil diff --git a/cmd/decrypt/postgres_test.go b/cmd/decrypt/postgres_test.go new file mode 100644 index 000000000..1e835ebb3 --- /dev/null +++ b/cmd/decrypt/postgres_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "testing" + + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +func TestPostgresExportReadsPathAndObservers(t *testing.T) { + dsn := pgtest.NewSchema(t) + writer, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { writer.Close() }) + if err := dbschema.Apply(writer, t.Logf); err != nil { + t.Fatal(err) + } + for _, query := range []string{ + `INSERT INTO transmissions(id,raw_hex,hash,first_seen,decoded_json) VALUES(1,'00','export-fixture','2026-01-01','{"path":{"hops":["ab","cd"]}}')`, + `INSERT INTO observers(rowid,id,name) VALUES(10,'observer','Example observer')`, + `INSERT INTO observations(transmission_id,observer_idx,snr,rssi,timestamp,path_json) VALUES(1,10,1.5,-90,1767225600,'["ab"]')`, + } { + if _, err := writer.Exec(query); err != nil { + t.Fatal(err) + } + } + reader, err := openExportDB(pgtest.ReadOnly(t, dsn)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { reader.Close() }) + path := getPathFromDB(reader, 1) + if len(path) != 2 || path[0] != "ab" || path[1] != "cd" { + t.Fatalf("PostgreSQL path query lost data: %v", path) + } + observers := getObservers(reader, 1) + if len(observers) != 1 || observers[0].Name != "Example observer" || observers[0].SNR != 1.5 { + t.Fatalf("PostgreSQL observer query lost data: %+v", observers) + } +} diff --git a/cmd/ingestor/README.md b/cmd/ingestor/README.md index 0ad2986b3..8655ed76e 100644 --- a/cmd/ingestor/README.md +++ b/cmd/ingestor/README.md @@ -1,25 +1,25 @@ # MeshCore MQTT Ingestor (Go) -Standalone MQTT ingestion service for CoreScope. Connects to MQTT brokers, decodes raw MeshCore packets, and writes to the same SQLite database used by the Node.js web server. +Standalone MQTT ingestion service for CoreScope. Connects to MQTT brokers, decodes raw MeshCore packets, and writes to PostgreSQL. The separate Go API server uses a restricted reader credential. -This is the first step of a larger Go rewrite — separating MQTT ingestion from the web server. +The migration command owns schema changes. The ingestor only connects to an initialized, ready telemetry schema. ## Architecture ``` -MQTT Broker(s) → Go Ingestor → SQLite DB ← Node.js Web Server +MQTT Broker(s) → Go Ingestor → PostgreSQL ← Go API Server (this binary) (shared) ``` -- **Single static binary** — no runtime dependencies, no CGO -- **SQLite** via `github.com/mattn/go-sqlite3` (cgo; cross-compiled with `zig cc`, see the root `Makefile`) +- **Single binary** — PostgreSQL runtime driver is pure Go +- **PostgreSQL 18** via pgx through `database/sql` - **MQTT** via `github.com/eclipse/paho.mqtt.golang` -- Runs **alongside** the Node.js server — they share the DB file -- Does NOT serve HTTP/WebSocket — that stays in Node.js +- Runs alongside the Go API server with separate writer and reader roles +- Does not serve HTTP/WebSocket — that stays in the Go API server ## Build -Requires Go 1.22+. +Requires Go 1.25+. ```bash cd cmd/ingestor @@ -38,16 +38,19 @@ GOOS=linux GOARCH=amd64 go build -o corescope-ingestor . ./corescope-ingestor -config /path/to/config.json ``` -The config file uses the same format as the Node.js `config.json`. The ingestor reads the `mqttSources` array (or legacy `mqtt` object) and `dbPath` fields. +The ingestor reads `mqttSources` (or the legacy `mqtt` object), `databaseURL` and `stateDir`. `-database-url` and `-state-dir` override configuration. `dbPath` and `-db` are never interpreted as database URLs: import existing SQLite snapshots offline first. Do not place connection URLs in command logs. ### Environment Variables | Variable | Description | Default | |----------|-------------|---------| -| `DB_PATH` | SQLite database path | `data/meshcore.db` | +| `CORESCOPE_WRITER_DATABASE_URL` | Ingestor-specific PostgreSQL writer URL; overrides the generic URL | required when no generic/config URL exists | +| `CORESCOPE_DATABASE_URL` | Generic PostgreSQL URL for standalone runs | config `databaseURL` | +| `CORESCOPE_STATE_DIR` | Local queue/statistics directory shared with the API server | `data` | +| `CORESCOPE_APPROVED_CHANNELS_DATABASE_URL` | Separate restricted account reader for `approved_channels` view | config `userManagement.approvedChannelsDatabaseURL` | | `MQTT_BROKER` | Single MQTT broker URL (overrides config) | — | | `MQTT_TOPIC` | MQTT topic (used with `MQTT_BROKER`) | `meshcore/#` | -| `CORESCOPE_INGESTOR_STATS` | Path to the per-second stats JSON file consumed by the server's `/api/perf/io` and `/api/perf/write-sources` endpoints (#1120) | `/tmp/corescope-ingestor-stats.json` | +| `CORESCOPE_INGESTOR_STATS` | Path to the per-second stats JSON file consumed by the server's `/api/perf/io` and `/api/perf/write-sources` endpoints (#1120) | `/ingestor-stats.json` | ### Stats file (`CORESCOPE_INGESTOR_STATS`) @@ -60,17 +63,15 @@ the Perf page so operators can self-diagnose write-volume anomalies. The writer uses `O_NOFOLLOW | O_CREAT | O_TRUNC` mode `0o600`, so a pre-planted symlink at the path cannot be used to clobber an arbitrary file. -**Security note:** the default lives in `/tmp`, which is world-writable on -most hosts (sticky bit only protects deletion, not creation). On -shared/multi-tenant hosts, override `CORESCOPE_INGESTOR_STATS` to point at a -private directory (e.g. `/var/lib/corescope/ingestor-stats.json`) that only -the corescope user can write to. +The state directory is created privately and stores small queue/statistics files. +Database URLs never become filesystem paths. ### Minimal Config ```json { - "dbPath": "data/meshcore.db", + "databaseURL": "postgres://telemetry_writer@localhost/telemetry", + "stateDir": "data", "mqttSources": [ { "name": "local", @@ -81,7 +82,7 @@ the corescope user can write to. } ``` -### Full Config (same as Node.js) +### Full Config The ingestor reads these fields from the existing `config.json`: @@ -93,13 +94,14 @@ The ingestor reads these fields from the existing `config.json`: - `iataFilter` — optional regional filter - `clientId`: optional MQTT ClientID. Default `corescope--`, new suffix per ingestor start. Two running ingestors must not share a value on one broker - `mqtt` — legacy single-broker config (auto-converted to `mqttSources`) -- `dbPath` — SQLite DB path (default: `data/meshcore.db`) +- `databaseURL` — restricted telemetry writer URL; schema-owner credentials are refused +- `stateDir` — local queue/statistics directory (default: `data`) ## Test ```bash cd cmd/ingestor -go test -v ./... +CORESCOPE_TEST_POSTGRES_URL=postgres://test_admin@localhost/postgres go test -v ./... ``` ## What It Does @@ -109,7 +111,7 @@ go test -v ./... 3. Receives raw hex packets via JSON messages (`{ "raw": "...", "SNR": ..., "RSSI": ... }`) 4. Decodes MeshCore packet headers, paths, and payloads (ported from `decoder.js`) 5. Computes content hashes (path-independent, SHA-256-based) -6. Writes to SQLite: `transmissions` + `observations` tables +6. Writes atomically to PostgreSQL: `transmissions` + `observations` tables 7. Upserts `nodes` from decoded ADVERT packets (with validation) 8. Upserts `observers` from MQTT topic metadata @@ -147,3 +149,10 @@ cmd/ingestor/ util.go — shared utilities go.mod / go.sum — Go module definition ``` + +PostgreSQL autovacuum, automatic analyze and checkpoints are managed by the +PostgreSQL service, not by runtime owner privileges. The previous SQLite +`db.vacuumOnStartup`, `db.incrementalVacuumPages` and `db.analysisLimit` +settings have no PostgreSQL effect. The migration/restore procedure must run +`ANALYZE` after bulk imports; runtime credentials cannot alter planner targets. +Packet retention remains bounded to 250 transmissions per transaction. diff --git a/cmd/ingestor/advert_evidence_migration_race_test.go b/cmd/ingestor/advert_evidence_migration_race_test.go index b7965e111..93ae8b91b 100644 --- a/cmd/ingestor/advert_evidence_migration_race_test.go +++ b/cmd/ingestor/advert_evidence_migration_race_test.go @@ -11,18 +11,24 @@ import ( func seedUnbackfilledAdvert(t *testing.T, canonical, observed string) (*Store, string) { t.Helper() path := filepath.Join(t.TempDir(), "legacy-advert.db") - s, err := OpenStore(path) + s, err := openPostgresTestStore(t, path) if err != nil { t.Fatal(err) } if err := s.UpsertObserver("fixture-observer", "Fixture observer", "", nil); err != nil { t.Fatal(err) } - if _, err := s.db.Exec(`INSERT INTO transmissions(id,hash,raw_hex,first_seen,payload_type,route_type) VALUES(1,'legacy-advert',?,'2026-01-01T00:00:00Z',4,?); - INSERT INTO observations(id,transmission_id,observer_idx,raw_hex,path_json,timestamp) VALUES(1,1,(SELECT rowid FROM observers WHERE id='fixture-observer'),?,'[]',1)`, canonical, int(canonical[1]-'0')&3, observed); err != nil { + if _, err := s.db.Exec(`INSERT INTO transmissions(id,hash,raw_hex,first_seen,payload_type,route_type) VALUES(1,'legacy-advert',$1,'2026-01-01T00:00:00Z',4,$2)`, canonical, int(canonical[1]-'0')&3); err != nil { s.Close() t.Fatal(err) } + if _, err := s.db.Exec(`INSERT INTO observations(id,transmission_id,observer_idx,raw_hex,path_json,timestamp) VALUES(1,1,(SELECT rowid FROM observers WHERE id='fixture-observer'),$1,'[]',1)`, observed); err != nil { + t.Fatal(err) + } + if _, err := testAdmin(t, s).Exec(`SELECT setval('observations_id_seq',1,true)`); err != nil { + t.Fatal(err) + } + return s, path } @@ -33,9 +39,9 @@ func TestAdvertRouteEvidenceLegacyProtectionErrorsDoNotDropIncomingRaw(t *testin defer s.Close() stmt := `DROP TABLE advert_evidence_backfill` if failure == "write" { - stmt = `CREATE TRIGGER fail_old_evidence BEFORE INSERT ON advert_route_evidence WHEN NEW.bit=2 BEGIN SELECT RAISE(ABORT,'fixture old evidence failure'); END` + stmt = `CREATE OR REPLACE FUNCTION fail_old_evidence_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.bit=2 THEN RAISE EXCEPTION 'fixture old evidence failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER fail_old_evidence BEFORE INSERT ON advert_route_evidence FOR EACH ROW EXECUTE FUNCTION fail_old_evidence_fn()` } - if _, err := s.db.Exec(stmt); err != nil { + if _, err := testAdmin(t, s).Exec(stmt); err != nil { t.Fatal(err) } _, err := s.InsertTransmission(&PacketData{Hash: "legacy-advert", ObserverID: "fixture-observer", PayloadType: 4, RouteType: 1, RawHex: "1100aa", PathJSON: "[]"}) @@ -75,19 +81,21 @@ func TestAdvertRouteEvidenceLegacyCoalescedPathAndIndex(t *testing.T) { if mask != 3 { t.Fatalf("coalesced path lookup lost old evidence: mask=%d", mask) } - rows, err := s.db.Query(`EXPLAIN QUERY PLAN `+legacyAdvertObservationSQL, 1, 1, "") + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } + rows, err := s.db.Query(`EXPLAIN (COSTS OFF) `+legacyAdvertObservationSQL, 1, 1, "") if err != nil { t.Fatal(err) } defer rows.Close() indexed := false for rows.Next() { - var id, parent, unused int var detail string - if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { + if err := rows.Scan(&detail); err != nil { t.Fatal(err) } - if strings.Contains(detail, "SEARCH observations USING INDEX idx_observations_dedup") { + if strings.Contains(detail, "idx_observations_dedup") { indexed = true } } @@ -111,7 +119,7 @@ func TestAdvertRouteEvidenceCompletionSkipsLookupAfterRestart(t *testing.T) { } s.Close() var err error - s, err = OpenStore(path) + s, err = openPostgresTestStore(t, path) if err != nil { t.Fatal(err) } @@ -120,7 +128,7 @@ func TestAdvertRouteEvidenceCompletionSkipsLookupAfterRestart(t *testing.T) { } // Any accidental legacy lookup now fails. Completed stores must incur // no extra SQL on this steady-state overwrite path. - if _, err := s.db.Exec(`DROP TABLE advert_evidence_backfill`); err != nil { + if _, err := testAdmin(t, s).Exec(`DROP TABLE advert_evidence_backfill`); err != nil { t.Fatal(err) } if _, err := s.InsertTransmission(&PacketData{Hash: "legacy-advert", ObserverID: "fixture-observer", PayloadType: 4, RouteType: 1, RawHex: "1100aa", PathJSON: "[]"}); err != nil { @@ -129,7 +137,7 @@ func TestAdvertRouteEvidenceCompletionSkipsLookupAfterRestart(t *testing.T) { } func BenchmarkAdvertEvidenceLegacyPreservation(b *testing.B) { - s, err := OpenStore(filepath.Join(b.TempDir(), "legacy-preservation.db")) + s, err := openPostgresTestStore(b, filepath.Join(b.TempDir(), "legacy-preservation.db")) if err != nil { b.Fatal(err) } @@ -151,7 +159,7 @@ func BenchmarkAdvertEvidenceLegacyPreservation(b *testing.B) { if state != "pending" { cursor = 1 } - if _, err := s.db.Exec(`INSERT INTO advert_evidence_backfill(id,obs_cursor) VALUES(1,?) ON CONFLICT(id) DO UPDATE SET obs_cursor=excluded.obs_cursor`, cursor); err != nil { + if _, err := s.db.Exec(`INSERT INTO advert_evidence_backfill(id,obs_cursor) VALUES(1,$1) ON CONFLICT(id) DO UPDATE SET obs_cursor=excluded.obs_cursor`, cursor); err != nil { b.Fatal(err) } s.advertEvidenceComplete.Store(state == "complete") @@ -160,7 +168,17 @@ func BenchmarkAdvertEvidenceLegacyPreservation(b *testing.B) { b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - if err := s.preserveLegacyAdvertObservation(1, observerIdx, "[]"); err != nil { + if err := func() error { + tx, e := beginWrite(s.db) + if e != nil { + return e + } + defer tx.Rollback() + if e = s.preserveLegacyAdvertObservation(tx, 1, observerIdx, "[]"); e != nil { + return e + } + return tx.Commit() + }(); err != nil { b.Fatal(err) } } @@ -196,7 +214,7 @@ func TestAdvertRouteEvidencePreservesLegacyConflictBeforeBackfill(t *testing.T) if restart { s.Close() var err error - s, err = OpenStore(path) + s, err = openPostgresTestStore(t, path) if err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/advert_evidence_scale_test.go b/cmd/ingestor/advert_evidence_scale_test.go index bd716a373..b6aabc767 100644 --- a/cmd/ingestor/advert_evidence_scale_test.go +++ b/cmd/ingestor/advert_evidence_scale_test.go @@ -25,7 +25,7 @@ func TestAdvertBackfillScale(t *testing.T) { } else if !os.IsNotExist(err) { t.Fatal(err) } - s, err := OpenStore(path) + s, err := openPostgresTestStore(t, path) if err != nil { t.Fatal(err) } @@ -42,17 +42,17 @@ func TestAdvertBackfillScale(t *testing.T) { { raw := "1100" + strings.Repeat("ab", 118) for start := 1; start <= 1000000; start += 10000 { - _, err = s.db.Exec(`WITH RECURSIVE ids(n) AS (VALUES(?) UNION ALL SELECT n+1 FROM ids WHERE n COALESCE((SELECT obs_cursor FROM advert_evidence_backfill WHERE id=1),0)` // Caller holds writerMu, including through the subsequent observation UPSERT. // The backfill commits evidence and obs_cursor together under that same lock. -func (s *Store) preserveLegacyAdvertObservation(txID, observerIdx int64, path string) error { +func (s *Store) preserveLegacyAdvertObservation(tx *sql.Tx, txID, observerIdx int64, path string) error { if s.advertEvidenceComplete.Load() { return nil } var raw sql.NullString - err := s.stmtGetLegacyAdvertObservation.QueryRow(txID, observerIdx, path).Scan(&raw) + err := tx.Stmt(s.stmtGetLegacyAdvertObservation).QueryRow(txID, observerIdx, path).Scan(&raw) if err == sql.ErrNoRows { return nil } @@ -34,7 +34,7 @@ func (s *Store) preserveLegacyAdvertObservation(txID, observerIdx int64, path st return err } if bit := packetpath.AdvertRouteEvidence(raw.String); bit != 0 { - _, err = s.stmtInsertAdvertEvidence.Exec(txID, bit, txID, bit) + _, err = tx.Stmt(s.stmtInsertAdvertEvidence).Exec(txID, bit, txID, bit) } return err } @@ -44,12 +44,12 @@ func (s *Store) preserveLegacyAdvertObservation(txID, observerIdx int64, path st // cursors and 500-row transactions make this cancellable/resumable; new traffic // records evidence synchronously, so rows beyond either scan need no replay. func (s *Store) backfillAdvertEvidence(ctx context.Context, db *sql.DB) error { - if _, err := db.ExecContext(ctx, `INSERT OR IGNORE INTO advert_evidence_backfill(id) VALUES(1)`); err != nil { + if _, err := db.ExecContext(ctx, `INSERT INTO advert_evidence_backfill(id) VALUES(1) ON CONFLICT DO NOTHING`); err != nil { return err } for _, source := range []struct{ cursor, table, query string }{ - {"tx_cursor", "transmissions", `SELECT id,id,COALESCE(raw_hex,''),payload_type FROM transmissions WHERE id>? AND id<=? ORDER BY id LIMIT 500`}, - {"obs_cursor", "observations", `SELECT o.id,o.transmission_id,COALESCE(o.raw_hex,''),t.payload_type FROM observations o JOIN transmissions t ON t.id=o.transmission_id WHERE o.id>? AND o.id<=? ORDER BY o.id LIMIT 500`}, + {"tx_cursor", "transmissions", `SELECT id,id,COALESCE(raw_hex,''),payload_type FROM transmissions WHERE id>$1 AND id<=$2 ORDER BY id LIMIT 500`}, + {"obs_cursor", "observations", `SELECT o.id,o.transmission_id,COALESCE(o.raw_hex,''),t.payload_type FROM observations o JOIN transmissions t ON t.id=o.transmission_id WHERE o.id>$1 AND o.id<=$2 ORDER BY o.id LIMIT 500`}, } { // A finite horizon prevents live traffic from extending this scan. var upper int64 @@ -101,17 +101,17 @@ func (s *Store) backfillAdvertEvidence(ctx context.Context, db *sql.DB) error { err = func() error { writerMu.Lock() defer writerMu.Unlock() - tx, err := db.BeginTx(ctx, nil) + tx, err := beginWriteContext(ctx, db) if err != nil { return err } defer tx.Rollback() for _, item := range batch { - if _, err := tx.ExecContext(ctx, insertAdvertEvidenceSQL+` AND EXISTS(SELECT 1 FROM transmissions WHERE id=?)`, item.txID, item.bit, item.txID, item.bit, item.txID); err != nil { + if _, err := tx.ExecContext(ctx, insertAdvertEvidenceSQL+` AND EXISTS(SELECT 1 FROM transmissions WHERE id=$5)`, item.txID, item.bit, item.txID, item.bit, item.txID); err != nil { return err } } - if _, err := tx.ExecContext(ctx, `UPDATE advert_evidence_backfill SET `+source.cursor+`=? WHERE id=1`, lastID); err != nil { + if _, err := tx.ExecContext(ctx, `UPDATE advert_evidence_backfill SET `+source.cursor+`=$1 WHERE id=1`, lastID); err != nil { return err } return tx.Commit() diff --git a/cmd/ingestor/advert_route_evidence_test.go b/cmd/ingestor/advert_route_evidence_test.go index bccbf8a11..d1a5cf80e 100644 --- a/cmd/ingestor/advert_route_evidence_test.go +++ b/cmd/ingestor/advert_route_evidence_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "path/filepath" + "strings" "testing" ) @@ -28,7 +29,7 @@ func TestAdvertRouteEvidenceSurvivesObservationUpsert(t *testing.T) { } { t.Run(sequence.name, func(t *testing.T) { path := filepath.Join(t.TempDir(), "evidence.db") - s, err := OpenStore(path) + s, err := openPostgresTestStore(t, path) if err != nil { t.Fatal(err) } @@ -64,7 +65,7 @@ func TestAdvertRouteEvidenceSurvivesObservationUpsert(t *testing.T) { if canonical != sequence.raws[0] { t.Fatalf("canonical raw changed: %s", canonical) } - if err := s.db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='advert_route_evidence'`).Scan(&count); err != nil { + if err := s.db.QueryRow(`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='advert_route_evidence'`).Scan(&count); err != nil { t.Fatal(err) } if count != 1 { @@ -94,14 +95,14 @@ func TestAdvertRouteEvidenceSurvivesObservationUpsert(t *testing.T) { t.Fatal("duplicate traffic appended route evidence") } var sequenceID int64 - if err := s.db.QueryRow(`SELECT seq FROM sqlite_sequence WHERE name='advert_route_evidence'`).Scan(&sequenceID); err != nil { + if err := s.db.QueryRow(`SELECT last_value FROM advert_route_evidence_id_seq`).Scan(&sequenceID); err != nil { t.Fatal(err) } if sequenceID != maxID { t.Fatalf("duplicate traffic advanced evidence sequence: %d, want %d", sequenceID, maxID) } s.Close() - s, err = OpenStore(path) + s, err = openPostgresTestStore(t, path) if err != nil { t.Fatal(err) } @@ -121,7 +122,7 @@ func TestAdvertRouteEvidenceSurvivesObservationUpsert(t *testing.T) { } func TestAdvertRouteEvidenceBackfillResumeAndLiveUnion(t *testing.T) { - s, err := OpenStore(filepath.Join(t.TempDir(), "backfill.db")) + s, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), "backfill.db")) if err != nil { t.Fatal(err) } @@ -136,10 +137,10 @@ func TestAdvertRouteEvidenceBackfillResumeAndLiveUnion(t *testing.T) { t.Fatal(err) } for id := 1; id <= 1200; id++ { - if _, err := tx.Exec(`INSERT INTO transmissions(id,hash,raw_hex,first_seen,payload_type,route_type) VALUES(?,?, '1100aa','2026-01-01T00:00:00Z',4,1)`, id, fmt.Sprintf("history-%d", id)); err != nil { + if _, err := tx.Exec(`INSERT INTO transmissions(id,hash,raw_hex,first_seen,payload_type,route_type) VALUES($1,$2, '1100aa','2026-01-01T00:00:00Z',4,1)`, id, fmt.Sprintf("history-%d", id)); err != nil { t.Fatal(err) } - if _, err := tx.Exec(`INSERT INTO observations(transmission_id,observer_idx,raw_hex,path_json,timestamp) VALUES(?,(SELECT rowid FROM observers WHERE id='fixture-observer'),'1200aa','[]',1)`, id); err != nil { + if _, err := tx.Exec(`INSERT INTO observations(transmission_id,observer_idx,raw_hex,path_json,timestamp) VALUES($1,(SELECT rowid FROM observers WHERE id='fixture-observer'),'1200aa','[]',1)`, id); err != nil { t.Fatal(err) } } @@ -153,7 +154,7 @@ func TestAdvertRouteEvidenceBackfillResumeAndLiveUnion(t *testing.T) { } // Abort after one committed batch; the failed batch must not move its // persisted cursor, so a retry can recover every remaining frame. - if _, err := s.db.Exec(`CREATE TRIGGER fail_evidence_batch BEFORE INSERT ON advert_route_evidence WHEN NEW.tx_id=501 BEGIN SELECT RAISE(ABORT,'fixture failure'); END`); err != nil { + if _, err := testAdmin(t, s).Exec(`CREATE OR REPLACE FUNCTION fail_evidence_batch_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.tx_id=501 THEN RAISE EXCEPTION 'fixture failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER fail_evidence_batch BEFORE INSERT ON advert_route_evidence FOR EACH ROW EXECUTE FUNCTION fail_evidence_batch_fn()`); err != nil { t.Fatal(err) } if err := s.backfillAdvertEvidence(context.Background(), s.db); err == nil { @@ -166,7 +167,7 @@ func TestAdvertRouteEvidenceBackfillResumeAndLiveUnion(t *testing.T) { if cursor != 500 { t.Fatalf("cursor=%d after failure, want committed batch boundary 500", cursor) } - if _, err := s.db.Exec(`DROP TRIGGER fail_evidence_batch`); err != nil { + if _, err := testAdmin(t, s).Exec(`DROP TRIGGER fail_evidence_batch ON advert_route_evidence`); err != nil { t.Fatal(err) } done := make(chan error, 1) @@ -187,13 +188,13 @@ func TestAdvertRouteEvidenceBackfillResumeAndLiveUnion(t *testing.T) { t.Fatalf("got %d evidence rows, want two per history transmission", count) } var before, after int64 - if err := s.db.QueryRow(`SELECT seq FROM sqlite_sequence WHERE name='advert_route_evidence'`).Scan(&before); err != nil { + if err := s.db.QueryRow(`SELECT last_value FROM advert_route_evidence_id_seq`).Scan(&before); err != nil { t.Fatal(err) } if err := s.backfillAdvertEvidence(context.Background(), s.db); err != nil { t.Fatal(err) } - if err := s.db.QueryRow(`SELECT seq FROM sqlite_sequence WHERE name='advert_route_evidence'`).Scan(&after); err != nil { + if err := s.db.QueryRow(`SELECT last_value FROM advert_route_evidence_id_seq`).Scan(&after); err != nil { t.Fatal(err) } if before != after { @@ -206,7 +207,7 @@ func TestAdvertRouteEvidenceBackfillResumeAndLiveUnion(t *testing.T) { func TestAdvertRouteEvidenceFailureKeepsCoreIngestion(t *testing.T) { for _, failure := range []string{"incoming-write", "legacy-read", "legacy-write"} { t.Run(failure, func(t *testing.T) { - s, err := OpenStore(filepath.Join(t.TempDir(), "write-failure.db")) + s, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), "write-failure.db")) if err != nil { t.Fatal(err) } @@ -226,14 +227,14 @@ func TestAdvertRouteEvidenceFailureKeepsCoreIngestion(t *testing.T) { if _, err := s.db.Exec(`DELETE FROM advert_route_evidence; UPDATE observations SET raw_hex='1200aa',resolved_path=NULL`); err != nil { t.Fatal(err) } - stmt := `CREATE TRIGGER fail_evidence BEFORE INSERT ON advert_route_evidence WHEN NEW.bit=1 BEGIN SELECT RAISE(ABORT,'fixture evidence failure'); END` + stmt := `CREATE OR REPLACE FUNCTION fail_evidence_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.bit=1 THEN RAISE EXCEPTION 'fixture evidence failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER fail_evidence BEFORE INSERT ON advert_route_evidence FOR EACH ROW EXECUTE FUNCTION fail_evidence_fn()` if failure == "legacy-read" { stmt = `DROP TABLE advert_evidence_backfill` } if failure == "legacy-write" { - stmt = `CREATE TRIGGER fail_evidence BEFORE INSERT ON advert_route_evidence WHEN NEW.bit=2 BEGIN SELECT RAISE(ABORT,'fixture evidence failure'); END` + stmt = `CREATE OR REPLACE FUNCTION fail_evidence_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.bit=2 THEN RAISE EXCEPTION 'fixture evidence failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER fail_evidence BEFORE INSERT ON advert_route_evidence FOR EACH ROW EXECUTE FUNCTION fail_evidence_fn()` } - if _, err := s.db.Exec(stmt); err != nil { + if _, err := testAdmin(t, s).Exec(stmt); err != nil { t.Fatal(err) } before := s.Stats.WriteErrors.Load() @@ -275,7 +276,7 @@ func TestAdvertRouteEvidenceFailureKeepsCoreIngestion(t *testing.T) { } func TestAdvertRouteEvidenceConstraintsAndFeedIndex(t *testing.T) { - s, err := OpenStore(filepath.Join(t.TempDir(), "constraints.db")) + s, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), "constraints.db")) if err != nil { t.Fatal(err) } @@ -288,17 +289,26 @@ func TestAdvertRouteEvidenceConstraintsAndFeedIndex(t *testing.T) { `INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,3)`, `INSERT INTO advert_route_evidence(tx_id,bit) VALUES(999,2)`, } { - if _, err := s.db.Exec(stmt); err == nil { + if _, err := testAdmin(t, s).Exec(stmt); err == nil { t.Errorf("constraint accepted: %s", stmt) } } - var id, parent, unused int - var detail string - if err := s.db.QueryRow(`EXPLAIN QUERY PLAN SELECT id,tx_id,bit FROM advert_route_evidence WHERE id>1 ORDER BY id LIMIT 500`).Scan(&id, &parent, &unused, &detail); err != nil { + s.db.Exec(`SET enable_seqscan=off`) + rows, err := s.db.Query(`EXPLAIN (COSTS OFF) SELECT id,tx_id,bit FROM advert_route_evidence WHERE id>1 ORDER BY id LIMIT 500`) + if err != nil { t.Fatal(err) } - if detail != "SEARCH advert_route_evidence USING INTEGER PRIMARY KEY (rowid>?)" { - t.Fatalf("feed must seek by primary key: %s", detail) + var plan string + for rows.Next() { + var line string + if err := rows.Scan(&line); err != nil { + t.Fatal(err) + } + plan += line + } + rows.Close() + if !strings.Contains(plan, "advert_route_evidence_pkey") { + t.Fatalf("feed must seek by primary key: %s", plan) } if _, err := s.db.Exec(`DELETE FROM observations; DELETE FROM transmissions`); err != nil { t.Fatal(err) @@ -316,7 +326,7 @@ func TestAdvertRouteEvidenceConstraintsAndFeedIndex(t *testing.T) { } func BenchmarkAdvertEvidenceRepeatedWrite(b *testing.B) { - s, err := OpenStore(filepath.Join(b.TempDir(), "repeat.db")) + s, err := openPostgresTestStore(b, filepath.Join(b.TempDir(), "repeat.db")) if err != nil { b.Fatal(err) } @@ -335,7 +345,7 @@ func BenchmarkAdvertEvidenceRepeatedWrite(b *testing.B) { } func TestAdvertRouteEvidenceRejectsUnprovenFrames(t *testing.T) { - s, err := OpenStore(filepath.Join(t.TempDir(), "evidence.db")) + s, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), "evidence.db")) if err != nil { t.Fatal(err) } @@ -347,7 +357,7 @@ func TestAdvertRouteEvidenceRejectsUnprovenFrames(t *testing.T) { } } var count int - if err := s.db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='advert_route_evidence'`).Scan(&count); err != nil { + if err := s.db.QueryRow(`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='advert_route_evidence'`).Scan(&count); err != nil { t.Fatal(err) } if count != 1 { diff --git a/cmd/ingestor/approved_channels.go b/cmd/ingestor/approved_channels.go index 3cd42e969..d4da85083 100644 --- a/cmd/ingestor/approved_channels.go +++ b/cmd/ingestor/approved_channels.go @@ -1,20 +1,19 @@ package main import ( + "context" "database/sql" "errors" - "fmt" "log" - "os" - "path/filepath" "sync/atomic" "time" "github.com/meshcore-analyzer/channel" + "github.com/meshcore-analyzer/pgutil" ) // Approved hashtag channels (docs/specs/2026-10-07-channel-proposals-design.md). -// The server owns users.db; the ingestor only reads it, read-only, with raw +// The server owns account data; the ingestor only reads its approved view // SQL, once per approvedChannelsRefresh, and never writes it (AGENTS.md, // read/write separation). @@ -22,9 +21,9 @@ const approvedChannelsRefresh = time.Minute // approvedChannelsQuery is pinned in internal/users/proposals_test.go // (ingestorApprovedQuery), which checks it against ApprovedSubjects. -const approvedChannelsQuery = `SELECT subject FROM proposals WHERE kind = 'hashtag_channel' AND status = 'approved' ORDER BY decided_at, id LIMIT ?` +const approvedChannelsQuery = `SELECT subject FROM approved_channels WHERE kind = 'hashtag_channel' ORDER BY decided_at, id LIMIT $1` -var errUsersDBMissing = errors.New("users.db not found") +var errUsersDBMissing = errors.New("approved channel database URL is not configured") // channelKeySet hands each message the current channel key map. Snapshots // are never mutated after they are stored, so decoders read them without a @@ -53,7 +52,7 @@ func (s *channelKeySet) Snapshot() map[string]string { return *s.cur.Load() } // refresh re-reads the approved names and swaps in configured + approved. // A failed read keeps the current snapshot: only a successful read changes -// the set, so a locked or missing users.db never drops a key. +// the set, so an unavailable or incomplete account store never drops a key. func (s *channelKeySet) refresh() { names, err := s.readApproved() if err != nil { @@ -64,7 +63,7 @@ func (s *channelKeySet) refresh() { return } if s.failing { - log.Printf("[channels] approved channels readable again from %s", s.path) + log.Print("[channels] approved channels readable again") s.failing = false } next, added := mergeApprovedKeys(s.configured, names) @@ -77,20 +76,47 @@ func (s *channelKeySet) refresh() { func (s *channelKeySet) readApproved() ([]string, error) { if s.db == nil { - if _, err := os.Stat(s.path); err != nil { - if os.IsNotExist(err) { - return nil, fmt.Errorf("%w at %s", errUsersDBMissing, s.path) - } - return nil, err + if s.path == "" { + return nil, errUsersDBMissing } - db, err := sql.Open("sqlite3", usersDBReadOnlyDSN(s.path)) + db, err := pgutil.Open(s.path, true) if err != nil { return nil, err } + if err = pgutil.AssertReadOnly(db); err != nil { + db.Close() + return nil, err + } + var credentialAccess bool + if err = db.QueryRow(`SELECT EXISTS(SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=current_schema() AND c.relkind IN ('r','p','m','f') AND c.relname <> 'corescope_schema' AND has_table_privilege(c.oid,'SELECT'))`).Scan(&credentialAccess); err != nil { + db.Close() + return nil, err + } + if credentialAccess { + db.Close() + return nil, errors.New("approved-channel reader has account-table access") + } db.SetMaxOpenConns(1) s.db = db } - rows, err := s.db.Query(approvedChannelsQuery, s.max) + // Read the gate and projection in one snapshot. The readiness marker is the + // only base table this role can read; an import must not expose partial keys. + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true}) + if err != nil { + return nil, err + } + defer tx.Rollback() + var version int + var ready bool + if err := tx.QueryRowContext(ctx, `SELECT version,ready FROM corescope_schema WHERE kind='accounts'`).Scan(&version, &ready); err != nil { + return nil, errors.New("approved-channel account schema is not initialized") + } + if version != 1 || !ready { + return nil, errors.New("approved-channel account schema is incomplete or unsupported") + } + rows, err := tx.QueryContext(ctx, approvedChannelsQuery, s.max) if err != nil { return nil, err } @@ -103,13 +129,16 @@ func (s *channelKeySet) readApproved() ([]string, error) { } out = append(out, name) } - return out, rows.Err() -} - -// usersDBReadOnlyDSN opens users.db read-only: this process must never -// take a write lock on the server's file. -func usersDBReadOnlyDSN(path string) string { - return "file:" + path + "?mode=ro&_busy_timeout=5000" + if err := rows.Err(); err != nil { + return nil, err + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := tx.Commit(); err != nil { + return nil, err + } + return out, nil } // mergeApprovedKeys returns a new map: the configured keys plus a derived key @@ -143,12 +172,7 @@ func (s *channelKeySet) Close() { } } -// logApprovedChannelsSource logs the users.db file the approved channels are -// read from, resolved to an absolute path so a relative one shows which file -// it means; the path as given when it cannot be resolved. -func logApprovedChannelsSource(path string) { - if abs, err := filepath.Abs(path); err == nil { - path = abs - } - log.Printf("[proposals] reading approved channels from %s", path) +// Connection strings contain credentials and must never appear in logs. +func logApprovedChannelsSource(databaseURL string) { + log.Print("[proposals] reading approved channels with restricted PostgreSQL reader") } diff --git a/cmd/ingestor/approved_channels_test.go b/cmd/ingestor/approved_channels_test.go index e97fb876b..9843552a3 100644 --- a/cmd/ingestor/approved_channels_test.go +++ b/cmd/ingestor/approved_channels_test.go @@ -3,10 +3,12 @@ package main import ( "bytes" "crypto/sha256" - "database/sql" "encoding/hex" "fmt" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" "log" + "net/url" "os" "path/filepath" "reflect" @@ -18,7 +20,7 @@ import ( // The columns cmd/server's users.db (internal/users schema v4) has; the // ingestor reads kind, subject, status, decided_at and id. const testProposalsDDL = `CREATE TABLE IF NOT EXISTS proposals ( - id INTEGER PRIMARY KEY, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, kind TEXT NOT NULL, subject TEXT NOT NULL, status TEXT NOT NULL, @@ -36,7 +38,7 @@ type testProposal struct { func execUsersDB(t *testing.T, path string, stmts ...string) { t.Helper() - db, err := sql.Open("sqlite3", path) + db, err := pgutil.Open(approvedFixture(t, path).owner, false) if err != nil { t.Fatal(err) } @@ -46,6 +48,20 @@ func execUsersDB(t *testing.T, path string, stmts ...string) { t.Fatalf("%s: %v", s, err) } } + if stmts[0] == testProposalsDDL { + f := approvedFixture(t, path) + for _, stmt := range []string{ + `CREATE TABLE IF NOT EXISTS corescope_schema(version INTEGER NOT NULL, kind TEXT PRIMARY KEY, ready BOOLEAN NOT NULL)`, + `INSERT INTO corescope_schema VALUES(1,'accounts',true) ON CONFLICT(kind) DO NOTHING`, + `CREATE OR REPLACE VIEW approved_channels AS SELECT kind,subject,decided_at,id FROM proposals WHERE status='approved'`, + `GRANT SELECT ON approved_channels,corescope_schema TO ` + f.role, + } { + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } + } + } + } func writeUsersDB(t *testing.T, path string, rows ...testProposal) { @@ -62,19 +78,42 @@ func configuredKeys() map[string]string { return map[string]string{"Public": "8b3387e9c5cdea6ac9e5edbaa115cd72", "#mesh": deriveHashtagChannelKey("#mesh")} } -func TestApprovedChannelsQueryMatchesUsersPackage(t *testing.T) { - src, err := os.ReadFile(filepath.Join("..", "..", "internal", "users", "proposals_test.go")) - if err != nil { +func TestApprovedChannelsQueryMatchesProjection(t *testing.T) { + path := "projection" + writeUsersDB(t, path, testProposal{"#fixture", "approved", 1}) + reader := newApprovedTestKeySet(t, configuredKeys(), path, 128) + defer reader.Close() + if _, err := reader.readApproved(); err != nil { t.Fatal(err) } - if !strings.Contains(string(src), approvedChannelsQuery) { - t.Fatalf("internal/users/proposals_test.go no longer pins %q; keep the two in step", approvedChannelsQuery) +} + +func TestApprovedChannelsRejectsIncompleteAccountImport(t *testing.T) { + path := "account-import-gate" + writeUsersDB(t, path, testProposal{"#fixture", "approved", 1}) + reader := newApprovedTestKeySet(t, configuredKeys(), path, 128) + defer reader.Close() + reader.refresh() + if _, ok := reader.Snapshot()["#fixture"]; !ok { + t.Fatal("projection reader with readiness metadata could not load a verified account store") + } + execUsersDB(t, path, `UPDATE corescope_schema SET ready=false WHERE kind='accounts'`, `DELETE FROM proposals`) + if _, err := reader.readApproved(); err == nil { + t.Fatal("partially imported account data was accepted") + } + reader.refresh() + if _, ok := reader.Snapshot()["#fixture"]; !ok { + t.Fatal("incomplete account import discarded the last good approved keys") + } + execUsersDB(t, path, `UPDATE corescope_schema SET ready=true,version=2 WHERE kind='accounts'`) + if _, err := reader.readApproved(); err == nil { + t.Fatal("future account schema version was accepted") } } func TestChannelKeySetOffReturnsConfiguredMap(t *testing.T) { cfgd := configuredKeys() - s := newChannelKeySet(cfgd, filepath.Join(t.TempDir(), "users.db"), 128) + s := newApprovedTestKeySet(t, cfgd, filepath.Join(t.TempDir(), "users.db"), 128) if reflect.ValueOf(s.Snapshot()).Pointer() != reflect.ValueOf(cfgd).Pointer() { t.Fatal("before any refresh the decoder must get the very map loadChannelKeys built") } @@ -85,7 +124,7 @@ func TestChannelKeySetLoadsApproved(t *testing.T) { writeUsersDB(t, path, testProposal{"#mycity", "approved", 10}, testProposal{"#waiting", "pending", 0}, testProposal{"#nope", "rejected", 11}, testProposal{"#gone", "revoked", 12}) - s := newChannelKeySet(configuredKeys(), path, 128) + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) defer s.Close() s.refresh() snap := s.Snapshot() @@ -107,7 +146,7 @@ func TestChannelKeySetConfiguredNameWins(t *testing.T) { writeUsersDB(t, path, testProposal{"#mesh", "approved", 1}) cfgd := configuredKeys() cfgd["#mesh"] = "00112233445566778899aabbccddeeff" - s := newChannelKeySet(cfgd, path, 128) + s := newApprovedTestKeySet(t, cfgd, path, 128) defer s.Close() s.refresh() if got := s.Snapshot()["#mesh"]; got != "00112233445566778899aabbccddeeff" { @@ -124,7 +163,7 @@ func TestChannelKeySetConfiguredNameWins(t *testing.T) { func TestChannelKeySetRevokeRemovesKeyOnNextRefresh(t *testing.T) { path := filepath.Join(t.TempDir(), "users.db") writeUsersDB(t, path, testProposal{"#mycity", "approved", 1}) - s := newChannelKeySet(configuredKeys(), path, 128) + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) defer s.Close() s.refresh() before := s.Snapshot() @@ -143,7 +182,7 @@ func TestChannelKeySetMissingFileKeepsConfiguredAndLogsOnce(t *testing.T) { log.SetOutput(&buf) defer log.SetOutput(os.Stderr) path := filepath.Join(t.TempDir(), "users.db") - s := newChannelKeySet(configuredKeys(), path, 128) + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) defer s.Close() s.refresh() s.refresh() @@ -166,10 +205,10 @@ func TestChannelKeySetMissingFileKeepsConfiguredAndLogsOnce(t *testing.T) { func TestChannelKeySetReadErrorKeepsLastGoodSet(t *testing.T) { path := filepath.Join(t.TempDir(), "users.db") writeUsersDB(t, path, testProposal{"#mycity", "approved", 1}) - s := newChannelKeySet(configuredKeys(), path, 128) + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) defer s.Close() s.refresh() - execUsersDB(t, path, `DROP TABLE proposals`) + execUsersDB(t, path, `DROP VIEW approved_channels`) s.refresh() if _, ok := s.Snapshot()["#mycity"]; !ok { t.Fatal("a failing read removed an approved key") @@ -181,7 +220,7 @@ func TestChannelKeySetSkipsNamesThatFailTheRules(t *testing.T) { writeUsersDB(t, path, testProposal{"#public", "approved", 1}, testProposal{"nohash", "approved", 2}, testProposal{"#" + strings.Repeat("a", 31), "approved", 3}, testProposal{"#ok", "approved", 4}) - s := newChannelKeySet(configuredKeys(), path, 128) + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) defer s.Close() s.refresh() snap := s.Snapshot() @@ -193,7 +232,7 @@ func TestChannelKeySetSkipsNamesThatFailTheRules(t *testing.T) { func TestChannelKeySetCapKeepsOldestApprovals(t *testing.T) { path := filepath.Join(t.TempDir(), "users.db") writeUsersDB(t, path, testProposal{"#c", "approved", 30}, testProposal{"#a", "approved", 10}, testProposal{"#b", "approved", 20}) - s := newChannelKeySet(configuredKeys(), path, 2) + s := newApprovedTestKeySet(t, configuredKeys(), path, 2) defer s.Close() s.refresh() snap := s.Snapshot() @@ -205,35 +244,37 @@ func TestChannelKeySetCapKeepsOldestApprovals(t *testing.T) { func TestChannelKeySetIsReadOnly(t *testing.T) { path := filepath.Join(t.TempDir(), "users.db") writeUsersDB(t, path, testProposal{"#mycity", "approved", 1}) - s := newChannelKeySet(configuredKeys(), path, 128) + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) defer s.Close() s.refresh() _, err := s.db.Exec(`INSERT INTO proposals (kind, subject, status, created_at) VALUES ('hashtag_channel', '#x', 'approved', 1)`) - if err == nil || !strings.Contains(strings.ToLower(err.Error()), "readonly") { + if err == nil || !strings.Contains(strings.ToLower(err.Error()), "read-only") { t.Fatalf("write through the ingestor's users.db handle: err = %v; want a read-only refusal", err) } - if !strings.Contains(usersDBReadOnlyDSN(path), "mode=ro") { - t.Fatal("DSN without mode=ro") + if _, err := s.db.Exec(`SET default_transaction_read_only=off`); err != nil { + t.Fatal(err) + } + if _, err := s.db.Query(`SELECT * FROM proposals`); err == nil { + t.Fatal("projection reader can read proposals table") } + } // The server keeps users.db in WAL mode with a writer connection open. func TestChannelKeySetReadsWALWithOpenWriter(t *testing.T) { path := filepath.Join(t.TempDir(), "users.db") writeUsersDB(t, path, testProposal{"#mycity", "approved", 1}) - w, err := sql.Open("sqlite3", path) + w, err := pgutil.Open(approvedFixture(t, path).owner, false) if err != nil { t.Fatal(err) } defer w.Close() w.SetMaxOpenConns(1) - if _, err := w.Exec(`PRAGMA journal_mode=WAL`); err != nil { - t.Fatal(err) - } + if _, err := w.Exec(`INSERT INTO proposals (kind, subject, status, created_at, decided_at) VALUES ('hashtag_channel', '#walcity', 'approved', 9, 9)`); err != nil { t.Fatal(err) } - s := newChannelKeySet(configuredKeys(), path, 128) + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) defer s.Close() s.refresh() if s.Snapshot()["#walcity"] == "" || s.Snapshot()["#mycity"] == "" { @@ -256,7 +297,7 @@ func grpTxtFor(t testing.TB, channelName, text string) []byte { func TestChannelKeySetConcurrentDecodeWhileSwapping(t *testing.T) { path := filepath.Join(t.TempDir(), "users.db") writeUsersDB(t, path, testProposal{"#mycity", "approved", 1}) - s := newChannelKeySet(configuredKeys(), path, 128) + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) defer s.Close() s.refresh() buf := grpTxtFor(t, "#mycity", "alice: hi") @@ -338,8 +379,8 @@ func TestApprovedChannelsConfig(t *testing.T) { return cfg } off := load(``) - if off.ApprovedChannelsEnabled() || off.ApprovedChannelsMax() != 128 || off.UsersDBPath() != filepath.Join(dir, "data", "users.db") { - t.Fatalf("no block: enabled=%v max=%d path=%q", off.ApprovedChannelsEnabled(), off.ApprovedChannelsMax(), off.UsersDBPath()) + if off.ApprovedChannelsEnabled() || off.ApprovedChannelsMax() != 128 || off.ApprovedChannelsURL() != "" { + t.Fatalf("no block: enabled=%v max=%d path=%q", off.ApprovedChannelsEnabled(), off.ApprovedChannelsMax(), off.ApprovedChannelsURL()) } on := load(`, "userManagement": {"enabled": true, "adminEmails": ["a@example.org"], "mail": {"provider": "brevo"}, "channelProposals": {"enabled": true}}`) if !on.ApprovedChannelsEnabled() || on.ApprovedChannelsMax() != 128 { @@ -354,9 +395,9 @@ func TestApprovedChannelsConfig(t *testing.T) { t.Errorf("%s resolved as on", um) } } - custom := load(`, "userManagement": {"enabled": true, "dbPath": "accounts/users.db", "channelProposals": {"enabled": true, "maxApproved": 7}}`) - if custom.UsersDBPath() != "accounts/users.db" || custom.ApprovedChannelsMax() != 7 { - t.Fatalf("custom: path=%q max=%d", custom.UsersDBPath(), custom.ApprovedChannelsMax()) + custom := load(`, "userManagement": {"enabled": true, "approvedChannelsDatabaseURL": "postgres://reader@localhost/accounts", "channelProposals": {"enabled": true, "maxApproved": 7}}`) + if custom.ApprovedChannelsURL() != "postgres://reader@localhost/accounts" || custom.ApprovedChannelsMax() != 7 { + t.Fatalf("custom: path=%q max=%d", custom.ApprovedChannelsURL(), custom.ApprovedChannelsMax()) } if load(`, "userManagement": {"enabled": true, "channelProposals": {"enabled": true, "maxApproved": -3}}`).ApprovedChannelsMax() != 128 { t.Fatal("negative maxApproved must fall back to 128") @@ -365,17 +406,51 @@ func TestApprovedChannelsConfig(t *testing.T) { // The startup line shows the absolute users.db path, so a relative path // resolved against another working directory than the server's is visible. -func TestApprovedChannelsSourceLogIsAbsolute(t *testing.T) { +func TestApprovedChannelsSourceLogRedactsCredentials(t *testing.T) { var buf bytes.Buffer prev := log.Writer() log.SetOutput(&buf) defer log.SetOutput(prev) - logApprovedChannelsSource("users.db") - want, err := filepath.Abs("users.db") + logApprovedChannelsSource("postgres://reader:secret-password@localhost/accounts") + if strings.Contains(buf.String(), "secret-password") || strings.Contains(buf.String(), "postgres://") { + t.Fatal("database URL leaked in log") + } +} + +type approvedFixtureDB struct{ owner, reader, role string } + +var approvedFixtures sync.Map + +func approvedFixture(t *testing.T, key string) approvedFixtureDB { + t.Helper() + key = t.Name() + "|" + key + if v, ok := approvedFixtures.Load(key); ok { + return v.(approvedFixtureDB) + } + owner := pgtest.NewSchema(t) + u, _ := url.Parse(owner) + schema := u.Query().Get("search_path") + role := schema + "_reader" + db, err := pgutil.Open(owner, false) if err != nil { t.Fatal(err) } - if !strings.Contains(buf.String(), "[proposals] reading approved channels from "+want+"\n") { - t.Fatalf("log line = %q, want the absolute path %q", buf.String(), want) + for _, stmt := range []string{`CREATE ROLE ` + role + ` LOGIN PASSWORD '` + role + `'`, `GRANT USAGE ON SCHEMA ` + schema + ` TO ` + role} { + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } } + u.User = url.UserPassword(role, role) + f := approvedFixtureDB{owner, u.String(), role} + approvedFixtures.Store(key, f) + t.Cleanup(func() { + approvedFixtures.Delete(key) + db.Exec(`DROP OWNED BY ` + role) + db.Exec(`DROP ROLE ` + role) + db.Close() + }) + return f +} +func newApprovedTestKeySet(t *testing.T, configured map[string]string, key string, max int) *channelKeySet { + return newChannelKeySet(configured, approvedFixture(t, key).reader, max) } diff --git a/cmd/ingestor/async_migration.go b/cmd/ingestor/async_migration.go index 32133db4e..431f12a8b 100644 --- a/cmd/ingestor/async_migration.go +++ b/cmd/ingestor/async_migration.go @@ -1,31 +1,7 @@ -// Async migration helper — runs schema/backfill work that may take minutes on -// large prod tables WITHOUT blocking ingestor startup. -// -// MIGRATION ANNOTATION CONVENTION (read this before touching migrations): -// -// Sync schema/data migrations (CREATE INDEX, ALTER TABLE, UPDATE ... WHERE) -// that run inline during OpenStore() block the ingestor from accepting -// packets until they finish. On an empty dev DB they return in milliseconds; -// at prod scale (1.9M+ observations, 80K+ adverts) they can pin the boot -// for minutes and trigger restart loops. This regression class has bitten us -// repeatedly (#791 resolved_path backfill, #1483 obs_observer_ts_idx_v1). -// -// ANY new CREATE INDEX / ALTER TABLE / data-rewrite migration MUST EITHER: -// 1. Run via Store.RunAsyncMigration(...) below (preferred for backfills -// and any work that may touch >1K rows). The migration is recorded as -// `pending_async` immediately, returns to the caller (boot proceeds), -// and completes in a goroutine. Status flips to `done` (or `failed` -// with an error message) when fn returns. -// 2. Carry the preflight annotation comment immediately above the -// migration block, e.g. -// // PREFLIGHT: async=true reason="" -// Use this for migrations that are genuinely cheap at any scale -// (e.g. ALTER TABLE ADD COLUMN, CREATE INDEX on a known-bounded -// table). The annotation is grepped by -// ~/.openclaw/skills/pr-preflight/scripts/check-async-migrations.sh -// — its absence on a touched migration block is a hard-fail gate. -// -// See MIGRATIONS.md in the repo root for the full policy and examples. +// Async data backfills resume through durable bookkeeping. Schema DDL belongs +// exclusively to the offline migration command; runtime callbacks must not +// create or alter tables/indexes. Long backfills use bounded transactions and +// yield between batches so live ingestion can continue. package main @@ -36,19 +12,16 @@ import ( "log" ) -// ensureAsyncMigrationsTable creates the bookkeeping table used by -// RunAsyncMigration / AsyncMigrationStatus. Idempotent. +// ensureAsyncMigrationsTable checks bootstrap-owned bookkeeping without DDL. func ensureAsyncMigrationsTable(db *sql.DB) error { - _, err := db.Exec(` - CREATE TABLE IF NOT EXISTS _async_migrations ( - name TEXT PRIMARY KEY, - status TEXT NOT NULL, -- pending_async | done | failed - started_at TEXT NOT NULL DEFAULT (datetime('now')), - ended_at TEXT, - error TEXT - ) - `) - return err + var present bool + if err := db.QueryRow(`SELECT to_regclass('_async_migrations') IS NOT NULL`).Scan(&present); err != nil { + return err + } + if !present { + return fmt.Errorf("missing migration bookkeeping; run the offline migration command") + } + return nil } // RunAsyncMigration registers `name` as a pending async migration and @@ -70,7 +43,7 @@ func (s *Store) RunAsyncMigration(ctx context.Context, name string, fn func(cont } var existing string - row := s.db.QueryRow(`SELECT status FROM _async_migrations WHERE name = ?`, name) + row := s.db.QueryRow(`SELECT status FROM _async_migrations WHERE name = $1`, name) switch err := row.Scan(&existing); err { case nil: if existing == "done" { @@ -79,13 +52,13 @@ func (s *Store) RunAsyncMigration(ctx context.Context, name string, fn func(cont // pending_async or failed → reset and retry. if _, err := s.db.Exec(` UPDATE _async_migrations - SET status = 'pending_async', started_at = datetime('now'), ended_at = NULL, error = NULL - WHERE name = ?`, name); err != nil { + SET status = 'pending_async', started_at = to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD HH24:MI:SS'), ended_at = NULL, error = NULL + WHERE name = $1`, name); err != nil { return fmt.Errorf("reset async migration %q: %w", name, err) } case sql.ErrNoRows: if _, err := s.db.Exec(` - INSERT INTO _async_migrations (name, status) VALUES (?, 'pending_async')`, + INSERT INTO _async_migrations (name, status) VALUES ($1, 'pending_async')`, name); err != nil { return fmt.Errorf("register async migration %q: %w", name, err) } @@ -105,8 +78,8 @@ func (s *Store) RunAsyncMigration(ctx context.Context, name string, fn func(cont if runErr != nil { if _, err := s.db.Exec(` UPDATE _async_migrations - SET status = 'failed', ended_at = datetime('now'), error = ? - WHERE name = ?`, runErr.Error(), name); err != nil { + SET status = 'failed', ended_at = to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD HH24:MI:SS'), error = $1 + WHERE name = $2`, runErr.Error(), name); err != nil { log.Printf("[async-migration] failed to record failure for %q: %v", name, err) } log.Printf("[async-migration] %q FAILED: %v", name, runErr) @@ -114,8 +87,8 @@ func (s *Store) RunAsyncMigration(ctx context.Context, name string, fn func(cont } if _, err := s.db.Exec(` UPDATE _async_migrations - SET status = 'done', ended_at = datetime('now'), error = NULL - WHERE name = ?`, name); err != nil { + SET status = 'done', ended_at = to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD HH24:MI:SS'), error = NULL + WHERE name = $1`, name); err != nil { log.Printf("[async-migration] failed to mark %q done: %v", name, err) return } @@ -136,7 +109,7 @@ func (s *Store) AsyncMigrationStatus(name string) (string, error) { return "", err } var status string - err := s.db.QueryRow(`SELECT status FROM _async_migrations WHERE name = ?`, name).Scan(&status) + err := s.db.QueryRow(`SELECT status FROM _async_migrations WHERE name = $1`, name).Scan(&status) return status, err } diff --git a/cmd/ingestor/async_migration_test.go b/cmd/ingestor/async_migration_test.go index 2a8a942d2..d7abd672c 100644 --- a/cmd/ingestor/async_migration_test.go +++ b/cmd/ingestor/async_migration_test.go @@ -110,7 +110,7 @@ func TestRunAsyncMigration_PanicCapture(t *testing.T) { } var errMsg sql.NullString - if err := s.db.QueryRow(`SELECT error FROM _async_migrations WHERE name = ?`, name).Scan(&errMsg); err != nil { + if err := s.db.QueryRow(`SELECT error FROM _async_migrations WHERE name = $1`, name).Scan(&errMsg); err != nil { t.Fatalf("error column lookup: %v", err) } if !errMsg.Valid || errMsg.String == "" { @@ -160,7 +160,7 @@ func TestRunAsyncMigration_RestartSafetyFailedIsRetried(t *testing.T) { if err := ensureAsyncMigrationsTable(s.db); err != nil { t.Fatalf("ensure table: %v", err) } - if _, err := s.db.Exec(`INSERT INTO _async_migrations (name, status, error) VALUES (?, 'failed', 'simulated prior crash')`, name); err != nil { + if _, err := s.db.Exec(`INSERT INTO _async_migrations (name, status, error) VALUES ($1, 'failed', 'simulated prior crash')`, name); err != nil { t.Fatalf("seed failed row: %v", err) } @@ -181,7 +181,7 @@ func TestRunAsyncMigration_RestartSafetyFailedIsRetried(t *testing.T) { // And the error column must be cleared on success. var errCol sql.NullString - if err := s.db.QueryRow(`SELECT error FROM _async_migrations WHERE name = ?`, name).Scan(&errCol); err != nil { + if err := s.db.QueryRow(`SELECT error FROM _async_migrations WHERE name = $1`, name).Scan(&errCol); err != nil { t.Fatalf("error col: %v", err) } if errCol.Valid && errCol.String != "" { @@ -200,7 +200,7 @@ func TestRunAsyncMigration_RestartSafetyPendingIsRetried(t *testing.T) { if err := ensureAsyncMigrationsTable(s.db); err != nil { t.Fatalf("ensure table: %v", err) } - if _, err := s.db.Exec(`INSERT INTO _async_migrations (name, status) VALUES (?, 'pending_async')`, name); err != nil { + if _, err := s.db.Exec(`INSERT INTO _async_migrations (name, status) VALUES ($1, 'pending_async')`, name); err != nil { t.Fatalf("seed pending row: %v", err) } @@ -243,7 +243,7 @@ func TestRunAsyncMigration_FnErrorRecorded(t *testing.T) { } var errCol sql.NullString - if err := s.db.QueryRow(`SELECT error FROM _async_migrations WHERE name = ?`, name).Scan(&errCol); err != nil { + if err := s.db.QueryRow(`SELECT error FROM _async_migrations WHERE name = $1`, name).Scan(&errCol); err != nil { t.Fatalf("error col: %v", err) } if !errCol.Valid || errCol.String == "" { diff --git a/cmd/ingestor/client_reception.go b/cmd/ingestor/client_reception.go index f3e73609d..611ed11cb 100644 --- a/cmd/ingestor/client_reception.go +++ b/cmd/ingestor/client_reception.go @@ -128,7 +128,7 @@ func (s *Store) UpsertClientObserver(pubkey, name, ts string) error { return nil } _, err := s.db.Exec(` - INSERT INTO client_observers (pubkey, name, last_seen) VALUES (?,?,?) + INSERT INTO client_observers (pubkey, name, last_seen) VALUES ($1,$2,$3) ON CONFLICT(pubkey) DO UPDATE SET name = excluded.name, last_seen = excluded.last_seen`, strings.ToLower(pubkey), name, ts) return err @@ -258,7 +258,7 @@ func (s *Store) InsertClientReception(r *ClientReception) (bool, error) { res, err := s.db.Exec(` INSERT INTO client_receptions (rx_pubkey, heard_key, heard_keylen, rssi, snr, lat, lon, pos_acc_m, rx_at, ingested_at, src) - VALUES (?,?,?,?,?,?,?,?,?,?,?) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) ON CONFLICT(rx_pubkey, heard_key, rx_at) DO NOTHING`, r.RxPubkey, r.HeardKey, r.HeardKeyLen, r.RSSI, r.SNR, r.Lat, r.Lon, r.PosAccM, r.RxAt, r.IngestedAt, r.Src) if err != nil { @@ -299,7 +299,7 @@ func (s *Store) InsertClientRfSample(o *ClientRfSample) (bool, error) { uptime_secs, battery_mv, queue_len, errors, noise_floor, last_rssi, last_snr, tx_air_secs, rx_air_secs, recv, sent, flood_rx, direct_rx, flood_tx, direct_tx, recv_errors) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23) ON CONFLICT(rx_pubkey, sampled_at) DO NOTHING`, o.RxPubkey, o.SampledAt, o.IngestedAt, o.Lat, o.Lon, o.PosAccM, boolToInt(o.Stationary), o.UptimeSecs, o.BatteryMV, o.QueueLen, o.Errors, o.NoiseFloor, o.LastRSSI, @@ -418,7 +418,7 @@ func (s *Store) InsertClientRxObservation(o *ClientRxObservation) (bool, error) (rx_pubkey, rx_at, ingested_at, pkt_hash, route_type, payload_type, code1, code2, scope_name, hash_size, hop_count, path_json, forwarder, snr, rssi, lat, lon, pos_acc_m) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18) ON CONFLICT(rx_pubkey, pkt_hash, rx_at) DO NOTHING`, o.RxPubkey, o.RxAt, o.IngestedAt, o.PktHash, o.RouteType, o.PayloadType, o.Code1, o.Code2, o.ScopeName, o.HashSize, o.HopCount, o.PathJSON, o.Forwarder, @@ -458,7 +458,7 @@ func (s *Store) InsertClientDeclaredRegions(o *ClientDeclaredRegions) (bool, err res, err := s.db.Exec(` INSERT INTO node_declared_regions (target, rx_pubkey, observed_at, ingested_at, regions_csv, truncated, lat, lon, pos_acc_m, repeater_clock) - VALUES (?,?,?,?,?,?,?,?,?,?) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) ON CONFLICT(target, rx_pubkey, observed_at) DO NOTHING`, o.Target, o.RxPubkey, o.ObservedAt, o.IngestedAt, o.RegionsCSV, boolToInt(o.Truncated), o.Lat, o.Lon, o.PosAccM, o.RepeaterClock) if err != nil { @@ -478,7 +478,7 @@ func (s *Store) CurrentDeclaredRegions(target string) (*ClientDeclaredRegions, e row := s.db.QueryRow(` SELECT target, rx_pubkey, observed_at, ingested_at, regions_csv, truncated, lat, lon, pos_acc_m, repeater_clock FROM node_declared_regions - WHERE target = ? + WHERE target = $1 ORDER BY observed_at DESC LIMIT 1`, target) var o ClientDeclaredRegions var truncated int diff --git a/cmd/ingestor/client_reception_test.go b/cmd/ingestor/client_reception_test.go index c29662bf4..a57724a96 100644 --- a/cmd/ingestor/client_reception_test.go +++ b/cmd/ingestor/client_reception_test.go @@ -54,16 +54,18 @@ func TestPruneOldClientReceptions(t *testing.T) { // this is a regression guard. func TestPruneClientRxObservationsUsesIndex(t *testing.T) { s := newTestStore(t) - rows, err := s.db.Query(`EXPLAIN QUERY PLAN DELETE FROM client_rx_observations WHERE rx_at < ?`, "2026-01-01T00:00:00Z") + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } + rows, err := s.db.Query(`EXPLAIN (COSTS OFF) DELETE FROM client_rx_observations WHERE rx_at < $1`, "2026-01-01T00:00:00Z") if err != nil { t.Fatal(err) } defer rows.Close() plan := "" for rows.Next() { - var id, parent, notused int var detail string - if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + if err := rows.Scan(&detail); err != nil { t.Fatal(err) } plan += detail + "\n" @@ -125,12 +127,12 @@ func TestPruneOldClientRxObservations(t *testing.T) { t.Fatalf("expected 2 observations remaining (recent + boundary), got %d", remaining) } var recentSurvived int - s.db.QueryRow(`SELECT COUNT(*) FROM client_rx_observations WHERE pkt_hash = ?`, "hash-recent").Scan(&recentSurvived) + s.db.QueryRow(`SELECT COUNT(*) FROM client_rx_observations WHERE pkt_hash = $1`, "hash-recent").Scan(&recentSurvived) if recentSurvived != 1 { t.Fatalf("the recent row must survive the prune; got %d", recentSurvived) } var boundarySurvived int - s.db.QueryRow(`SELECT COUNT(*) FROM client_rx_observations WHERE pkt_hash = ?`, "hash-boundary").Scan(&boundarySurvived) + s.db.QueryRow(`SELECT COUNT(*) FROM client_rx_observations WHERE pkt_hash = $1`, "hash-boundary").Scan(&boundarySurvived) if boundarySurvived != 1 { t.Fatalf("boundary row (500ms after cutoff instant) must survive; an RFC3339 (no-ms) cutoff would wrongly delete it because '.' < 'Z' lexicographically — got %d", boundarySurvived) } @@ -139,7 +141,7 @@ func TestPruneOldClientRxObservations(t *testing.T) { func TestClientReceptionsTableExists(t *testing.T) { s := newTestStore(t) cols := map[string]bool{} - rows, err := s.db.Query(`PRAGMA table_info(client_receptions)`) + rows, err := s.db.Query(`SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='client_receptions' ORDER BY ordinal_position`) if err != nil { t.Fatalf("PRAGMA failed: %v", err) } @@ -168,12 +170,13 @@ func crI(i int) *int { return &i } // per-node coverage query (sargable heard_key IN-list + bbox, mirroring // cmd/server coverageHeardKeyCandidates) seeks the heard_key composite index // rather than scanning the table. Without idx_client_recept_heard_geo the plan -// is "SCAN client_receptions". +// is "Seq Scan on client_receptions". func TestClientReceptionsCoverageQueryUsesIndex(t *testing.T) { s := newTestStore(t) - q := `EXPLAIN QUERY PLAN SELECT lat, lon, snr, rssi, heard_key, rx_at + q := `EXPLAIN (COSTS OFF) SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions - WHERE heard_key IN (?,?,?) AND lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?` + WHERE heard_key IN ($1,$2,$3) AND lat BETWEEN $4 AND $5 AND lon BETWEEN $6 AND $7` + s.db.Exec(`SET enable_seqscan=off`) rows, err := s.db.Query(q, "aabbccddeeff00112233", "aabbcc", "aabb", 50.0, 52.0, 3.0, 4.0) if err != nil { t.Fatal(err) @@ -181,17 +184,16 @@ func TestClientReceptionsCoverageQueryUsesIndex(t *testing.T) { defer rows.Close() plan := "" for rows.Next() { - var id, parent, notused int var detail string - if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + if err := rows.Scan(&detail); err != nil { t.Fatal(err) } plan += detail + "\n" } - if !strings.Contains(plan, "USING INDEX idx_client_recept") { + if !strings.Contains(plan, "idx_client_recept") { t.Fatalf("coverage query should use a client_recept index, plan was:\n%s", plan) } - if strings.Contains(plan, "SCAN client_receptions") { + if strings.Contains(plan, "Seq Scan on client_receptions") { t.Fatalf("coverage query should not full-scan, plan was:\n%s", plan) } } @@ -201,16 +203,18 @@ func TestClientReceptionsCoverageQueryUsesIndex(t *testing.T) { // index rather than full-scanning under the writer lock (polish review). func TestClientReceptionsRetentionUsesRxAtIndex(t *testing.T) { s := newTestStore(t) - rows, err := s.db.Query(`EXPLAIN QUERY PLAN DELETE FROM client_receptions WHERE rx_at < ?`, "2026-01-01T00:00:00Z") + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } + rows, err := s.db.Query(`EXPLAIN (COSTS OFF) DELETE FROM client_receptions WHERE rx_at < $1`, "2026-01-01T00:00:00Z") if err != nil { t.Fatal(err) } defer rows.Close() plan := "" for rows.Next() { - var id, parent, notused int var detail string - if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + if err := rows.Scan(&detail); err != nil { t.Fatal(err) } plan += detail + "\n" @@ -230,22 +234,24 @@ func TestClientReceptionsRetentionUsesRxAtIndex(t *testing.T) { // index-backed). func TestRxLeaderboardQueryIsIndexBacked(t *testing.T) { s := newTestStore(t) - rows, err := s.db.Query(`EXPLAIN QUERY PLAN + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } + rows, err := s.db.Query(`EXPLAIN (COSTS OFF) SELECT cr.rx_pubkey, COUNT(*), COUNT(DISTINCT cr.heard_key) FROM client_receptions cr - WHERE cr.rx_at >= ? + WHERE cr.rx_at >= $1 GROUP BY cr.rx_pubkey ORDER BY COUNT(*) DESC - LIMIT ?`, "2026-01-01T00:00:00Z", 100) + LIMIT $2`, "2026-01-01T00:00:00Z", 100) if err != nil { t.Fatal(err) } defer rows.Close() plan := "" for rows.Next() { - var id, parent, notused int var detail string - if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + if err := rows.Scan(&detail); err != nil { t.Fatal(err) } plan += detail + "\n" @@ -254,7 +260,7 @@ func TestRxLeaderboardQueryIsIndexBacked(t *testing.T) { // The concern is a bare table-heap scan, not which specific index wins. The // plan must stay index-backed (covering or search) — a regression to a bare // "SCAN cr" without an index fails here. - if !strings.Contains(plan, "INDEX") { + if !strings.Contains(strings.ToLower(plan), "index") { t.Fatalf("leaderboard SELECT must stay index-backed (no full table-heap scan), plan was:\n%s", plan) } } @@ -380,7 +386,7 @@ func TestHandleClientPacketRelayedAdvertWritesReception(t *testing.T) { handleClientPacket(s, &Config{}, "test", testCompanionPK, msg, nil, nil) var obsName string - s.db.QueryRow(`SELECT name FROM client_observers WHERE pubkey=?`, testCompanionPK).Scan(&obsName) + s.db.QueryRow(`SELECT name FROM client_observers WHERE pubkey=$1`, testCompanionPK).Scan(&obsName) if obsName != "MyMob" { t.Fatalf("expected client_observers name 'MyMob', got %q", obsName) } @@ -390,7 +396,7 @@ func TestHandleClientPacketRelayedAdvertWritesReception(t *testing.T) { // The 0-hop advert→full-pubkey branch is covered by TestDeriveHeardKey. var n, keylen int var src string - if err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(MAX(heard_keylen),0), COALESCE(MAX(src),'') FROM client_receptions WHERE rx_pubkey=?`, testCompanionPK).Scan(&n, &keylen, &src); err != nil { + if err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(MAX(heard_keylen),0), COALESCE(MAX(src),'') FROM client_receptions WHERE rx_pubkey=$1`, testCompanionPK).Scan(&n, &keylen, &src); err != nil { t.Fatal(err) } if n != 1 || keylen < 2 || src != "rxlog" { @@ -401,7 +407,7 @@ func TestHandleClientPacketRelayedAdvertWritesReception(t *testing.T) { const companion2 = "b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3" handleClientPacket(s, &Config{}, "test", companion2, map[string]interface{}{"raw": advertHex, "direction": "rx"}, nil, nil) var n2 int - s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions WHERE rx_pubkey=?`, companion2).Scan(&n2) + s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions WHERE rx_pubkey=$1`, companion2).Scan(&n2) if n2 != 0 { t.Fatalf("packet without gps must be dropped, got %d rows", n2) } @@ -432,7 +438,7 @@ func TestHandleClientPacketZeroHopAdvertWritesReception(t *testing.T) { var keylen int var snr sql.NullFloat64 var lat, lon float64 - if err := s.db.QueryRow(`SELECT heard_key, heard_keylen, src, snr, lat, lon FROM client_receptions WHERE rx_pubkey=?`, testCompanionPK). + if err := s.db.QueryRow(`SELECT heard_key, heard_keylen, src, snr, lat, lon FROM client_receptions WHERE rx_pubkey=$1`, testCompanionPK). Scan(&heardKey, &keylen, &src, &snr, &lat, &lon); err != nil { t.Fatalf("expected a 0-hop advert reception: %v", err) } @@ -560,7 +566,7 @@ func TestHandleClientPacketDiscoverRespWritesReception(t *testing.T) { var heardKey, src string var keylen int - if err := s.db.QueryRow(`SELECT heard_key, heard_keylen, src FROM client_receptions WHERE rx_pubkey=?`, "aa11"). + if err := s.db.QueryRow(`SELECT heard_key, heard_keylen, src FROM client_receptions WHERE rx_pubkey=$1`, "aa11"). Scan(&heardKey, &keylen, &src); err != nil { t.Fatalf("expected a discover-response reception: %v", err) } @@ -709,7 +715,7 @@ func TestClientObservationScopeNameFromTransportCode(t *testing.T) { // same content dedups across scopes), so raw and raw2 below share one // pkt_hash — distinguish the two inserted rows by rx_at, not pkt_hash. var code1, code2, scopeName sql.NullString - if err := s.db.QueryRow(`SELECT code1, code2, scope_name FROM client_rx_observations WHERE rx_at = ?`, ts1). + if err := s.db.QueryRow(`SELECT code1, code2, scope_name FROM client_rx_observations WHERE rx_at = $1`, ts1). Scan(&code1, &code2, &scopeName); err != nil { t.Fatal(err) } @@ -735,7 +741,7 @@ func TestClientObservationScopeNameFromTransportCode(t *testing.T) { handleClientPacket(s, cfgWithObservations(), "test", "aa11", msg2, nil, regionKeySetFromKeys(regionKeys)) var code1b, scopeNameB sql.NullString - if err := s.db.QueryRow(`SELECT code1, scope_name FROM client_rx_observations WHERE rx_at = ?`, ts2). + if err := s.db.QueryRow(`SELECT code1, scope_name FROM client_rx_observations WHERE rx_at = $1`, ts2). Scan(&code1b, &scopeNameB); err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/client_regions_test.go b/cmd/ingestor/client_regions_test.go index f8ee7e1b0..5d7485a9b 100644 --- a/cmd/ingestor/client_regions_test.go +++ b/cmd/ingestor/client_regions_test.go @@ -442,7 +442,7 @@ func TestDeclaredRegionsDifferentReportersPersistBoth(t *testing.T) { } var n int - if err := s.db.QueryRow(`SELECT COUNT(*) FROM node_declared_regions WHERE target = ? AND observed_at = ?`, "bb11", at).Scan(&n); err != nil { + if err := s.db.QueryRow(`SELECT COUNT(*) FROM node_declared_regions WHERE target = $1 AND observed_at = $2`, "bb11", at).Scan(&n); err != nil { t.Fatal(err) } if n != 2 { @@ -480,16 +480,18 @@ func TestCurrentDeclaredRegionsNormalizesTargetCase(t *testing.T) { // index seek on idx_ndr_prune rather than a full table scan. func TestPruneClientDeclaredRegionsUsesIndex(t *testing.T) { s := newTestStore(t) - rows, err := s.db.Query(`EXPLAIN QUERY PLAN DELETE FROM node_declared_regions WHERE observed_at < ?`, "2026-01-01T00:00:00Z") + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } + rows, err := s.db.Query(`EXPLAIN (COSTS OFF) DELETE FROM node_declared_regions WHERE observed_at < $1`, "2026-01-01T00:00:00Z") if err != nil { t.Fatal(err) } defer rows.Close() plan := "" for rows.Next() { - var id, parent, notused int var detail string - if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + if err := rows.Scan(&detail); err != nil { t.Fatal(err) } plan += detail + "\n" @@ -592,7 +594,7 @@ func TestPruneOldClientDeclaredRegionsAtMillisecondBoundary(t *testing.T) { t.Fatalf("expected 1 row pruned (1ms before the cutoff), got %d", n) } var boundarySurvived int - s.db.QueryRow(`SELECT COUNT(*) FROM node_declared_regions WHERE observed_at = ?`, boundary).Scan(&boundarySurvived) + s.db.QueryRow(`SELECT COUNT(*) FROM node_declared_regions WHERE observed_at = $1`, boundary).Scan(&boundarySurvived) if boundarySurvived != 1 { t.Fatalf("boundary row (1ms after a cutoff with a non-zero millisecond component) must survive; an RFC3339 (no-ms) cutoff would wrongly delete it because '.' < 'Z' lexicographically — got %d", boundarySurvived) } diff --git a/cmd/ingestor/client_rf_sample.go b/cmd/ingestor/client_rf_sample.go index 46a8f5a26..1ecc4c300 100644 --- a/cmd/ingestor/client_rf_sample.go +++ b/cmd/ingestor/client_rf_sample.go @@ -134,7 +134,7 @@ func (s *Store) ClientRfDeltas(rxPubkey, from, to string) ([]ClientRfDelta, erro LAG(recv) OVER w AS prev_recv, LAG(recv_errors) OVER w AS prev_errs FROM client_rf_samples - WHERE rx_pubkey = ? AND sampled_at >= ? AND sampled_at < ? + WHERE rx_pubkey = $1 AND sampled_at >= $2 AND sampled_at < $3 WINDOW w AS (ORDER BY sampled_at) ORDER BY sampled_at`, rxPubkey, from, to) if err != nil { diff --git a/cmd/ingestor/config.go b/cmd/ingestor/config.go index 7bb6932b5..7b8a26379 100644 --- a/cmd/ingestor/config.go +++ b/cmd/ingestor/config.go @@ -6,7 +6,6 @@ import ( "fmt" "log" "os" - "path/filepath" "strings" "sync" "time" @@ -46,8 +45,10 @@ type MQTTLegacy struct { Topic string `json:"topic"` } -// Config holds the ingestor configuration, compatible with the Node.js config.json format. +// Config holds the shared Go-service configuration and ingestor-only settings. type Config struct { + DatabaseURL string `json:"databaseURL"` + StateDir string `json:"stateDir"` DBPath string `json:"dbPath"` MQTT *MQTTLegacy `json:"mqtt,omitempty"` MQTTSources []MQTTSource `json:"mqttSources,omitempty"` @@ -107,7 +108,7 @@ type Config struct { NeighborEdgesMaxAgeDays int `json:"neighborEdgesMaxAgeDays,omitempty"` // IngestBufferSize caps the in-memory queue (number of MQTT messages) held - // while the single SQLite writer is blocked by startup migrations/prunes + // while the single serialized writer is blocked by startup migrations/prunes // (#1608). Received messages are drained once the write path is ready. // 0 / unset => default. Bounded memory. IngestBufferSize int `json:"ingestBufferSize,omitempty"` @@ -305,10 +306,10 @@ type RuntimeConfig struct { MaxMemoryMB int `json:"maxMemoryMB"` } -// DBConfig is the shared SQLite vacuum/maintenance config (#919, #921). +// DBConfig shares startup-load settings and accepts obsolete SQLite config keys. type DBConfig = dbconfig.DBConfig -// IncrementalVacuumPages returns the configured pages per vacuum or 1024 default. +// IncrementalVacuumPages preserves legacy config decoding; PostgreSQL does not use it. func (c *Config) IncrementalVacuumPages() int { if c.DB != nil && c.DB.IncrementalVacuumPages > 0 { return c.DB.IncrementalVacuumPages @@ -316,16 +317,8 @@ func (c *Config) IncrementalVacuumPages() int { return 1024 } -// AnalysisLimit returns the per-index row cap for the planner stats refresh -// (#2058). A negative setting disables the refresh; zero means unset, matching -// IncrementalVacuumPages above. -// -// 10000 rather than the 400 SQLite's documentation offers: measured on the -// 9.4 GB staging database, 400 and 1000 leave the channel-query plan exactly as -// it was, 10000 produces the same plan as an unbounded ANALYZE, and it costs 2.0s -// against that ANALYZE's 242.9s, both timed warm. The full ladder, and the cold -// figure that matters at startup, are in Store.RefreshPlannerStats and -// Store.EnsurePlannerStats. +// AnalysisLimit preserves the old SQLite setting for configuration compatibility. +// PostgreSQL owns planner statistics; runtime maintenance does not use this limit. func (c *Config) AnalysisLimit() int { if c.DB != nil && c.DB.AnalysisLimit != 0 { return c.DB.AnalysisLimit @@ -541,10 +534,26 @@ func LoadConfig(path string) (*Config, error) { }} } - // Default DB path - if cfg.DBPath == "" { - cfg.DBPath = "data/meshcore.db" + if v := os.Getenv("CORESCOPE_DATABASE_URL"); v != "" { + cfg.DatabaseURL = v + } + if v := os.Getenv("CORESCOPE_WRITER_DATABASE_URL"); v != "" { + cfg.DatabaseURL = v + } + if v := os.Getenv("CORESCOPE_STATE_DIR"); v != "" { + cfg.StateDir = v + } + if cfg.StateDir == "" { + cfg.StateDir = "data" } + if v := os.Getenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL"); v != "" { + if cfg.UserManagement == nil { + cfg.UserManagement = &UserManagementConfig{} + } + cfg.UserManagement.ApprovedChannelsDatabaseURL = v + } + // Legacy DBPath is never interpreted as a connection string. + // Default DB path // Normalize: convert legacy single mqtt config to mqttSources if len(cfg.MQTTSources) == 0 && cfg.MQTT != nil && cfg.MQTT.Broker != "" { @@ -621,9 +630,10 @@ const autoRegionKeysDefaultRefreshMinutes = 15 // UserManagementConfig is the ingestor's view of userManagement. type UserManagementConfig struct { - Enabled bool `json:"enabled"` - DBPath string `json:"dbPath,omitempty"` - ChannelProposals *ChannelProposalsConfig `json:"channelProposals,omitempty"` + Enabled bool `json:"enabled"` + ApprovedChannelsDatabaseURL string `json:"approvedChannelsDatabaseURL,omitempty"` + DBPath string `json:"dbPath,omitempty"` + ChannelProposals *ChannelProposalsConfig `json:"channelProposals,omitempty"` } // ChannelProposalsConfig is the ingestor's view of @@ -652,15 +662,12 @@ func (c *Config) ApprovedChannelsMax() int { return u.ChannelProposals.MaxApproved } -// UsersDBPath resolves users.db with the server's rule: userManagement.dbPath, -// else users.db next to the analyzer database. -func (c *Config) UsersDBPath() string { - if c.UserManagement != nil { - if p := strings.TrimSpace(c.UserManagement.DBPath); p != "" { - return p - } +// ApprovedChannelsURL is a separate restricted account-reader credential. +func (c *Config) ApprovedChannelsURL() string { + if c.UserManagement == nil { + return "" } - return filepath.Join(filepath.Dir(c.DBPath), "users.db") + return strings.TrimSpace(c.UserManagement.ApprovedChannelsDatabaseURL) } // AutoRegionKeysEnabled reports whether region keys may be derived from diff --git a/cmd/ingestor/config_test.go b/cmd/ingestor/config_test.go index e5d79e3dc..5d9a37144 100644 --- a/cmd/ingestor/config_test.go +++ b/cmd/ingestor/config_test.go @@ -40,8 +40,8 @@ func TestLoadConfigMissingFile(t *testing.T) { if err != nil { t.Fatalf("missing config should not error (zero-config mode), got: %v", err) } - if cfg.DBPath != "data/meshcore.db" { - t.Errorf("dbPath=%s, want data/meshcore.db", cfg.DBPath) + if cfg.DBPath != "" || cfg.DatabaseURL != "" || cfg.StateDir != "data" { + t.Errorf("unexpected database/state defaults") } // Should default to localhost MQTT if len(cfg.MQTTSources) != 1 { @@ -193,8 +193,8 @@ func TestLoadConfigDefaultDBPath(t *testing.T) { if err != nil { t.Fatal(err) } - if cfg.DBPath != "data/meshcore.db" { - t.Errorf("dbPath=%s, want data/meshcore.db", cfg.DBPath) + if cfg.DBPath != "" || cfg.DatabaseURL != "" || cfg.StateDir != "data" { + t.Errorf("unexpected database/state defaults") } } diff --git a/cmd/ingestor/coverage_boost_test.go b/cmd/ingestor/coverage_boost_test.go index fc2daa889..fb77d5e56 100644 --- a/cmd/ingestor/coverage_boost_test.go +++ b/cmd/ingestor/coverage_boost_test.go @@ -635,7 +635,9 @@ func TestDecryptChannelMessageBadCiphertextHex(t *testing.T) { func TestCheckpointDoesNotPanic(t *testing.T) { store := newTestStore(t) - store.Checkpoint() + if _, err := store.db.Exec(`CHECKPOINT`); err == nil { + t.Fatal("runtime role may force server checkpoints") + } } func TestLogStatsDoesNotPanic(t *testing.T) { @@ -762,7 +764,7 @@ func TestOpenStoreInvalidPath(t *testing.T) { t.Fatalf("setup: %v", err) } f.Close() - _, err = OpenStore(filepath.Join(f.Name(), "db.sqlite")) + _, err = openPostgresTestStore(t, filepath.Join(f.Name(), "db.sqlite")) if err == nil { t.Error("should error on impossible path") } @@ -1025,28 +1027,28 @@ func TestApplySchemaMigrationsOnFreshDB(t *testing.T) { // Check that migrations were recorded var count int - if err := store.db.QueryRow("SELECT COUNT(*) FROM _migrations").Scan(&count); err != nil { + if err := store.db.QueryRow("SELECT COUNT(*) FROM corescope_schema WHERE kind='telemetry' AND ready=true").Scan(&count); err != nil { t.Fatal(err) } - if count < 3 { - t.Errorf("expected at least 3 migrations recorded, got %d", count) + if count != 1 { + t.Errorf("expected one ready telemetry schema, got %d", count) } // Check observations table exists with dedup index var tblName string - err := store.db.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name='observations'").Scan(&tblName) + err := store.db.QueryRow("SELECT table_name FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='observations'").Scan(&tblName) if err != nil { t.Error("observations table should exist") } // Check inactive_nodes table exists - err = store.db.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name='inactive_nodes'").Scan(&tblName) + err = store.db.QueryRow("SELECT table_name FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='inactive_nodes'").Scan(&tblName) if err != nil { t.Error("inactive_nodes table should exist") } // Check packets_v view exists - err = store.db.QueryRow("SELECT name FROM sqlite_master WHERE type='view' AND name='packets_v'").Scan(&tblName) + err = store.db.QueryRow("SELECT table_name FROM information_schema.views WHERE table_schema=current_schema() AND table_name='packets_v'").Scan(&tblName) if err != nil { t.Error("packets_v view should exist") } @@ -1058,14 +1060,14 @@ func TestOpenStoreExistingDB(t *testing.T) { dbPath := dir + "/test.db" // Open and close - s1, err := OpenStore(dbPath) + s1, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } s1.Close() // Re-open — should skip migrations (already applied) - s2, err := OpenStore(dbPath) + s2, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } @@ -1078,7 +1080,7 @@ func TestMoveStaleNodesReplace(t *testing.T) { pk := "stale_node_replace_0000000000000000000000000001" // Insert into inactive_nodes first - if _, err := store.db.Exec("INSERT INTO inactive_nodes (public_key, name, role, last_seen, first_seen) VALUES (?, 'Old', 'companion', '2019-01-01T00:00:00Z', '2019-01-01T00:00:00Z')", pk); err != nil { + if _, err := store.db.Exec("INSERT INTO inactive_nodes (public_key, name, role, last_seen, first_seen) VALUES ($1, 'Old', 'companion', '2019-01-01T00:00:00Z', '2019-01-01T00:00:00Z')", pk); err != nil { t.Fatal(err) } @@ -1095,7 +1097,7 @@ func TestMoveStaleNodesReplace(t *testing.T) { // Should have replaced the inactive node var name string - if err := store.db.QueryRow("SELECT name FROM inactive_nodes WHERE public_key = ?", pk).Scan(&name); err != nil { + if err := store.db.QueryRow("SELECT name FROM inactive_nodes WHERE public_key = $1", pk).Scan(&name); err != nil { t.Fatal(err) } if name != "StaleNode" { @@ -1165,7 +1167,7 @@ func TestRemoveStaleObservers(t *testing.T) { } // Override last_seen to 30 days ago cutoff := time.Now().UTC().AddDate(0, 0, -30).Format(time.RFC3339) - _, err = store.db.Exec("UPDATE observers SET last_seen = ? WHERE id = ?", cutoff, "obs-old") + _, err = store.db.Exec("UPDATE observers SET last_seen = $1 WHERE id = $2", cutoff, "obs-old") if err != nil { t.Fatal(err) } @@ -1195,7 +1197,7 @@ func TestRemoveStaleObservers(t *testing.T) { // Check that the old observer is marked inactive var inactive int - if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-old").Scan(&inactive); err != nil { + if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = $1", "obs-old").Scan(&inactive); err != nil { t.Fatal(err) } if inactive != 1 { @@ -1204,7 +1206,7 @@ func TestRemoveStaleObservers(t *testing.T) { // Check that the recent observer is still active var newInactive int - if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-new").Scan(&newInactive); err != nil { + if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = $1", "obs-new").Scan(&newInactive); err != nil { t.Fatal(err) } if newInactive != 0 { @@ -1233,7 +1235,7 @@ func TestRemoveStaleObserversKeepForever(t *testing.T) { t.Fatal(err) } cutoff := time.Now().UTC().AddDate(0, 0, -365).Format(time.RFC3339) - _, err = store.db.Exec("UPDATE observers SET last_seen = ? WHERE id = ?", cutoff, "obs-ancient") + _, err = store.db.Exec("UPDATE observers SET last_seen = $1 WHERE id = $2", cutoff, "obs-ancient") if err != nil { t.Fatal(err) } @@ -1257,7 +1259,7 @@ func TestRemoveStaleObserversKeepForever(t *testing.T) { // Observer should NOT be marked inactive var inactive int - if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-ancient").Scan(&inactive); err != nil { + if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = $1", "obs-ancient").Scan(&inactive); err != nil { t.Fatal(err) } if inactive != 0 { @@ -1274,7 +1276,7 @@ func TestRemoveStaleObserversReactivation(t *testing.T) { t.Fatal(err) } cutoff := time.Now().UTC().AddDate(0, 0, -30).Format(time.RFC3339) - _, err = store.db.Exec("UPDATE observers SET last_seen = ? WHERE id = ?", cutoff, "obs-test") + _, err = store.db.Exec("UPDATE observers SET last_seen = $1 WHERE id = $2", cutoff, "obs-test") if err != nil { t.Fatal(err) } @@ -1289,7 +1291,7 @@ func TestRemoveStaleObserversReactivation(t *testing.T) { // Verify it's inactive var inactive int - if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-test").Scan(&inactive); err != nil { + if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = $1", "obs-test").Scan(&inactive); err != nil { t.Fatal(err) } if inactive != 1 { @@ -1302,7 +1304,7 @@ func TestRemoveStaleObserversReactivation(t *testing.T) { t.Fatal(err) } - if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-test").Scan(&inactive); err != nil { + if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = $1", "obs-test").Scan(&inactive); err != nil { t.Fatal(err) } if inactive != 0 { diff --git a/cmd/ingestor/coverage_gate_test.go b/cmd/ingestor/coverage_gate_test.go index 0ba2f3cbb..4ef5eade2 100644 --- a/cmd/ingestor/coverage_gate_test.go +++ b/cmd/ingestor/coverage_gate_test.go @@ -91,7 +91,7 @@ func TestClientRxCoverageGateOffDoesNotFallThroughToObserverPath(t *testing.T) { t.Fatalf("feature OFF: expected 0 client_receptions rows, got %d", n) } var observerRows int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = ?`, testCompanionPK).Scan(&observerRows); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = $1`, testCompanionPK).Scan(&observerRows); err != nil { t.Fatal(err) } if observerRows != 0 { diff --git a/cmd/ingestor/coverage_query_bench_test.go b/cmd/ingestor/coverage_query_bench_test.go index 078ec8643..e8a928816 100644 --- a/cmd/ingestor/coverage_query_bench_test.go +++ b/cmd/ingestor/coverage_query_bench_test.go @@ -12,9 +12,9 @@ import ( // match on the heard node. const coverageBenchSQL = `SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions - WHERE lat BETWEEN ? AND ? AND lon BETWEEN ? AND ? - AND ( (heard_keylen = 32 AND heard_key = ?) - OR (heard_keylen IN (2,3) AND substr(?, 1, heard_keylen*2) = heard_key) )` + WHERE lat BETWEEN $1 AND $2 AND lon BETWEEN $3 AND $4 + AND ( (heard_keylen = 32 AND heard_key = $5) + OR (heard_keylen IN (2,3) AND substr($6, 1, heard_keylen*2) = heard_key) )` // BenchmarkCoverageQuery seeds ~1M receptions across a metro-area bbox and times // the coverage query with the indexes (#5/#18) versus a forced full table scan. @@ -24,7 +24,7 @@ func BenchmarkCoverageQuery(b *testing.B) { const prefixPool = 2000 // distinct 3-byte heard_key prefixes dir := b.TempDir() - s, err := OpenStore(dir + "/bench.db") + s, err := openPostgresTestStore(b, dir+"/bench.db") if err != nil { b.Fatal(err) } @@ -37,7 +37,7 @@ func BenchmarkCoverageQuery(b *testing.B) { } stmt, err := tx.Prepare(`INSERT INTO client_receptions (rx_pubkey,heard_key,heard_keylen,snr,lat,lon,rx_at,ingested_at,src) - VALUES (?,?,?,?,?,?,?,?,?)`) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`) if err != nil { b.Fatal(err) } @@ -78,7 +78,7 @@ func BenchmarkCoverageQuery(b *testing.B) { // full pubkey and its 2/3-byte prefixes, so an IN-list seeks them via the // heard_key-leading composite instead of scanning the bbox. inListSQL := `SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions - WHERE heard_key IN (?,?,?) AND lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?` + WHERE heard_key IN ($1,$2,$3) AND lat BETWEEN $4 AND $5 AND lon BETWEEN $6 AND $7` runIN := func(b *testing.B) { for i := 0; i < b.N; i++ { rows, err := s.db.Query(inListSQL, target, target[:4], target[:6], 51.1, 51.3, 3.6, 3.8) diff --git a/cmd/ingestor/db.go b/cmd/ingestor/db.go index 418230db7..907cf6512 100644 --- a/cmd/ingestor/db.go +++ b/cmd/ingestor/db.go @@ -14,9 +14,9 @@ import ( "sync/atomic" "time" - _ "github.com/mattn/go-sqlite3" "github.com/meshcore-analyzer/dbschema" "github.com/meshcore-analyzer/packetpath" + "github.com/meshcore-analyzer/pgutil" ) // DBStats tracks operational metrics for the ingestor database. @@ -65,11 +65,12 @@ func (s *DBStats) SnapshotBackfills() map[string]int64 { return out } -// Store wraps the SQLite database for packet ingestion. +// Store wraps the PostgreSQL database for packet ingestion. type Store struct { - db *sql.DB - path string // filesystem path to the SQLite DB (used to resolve queue dirs) - Stats DBStats + db *sql.DB + path string // compatibility anchor for local file queues; never a database URL + stateDir string + Stats DBStats stmtGetTxByHash *sql.Stmt stmtInsertTransmission *sql.Stmt @@ -122,796 +123,65 @@ const relayTouchDebounce = 5 * time.Minute // an extra UPDATE, never a missed one. const relayTouchedMaxEntries = 50000 -// OpenStore opens or creates a SQLite DB at the given path, applying the -// v3 schema that is compatible with the Node.js server. -func OpenStore(dbPath string) (*Store, error) { - return OpenStoreWithInterval(dbPath, 300) +// OpenStore connects with a restricted telemetry writer credential. +func OpenStore(databaseURL string) (*Store, error) { return OpenStoreWithInterval(databaseURL, 300) } +func OpenStoreWithInterval(databaseURL string, sampleIntervalSec int) (*Store, error) { + return OpenStoreWithState(databaseURL, "data", sampleIntervalSec) } - -// OpenStoreWithInterval opens or creates a SQLite DB with a configurable sample interval. -func OpenStoreWithInterval(dbPath string, sampleIntervalSec int) (*Store, error) { - dir := filepath.Dir(dbPath) - if err := os.MkdirAll(dir, 0o755); err != nil { - return nil, fmt.Errorf("creating data dir: %w", err) - } - - db, err := sql.Open("sqlite3", dbschema.WriterDSN(dbPath)) +func OpenStoreWithState(databaseURL, stateDir string, sampleIntervalSec int) (*Store, error) { + db, err := pgutil.Open(databaseURL, false) if err != nil { - return nil, fmt.Errorf("opening db: %w", err) + return nil, err } - - if err := db.Ping(); err != nil { - return nil, fmt.Errorf("pinging db: %w", err) - } - + ok := false + defer func() { + if !ok { + db.Close() + } + }() db.SetMaxOpenConns(1) db.SetMaxIdleConns(1) - log.Printf("SQLite config: busy_timeout=5000ms, max_open_conns=1, max_idle_conns=1, journal=WAL, synchronous=FULL") - - if err := applySchema(db); err != nil { - return nil, fmt.Errorf("applying schema: %w", err) + var synchronous string + if err = db.QueryRow(`SHOW synchronous_commit`).Scan(&synchronous); err != nil { + return nil, err } - - // Apply the additional server-originated migrations (now owned by - // the ingestor per #1287). Adds the indexes/columns that used to live - // in cmd/server/ensure_*.go: server now ASSERTS these exist. - if err := dbschema.Apply(db, log.Printf); err != nil { - return nil, fmt.Errorf("dbschema.Apply: %w", err) + if synchronous != "on" { + return nil, fmt.Errorf("telemetry writer requires synchronous_commit=on") } - s := &Store{db: db, path: dbPath, sampleIntervalSec: sampleIntervalSec} - if err := s.prepareStatements(); err != nil { - return nil, fmt.Errorf("preparing statements: %w", err) + if err = dbschema.AssertReady(db); err != nil { + return nil, err } - - // Continue the scope-match tally from where the previous process left - // off. Not fatal: a tally that cannot be read costs an observability - // number, and refusing to ingest over it would be the worse trade. - if err := s.LoadScopeMatchTotals(); err != nil { - log.Printf("[regions] restoring scope-match tally: %v", err) + if err = dbschema.AssertWriter(db); err != nil { + return nil, err } - - // Schedule async migrations. These must NOT block boot. See - // async_migration.go for the convention. - // PREFLIGHT: async=true reason="composite index build on observations (1.9M+ rows in prod) — converted from sync after v3.8.3" - var idxDone int - if s.db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'obs_observer_ts_idx_v1'").Scan(&idxDone) != nil { - if err := s.RunAsyncMigration(context.Background(), "obs_observer_ts_idx_v1", - func(ctx context.Context, d *sql.DB) error { - log.Println("[migration/async] Building (observer_idx, timestamp) composite index on observations...") - if _, err := d.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_observations_observer_idx_timestamp ON observations(observer_idx, timestamp)`); err != nil { - return err - } - if _, err := d.ExecContext(ctx, `INSERT OR IGNORE INTO _migrations (name) VALUES ('obs_observer_ts_idx_v1')`); err != nil { - return err - } - log.Println("[migration/async] observations(observer_idx, timestamp) index created") - return nil - }); err != nil { - log.Printf("[migration/async] scheduling obs_observer_ts_idx_v1 failed: %v", err) - } + if stateDir == "" { + stateDir = "data" } - - // #1690: backfill transmissions.last_seen from MAX(observations.timestamp) - // per transmission. The column is added inline by dbschema.Apply (cheap - // metadata-only ALTER); the populate query is potentially expensive - // (full obs scan + group) so we run it async. Subsequent observation - // inserts maintain the column inline (see InsertTransmission below). - // PREFLIGHT: async=true reason="full-table backfill JOIN (1.9M+ obs × 86k+ tx in prod) — must not block ingestor boot" - if err := s.RunAsyncMigration(context.Background(), "tx_last_seen_backfill_v1", - func(ctx context.Context, d *sql.DB) error { - log.Println("[migration/async] Backfilling transmissions.last_seen from MAX(observations.timestamp)...") - res, err := d.ExecContext(ctx, ` - UPDATE transmissions - SET last_seen = COALESCE(( - SELECT MAX(timestamp) FROM observations WHERE transmission_id = transmissions.id - ), last_seen) - WHERE last_seen = 0 - `) - if err != nil { - return err - } - n, _ := res.RowsAffected() - log.Printf("[migration/async] transmissions.last_seen backfill complete: %d rows updated", n) - return nil - }); err != nil { - log.Printf("[migration/async] scheduling tx_last_seen_backfill_v1 failed: %v", err) + if err = os.MkdirAll(stateDir, 0700); err != nil { + return nil, fmt.Errorf("create state directory: %w", err) + } + s := &Store{db: db, path: filepath.Join(stateDir, "meshcore"), stateDir: stateDir, sampleIntervalSec: sampleIntervalSec} + if err = s.prepareStatements(); err != nil { + return nil, fmt.Errorf("prepare writer statements: %w", err) + } + if err = s.LoadScopeMatchTotals(); err != nil { + log.Printf("[regions] restoring scope-match tally: %v", err) } - - // A missing/failed completion lookup leaves preservation enabled. This - // lifecycle state is restored on restart, independently of main's startup. var evidenceStatus string _ = db.QueryRow(`SELECT status FROM _async_migrations WHERE name='advert_route_evidence_v1'`).Scan(&evidenceStatus) s.advertEvidenceComplete.Store(evidenceStatus == "done") - return s, nil -} - -func applySchema(db *sql.DB) error { - // auto_vacuum=INCREMENTAL is set via DSN pragma (must be before journal_mode). - // Logging of current mode is handled by CheckAutoVacuum — no duplicate log here. - - schema := ` - CREATE TABLE IF NOT EXISTS nodes ( - public_key TEXT PRIMARY KEY, - name TEXT, - role TEXT, - lat REAL, - lon REAL, - last_seen TEXT, - first_seen TEXT, - advert_count INTEGER DEFAULT 0, - battery_mv INTEGER, - temperature_c REAL, - foreign_advert INTEGER DEFAULT 0 - ); - - CREATE TABLE IF NOT EXISTS scope_match_totals ( - id INTEGER PRIMARY KEY CHECK (id = 1), - since_unix INTEGER NOT NULL, - unique_matches INTEGER NOT NULL, - explicit_over_derived INTEGER NOT NULL, - ambiguous INTEGER NOT NULL, - none_matches INTEGER NOT NULL, - updated_unix INTEGER NOT NULL - ); - - CREATE TABLE IF NOT EXISTS observers ( - id TEXT PRIMARY KEY, - name TEXT, - iata TEXT, - last_seen TEXT, - first_seen TEXT, - packet_count INTEGER DEFAULT 0, - model TEXT, - firmware TEXT, - client_version TEXT, - radio TEXT, - battery_mv INTEGER, - uptime_secs INTEGER, - noise_floor REAL, - inactive INTEGER DEFAULT 0, - last_packet_at TEXT DEFAULT NULL, - clock_skew_seconds INTEGER DEFAULT NULL, - clock_skew_count_24h INTEGER DEFAULT 0, - clock_last_naive_at TEXT DEFAULT NULL, - can_relay INTEGER DEFAULT 1, - can_relay_seen INTEGER DEFAULT 0 - ); - - CREATE INDEX IF NOT EXISTS idx_nodes_last_seen ON nodes(last_seen); - CREATE INDEX IF NOT EXISTS idx_observers_last_seen ON observers(last_seen); - - CREATE TABLE IF NOT EXISTS inactive_nodes ( - public_key TEXT PRIMARY KEY, - name TEXT, - role TEXT, - lat REAL, - lon REAL, - last_seen TEXT, - first_seen TEXT, - advert_count INTEGER DEFAULT 0, - battery_mv INTEGER, - temperature_c REAL, - foreign_advert INTEGER DEFAULT 0 - ); - - CREATE INDEX IF NOT EXISTS idx_inactive_nodes_last_seen ON inactive_nodes(last_seen); - - CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raw_hex TEXT NOT NULL, - hash TEXT NOT NULL UNIQUE, - first_seen TEXT NOT NULL, - route_type INTEGER, - payload_type INTEGER, - payload_version INTEGER, - decoded_json TEXT, - from_pubkey TEXT, - last_seen INTEGER NOT NULL DEFAULT 0, - created_at TEXT DEFAULT (datetime('now')), - code1 TEXT, - code2 TEXT - ); - - CREATE INDEX IF NOT EXISTS idx_transmissions_hash ON transmissions(hash); - CREATE INDEX IF NOT EXISTS idx_transmissions_first_seen ON transmissions(first_seen); - CREATE INDEX IF NOT EXISTS idx_transmissions_payload_type ON transmissions(payload_type); - -- idx_tx_code1 is created by the transport_codes_v1 migration below, - -- after the ALTER runs — same reasoning as idx_transmissions_from_pubkey - -- and idx_tx_last_seen_zero above: a legacy DB (table-exists, - -- column-missing) would trip on this CREATE INDEX referencing code1 - -- before the column exists, since CREATE TABLE IF NOT EXISTS is a - -- no-op against a pre-existing table. - -- idx_transmissions_from_pubkey is created by the from_pubkey_v1 - -- migration after the column is added on legacy DBs (#1143). - -- idx_tx_last_seen_zero (partial, WHERE last_seen=0) is created by - -- dbschema.Apply after ensuring the last_seen column exists (#1690, - -- partial-index swap #1740) — keep it OUT of this base schema block - -- so legacy DBs (table-exists, column-missing) don't trip on the - -- CREATE INDEX before the ALTER runs. - - -- Mobile client RX coverage: a roaming companion = a mobile observer - -- with a moving GPS position, so it gets its own table rather than - -- observations (which assumes a fixed observer/location). - CREATE TABLE IF NOT EXISTS client_receptions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - rx_pubkey TEXT NOT NULL, - heard_key TEXT NOT NULL, - heard_keylen INTEGER NOT NULL, - rssi INTEGER, - snr REAL, - lat REAL NOT NULL, - lon REAL NOT NULL, - pos_acc_m REAL, - rx_at TEXT NOT NULL, - ingested_at TEXT NOT NULL, - src TEXT NOT NULL, - UNIQUE(rx_pubkey, heard_key, rx_at) - ); - -- Coverage queries filter by bbox AND match the heard node either by full - -- key (heard_keylen=32 AND heard_key=?) or by 2-3 byte prefix. The composite - -- (heard_key, heard_keylen, lat, lon) serves the heard_key-equality seek and - -- carries lat/lon so the bbox range is satisfied from the index; it also - -- supersedes the old single-column heard_key index. idx_client_recept_latlon - -- lets the planner instead drive from a selective bbox. (#5, #18) - CREATE INDEX IF NOT EXISTS idx_client_recept_heard_geo ON client_receptions(heard_key, heard_keylen, lat, lon); - CREATE INDEX IF NOT EXISTS idx_client_recept_latlon ON client_receptions(lat, lon); - -- rx_at backs both the retention reaper (DELETE WHERE rx_at < ?) and the - -- leaderboard, which range-scans WHERE rx_at >= ? and aggregates per - -- rx_pubkey in Go (see rxLeaderboard's frontier-weighted scoring). Without - -- this index either would full-scan the table under the writer lock - -- (verified by an EXPLAIN test). A dedicated rx_pubkey index stays - -- redundant — the leaderboard no longer groups by rx_pubkey in SQL. - CREATE INDEX IF NOT EXISTS idx_client_recept_rxat ON client_receptions(rx_at); - DROP INDEX IF EXISTS idx_client_recept_rxpk; - - -- Self-reported name of each mobile client (companion), from the SELF_INFO - -- name the app sends as "origin". Lets the leaderboard show a name even - -- when the companion never advertised (so it isn't in the nodes table). - CREATE TABLE IF NOT EXISTS client_observers ( - pubkey TEXT PRIMARY KEY, - name TEXT, - last_seen TEXT - ); - - -- Diagnostic RF observations from mobile clients. Unlike client_receptions - -- this holds EVERY decodable packet, attributable or not, so it must never - -- be used for coverage. pkt_hash is ComputeContentHash() — identical to - -- transmissions.hash — so dark-traffic queries are a plain equality join. - CREATE TABLE IF NOT EXISTS client_rx_observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - rx_pubkey TEXT NOT NULL, - rx_at TEXT NOT NULL, - ingested_at TEXT NOT NULL, - pkt_hash TEXT NOT NULL, - route_type INTEGER NOT NULL, - payload_type INTEGER NOT NULL, - code1 TEXT, - code2 TEXT, - scope_name TEXT, - hash_size INTEGER NOT NULL, - hop_count INTEGER NOT NULL, - path_json TEXT, - forwarder TEXT, - snr REAL, - rssi INTEGER, - lat REAL NOT NULL, - lon REAL NOT NULL, - pos_acc_m REAL, - UNIQUE(rx_pubkey, pkt_hash, rx_at) - ); - CREATE INDEX IF NOT EXISTS idx_cro_prune ON client_rx_observations(rx_at); - CREATE INDEX IF NOT EXISTS idx_cro_hash ON client_rx_observations(pkt_hash, rx_at); - CREATE INDEX IF NOT EXISTS idx_cro_forwarder ON client_rx_observations(forwarder, rx_at); - CREATE INDEX IF NOT EXISTS idx_cro_scope ON client_rx_observations(scope_name, rx_at); - - -- RF environment samples from mobile clients: radio counters paired with - -- a GPS point. Absolutes only — deltas are computed at query time, and a - -- decrease in uptime_secs (reboot) or any counter (wrap) breaks the chain. - CREATE TABLE IF NOT EXISTS client_rf_samples ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - rx_pubkey TEXT NOT NULL, - sampled_at TEXT NOT NULL, - ingested_at TEXT NOT NULL, - lat REAL NOT NULL, - lon REAL NOT NULL, - pos_acc_m REAL, - stationary INTEGER NOT NULL DEFAULT 0, - uptime_secs INTEGER NOT NULL, - battery_mv INTEGER, - queue_len INTEGER, - errors INTEGER, - noise_floor INTEGER, - last_rssi INTEGER, - last_snr REAL, - tx_air_secs INTEGER, - rx_air_secs INTEGER, - recv INTEGER, - sent INTEGER, - flood_rx INTEGER, - direct_rx INTEGER, - flood_tx INTEGER, - direct_tx INTEGER, - recv_errors INTEGER, - UNIQUE(rx_pubkey, sampled_at) - ); - CREATE INDEX IF NOT EXISTS idx_crf_prune ON client_rf_samples(sampled_at); - CREATE INDEX IF NOT EXISTS idx_crf_track ON client_rf_samples(rx_pubkey, sampled_at); - - -- Declared region lists reported by repeaters via ANON_REQ_TYPE_REGIONS. - -- Observations, never state: "current" is the greatest observed_at for a - -- target, NOT the greatest ingested_at — a drive buffered offline can - -- arrive days late and must not overwrite a fresher reading. - CREATE TABLE IF NOT EXISTS node_declared_regions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - target TEXT NOT NULL, - rx_pubkey TEXT NOT NULL, - observed_at TEXT NOT NULL, - ingested_at TEXT NOT NULL, - regions_csv TEXT NOT NULL, - truncated INTEGER NOT NULL DEFAULT 0, - lat REAL, - lon REAL, - pos_acc_m REAL, - repeater_clock INTEGER, - UNIQUE(target, rx_pubkey, observed_at) - ); - CREATE INDEX IF NOT EXISTS idx_ndr_target ON node_declared_regions(target, observed_at); - CREATE INDEX IF NOT EXISTS idx_ndr_prune ON node_declared_regions(observed_at); - ` - if _, err := db.Exec(schema); err != nil { - return fmt.Errorf("base schema: %w", err) - } - - // Create observations table (v3 schema) - obsExists := false - row := db.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name='observations'") - var dummy string - if row.Scan(&dummy) == nil { - obsExists = true - } - - if !obsExists { - obs := ` - CREATE TABLE observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - transmission_id INTEGER NOT NULL REFERENCES transmissions(id), - observer_idx INTEGER, - direction TEXT, - snr REAL, - rssi REAL, - score INTEGER, - path_json TEXT, - timestamp INTEGER NOT NULL - ); - CREATE INDEX idx_observations_transmission_id ON observations(transmission_id); - CREATE INDEX idx_observations_observer_idx ON observations(observer_idx); - CREATE INDEX idx_observations_timestamp ON observations(timestamp); - CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ON observations(transmission_id, observer_idx, COALESCE(path_json, '')); - ` - if _, err := db.Exec(obs); err != nil { - return fmt.Errorf("observations schema: %w", err) - } + // The data backfill is resumable bookkeeping, never runtime DDL. It does + // not block readiness, and only rows not maintained by live ingest match. + if err = s.RunAsyncMigration(context.Background(), "tx_last_seen_backfill_v1", func(ctx context.Context, d *sql.DB) error { + _, e := d.ExecContext(ctx, `UPDATE transmissions SET last_seen=COALESCE((SELECT MAX(timestamp) FROM observations WHERE transmission_id=transmissions.id),last_seen) WHERE last_seen=0`) + return e + }); err != nil { + return nil, err } - // Create/rebuild packets_v view (v3 schema: observer_idx → observers.rowid) - // The Go server reads this view; without it fresh installs get "no such table: packets_v". - db.Exec(`DROP VIEW IF EXISTS packets_v`) - _, vErr := db.Exec(` - CREATE VIEW packets_v AS - SELECT o.id, COALESCE(o.raw_hex, t.raw_hex) AS raw_hex, - datetime(o.timestamp, 'unixepoch') AS timestamp, - obs.id AS observer_id, obs.name AS observer_name, - o.direction, o.snr, o.rssi, o.score, t.hash, t.route_type, - t.payload_type, t.payload_version, o.path_json, t.decoded_json, - t.created_at - FROM observations o - JOIN transmissions t ON t.id = o.transmission_id - LEFT JOIN observers obs ON obs.rowid = o.observer_idx AND (obs.inactive IS NULL OR obs.inactive = 0) - `) - if vErr != nil { - return fmt.Errorf("packets_v view: %w", vErr) - } - - // One-time migration: recalculate advert_count to count unique transmissions only - db.Exec(`CREATE TABLE IF NOT EXISTS _migrations (name TEXT PRIMARY KEY)`) - var migDone int - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'advert_count_unique_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Recalculating advert_count (unique transmissions only)...") - // Note: this migration is gated on a one-shot _migrations row, so it - // runs at most once per DB. The historical version used a LIKE-on-JSON - // substring match (#1143). Switching to from_pubkey here is safe even - // though the column may not yet be backfilled on legacy DBs: the - // migration is already marked done on those DBs and won't re-run. - db.Exec(` - UPDATE nodes SET advert_count = ( - SELECT COUNT(*) FROM transmissions t - WHERE t.payload_type = 4 - AND t.from_pubkey = nodes.public_key - ) - `) - db.Exec(`INSERT INTO _migrations (name) VALUES ('advert_count_unique_v1')`) - log.Println("[migration] advert_count recalculated") - } - - // One-time migration: change noise_floor from INTEGER to REAL affinity. - // SQLite doesn't support ALTER COLUMN, but existing float values are stored - // as REAL regardless of column affinity. New table definition already uses REAL. - // This migration casts any integer-stored noise_floor values to real. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'noise_floor_real_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Ensuring noise_floor values are stored as REAL...") - db.Exec(`UPDATE observers SET noise_floor = CAST(noise_floor AS REAL) WHERE noise_floor IS NOT NULL AND typeof(noise_floor) = 'integer'`) - db.Exec(`INSERT INTO _migrations (name) VALUES ('noise_floor_real_v1')`) - log.Println("[migration] noise_floor migration complete") - } - - // One-time migration: add telemetry columns to nodes and inactive_nodes tables. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'node_telemetry_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding telemetry columns to nodes/inactive_nodes...") - - // checkAndAddColumn checks whether `column` already exists in `table` - // using PRAGMA table_info, and adds it if missing. All call sites pass - // hardcoded table/column/type literals so there is no SQL injection risk. - checkAndAddColumn := func(table, column, colType string) error { - rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%s)", table)) - if err != nil { - return fmt.Errorf("querying table info for %s: %w", table, err) - } - defer rows.Close() - - exists := false - for rows.Next() { - var cid int - var name, ctype string - var notnull, pk int - var dfltValue sql.NullString - if err := rows.Scan(&cid, &name, &ctype, ¬null, &dfltValue, &pk); err != nil { - return fmt.Errorf("scanning table info for %s: %w", table, err) - } - if name == column { - exists = true - break - } - } - if err := rows.Err(); err != nil { - return fmt.Errorf("iterating table info for %s: %w", table, err) - } - if exists { - return nil - } - if _, err := db.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, column, colType)); err != nil { - return fmt.Errorf("adding column %s to %s: %w", column, table, err) - } - return nil - } - - if err := checkAndAddColumn("nodes", "battery_mv", "INTEGER"); err != nil { - return err - } - if err := checkAndAddColumn("nodes", "temperature_c", "REAL"); err != nil { - return err - } - if err := checkAndAddColumn("inactive_nodes", "battery_mv", "INTEGER"); err != nil { - return err - } - if err := checkAndAddColumn("inactive_nodes", "temperature_c", "REAL"); err != nil { - return err - } - if _, err := db.Exec(`INSERT INTO _migrations (name) VALUES ('node_telemetry_v1')`); err != nil { - return fmt.Errorf("recording node_telemetry_v1 migration: %w", err) - } - log.Println("[migration] node telemetry columns added") - } - - // One-time migration: add timestamp index on observations for fast stats queries. - // Older databases created before this index was added suffer from full table scans - // on COUNT(*) WHERE timestamp > ?, causing /api/stats to take 30s+. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'obs_timestamp_index_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding timestamp index on observations...") - db.Exec(`CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp)`) - db.Exec(`INSERT INTO _migrations (name) VALUES ('obs_timestamp_index_v1')`) - log.Println("[migration] observations timestamp index created") - } - - // #1481 P0-3: covering index for GetObserverPacketCounts. The query - // joins observations → observers and GROUP BYs observer_idx with a - // timestamp WHERE filter; a composite (observer_idx, timestamp) - // index lets SQLite resolve the grouping + range filter from the - // index alone instead of a 1.9M-row scan. - // - // CONVERTED TO ASYNC (preflight-async-migration-gate). Scheduling - // happens in OpenStore() once the real *Store exists so the - // backfill WaitGroup is shared with the rest of the ingestor. - // The legacy `_migrations` gate is preserved by the async fn so - // DBs that already completed the sync build stay no-op. - - // #1483: normalize nodes.public_key to lowercase. The server's - // GetNodeLocationsByKeys lookup dropped LOWER(public_key) for perf - // (#1481 P0-3) and now relies on stored keys being lowercase. The - // decoder writes lowercase today, but legacy/admin/API inserts may - // have left mixed-case rows. Idempotent: counts and lowers any - // non-lowercase rows on every boot, runs once via _migrations gate - // for the bulk fix. Re-running stays cheap because subsequent - // passes match zero rows. - if r := db.QueryRow("SELECT COUNT(*) FROM nodes WHERE public_key != lower(public_key)"); r != nil { - var n int64 - _ = r.Scan(&n) - if n > 0 { - log.Printf("[migration] Normalizing %d nodes.public_key row(s) to lowercase (#1483)...", n) - if _, err := db.Exec(`UPDATE nodes SET public_key = lower(public_key) WHERE public_key != lower(public_key)`); err != nil { - log.Printf("[migration] public_key lowercase normalize failed: %v", err) - } else { - log.Printf("[migration] public_key lowercase normalize complete (%d rows)", n) - } - } - } - - // observer_metrics table for RF health dashboard - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observer_metrics_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Creating observer_metrics table...") - _, err := db.Exec(` - CREATE TABLE IF NOT EXISTS observer_metrics ( - observer_id TEXT NOT NULL, - timestamp TEXT NOT NULL, - noise_floor REAL, - tx_air_secs INTEGER, - rx_air_secs INTEGER, - recv_errors INTEGER, - battery_mv INTEGER, - PRIMARY KEY (observer_id, timestamp) - ) - `) - if err != nil { - return fmt.Errorf("observer_metrics schema: %w", err) - } - db.Exec(`INSERT INTO _migrations (name) VALUES ('observer_metrics_v1')`) - log.Println("[migration] observer_metrics table created") - } - - // Migration: add timestamp index for cross-observer time-range queries - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observer_metrics_ts_idx'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Creating observer_metrics timestamp index...") - _, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_observer_metrics_timestamp ON observer_metrics(timestamp)`) - if err != nil { - return fmt.Errorf("observer_metrics timestamp index: %w", err) - } - db.Exec(`INSERT INTO _migrations (name) VALUES ('observer_metrics_ts_idx')`) - log.Println("[migration] observer_metrics timestamp index created") - } - - // Migration: add inactive column to observers for soft-delete retention - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observers_inactive_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding inactive column to observers...") - _, err := db.Exec(`ALTER TABLE observers ADD COLUMN inactive INTEGER DEFAULT 0`) - if err != nil { - // Column may already exist (e.g. fresh install with schema above) - log.Printf("[migration] observers.inactive: %v (may already exist)", err) - } - db.Exec(`INSERT INTO _migrations (name) VALUES ('observers_inactive_v1')`) - log.Println("[migration] observers.inactive column added") - } - - // Migration: add packets_sent and packets_recv columns to observer_metrics - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observer_metrics_packets_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding packets_sent/packets_recv columns to observer_metrics...") - db.Exec(`ALTER TABLE observer_metrics ADD COLUMN packets_sent INTEGER`) - db.Exec(`ALTER TABLE observer_metrics ADD COLUMN packets_recv INTEGER`) - db.Exec(`INSERT INTO _migrations (name) VALUES ('observer_metrics_packets_v1')`) - log.Println("[migration] packets_sent/packets_recv columns added") - } - - // Migration: add channel_hash column for fast channel queries (#762) - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'channel_hash_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding channel_hash column to transmissions...") - db.Exec(`ALTER TABLE transmissions ADD COLUMN channel_hash TEXT DEFAULT NULL`) - db.Exec(`CREATE INDEX IF NOT EXISTS idx_tx_channel_hash ON transmissions(channel_hash) WHERE payload_type = 5`) - // Backfill: extract channel name for decrypted (CHAN) packets - res, err := db.Exec(`UPDATE transmissions SET channel_hash = json_extract(decoded_json, '$.channel') WHERE payload_type = 5 AND channel_hash IS NULL AND json_extract(decoded_json, '$.type') = 'CHAN'`) - if err == nil { - n, _ := res.RowsAffected() - log.Printf("[migration] Backfilled channel_hash for %d CHAN packets", n) - } - // Backfill: extract channelHashHex for encrypted (GRP_TXT) packets, prefixed with 'enc_' - res, err = db.Exec(`UPDATE transmissions SET channel_hash = 'enc_' || json_extract(decoded_json, '$.channelHashHex') WHERE payload_type = 5 AND channel_hash IS NULL AND json_extract(decoded_json, '$.type') = 'GRP_TXT'`) - if err == nil { - n, _ := res.RowsAffected() - log.Printf("[migration] Backfilled channel_hash for %d GRP_TXT packets", n) - } - db.Exec(`INSERT INTO _migrations (name) VALUES ('channel_hash_v1')`) - log.Println("[migration] channel_hash column added and backfilled") - } - - // Migration: dropped_packets table for signature validation failures (#793) - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'dropped_packets_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Creating dropped_packets table...") - _, err := db.Exec(` - CREATE TABLE IF NOT EXISTS dropped_packets ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - hash TEXT, - raw_hex TEXT, - reason TEXT NOT NULL, - observer_id TEXT, - observer_name TEXT, - node_pubkey TEXT, - node_name TEXT, - dropped_at DATETIME DEFAULT CURRENT_TIMESTAMP - ); - CREATE INDEX IF NOT EXISTS idx_dropped_observer ON dropped_packets(observer_id); - CREATE INDEX IF NOT EXISTS idx_dropped_node ON dropped_packets(node_pubkey); - `) - if err != nil { - return fmt.Errorf("dropped_packets schema: %w", err) - } - db.Exec(`INSERT INTO _migrations (name) VALUES ('dropped_packets_v1')`) - log.Println("[migration] dropped_packets table created") - } - - // Migration: observations.raw_hex (#881) is now owned by - // internal/dbschema/dbschema.go (#1321). The server PRAGMA-detects - // this column as hasObsRawHex; keeping a single canonical Apply - // path closes the startup race where the server's detector ran - // before this ALTER finished. - - // Migration: transmissions.scope_name (#899) is now owned by - // internal/dbschema/dbschema.go (#1321). See above. - - // Migration: add last_packet_at column to observers (#last-packet-at) - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observers_last_packet_at_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding last_packet_at column to observers...") - _, alterErr := db.Exec(`ALTER TABLE observers ADD COLUMN last_packet_at TEXT DEFAULT NULL`) - if alterErr != nil && !strings.Contains(alterErr.Error(), "duplicate column") { - return fmt.Errorf("observers last_packet_at ALTER: %w", alterErr) - } - // Backfill: set last_packet_at = last_seen only for observers that actually have - // observation rows (packet_count alone is unreliable — UpsertObserver sets it to 1 - // on INSERT even for status-only observers). - res, err := db.Exec(`UPDATE observers SET last_packet_at = last_seen - WHERE last_packet_at IS NULL - AND rowid IN (SELECT DISTINCT observer_idx FROM observations WHERE observer_idx IS NOT NULL)`) - if err == nil { - n, _ := res.RowsAffected() - log.Printf("[migration] Backfilled last_packet_at for %d observers with packets", n) - } - db.Exec(`INSERT INTO _migrations (name) VALUES ('observers_last_packet_at_v1')`) - log.Println("[migration] observers.last_packet_at column added") - } - - // Migration: per-observer naive-clock skew tracking (#1478). - // When the ingestor clamps a packet's envelope timestamp because the - // observer emitted a zone-less local-time string off from UTC by >15min - // (resolveRxTime in main.go), we record the event here so the UI can - // surface a ⚠️ chip + banner. Decays after 24h via server-side read sweep. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observers_clock_naive_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding clock-naive columns to observers (#1478)...") - // Each ALTER is independent — ignore "duplicate column" so reruns are safe. - for _, stmt := range []string{ - `ALTER TABLE observers ADD COLUMN clock_skew_seconds INTEGER DEFAULT NULL`, - `ALTER TABLE observers ADD COLUMN clock_skew_count_24h INTEGER DEFAULT 0`, - `ALTER TABLE observers ADD COLUMN clock_last_naive_at TEXT DEFAULT NULL`, - } { - if _, err := db.Exec(stmt); err != nil && !strings.Contains(err.Error(), "duplicate column") { - return fmt.Errorf("clock_naive migration: %w", err) - } - } - db.Exec(`INSERT INTO _migrations (name) VALUES ('observers_clock_naive_v1')`) - log.Println("[migration] observers.clock_naive columns added") - } - - // Migration: backfill observations.path_json from raw_hex (#888) - // NOTE: This runs ASYNC via BackfillPathJSONAsync() to avoid blocking MQTT startup. - // See staging outage where ~502K rows blocked ingest for 15+ hours. - - // One-time cleanup: delete legacy packets with empty hash or empty first_seen (#994) - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'cleanup_legacy_null_hash_ts'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Cleaning up legacy packets with empty hash/timestamp...") - db.Exec(`DELETE FROM observations WHERE transmission_id IN (SELECT id FROM transmissions WHERE hash = '' OR first_seen = '')`) - res, err := db.Exec(`DELETE FROM transmissions WHERE hash = '' OR first_seen = ''`) - if err == nil { - deleted, _ := res.RowsAffected() - log.Printf("[migration] deleted %d legacy packets with empty hash/timestamp", deleted) - } - db.Exec(`INSERT INTO _migrations (name) VALUES ('cleanup_legacy_null_hash_ts')`) - } - - // Migration: foreign_advert column on nodes/inactive_nodes (#730) - // Marks nodes whose ADVERT GPS lies outside the configured geofilter polygon. - // Default 0; set to 1 by the ingestor when GeoFilter is configured and - // PassesFilter() returns false. Allows operators to surface bridged/leaked - // adverts without silently dropping them. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'foreign_advert_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding foreign_advert column to nodes/inactive_nodes...") - if _, err := db.Exec(`ALTER TABLE nodes ADD COLUMN foreign_advert INTEGER DEFAULT 0`); err != nil { - log.Printf("[migration] nodes.foreign_advert: %v (may already exist)", err) - } - if _, err := db.Exec(`ALTER TABLE inactive_nodes ADD COLUMN foreign_advert INTEGER DEFAULT 0`); err != nil { - log.Printf("[migration] inactive_nodes.foreign_advert: %v (may already exist)", err) - } - db.Exec(`CREATE INDEX IF NOT EXISTS idx_nodes_foreign_advert ON nodes(foreign_advert) WHERE foreign_advert = 1`) - db.Exec(`INSERT INTO _migrations (name) VALUES ('foreign_advert_v1')`) - log.Println("[migration] foreign_advert column added") - } - - // Migration: from_pubkey column on transmissions (#1143). - // Replaces the unsound `decoded_json LIKE '%pubkey%'` attribution path with - // an exact-match indexed column. Synchronously adds the column + index; - // row-level backfill is run by the SERVER asynchronously - // (cmd/server/from_pubkey_migration.go) so we don't block ingestor boot. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'from_pubkey_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding from_pubkey column + index to transmissions (#1143)...") - if _, err := db.Exec(`ALTER TABLE transmissions ADD COLUMN from_pubkey TEXT`); err != nil { - log.Printf("[migration] transmissions.from_pubkey: %v (may already exist)", err) - } - if _, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey)`); err != nil { - log.Printf("[migration] idx_transmissions_from_pubkey: %v", err) - } - db.Exec(`INSERT INTO _migrations (name) VALUES ('from_pubkey_v1')`) - log.Println("[migration] from_pubkey column + index added") - } - - // Migration: nodes.default_scope (#899 Feature 3) is now owned by - // internal/dbschema/dbschema.go (#1321). The server PRAGMA-detects - // this column as hasDefaultScope; keeping a single canonical Apply - // path closes the startup race that #1321 documented. - - // Migration: normalize known channel_hash values for existing rows. - // Before this PR, config key "public" was stored as channel_hash="public". - // After this PR, new rows use channel_hash="Public". Without backfill, - // channel grouping queries split into two buckets across the upgrade boundary. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'channel_hash_casing_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Normalizing known channel_hash values...") - res, err := db.Exec(`UPDATE transmissions SET channel_hash = 'Public' WHERE channel_hash = 'public' AND payload_type = 5`) - if err != nil { - log.Printf("[migration] ERROR: failed to normalize channel_hash: %v", err) - return fmt.Errorf("migration channel_hash_casing_v1 UPDATE failed: %w", err) - } - n, _ := res.RowsAffected() - log.Printf("[migration] Normalized %d channel_hash rows from 'public' to 'Public'", n) - if _, err := db.Exec(`INSERT OR IGNORE INTO _migrations (name) VALUES ('channel_hash_casing_v1')`); err != nil { - log.Printf("[migration] WARNING: failed to record migration: %v", err) - } - log.Println("[migration] channel_hash casing normalization complete") - } - - // Migration: transmissions.code1/code2 — the transport codes were decoded - // and discarded; storing them makes scope forwarding queryable per repeater. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'transport_codes_v1'") - if row.Scan(new(int)) != nil { - log.Println("[migration] Adding code1/code2 columns to transmissions...") - // Each ALTER is independent — ignore "duplicate column" so reruns (and - // the fresh-database path, where the base schema already created the - // columns) are safe. Any other failure must stop before the guard row - // is inserted, or prepareStatements' code1/code2 INSERT fails forever - // on every subsequent restart with no way to retry the migration. - for _, stmt := range []string{ - `ALTER TABLE transmissions ADD COLUMN code1 TEXT DEFAULT NULL`, - `ALTER TABLE transmissions ADD COLUMN code2 TEXT DEFAULT NULL`, - } { - if _, err := db.Exec(stmt); err != nil && !strings.Contains(err.Error(), "duplicate column") { - return fmt.Errorf("transport_codes_v1 migration: %w", err) - } - } - db.Exec(`CREATE INDEX IF NOT EXISTS idx_tx_code1 ON transmissions(code1) WHERE code1 IS NOT NULL`) - db.Exec(`INSERT INTO _migrations (name) VALUES ('transport_codes_v1')`) - log.Println("[migration] code1/code2 columns added") - } - - return nil + ok = true + return s, nil } func (s *Store) prepareStatements() error { @@ -925,20 +195,20 @@ func (s *Store) prepareStatements() error { return err } - s.stmtGetTxByHash, err = s.db.Prepare("SELECT id, first_seen FROM transmissions WHERE hash = ?") + s.stmtGetTxByHash, err = s.db.Prepare("SELECT id, first_seen FROM transmissions WHERE hash = $1") if err != nil { return err } s.stmtInsertTransmission, err = s.db.Prepare(` INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, channel_hash, scope_name, from_pubkey, last_seen, code1, code2) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) RETURNING id `) if err != nil { return err } - s.stmtUpdateTxFirstSeen, err = s.db.Prepare("UPDATE transmissions SET first_seen = ? WHERE id = ?") + s.stmtUpdateTxFirstSeen, err = s.db.Prepare("UPDATE transmissions SET first_seen = $1 WHERE id = $2") if err != nil { return err } @@ -953,25 +223,25 @@ func (s *Store) prepareStatements() error { // "tx_last_seen_backfill_v1". // PREFLIGHT: async=true reason="prepared-statement row-level UPDATE BY PRIMARY KEY (transmissions.id) — single-row touch per observation, indexed by PK, constant-time at any scale. Not a migration." s.stmtTouchNodeLastSeen, err = s.db.Prepare( - "UPDATE nodes SET last_seen = ? WHERE public_key = ? AND (last_seen IS NULL OR last_seen < ?)") + "UPDATE nodes SET last_seen = $1 WHERE public_key = $2 AND (last_seen IS NULL OR last_seen < $3)") if err != nil { return fmt.Errorf("preparing touch node last_seen: %w", err) } - s.stmtBumpTxLastSeen, err = s.db.Prepare("UPDATE transmissions SET last_seen = ? WHERE id = ? AND last_seen < ?") + s.stmtBumpTxLastSeen, err = s.db.Prepare("UPDATE transmissions SET last_seen = $1 WHERE id = $2 AND last_seen < $3") if err != nil { return err } s.stmtInsertObservation, err = s.db.Prepare(` INSERT INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT(transmission_id, observer_idx, COALESCE(path_json, '')) DO UPDATE SET - snr = COALESCE(excluded.snr, snr), - rssi = COALESCE(excluded.rssi, rssi), - score = COALESCE(excluded.score, score), - raw_hex = COALESCE(excluded.raw_hex, raw_hex), - resolved_path = COALESCE(excluded.resolved_path, resolved_path) + snr = COALESCE(excluded.snr, observations.snr), + rssi = COALESCE(excluded.rssi, observations.rssi), + score = COALESCE(excluded.score, observations.score), + raw_hex = COALESCE(excluded.raw_hex, observations.raw_hex), + resolved_path = COALESCE(excluded.resolved_path, observations.resolved_path) `) if err != nil { return err @@ -979,20 +249,20 @@ func (s *Store) prepareStatements() error { s.stmtUpsertNode, err = s.db.Prepare(` INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen) - VALUES (?, ?, ?, ?, ?, ?, ?) + VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT(public_key) DO UPDATE SET - name = COALESCE(?, name), - role = COALESCE(?, role), - lat = COALESCE(?, lat), - lon = COALESCE(?, lon), - last_seen = MAX(MIN(COALESCE(last_seen, ''), ?), ?) + name = COALESCE($8, nodes.name), + role = COALESCE($9, nodes.role), + lat = COALESCE($10, nodes.lat), + lon = COALESCE($11, nodes.lon), + last_seen = GREATEST(LEAST(COALESCE(nodes.last_seen, ''), $12), $13) `) if err != nil { return err } s.stmtIncrementAdvertCount, err = s.db.Prepare(` - UPDATE nodes SET advert_count = advert_count + 1 WHERE public_key = ? + UPDATE nodes SET advert_count = advert_count + 1 WHERE public_key = $1 `) if err != nil { return err @@ -1000,27 +270,27 @@ func (s *Store) prepareStatements() error { s.stmtUpsertObserver, err = s.db.Prepare(` INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, can_relay, can_relay_seen) - VALUES (?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, COALESCE(?, 1), CASE WHEN ? IS NULL THEN 0 ELSE 1 END) + VALUES ($1, $2, $3, $4, $5, 1, $6, $7, $8, $9, $10, $11, $12, COALESCE($13, 1), CASE WHEN $14::bigint IS NULL THEN 0 ELSE 1 END) ON CONFLICT(id) DO UPDATE SET - name = COALESCE(?, name), - iata = COALESCE(?, iata), - last_seen = MAX(MIN(COALESCE(last_seen, ''), ?), ?), - packet_count = packet_count + 1, - model = COALESCE(?, model), - firmware = COALESCE(?, firmware), - client_version = COALESCE(?, client_version), - radio = COALESCE(?, radio), - battery_mv = COALESCE(?, battery_mv), - uptime_secs = COALESCE(?, uptime_secs), - noise_floor = COALESCE(?, noise_floor), - can_relay = COALESCE(?, can_relay), - can_relay_seen = CASE WHEN ? IS NULL THEN can_relay_seen ELSE 1 END + name = COALESCE($15, observers.name), + iata = COALESCE($16, observers.iata), + last_seen = GREATEST(LEAST(COALESCE(observers.last_seen, ''), $17), $18), + packet_count = observers.packet_count + 1, + model = COALESCE($19, observers.model), + firmware = COALESCE($20, observers.firmware), + client_version = COALESCE($21, observers.client_version), + radio = COALESCE($22, observers.radio), + battery_mv = COALESCE($23, observers.battery_mv), + uptime_secs = COALESCE($24, observers.uptime_secs), + noise_floor = COALESCE($25, observers.noise_floor), + can_relay = COALESCE($26, observers.can_relay), + can_relay_seen = CASE WHEN $27::bigint IS NULL THEN observers.can_relay_seen ELSE 1 END `) if err != nil { return err } - s.stmtGetObserverRowid, err = s.db.Prepare("SELECT rowid FROM observers WHERE id = ?") + s.stmtGetObserverRowid, err = s.db.Prepare("SELECT rowid FROM observers WHERE id = $1") if err != nil { return err } @@ -1030,27 +300,30 @@ func (s *Store) prepareStatements() error { // taking MAX with rxTime, so the guard never locks in a past bug's stale future. s.stmtUpdateObserverLastSeen, err = s.db.Prepare(` UPDATE observers SET - last_seen = MAX(MIN(COALESCE(last_seen, ''), ?), ?), - last_packet_at = MAX(MIN(COALESCE(last_packet_at, ''), ?), ?) - WHERE rowid = ?`) + last_seen = GREATEST(LEAST(COALESCE(last_seen, ''), $1), $2), + last_packet_at = GREATEST(LEAST(COALESCE(last_packet_at, ''), $3), $4) + WHERE rowid = $5`) if err != nil { return err } s.stmtUpdateNodeTelemetry, err = s.db.Prepare(` UPDATE nodes SET - battery_mv = COALESCE(?, battery_mv), - temperature_c = COALESCE(?, temperature_c) - WHERE public_key = ? + battery_mv = COALESCE($1, battery_mv), + temperature_c = COALESCE($2, temperature_c) + WHERE public_key = $3 `) if err != nil { return err } s.stmtUpsertMetrics, err = s.db.Prepare(` - INSERT OR REPLACE INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) - `) + INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT(observer_id,timestamp) DO UPDATE SET noise_floor=excluded.noise_floor, + tx_air_secs=excluded.tx_air_secs,rx_air_secs=excluded.rx_air_secs,recv_errors=excluded.recv_errors, + battery_mv=excluded.battery_mv,packets_sent=excluded.packets_sent,packets_recv=excluded.packets_recv + `) if err != nil { return err } @@ -1061,158 +334,131 @@ func (s *Store) prepareStatements() error { // InsertTransmission inserts a decoded packet into transmissions + observations. // Returns true if a new transmission was created (not a duplicate hash). func (s *Store) InsertTransmission(data *PacketData) (bool, error) { - hash := data.Hash - if hash == "" { + if data.Hash == "" { return false, nil } - - // Wait/hold instrumentation (#1340). The hot path uses prepared - // statements that auto-commit; gate the whole function under - // writerMu so concurrent mqtt_handler inserts queue behind any - // other writer (vacuum, prune, neighbor-builder) and the wait is - // Go-visible. - mqttWaitStart := time.Now() + waitStart := time.Now() writerMu.Lock() - mqttWait := time.Since(mqttWaitStart) - mqttHoldStart := time.Now() + wait := time.Since(waitStart) + holdStart := time.Now() defer func() { - mqttHold := time.Since(mqttHoldStart) writerMu.Unlock() - recordWriterTiming("mqtt_handler", mqttWait, mqttHold, "InsertTransmission") + recordWriterTiming("mqtt_handler", wait, time.Since(holdStart), "InsertTransmission") }() - - rxTime := data.Timestamp + tx, err := beginWrite(s.db) + if err != nil { + return false, err + } + defer tx.Rollback() ingestNow := time.Now().UTC().Format(time.RFC3339) + rxTime := data.Timestamp if rxTime == "" { rxTime = ingestNow } - var txID int64 - isNew := false - - // Check for existing transmission - var existingID int64 - var existingFirstSeen string - err := s.stmtGetTxByHash.QueryRow(hash).Scan(&existingID, &existingFirstSeen) - if err == nil { - // Existing transmission - txID = existingID - if rxTime < existingFirstSeen { - _, _ = s.stmtUpdateTxFirstSeen.Exec(rxTime, txID) - } - } else { - // New transmission - isNew = true - result, err := s.stmtInsertTransmission.Exec( - data.RawHex, hash, rxTime, - data.RouteType, data.PayloadType, data.PayloadVersion, - data.DecodedJSON, nilIfEmpty(data.ChannelHash), - scopeNameForDB(data), - nilIfEmpty(data.FromPubkey), - epochSecondsForLastSeen(rxTime), - nilIfEmpty(data.Code1), - nilIfEmpty(data.Code2), - ) - if err != nil { - s.Stats.WriteErrors.Add(1) - return false, fmt.Errorf("insert transmission: %w", err) - } - txID, _ = result.LastInsertId() - s.Stats.TransmissionsInserted.Add(1) + var firstSeen string + err = tx.Stmt(s.stmtGetTxByHash).QueryRow(data.Hash).Scan(&txID, &firstSeen) + isNew := err == sql.ErrNoRows + if err != nil && !isNew { + return false, err } - - if !isNew { - s.Stats.DuplicateTransmissions.Add(1) + if isNew { + err = tx.Stmt(s.stmtInsertTransmission).QueryRow(data.RawHex, data.Hash, rxTime, data.RouteType, data.PayloadType, data.PayloadVersion, data.DecodedJSON, nilIfEmpty(data.ChannelHash), scopeNameForDB(data), nilIfEmpty(data.FromPubkey), epochSecondsForLastSeen(rxTime), nilIfEmpty(data.Code1), nilIfEmpty(data.Code2)).Scan(&txID) + } else if rxTime < firstSeen { + _, err = tx.Stmt(s.stmtUpdateTxFirstSeen).Exec(rxTime, txID) + } + if err != nil { + s.Stats.WriteErrors.Add(1) + return false, fmt.Errorf("write transmission: %w", err) } - // Capture route evidence BEFORE the observation conflict update can erase - // a different route. Duplicate evidence is a read-only indexed probe. - // Analytics failures must not drop core observations or liveness updates; - // known evidence is a lower bound when an evidence read/write fails. if data.PayloadType == 4 { if bit := packetpath.AdvertRouteEvidence(data.RawHex); bit != 0 { - if _, err := s.stmtInsertAdvertEvidence.Exec(txID, bit, txID, bit); err != nil { + if err := optionalWrite(tx, func() error { _, e := tx.Stmt(s.stmtInsertAdvertEvidence).Exec(txID, bit, txID, bit); return e }); err != nil { s.Stats.WriteErrors.Add(1) log.Printf("[db] record advert route evidence (non-fatal): %v", err) } } } - - // Resolve observer_idx and update last_seen var observerIdx *int64 if data.ObserverID != "" { - var rowid int64 - err := s.stmtGetObserverRowid.QueryRow(data.ObserverID).Scan(&rowid) + var id int64 + err = tx.Stmt(s.stmtGetObserverRowid).QueryRow(data.ObserverID).Scan(&id) if err == nil { - observerIdx = &rowid - // observer.last_seen and last_packet_at answer "when did the analyzer - // last hear from this observer" — both are ingest-time questions. - // Per-packet rxTime is stored separately on observations/transmissions - // using envelope time (see InsertTransmission above). See #1465. - _, _ = s.stmtUpdateObserverLastSeen.Exec(ingestNow, ingestNow, ingestNow, ingestNow, rowid) + observerIdx = &id + _, err = tx.Stmt(s.stmtUpdateObserverLastSeen).Exec(ingestNow, ingestNow, ingestNow, ingestNow, id) + } + if err != nil && err != sql.ErrNoRows { + return false, err } } - - // Until backfill commits this observation's evidence, preserve its old - // frame before UPSERT can destroy it. writerMu also guards checkpoints. - // Run even for malformed incoming raw: the surviving old frame is valid - // evidence independently of whether the new frame contributes a bit. if !isNew && data.PayloadType == 4 && observerIdx != nil && data.RawHex != "" { - if err := s.preserveLegacyAdvertObservation(txID, *observerIdx, data.PathJSON); err != nil { + if err := optionalWrite(tx, func() error { return s.preserveLegacyAdvertObservation(tx, txID, *observerIdx, data.PathJSON) }); err != nil { s.Stats.WriteErrors.Add(1) log.Printf("[db] preserve legacy advert evidence (non-fatal): %v", err) } } - - // Insert observation epochTs := time.Now().Unix() - if t, err := time.Parse(time.RFC3339, rxTime); err == nil { + if t, e := time.Parse(time.RFC3339, rxTime); e == nil { epochTs = t.Unix() } - - // Resolve hop prefixes to full pubkeys for `observations.resolved_path`. - // Per #1547: this writer was lost in the #1289 refactor and lives in - // the ingestor now. Per #1560: use the context-aware resolver so - // 1-byte prefix collisions are disambiguated via NeighborGraph - // adjacency (anchored on from_pubkey for ADVERTs, previous hop - // otherwise). Empty resolved JSON → NULL via nilIfEmpty. - resolved := resolvePathWithContext( - parsePathArray(data.PathJSON), - strings.ToLower(data.FromPubkey), - s.neighborGraph.load(), - s.prefixIdx.load(), - ) - resolvedJSON := marshalResolvedPath(resolved) - - _, err = s.stmtInsertObservation.Exec( - txID, observerIdx, data.Direction, - data.SNR, data.RSSI, data.Score, - data.PathJSON, epochTs, nilIfEmpty(data.RawHex), - nilIfEmpty(resolvedJSON), - ) + resolved := resolvePathWithContext(parsePathArray(data.PathJSON), strings.ToLower(data.FromPubkey), s.neighborGraph.load(), s.prefixIdx.load()) + _, err = tx.Stmt(s.stmtInsertObservation).Exec(txID, observerIdx, data.Direction, data.SNR, data.RSSI, data.Score, data.PathJSON, epochTs, nilIfEmpty(data.RawHex), nilIfEmpty(marshalResolvedPath(resolved))) if err != nil { s.Stats.WriteErrors.Add(1) - log.Printf("[db] observation insert (non-fatal): %v", err) + return false, fmt.Errorf("insert observation: %w", err) + } + if _, err = tx.Stmt(s.stmtBumpTxLastSeen).Exec(epochTs, txID, epochTs); err != nil { + return false, err + } + if err = tx.Commit(); err != nil { + s.Stats.WriteErrors.Add(1) + return false, fmt.Errorf("commit packet: %w", err) + } + if isNew { + s.Stats.TransmissionsInserted.Add(1) } else { - s.Stats.ObservationsInserted.Add(1) - // #1598: a resolved hop proves the node was forwarding traffic at - // rxTime. Refresh its last_seen so staleness/eviction logic sees - // relay activity, not just ADVERTs. - s.touchRelayNodesLocked(resolvedPubkeys(resolved), rxTime) - // #1690: bump transmissions.last_seen so cold-load can filter on - // effective recency. Conditional `last_seen < ?` so we never go - // backwards on out-of-order ingest. - if _, err := s.stmtBumpTxLastSeen.Exec(epochTs, txID, epochTs); err != nil { - log.Printf("[db] tx last_seen bump (non-fatal): %v", err) - } + s.Stats.DuplicateTransmissions.Add(1) } - - // Each prepared-stmt Exec auto-commits. Count one WAL commit per - // successful InsertTransmission so the perf page sees commit pressure. + s.Stats.ObservationsInserted.Add(1) s.Stats.WALCommits.Add(1) - + s.touchRelayNodesLocked(resolvedPubkeys(resolved), rxTime) return isNew, nil } +// beginWrite serializes ID allocation before any statement can consume an +// identity value. A transaction lock releases only after commit or rollback. +func beginWrite(db *sql.DB) (*sql.Tx, error) { return beginWriteContext(context.Background(), db) } +func beginWriteContext(ctx context.Context, db *sql.DB) (*sql.Tx, error) { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + if _, err = tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended(current_schema(),$1))`, dbschema.WriterLockKey); err != nil { + tx.Rollback() + return nil, err + } + return tx, nil +} + +// Analytics are best effort; a failed PostgreSQL statement otherwise poisons +// the packet transaction. Savepoints preserve the existing non-fatal behavior. +func optionalWrite(tx *sql.Tx, fn func() error) error { + if _, err := tx.Exec(`SAVEPOINT packet_analytics`); err != nil { + return err + } + err := fn() + if err != nil { + if _, rollbackErr := tx.Exec(`ROLLBACK TO SAVEPOINT packet_analytics`); rollbackErr != nil { + return rollbackErr + } + } + _, releaseErr := tx.Exec(`RELEASE SAVEPOINT packet_analytics`) + if err != nil { + return err + } + return releaseErr +} + // UpsertNode inserts or updates a node. func (s *Store) UpsertNode(pubKey, name, role string, lat, lon *float64, lastSeen string) error { ingestNow := time.Now().UTC().Format(time.RFC3339) @@ -1246,7 +492,7 @@ func (s *Store) MarkNodeForeign(pubKey string) error { if pubKey == "" { return nil } - _, err := s.db.Exec(`UPDATE nodes SET foreign_advert = 1 WHERE public_key = ?`, pubKey) + _, err := s.db.Exec(`UPDATE nodes SET foreign_advert = 1 WHERE public_key = $1`, pubKey) if err != nil { s.Stats.WriteErrors.Add(1) } @@ -1324,7 +570,7 @@ func (s *Store) UpsertObserverAt(id, name, iata string, meta *ObserverMeta, last s.Stats.ObserverUpserts.Add(1) // Reactivate if this observer was previously marked inactive - s.db.Exec(`UPDATE observers SET inactive = 0 WHERE id = ? AND inactive = 1`, id) + s.db.Exec(`UPDATE observers SET inactive = 0 WHERE id = $1 AND inactive = 1`, id) return nil } @@ -1347,18 +593,18 @@ func (s *Store) UpsertObserverRetained(id, name, iata string, meta *ObserverMeta _, err := s.db.Exec(` UPDATE observers SET - name = COALESCE(?, name), - iata = COALESCE(?, iata), - model = COALESCE(?, model), - firmware = COALESCE(?, firmware), - client_version = COALESCE(?, client_version), - radio = COALESCE(?, radio), - battery_mv = COALESCE(?, battery_mv), - uptime_secs = COALESCE(?, uptime_secs), - noise_floor = COALESCE(?, noise_floor), - can_relay = COALESCE(?, can_relay), - can_relay_seen = CASE WHEN ? IS NULL THEN can_relay_seen ELSE 1 END - WHERE id = ?`, + name = COALESCE($1, name), + iata = COALESCE($2, iata), + model = COALESCE($3, model), + firmware = COALESCE($4, firmware), + client_version = COALESCE($5, client_version), + radio = COALESCE($6, radio), + battery_mv = COALESCE($7, battery_mv), + uptime_secs = COALESCE($8, uptime_secs), + noise_floor = COALESCE($9, noise_floor), + can_relay = COALESCE($10, can_relay), + can_relay_seen = CASE WHEN $11::bigint IS NULL THEN can_relay_seen ELSE 1 END + WHERE id = $12`, name, normalizedIATA, model, firmware, clientVersion, radio, batteryMv, uptimeSecs, noiseFloor, canRelay, canRelay, id, ) @@ -1410,10 +656,9 @@ func observerMetaColumns(meta *ObserverMeta) (model, firmware, clientVersion, ra return model, firmware, clientVersion, radio, batteryMv, uptimeSecs, noiseFloor, canRelay } -// Close checkpoints the WAL and closes the database. +// Close waits for data backfills and closes the database. func (s *Store) Close() error { s.backfillWg.Wait() - s.Checkpoint() return s.db.Close() } @@ -1480,7 +725,7 @@ func (s *Store) InsertMetrics(data *MetricsData) error { func (s *Store) PruneOldMetrics(retentionDays int) (int64, error) { cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays).Format(time.RFC3339) // Tagged for /api/perf writer-lock visibility (#1340). - result, err := s.instrumentedExec("prune_metrics", `DELETE FROM observer_metrics WHERE timestamp < ?`, cutoff) + result, err := s.instrumentedExec("prune_metrics", `DELETE FROM observer_metrics WHERE timestamp < $1`, cutoff) if err != nil { return 0, fmt.Errorf("prune metrics: %w", err) } @@ -1491,192 +736,6 @@ func (s *Store) PruneOldMetrics(retentionDays int) (int64, error) { return n, nil } -// CheckAutoVacuum inspects the current auto_vacuum mode and logs a warning -// if not INCREMENTAL. Performs opt-in full VACUUM if db.vacuumOnStartup is set (#919). -func (s *Store) CheckAutoVacuum(cfg *Config) { - var autoVacuum int - if err := s.db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { - log.Printf("[db] warning: could not read auto_vacuum: %v", err) - return - } - - if autoVacuum == 2 { - log.Printf("[db] auto_vacuum=INCREMENTAL") - return - } - - modes := map[int]string{0: "NONE", 1: "FULL", 2: "INCREMENTAL"} - mode := modes[autoVacuum] - if mode == "" { - mode = fmt.Sprintf("UNKNOWN(%d)", autoVacuum) - } - - log.Printf("[db] auto_vacuum=%s — DB needs one-time VACUUM to enable incremental auto-vacuum. "+ - "Set db.vacuumOnStartup: true in config to migrate (will block startup for several minutes on large DBs). "+ - "See https://github.com/Kpa-clawbot/CoreScope/issues/919", mode) - - if cfg.DB != nil && cfg.DB.VacuumOnStartup { - // WARNING: Full VACUUM creates a temporary copy of the entire DB file. - // Requires ~2× the DB file size in free disk space or it will fail. - log.Printf("[db] vacuumOnStartup=true — starting one-time full VACUUM (ensure 2x DB size free disk space)...") - start := time.Now() - - if _, err := s.instrumentedExec("vacuum", "PRAGMA auto_vacuum = INCREMENTAL"); err != nil { - log.Printf("[db] VACUUM failed: could not set auto_vacuum: %v", err) - return - } - if _, err := s.instrumentedExec("vacuum", "VACUUM"); err != nil { - log.Printf("[db] VACUUM failed: %v", err) - return - } - - elapsed := time.Since(start) - log.Printf("[db] VACUUM complete in %v — auto_vacuum is now INCREMENTAL", elapsed.Round(time.Millisecond)) - } -} - -// RunIncrementalVacuum returns free pages to the OS (#919). -// Safe to call on auto_vacuum=NONE databases (noop). -func (s *Store) RunIncrementalVacuum(pages int) { - // Tagged for /api/perf writer-lock visibility (#1340). - if _, err := s.instrumentedExec("vacuum", fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pages)); err != nil { - log.Printf("[vacuum] incremental_vacuum error: %v", err) - } -} - -// RefreshPlannerStats rebuilds the query planner's cardinality statistics -// (#2058). -// -// Without a sqlite_stat1 table the planner works from built-in guesses, and on -// the channel queries it guesses wrong: it drives from the plain -// idx_transmissions_payload_type rather than idx_tx_channel_hash, the partial -// index (WHERE payload_type = 5) this schema already carries for that exact -// filter. -// -// Measured on the 9.4 GB staging database (1,250,489 transmissions, 14,169,329 -// observations), region-filtered GetChannels, counting page-cache misses -// because wall time there is dominated by the OS page cache (56.7s cold, 0.80s -// warm, for the same query and plan): -// -// analysis_limit ANALYZE driving index page misses -// none (no stats) - idx_transmissions_payload_type 143,442 -// 400 171ms idx_transmissions_payload_type 143,449 -// 1000 171ms idx_transmissions_payload_type 143,450 -// 10000 2.0s* idx_tx_channel_hash 107,429 -// 0 (unbounded) 242.9s idx_tx_channel_hash 107,429 -// -// (*) Every ANALYZE duration in that table was timed on a warm page cache, one -// after another. The same statement at limit 10000 took 3m43.9s cold, on a -// freshly started container. See EnsurePlannerStats. -// -// So 10000 buys the whole plan change, and the four-minute unbounded ANALYZE -// buys nothing beyond it. 400, the value SQLite's documentation offers for the -// bounded form, changes nothing at all on this data: it samples too few rows to -// separate the 126,336-row partial index from the 920,700-row plain one. -// -// ANALYZE, not PRAGMA optimize. Measured on the same database: optimize is a -// no-op here, because it only analyzes tables that the calling connection has -// itself queried during the session, and a maintenance call has queried none. -// PRAGMA optimize(0x03) returned no statements and sqlite_stat1 was not -// created. -// -// Note that analysis_limit=0 means *no* limit to SQLite, not "use a default". -// Config.AnalysisLimit maps an unset config to 10000 for that reason, and a -// negative value here disables the refresh. -// -// Returns whether the statistics were refreshed. This function owns its -// logging; callers need add nothing. -func (s *Store) RefreshPlannerStats(analysisLimit int) bool { - if analysisLimit < 0 { - return false - } - first := !s.hasPlannerStats() - start := time.Now() - // Tagged for /api/perf writer-lock visibility (#1340). - if _, err := s.instrumentedExec("analyze", fmt.Sprintf("PRAGMA analysis_limit=%d", analysisLimit)); err != nil { - log.Printf("[analyze] could not set analysis_limit: %v", err) - return false - } - if _, err := s.instrumentedExec("analyze", "ANALYZE"); err != nil { - log.Printf("[analyze] ANALYZE failed: %v", err) - return false - } - elapsed := time.Since(start).Round(time.Millisecond) - if first { - log.Printf("[analyze] planner statistics built in %v (analysis_limit=%d, first run against this database)", elapsed, analysisLimit) - } else { - log.Printf("[analyze] planner statistics refreshed in %v (analysis_limit=%d)", elapsed, analysisLimit) - } - return true -} - -// EnsurePlannerStats builds planner statistics when the database has none, and -// reports whether it did (#2058). -// -// This exists to close the window the startup stagger opens. The refresh ticker -// waits 2 minutes before its first run, and a query arriving in that window -// against a database with no statistics at all gets the plan measured in -// RefreshPlannerStats above: 143,442 pages read, which timed at 56.7s cold on -// the 9.4 GB staging file. -// -// It fires once per database, not once per restart. sqlite_stat1 is an ordinary -// table, so once written it stays in the file and a fresh read-only connection -// reads it back (verified across connection close, and through a mode=ro -// handle). Every later start therefore costs one query against sqlite_master and -// leaves the work to the ticker. -// -// The trade is a one-time ANALYZE early in startup, and it is not cheap on a -// cold page cache. Observed on staging at 9.4 GB: 3m43.9s, against the 2.0s the -// same statement takes warm. For those 3m44s it holds the store's single write -// connection (SetMaxOpenConns(1), db.go:142), so ingest stalls and buffers: the -// observations table took zero rows for four minutes and then 1027 in the minute -// the ANALYZE finished, against about 130 a minute either side, with nothing -// dropped. Hence the warning below, so an operator watching a first deploy can -// tell this apart from a hang. -// -// That cost belongs to the first ANALYZE, not to running it here. The refresh -// ticker would pay exactly the same 3m44s two minutes later; this only moves it -// earlier, where it overlaps the startup burst the ingest buffer is already -// sized for. -func (s *Store) EnsurePlannerStats(analysisLimit int) bool { - if s.hasPlannerStats() { - return false - } - log.Printf("[analyze] this database has no planner statistics; building them now. " + - "ANALYZE holds the single write connection until it finishes (3m43.9s measured on 9.4 GB, cold), " + - "so ingest will buffer and catch up. Once per database, not once per restart.") - return s.RefreshPlannerStats(analysisLimit) -} - -// hasPlannerStats reports whether ANALYZE has ever run against this database. -// A query error reads as "no stats", which is the safe direction for both -// callers: it costs the log line a wrong word, and costs EnsurePlannerStats one -// ANALYZE that was not needed, rather than skipping one that was. -func (s *Store) hasPlannerStats() bool { - var n int - if err := s.db.QueryRow( - `SELECT count(*) FROM sqlite_master WHERE type='table' AND name='sqlite_stat1'`).Scan(&n); err != nil { - return false - } - return n > 0 -} - -// Checkpoint runs a WAL checkpoint (TRUNCATE mode). -// Returns the number of WAL frames checkpointed (0 if WAL was already empty). -// TRUNCATE resets the WAL file to zero bytes when all frames are checkpointed; -// if active readers hold frames, it checkpoints what it can and leaves the rest. -func (s *Store) Checkpoint() int { - var busy, walFrames, checkpointed int - if err := s.db.QueryRow("PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &walFrames, &checkpointed); err != nil { - log.Printf("[db] WAL checkpoint error: %v", err) - return 0 - } - if walFrames > 0 { - log.Printf("[db] WAL checkpoint: %d/%d frames checkpointed (blocked=%v)", checkpointed, walFrames, busy != 0) - } - return checkpointed -} - // BackfillPathJSONAsync launches the path_json backfill in a background goroutine. // It processes observations with NULL/empty path_json that have raw_hex available, // decoding hop paths and updating the column. Safe to run concurrently with ingest @@ -1713,7 +772,7 @@ func (s *Store) BackfillPathJSONAsync() { -- to prevent the infinite re-UPDATE loop fixed in #1119. AND (o.path_json IS NULL OR o.path_json = '') AND t.payload_type != 9 - LIMIT ?`, batchSize) + LIMIT $1`, batchSize) if err != nil { log.Printf("[backfill] path_json query error: %v", err) errored = true @@ -1737,7 +796,7 @@ func (s *Store) BackfillPathJSONAsync() { for _, r := range batch { hops, err := packetpath.DecodePathFromRawHex(r.rawHex) if err != nil || len(hops) == 0 { - if _, execErr := s.db.Exec(`UPDATE observations SET path_json = '[]' WHERE id = ?`, r.id); execErr != nil { + if _, execErr := s.db.Exec(`UPDATE observations SET path_json = '[]' WHERE id = $1`, r.id); execErr != nil { log.Printf("[backfill] write error (id=%d): %v", r.id, execErr) } else { s.Stats.IncBackfill("path_json") @@ -1745,7 +804,7 @@ func (s *Store) BackfillPathJSONAsync() { continue } b, _ := json.Marshal(hops) - if _, execErr := s.db.Exec(`UPDATE observations SET path_json = ? WHERE id = ?`, string(b), r.id); execErr != nil { + if _, execErr := s.db.Exec(`UPDATE observations SET path_json = $1 WHERE id = $2`, string(b), r.id); execErr != nil { log.Printf("[backfill] write error (id=%d): %v", r.id, execErr) } else { updated++ @@ -1826,7 +885,7 @@ func (s *Store) BackfillDefaultScopeAsync(regionSet *regionKeySet) { // inserted before the transport_codes_v1 migration, by re-parsing raw_hex. // Runs in a background goroutine so it does not block MQTT startup. // -// Batched: SQLite holds a single write connection (SetMaxOpenConns(1)), so a +// Batched: the ingestor retains a single write connection, so a // multi-million-row UPDATE in one transaction would stall live ingest for the // whole run. 5k-row batches keep each transaction short. func (s *Store) BackfillTransportCodesAsync() { @@ -1876,9 +935,9 @@ func (s *Store) backfillTransportCodes(batchSize int) (int, error) { for { rows, err := s.db.Query(` SELECT id, raw_hex FROM transmissions - WHERE id > ? AND code1 IS NULL AND route_type IN (0, 3) + WHERE id > $1 AND code1 IS NULL AND route_type IN (0, 3) ORDER BY id - LIMIT ?`, lastID, batchSize) + LIMIT $2`, lastID, batchSize) if err != nil { return total, fmt.Errorf("transport_codes select: %w", err) } @@ -1909,11 +968,11 @@ func (s *Store) backfillTransportCodes(batchSize int) (int, error) { } if len(items) > 0 { - tx, err := s.db.Begin() + tx, err := beginWrite(s.db) if err != nil { return total, fmt.Errorf("transport_codes begin: %w", err) } - stmt, err := tx.Prepare(`UPDATE transmissions SET code1 = ?, code2 = ? WHERE id = ?`) + stmt, err := tx.Prepare(`UPDATE transmissions SET code1 = $1, code2 = $2 WHERE id = $3`) if err != nil { tx.Rollback() return total, fmt.Errorf("transport_codes prepare: %w", err) @@ -1960,7 +1019,7 @@ func (s *Store) LogStats() { // carries the requirement for future callers. // // Ownership (#1283/#1287/#1289): nodes is written by the ingestor only. -// The server opens SQLite mode=ro; its former touchRelayLastSeen has been +// The server uses a restricted PostgreSQL reader; its former touchRelayLastSeen has been // failing on every call since that refactor and is removed in this change. // cmd/server/readonly_invariant_test.go now guards against reintroduction. // @@ -1972,7 +1031,7 @@ func (s *Store) LogStats() { // Never inserts: the UPDATE matches an existing row or does nothing. // Never rewinds: the last_seen guard makes out-of-order ingest a no-op. // UpsertNode's ON CONFLICT clause is monotonic in the same direction -// (MAX(MIN(last_seen, ingestNow), rxTime) reduces to MAX(last_seen, rxTime) +// (GREATEST(LEAST(last_seen, ingestNow), rxTime) reduces to MAX(last_seen, rxTime) // for any non-future stored value), so an ADVERT cannot undo a touch. func (s *Store) touchRelayNodesLocked(pubkeys []string, rxTime string) { if len(pubkeys) == 0 || s.stmtTouchNodeLastSeen == nil { @@ -2036,17 +1095,17 @@ func (s *Store) compactRelayTouched(now time.Time) { // Returns the number of nodes moved. func (s *Store) MoveStaleNodes(nodeDays int) (int64, error) { cutoff := time.Now().UTC().AddDate(0, 0, -nodeDays).Format(time.RFC3339) - tx, err := s.db.Begin() + tx, err := beginWrite(s.db) if err != nil { return 0, fmt.Errorf("begin tx: %w", err) } defer tx.Rollback() - _, err = tx.Exec(`INSERT OR REPLACE INTO inactive_nodes SELECT * FROM nodes WHERE last_seen < ?`, cutoff) + _, err = tx.Exec(`INSERT INTO inactive_nodes (public_key,name,role,lat,lon,last_seen,first_seen,advert_count,battery_mv,temperature_c,foreign_advert,default_scope,configured_scope,configured_scope_at,multibyte_sup,multibyte_evidence) SELECT public_key,name,role,lat,lon,last_seen,first_seen,advert_count,battery_mv,temperature_c,foreign_advert,default_scope,configured_scope,configured_scope_at,multibyte_sup,multibyte_evidence FROM nodes WHERE last_seen < $1 ON CONFLICT(public_key) DO UPDATE SET name=excluded.name,role=excluded.role,lat=excluded.lat,lon=excluded.lon,last_seen=excluded.last_seen,first_seen=excluded.first_seen,advert_count=excluded.advert_count,battery_mv=excluded.battery_mv,temperature_c=excluded.temperature_c,foreign_advert=excluded.foreign_advert,default_scope=excluded.default_scope,configured_scope=excluded.configured_scope,configured_scope_at=excluded.configured_scope_at,multibyte_sup=excluded.multibyte_sup,multibyte_evidence=excluded.multibyte_evidence`, cutoff) if err != nil { return 0, fmt.Errorf("insert inactive: %w", err) } - result, err := tx.Exec(`DELETE FROM nodes WHERE last_seen < ?`, cutoff) + result, err := tx.Exec(`DELETE FROM nodes WHERE last_seen < $1`, cutoff) if err != nil { return 0, fmt.Errorf("delete stale: %w", err) } @@ -2071,7 +1130,7 @@ func (s *Store) RemoveStaleObservers(observerDays int) (int64, error) { } cutoff := time.Now().UTC().AddDate(0, 0, -observerDays).Format(time.RFC3339) // Tagged for /api/perf writer-lock visibility (#1340). - result, err := s.instrumentedExec("prune_observers", `UPDATE observers SET inactive = 1 WHERE last_seen < ? AND (inactive IS NULL OR inactive = 0)`, cutoff) + result, err := s.instrumentedExec("prune_observers", `UPDATE observers SET inactive = 1 WHERE last_seen < $1 AND (inactive IS NULL OR inactive = 0)`, cutoff) if err != nil { return 0, fmt.Errorf("mark stale observers inactive: %w", err) } @@ -2106,7 +1165,7 @@ func (s *Store) PurgeStaleObservers(purgeDays int) (int64, error) { result, err := s.instrumentedExec("purge_observers", ` DELETE FROM observers WHERE inactive = 1 - AND last_seen < ? + AND last_seen < $1 AND NOT EXISTS (SELECT 1 FROM observations o WHERE o.observer_idx = observers.rowid) AND NOT EXISTS (SELECT 1 FROM observer_metrics m WHERE m.observer_id = observers.id) AND NOT EXISTS (SELECT 1 FROM dropped_packets d WHERE d.observer_id = observers.id)`, cutoff) @@ -2134,7 +1193,7 @@ type DroppedPacket struct { // InsertDroppedPacket records a rejected packet in the dropped_packets table. func (s *Store) InsertDroppedPacket(dp *DroppedPacket) error { _, err := s.db.Exec( - `INSERT INTO dropped_packets (hash, raw_hex, reason, observer_id, observer_name, node_pubkey, node_name) VALUES (?, ?, ?, ?, ?, ?, ?)`, + `INSERT INTO dropped_packets (hash, raw_hex, reason, observer_id, observer_name, node_pubkey, node_name) VALUES ($1, $2, $3, $4, $5, $6, $7)`, dp.Hash, dp.RawHex, dp.Reason, dp.ObserverID, dp.ObserverName, dp.NodePubKey, dp.NodeName, ) if err != nil { @@ -2151,7 +1210,7 @@ func (s *Store) PruneDroppedPackets(retentionDays int) (int64, error) { return 0, nil } cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays).Format(time.RFC3339) - result, err := s.db.Exec(`DELETE FROM dropped_packets WHERE dropped_at < ?`, cutoff) + result, err := s.db.Exec(`DELETE FROM dropped_packets WHERE dropped_at < $1`, cutoff) if err != nil { return 0, fmt.Errorf("prune dropped packets: %w", err) } @@ -2196,7 +1255,7 @@ func nilIfEmpty(s string) interface{} { return s } -// boolToInt converts a bool to SQLite's 0/1 INTEGER representation. +// boolToInt converts a bool to the retained 0/1 integer representation. func boolToInt(b bool) int { if b { return 1 @@ -2230,15 +1289,15 @@ func (s *Store) UpdateNodeDefaultScope(pubkey, scope string) error { } // Short-circuit: skip if already stored. var cur sql.NullString - row := s.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = ?`, pubkey) + row := s.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = $1`, pubkey) if row.Scan(&cur) == nil && cur.Valid && cur.String == scope { return nil } - if _, err := s.db.Exec(`UPDATE nodes SET default_scope = ? WHERE public_key = ?`, scope, pubkey); err != nil { + if _, err := s.db.Exec(`UPDATE nodes SET default_scope = $1 WHERE public_key = $2`, scope, pubkey); err != nil { return err } // Mirror to inactive_nodes (node may be there if recently moved by retention). - _, err := s.db.Exec(`UPDATE inactive_nodes SET default_scope = ? WHERE public_key = ?`, scope, pubkey) + _, err := s.db.Exec(`UPDATE inactive_nodes SET default_scope = $1 WHERE public_key = $2`, scope, pubkey) return err } @@ -2337,19 +1396,19 @@ func (s *Store) UpdateNodeConfiguredScope(pubkey, scope, reportedAt string) erro // Last-write-wins: skip if the stored confirmation is newer-or-equal. if reportedAt != "" { var curAt sql.NullString - row := s.db.QueryRow(`SELECT configured_scope_at FROM nodes WHERE public_key = ?`, pubkey) + row := s.db.QueryRow(`SELECT configured_scope_at FROM nodes WHERE public_key = $1`, pubkey) if row.Scan(&curAt) == nil && curAt.Valid && curAt.String != "" && curAt.String >= reportedAt { return nil } } if _, err := s.db.Exec( - `UPDATE nodes SET configured_scope = ?, configured_scope_at = ? WHERE public_key = ?`, + `UPDATE nodes SET configured_scope = $1, configured_scope_at = $2 WHERE public_key = $3`, scope, reportedAt, pubkey); err != nil { return err } // Mirror to inactive_nodes (node may be there if recently moved by retention). _, err := s.db.Exec( - `UPDATE inactive_nodes SET configured_scope = ?, configured_scope_at = ? WHERE public_key = ?`, + `UPDATE inactive_nodes SET configured_scope = $1, configured_scope_at = $2 WHERE public_key = $3`, scope, reportedAt, pubkey) return err } @@ -2374,14 +1433,14 @@ func (s *Store) RecordNaiveSkew(observerID string, deltaSec int64, now time.Time // increments it. _, err := s.db.Exec(` INSERT INTO observers (id, clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at) - VALUES (?, ?, 1, ?) + VALUES ($1, $2, 1, $3) ON CONFLICT(id) DO UPDATE SET clock_skew_seconds = excluded.clock_skew_seconds, clock_last_naive_at = excluded.clock_last_naive_at, clock_skew_count_24h = CASE - WHEN clock_last_naive_at IS NULL OR clock_last_naive_at < ? + WHEN observers.clock_last_naive_at IS NULL OR observers.clock_last_naive_at < $4 THEN 1 - ELSE COALESCE(clock_skew_count_24h, 0) + 1 + ELSE COALESCE(observers.clock_skew_count_24h, 0) + 1 END `, observerID, deltaSec, nowStr, cutoff) return err @@ -2483,7 +1542,7 @@ func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID, // ─── Writer-lock instrumentation (issue #1340) ──────────────────────────── // -// Make SQLite writer-lock starvation visible to operators. Per-component +// Make serialized writer-lock starvation visible to operators. Per-component // wait_ms / hold_ms / contention_total histograms, surfaced via // /api/perf/write-sources under the "writer_perf" key. Component tags: // neighbor_builder, mqtt_handler, prune_packets, prune_observers, @@ -2495,7 +1554,7 @@ func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID, // every wrapped call site through the same package-level mutex. // WriterStatsSnapshot is a per-component wait/hold latency snapshot -// surfaced via /api/perf to make SQLite writer-lock starvation visible +// surfaced via /api/perf to make serialized writer-lock starvation visible // to operators (issue #1340). Times are in milliseconds. type WriterStatsSnapshot struct { Count int64 `json:"count"` @@ -2711,7 +1770,7 @@ func recordWriterTiming(component string, wait, hold time.Duration, queryForLog // writerMu serialises every wrapped writer call so the wait the next // caller sees is the wait the perf snapshot can attribute. The -// SQLite driver also enforces serial writes (SetMaxOpenConns(1)), +// connection pool also serializes writes (SetMaxOpenConns(1)), // but the wait inside the driver is invisible to Go — writerMu makes // it Go-visible. var writerMu sync.Mutex @@ -2737,7 +1796,7 @@ func (s *Store) WriterTx(component string, fn func(*sql.Tx) error) error { writerMu.Lock() wait := time.Since(waitStart) holdStart := time.Now() - tx, err := s.db.Begin() + tx, err := beginWrite(s.db) if err != nil { hold := time.Since(holdStart) writerMu.Unlock() diff --git a/cmd/ingestor/db_test.go b/cmd/ingestor/db_test.go index 9cd4fbfef..28f76f57f 100644 --- a/cmd/ingestor/db_test.go +++ b/cmd/ingestor/db_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/meshcore-analyzer/packetpath" + "github.com/meshcore-analyzer/pgutil" ) func tempDBPath(t *testing.T) string { @@ -33,14 +34,14 @@ func tempDBPath(t *testing.T) string { } func TestOpenStore(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } defer s.Close() // Verify tables exist - rows, err := s.db.Query("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name") + rows, err := s.db.Query("SELECT table_name FROM information_schema.tables WHERE table_schema=current_schema() ORDER BY table_name") if err != nil { t.Fatal(err) } @@ -69,7 +70,7 @@ func TestOpenStore(t *testing.T) { // Verify packets_v view exists var viewCount int - err = s.db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE type='view' AND name='packets_v'").Scan(&viewCount) + err = s.db.QueryRow("SELECT COUNT(*) FROM information_schema.views WHERE table_schema=current_schema() AND table_name='packets_v'").Scan(&viewCount) if err != nil { t.Fatal(err) } @@ -79,7 +80,7 @@ func TestOpenStore(t *testing.T) { } func TestInsertTransmission(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -129,7 +130,7 @@ func TestInsertTransmission(t *testing.T) { } func TestPacketsViewQueryable(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -177,7 +178,7 @@ func TestPacketsViewQueryable(t *testing.T) { } func TestUpsertNode(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -216,7 +217,7 @@ func TestUpsertNode(t *testing.T) { } func TestUpsertObserver(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -237,7 +238,7 @@ func TestUpsertObserver(t *testing.T) { } func TestUpsertObserverWithMeta(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -298,21 +299,21 @@ func TestUpsertObserverWithMeta(t *testing.T) { // Verify typeof returns correct SQLite types var typBattery, typUptime, typNoise string - s.db.QueryRow("SELECT typeof(battery_mv), typeof(uptime_secs), typeof(noise_floor) FROM observers WHERE id = 'obs1'"). + s.db.QueryRow("SELECT pg_typeof(battery_mv)::text, pg_typeof(uptime_secs)::text, pg_typeof(noise_floor)::text FROM observers WHERE id = 'obs1'"). Scan(&typBattery, &typUptime, &typNoise) - if typBattery != "integer" { + if typBattery != "bigint" { t.Errorf("typeof(battery_mv)=%s, want integer", typBattery) } - if typUptime != "integer" { + if typUptime != "bigint" { t.Errorf("typeof(uptime_secs)=%s, want integer", typUptime) } - if typNoise != "real" { + if typNoise != "double precision" { t.Errorf("typeof(noise_floor)=%s, want real", typNoise) } } func TestUpsertObserverMetaPreservesExisting(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -453,14 +454,14 @@ func TestExtractObserverMeta(t *testing.T) { } func TestSchemaNoiseFloorIsReal(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } defer s.Close() // Check column type affinity via PRAGMA - rows, err := s.db.Query("PRAGMA table_info(observers)") + rows, err := s.db.Query(`SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='observers' ORDER BY ordinal_position`) if err != nil { t.Fatal(err) } @@ -486,7 +487,7 @@ func TestSchemaNoiseFloorIsReal(t *testing.T) { } func TestInsertTransmissionWithObserver(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -522,7 +523,7 @@ func TestInsertTransmissionWithObserver(t *testing.T) { // #463: Verify that inserting a packet updates the observer's last_seen, // so low-traffic observers don't incorrectly appear offline. func TestInsertTransmissionUpdatesObserverLastSeen(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -534,11 +535,11 @@ func TestInsertTransmissionUpdatesObserverLastSeen(t *testing.T) { } // Backdate last_seen to 2 hours ago oldTime := "2026-03-24T22:00:00Z" - s.db.Exec("UPDATE observers SET last_seen = ? WHERE id = ?", oldTime, "obs1") + s.db.Exec("UPDATE observers SET last_seen = $1 WHERE id = $2", oldTime, "obs1") // Verify it was backdated var lastSeenBefore string - s.db.QueryRow("SELECT last_seen FROM observers WHERE id = ?", "obs1").Scan(&lastSeenBefore) + s.db.QueryRow("SELECT last_seen FROM observers WHERE id = $1", "obs1").Scan(&lastSeenBefore) if lastSeenBefore != oldTime { t.Fatalf("expected last_seen=%s, got %s", oldTime, lastSeenBefore) } @@ -562,7 +563,7 @@ func TestInsertTransmissionUpdatesObserverLastSeen(t *testing.T) { // Verify last_seen was updated to INGEST time, not envelope time (#1465). var lastSeenAfter string - s.db.QueryRow("SELECT last_seen FROM observers WHERE id = ?", "obs1").Scan(&lastSeenAfter) + s.db.QueryRow("SELECT last_seen FROM observers WHERE id = $1", "obs1").Scan(&lastSeenAfter) if lastSeenAfter == oldTime { t.Error("observer last_seen was NOT updated after packet insertion — low-traffic observers will appear offline") } @@ -578,7 +579,7 @@ func TestInsertTransmissionUpdatesObserverLastSeen(t *testing.T) { } func TestLastPacketAtUpdatedOnPacketOnly(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -590,7 +591,7 @@ func TestLastPacketAtUpdatedOnPacketOnly(t *testing.T) { } var lastPacketAt sql.NullString - s.db.QueryRow("SELECT last_packet_at FROM observers WHERE id = ?", "obs1").Scan(&lastPacketAt) + s.db.QueryRow("SELECT last_packet_at FROM observers WHERE id = $1", "obs1").Scan(&lastPacketAt) if lastPacketAt.Valid { t.Fatalf("expected last_packet_at to be NULL after UpsertObserver, got %s", lastPacketAt.String) } @@ -612,7 +613,7 @@ func TestLastPacketAtUpdatedOnPacketOnly(t *testing.T) { } after := time.Now().Unix() - s.db.QueryRow("SELECT last_packet_at FROM observers WHERE id = ?", "obs1").Scan(&lastPacketAt) + s.db.QueryRow("SELECT last_packet_at FROM observers WHERE id = $1", "obs1").Scan(&lastPacketAt) if !lastPacketAt.Valid { t.Fatal("expected last_packet_at to be non-NULL after InsertTransmission") } @@ -634,14 +635,14 @@ func TestLastPacketAtUpdatedOnPacketOnly(t *testing.T) { } var lastPacketAtAfterStatus sql.NullString - s.db.QueryRow("SELECT last_packet_at FROM observers WHERE id = ?", "obs1").Scan(&lastPacketAtAfterStatus) + s.db.QueryRow("SELECT last_packet_at FROM observers WHERE id = $1", "obs1").Scan(&lastPacketAtAfterStatus) if !lastPacketAtAfterStatus.Valid || lastPacketAtAfterStatus.String != lastPacketAt.String { t.Errorf("UpsertObserver should not change last_packet_at; expected %s, got %v", lastPacketAt.String, lastPacketAtAfterStatus) } } func TestEndToEndIngest(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -677,7 +678,7 @@ func TestEndToEndIngest(t *testing.T) { // Verify node was created var nodeName string - err = s.db.QueryRow("SELECT name FROM nodes WHERE public_key = ?", decoded.Payload.PubKey).Scan(&nodeName) + err = s.db.QueryRow("SELECT name FROM nodes WHERE public_key = $1", decoded.Payload.PubKey).Scan(&nodeName) if err != nil { t.Fatal(err) } @@ -687,7 +688,7 @@ func TestEndToEndIngest(t *testing.T) { } func TestInsertTransmissionEmptyHash(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -711,7 +712,7 @@ func TestInsertTransmissionEmptyHash(t *testing.T) { } func TestInsertTransmissionEmptyTimestamp(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -729,14 +730,14 @@ func TestInsertTransmissionEmptyTimestamp(t *testing.T) { } var firstSeen string - s.db.QueryRow("SELECT first_seen FROM transmissions WHERE hash = ?", data.Hash).Scan(&firstSeen) + s.db.QueryRow("SELECT first_seen FROM transmissions WHERE hash = $1", data.Hash).Scan(&firstSeen) if firstSeen == "" { t.Error("first_seen should be set even with empty timestamp") } } func TestInsertTransmissionEarlierFirstSeen(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -765,14 +766,14 @@ func TestInsertTransmissionEarlierFirstSeen(t *testing.T) { } var firstSeen string - s.db.QueryRow("SELECT first_seen FROM transmissions WHERE hash = ?", data.Hash).Scan(&firstSeen) + s.db.QueryRow("SELECT first_seen FROM transmissions WHERE hash = $1", data.Hash).Scan(&firstSeen) if firstSeen != "2026-03-25T06:00:00Z" { t.Errorf("first_seen=%s, want 2026-03-25T06:00:00Z (earlier timestamp)", firstSeen) } } func TestInsertTransmissionLaterFirstSeenNotUpdated(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -800,14 +801,14 @@ func TestInsertTransmissionLaterFirstSeenNotUpdated(t *testing.T) { } var firstSeen string - s.db.QueryRow("SELECT first_seen FROM transmissions WHERE hash = ?", data.Hash).Scan(&firstSeen) + s.db.QueryRow("SELECT first_seen FROM transmissions WHERE hash = $1", data.Hash).Scan(&firstSeen) if firstSeen != "2026-03-25T06:00:00Z" { t.Errorf("first_seen=%s should not change to later time", firstSeen) } } func TestInsertTransmissionNilSNRRSSI(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -926,7 +927,7 @@ func TestBuildPacketDataNilSNRRSSI(t *testing.T) { } func TestUpsertNodeEmptyLastSeen(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -949,14 +950,14 @@ func TestUpsertNodeEmptyLastSeen(t *testing.T) { func TestOpenStoreTwice(t *testing.T) { // Opening same DB twice tests the "observations already exists" path in applySchema path := tempDBPath(t) - s1, err := OpenStore(path) + s1, err := openPostgresTestStore(t, path) if err != nil { t.Fatal(err) } s1.Close() // Second open — observations table already exists - s2, err := OpenStore(path) + s2, err := openPostgresTestStore(t, path) if err != nil { t.Fatal(err) } @@ -971,7 +972,7 @@ func TestOpenStoreTwice(t *testing.T) { } func TestInsertTransmissionDedupObservation(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1005,7 +1006,7 @@ func TestInsertTransmissionDedupObservation(t *testing.T) { } func TestSchemaCompatibility(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1013,7 +1014,7 @@ func TestSchemaCompatibility(t *testing.T) { // Verify column names match what Node.js expects expectedTxCols := []string{"id", "raw_hex", "hash", "first_seen", "route_type", "payload_type", "payload_version", "decoded_json", "created_at"} - rows, _ := s.db.Query("PRAGMA table_info(transmissions)") + rows, _ := s.db.Query(`SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='transmissions' ORDER BY ordinal_position`) var txCols []string for rows.Next() { var cid int @@ -1041,7 +1042,7 @@ func TestSchemaCompatibility(t *testing.T) { // Verify observations columns expectedObsCols := []string{"id", "transmission_id", "observer_idx", "direction", "snr", "rssi", "score", "path_json", "timestamp"} - rows, _ = s.db.Query("PRAGMA table_info(observations)") + rows, _ = s.db.Query(`SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='observations' ORDER BY ordinal_position`) var obsCols []string for rows.Next() { var cid int @@ -1069,7 +1070,7 @@ func TestSchemaCompatibility(t *testing.T) { } func TestConcurrentWrites(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1167,7 +1168,7 @@ func TestConcurrentWrites(t *testing.T) { } func TestDBStats(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1231,7 +1232,7 @@ func TestDBStats(t *testing.T) { } func TestLoadTestThroughput(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1372,7 +1373,7 @@ func TestLoadTestThroughput(t *testing.T) { } func TestUpdateNodeTelemetry(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1420,7 +1421,7 @@ func TestUpdateNodeTelemetry(t *testing.T) { } func TestTelemetryMigrationAddsColumns(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1437,9 +1438,9 @@ func TestTelemetryMigrationAddsColumns(t *testing.T) { } var count int - s.db.QueryRow("SELECT COUNT(*) FROM _migrations WHERE name = 'node_telemetry_v1'").Scan(&count) + s.db.QueryRow("SELECT COUNT(*) FROM corescope_schema WHERE kind='telemetry' AND ready=true").Scan(&count) if count != 1 { - t.Errorf("migration node_telemetry_v1 should be recorded, count=%d", count) + t.Errorf("canonical telemetry schema should be ready, count=%d", count) } } @@ -1591,7 +1592,7 @@ func TestObsTimestampIndexMigration(t *testing.T) { // Case 1: new DB — OpenStore should create idx_observations_timestamp as part // of the observations table schema. t.Run("NewDB", func(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1599,7 +1600,7 @@ func TestObsTimestampIndexMigration(t *testing.T) { var count int err = s.db.QueryRow( - "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_timestamp'", + "SELECT COUNT(*) FROM pg_indexes WHERE schemaname=current_schema() AND indexname='idx_observations_timestamp'", ).Scan(&count) if err != nil { t.Fatal(err) @@ -1622,83 +1623,23 @@ func TestObsTimestampIndexMigration(t *testing.T) { // Case 2: existing DB that has the observations table but lacks the index // and lacks the _migrations entry — simulates an older installation. - t.Run("MigrationPath", func(t *testing.T) { - path := tempDBPath(t) - - // Build a bare-bones DB that mimics an old installation: - // observations table exists but idx_observations_timestamp does NOT. - db, err := sql.Open("sqlite3", path) + t.Run("MissingIndexIsRefused", func(t *testing.T) { + key := tempDBPath(t) + store, err := openPostgresTestStore(t, key) if err != nil { t.Fatal(err) } - _, err = db.Exec(` - CREATE TABLE IF NOT EXISTS _migrations (name TEXT PRIMARY KEY); - CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raw_hex TEXT NOT NULL, - hash TEXT NOT NULL UNIQUE, - first_seen TEXT NOT NULL, - route_type INTEGER, - payload_type INTEGER, - payload_version INTEGER, - decoded_json TEXT, - created_at TEXT DEFAULT (datetime('now')) - ); - CREATE TABLE IF NOT EXISTS observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - transmission_id INTEGER NOT NULL REFERENCES transmissions(id), - observer_idx INTEGER, - direction TEXT, - snr REAL, - rssi REAL, - score INTEGER, - path_json TEXT, - timestamp INTEGER NOT NULL - ); - `) - if err != nil { - db.Close() + owner := testAdmin(t, store) + if _, err := owner.Exec(`DROP INDEX idx_observations_timestamp`); err != nil { t.Fatal(err) } - // Confirm the index is absent before OpenStore runs. - var preCount int - db.QueryRow( - "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_timestamp'", - ).Scan(&preCount) - db.Close() - if preCount != 0 { - t.Fatalf("pre-condition failed: idx_observations_timestamp should not exist yet, got count=%d", preCount) - } - - // Now open via OpenStore — the migration should add the index. - s, err := OpenStore(path) - if err != nil { - t.Fatal(err) - } - defer s.Close() - - var idxCount int - err = s.db.QueryRow( - "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_timestamp'", - ).Scan(&idxCount) - if err != nil { - t.Fatal(err) - } - if idxCount != 1 { - t.Error("idx_observations_timestamp should exist after migration on old DB") - } - - var migCount int - err = s.db.QueryRow( - "SELECT COUNT(*) FROM _migrations WHERE name='obs_timestamp_index_v1'", - ).Scan(&migCount) - if err != nil { - t.Fatal(err) - } - if migCount != 1 { - t.Errorf("migration obs_timestamp_index_v1 should be recorded, got count=%d", migCount) + store.Close() + if next, err := openPostgresTestStore(t, key); err == nil { + next.Close() + t.Fatal("runtime accepted missing canonical index") } }) + } func TestBuildPacketDataScoreAndDirection(t *testing.T) { @@ -1739,7 +1680,7 @@ func TestBuildPacketDataNilScoreDirection(t *testing.T) { } func TestInsertTransmissionWithScoreAndDirection(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1805,7 +1746,7 @@ func TestRoundToInterval(t *testing.T) { } func TestInsertMetrics(t *testing.T) { - store, err := OpenStore(tempDBPath(t)) + store, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1849,7 +1790,7 @@ func TestInsertMetrics(t *testing.T) { } func TestInsertMetricsIdempotent(t *testing.T) { - store, err := OpenStoreWithInterval(tempDBPath(t), 300) + store, err := openPostgresTestStoreInterval(t, tempDBPath(t), 300) if err != nil { t.Fatal(err) } @@ -1879,7 +1820,7 @@ func TestInsertMetricsIdempotent(t *testing.T) { } func TestInsertMetricsNullFields(t *testing.T) { - store, err := OpenStore(tempDBPath(t)) + store, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1908,7 +1849,7 @@ func TestInsertMetricsNullFields(t *testing.T) { } func TestPruneOldMetrics(t *testing.T) { - store, err := OpenStore(tempDBPath(t)) + store, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1917,8 +1858,8 @@ func TestPruneOldMetrics(t *testing.T) { // Insert old and new metrics directly oldTs := time.Now().UTC().AddDate(0, 0, -40).Format(time.RFC3339) newTs := time.Now().UTC().Format(time.RFC3339) - store.db.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES (?, ?, ?)", "obs1", oldTs, -110.0) - store.db.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES (?, ?, ?)", "obs1", newTs, -112.0) + store.db.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES ($1, $2, $3)", "obs1", oldTs, -110.0) + store.db.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES ($1, $2, $3)", "obs1", newTs, -112.0) n, err := store.PruneOldMetrics(30) if err != nil { @@ -1975,7 +1916,7 @@ func TestExtractObserverMetaNewFields(t *testing.T) { // idx_observations_dedup CREATE UNIQUE INDEX statements elsewhere in this file.) func TestInsertObservationSNRFillIn(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -2058,7 +1999,7 @@ func TestInsertObservationSNRFillIn(t *testing.T) { // TestPerObservationRawHex verifies that two MQTT packets for the same hash // from different observers store distinct raw_hex per observation (#881). func TestPerObservationRawHex(t *testing.T) { - store, err := OpenStore(tempDBPath(t)) + store, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -2210,14 +2151,14 @@ func TestBuildPacketData_NonTracePathJSON(t *testing.T) { func TestScopeNameMigration(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } defer store.Close() // Verify column exists - rows, err := store.db.Query("PRAGMA table_info(transmissions)") + rows, err := store.db.Query(`SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='transmissions' ORDER BY ordinal_position`) if err != nil { t.Fatalf("PRAGMA: %v", err) } @@ -2266,7 +2207,7 @@ func TestScopeNameMigration(t *testing.T) { } func TestTransportCodesStored(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -2307,7 +2248,7 @@ func TestTransportCodesStored(t *testing.T) { } var c1, c2 *string - if err := s.db.QueryRow(`SELECT code1, code2 FROM transmissions WHERE hash = ?`, + if err := s.db.QueryRow(`SELECT code1, code2 FROM transmissions WHERE hash = $1`, ComputeContentHash(rawScoped)).Scan(&c1, &c2); err != nil { t.Fatalf("read scoped: %v", err) } @@ -2315,7 +2256,7 @@ func TestTransportCodesStored(t *testing.T) { t.Errorf("scoped codes = %v/%v, want AABB/CCDD", c1, c2) } - if err := s.db.QueryRow(`SELECT code1, code2 FROM transmissions WHERE hash = ?`, + if err := s.db.QueryRow(`SELECT code1, code2 FROM transmissions WHERE hash = $1`, ComputeContentHash(rawUnscoped)).Scan(&c1, &c2); err != nil { t.Fatalf("read unscoped: %v", err) } @@ -2323,7 +2264,7 @@ func TestTransportCodesStored(t *testing.T) { t.Errorf("unscoped codes = %v/%v, want NULL/NULL", c1, c2) } - if err := s.db.QueryRow(`SELECT code1, code2 FROM transmissions WHERE hash = ?`, + if err := s.db.QueryRow(`SELECT code1, code2 FROM transmissions WHERE hash = $1`, ComputeContentHash(rawZeroCodes)).Scan(&c1, &c2); err != nil { t.Fatalf("read zero-codes: %v", err) } @@ -2349,7 +2290,7 @@ func TestBackfillTransportCodes(t *testing.T) { }{{rawScoped, 0}, {rawFlood, 1}} { if _, err := s.db.Exec( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES (?,?,?,?,?)`, + VALUES ($1,$2,$3,$4,$5)`, tc.raw, ComputeContentHash(tc.raw), "2026-08-01T00:00:00Z", tc.routeType, 5); err != nil { t.Fatalf("seed: %v", err) } @@ -2359,7 +2300,7 @@ func TestBackfillTransportCodes(t *testing.T) { s.backfillWg.Wait() var c1 *string - if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = ?`, + if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = $1`, ComputeContentHash(rawScoped)).Scan(&c1); err != nil { t.Fatalf("read: %v", err) } @@ -2367,7 +2308,7 @@ func TestBackfillTransportCodes(t *testing.T) { t.Errorf("backfilled code1 = %v, want AABB", c1) } - if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = ?`, + if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = $1`, ComputeContentHash(rawFlood)).Scan(&c1); err != nil { t.Fatalf("read flood: %v", err) } @@ -2415,14 +2356,14 @@ func TestBackfillTransportCodes_MultiBatchWithUndecodableRow(t *testing.T) { // good2}, batch 2 = {good3, good4, good5}, batch 3 = {good6}. if _, err := s.db.Exec( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES (?,?,?,?,?)`, + VALUES ($1,$2,$3,$4,$5)`, badRaw, ComputeContentHash(badRaw), "2026-08-01T00:00:00Z", 0, 5); err != nil { t.Fatalf("seed bad row: %v", err) } for _, raw := range goodRaws { if _, err := s.db.Exec( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES (?,?,?,?,?)`, + VALUES ($1,$2,$3,$4,$5)`, raw, ComputeContentHash(raw), "2026-08-01T00:00:00Z", 0, 5); err != nil { t.Fatalf("seed good row: %v", err) } @@ -2439,7 +2380,7 @@ func TestBackfillTransportCodes_MultiBatchWithUndecodableRow(t *testing.T) { for i, raw := range goodRaws { wantCode1 := fmt.Sprintf("%04X", i+1) var c1 *string - if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = ?`, + if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = $1`, ComputeContentHash(raw)).Scan(&c1); err != nil { t.Fatalf("read good row %d: %v", i, err) } @@ -2449,7 +2390,7 @@ func TestBackfillTransportCodes_MultiBatchWithUndecodableRow(t *testing.T) { } var badC1 *string - if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = ?`, + if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = $1`, ComputeContentHash(badRaw)).Scan(&badC1); err != nil { t.Fatalf("read bad row: %v", err) } @@ -2470,7 +2411,7 @@ func TestBackfillTransportCodes_ErrorWithholdsGuardRow(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store1, err := OpenStore(dbPath) + store1, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -2478,7 +2419,7 @@ func TestBackfillTransportCodes_ErrorWithholdsGuardRow(t *testing.T) { raw := "14" + "AABB" + "CCDD" + "00" + strings.Repeat("11", 32) if _, err := store1.db.Exec( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES (?,?,?,?,?)`, + VALUES ($1,$2,$3,$4,$5)`, raw, ComputeContentHash(raw), "2026-08-01T00:00:00Z", 0, 5); err != nil { t.Fatalf("seed: %v", err) } @@ -2490,7 +2431,7 @@ func TestBackfillTransportCodes_ErrorWithholdsGuardRow(t *testing.T) { t.Fatal("backfillTransportCodes on a closed store returned nil error, want non-nil") } - store2, err := OpenStore(dbPath) + store2, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("reopen: %v", err) } @@ -2508,7 +2449,7 @@ func TestBackfillTransportCodes_ErrorWithholdsGuardRow(t *testing.T) { store2.backfillWg.Wait() var c1 *string - if err := store2.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = ?`, + if err := store2.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = $1`, ComputeContentHash(raw)).Scan(&c1); err != nil { t.Fatalf("read: %v", err) } @@ -2526,7 +2467,7 @@ func TestBackfillTransportCodes_ErrorWithholdsGuardRow(t *testing.T) { func TestUpdateNodeDefaultScope(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -2592,7 +2533,7 @@ func TestUpdateNodeDefaultScope(t *testing.T) { func TestBackfillPathJsonFromRawHex(t *testing.T) { dbPath := tempDBPath(t) - s, err := OpenStore(dbPath) + s, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } @@ -2601,26 +2542,26 @@ func TestBackfillPathJsonFromRawHex(t *testing.T) { // raw_hex: header 0x05 (route FLOOD, payload 0x01), path byte 0x42 (hash_size=2, count=2), // hops: AABB, CCDD, then some payload bytes rawHex := "0542AABBCCDD0000000000000000000000000000" - s.db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES (?, 'h1', '2025-01-01T00:00:00Z', 1)`, rawHex) + s.db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES ($1, 'h1', '2025-01-01T00:00:00Z', 1)`, rawHex) // Insert observation with raw_hex but empty path_json - s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (1, 1000, ?, '[]')`, rawHex) + s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (1, 1000, $1, '[]')`, rawHex) // Insert observation with raw_hex and NULL path_json - s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (1, 1001, ?, NULL)`, rawHex) + s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (1, 1001, $1, NULL)`, rawHex) // Insert observation with existing path_json (should NOT be overwritten) - s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (1, 1002, ?, '["XX","YY"]')`, rawHex) + s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (1, 1002, $1, '["XX","YY"]')`, rawHex) // Insert a TRACE transmission (payload_type = 0x09) — should be skipped traceRaw := "2604302D0D2359FEE7B100000000006733D63367" - s.db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES (?, 'h2', '2025-01-01T00:00:00Z', 9)`, traceRaw) - s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (2, 1003, ?, '[]')`, traceRaw) + s.db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES ($1, 'h2', '2025-01-01T00:00:00Z', 9)`, traceRaw) + s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (2, 1003, $1, '[]')`, traceRaw) // Remove the migration marker so it runs again on reopen s.db.Exec(`DELETE FROM _migrations WHERE name = 'backfill_path_json_from_raw_hex_v1'`) s.Close() // Reopen — backfill is now async, must trigger explicitly - s2, err := OpenStore(dbPath) + s2, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } @@ -2672,107 +2613,6 @@ func TestBackfillPathJsonFromRawHex(t *testing.T) { } } -func TestCleanupLegacyNullHashTimestamp(t *testing.T) { - path := tempDBPath(t) - - // Create a bare-bones DB with legacy bad data - db, err := sql.Open("sqlite3", path+"?_journal_mode=WAL&_busy_timeout=5000") - if err != nil { - t.Fatal(err) - } - db.Exec(`CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raw_hex TEXT NOT NULL, - hash TEXT NOT NULL, - first_seen TEXT NOT NULL, - route_type INTEGER, - payload_type INTEGER, - payload_version INTEGER, - decoded_json TEXT, - created_at TEXT DEFAULT (datetime('now')), - channel_hash TEXT DEFAULT NULL - )`) - db.Exec(`CREATE TABLE IF NOT EXISTS observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - transmission_id INTEGER NOT NULL REFERENCES transmissions(id), - observer_idx INTEGER, - direction TEXT, - snr REAL, - rssi REAL, - score INTEGER, - path_json TEXT, - timestamp INTEGER NOT NULL - )`) - db.Exec(`CREATE TABLE IF NOT EXISTS _migrations (name TEXT PRIMARY KEY)`) - db.Exec(`CREATE TABLE IF NOT EXISTS nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL)`) - db.Exec(`CREATE TABLE IF NOT EXISTS observers (id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT, packet_count INTEGER DEFAULT 0, model TEXT, firmware TEXT, client_version TEXT, radio TEXT, battery_mv INTEGER, uptime_secs INTEGER, noise_floor REAL, inactive INTEGER DEFAULT 0, last_packet_at TEXT DEFAULT NULL)`) - - // Insert good transmission - db.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, 'aabb', 'abc123', '2024-01-01T00:00:00Z')`) - db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (1, 1, 1704067200)`) - - // Insert bad: empty hash - db.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (2, 'ccdd', '', '2024-01-01T00:00:00Z')`) - db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (2, 1, 1704067200)`) - - // Insert bad: empty first_seen - db.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (3, 'eeff', 'def456', '')`) - db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (3, 2, 1704067200)`) - - db.Close() - - // Now open via OpenStore which should run the migration - s, err := OpenStore(path) - if err != nil { - t.Fatal(err) - } - defer s.Close() - - // Good transmission should remain - var count int - s.db.QueryRow("SELECT COUNT(*) FROM transmissions WHERE id = 1").Scan(&count) - if count != 1 { - t.Error("good transmission should not be deleted") - } - - // Bad transmissions should be gone - s.db.QueryRow("SELECT COUNT(*) FROM transmissions WHERE id = 2").Scan(&count) - if count != 0 { - t.Errorf("transmission with empty hash should be deleted, got count=%d", count) - } - s.db.QueryRow("SELECT COUNT(*) FROM transmissions WHERE id = 3").Scan(&count) - if count != 0 { - t.Errorf("transmission with empty first_seen should be deleted, got count=%d", count) - } - - // Observations for bad transmissions should be gone - s.db.QueryRow("SELECT COUNT(*) FROM observations WHERE transmission_id IN (2, 3)").Scan(&count) - if count != 0 { - t.Errorf("observations for bad transmissions should be deleted, got count=%d", count) - } - - // Observation for good transmission should remain - s.db.QueryRow("SELECT COUNT(*) FROM observations WHERE transmission_id = 1").Scan(&count) - if count != 1 { - t.Error("observation for good transmission should remain") - } - - // Migration marker should exist - var migCount int - s.db.QueryRow("SELECT COUNT(*) FROM _migrations WHERE name = 'cleanup_legacy_null_hash_ts'").Scan(&migCount) - if migCount != 1 { - t.Error("migration marker cleanup_legacy_null_hash_ts should be recorded") - } - - // Idempotent: opening again should not error - s.Close() - s2, err := OpenStore(path) - if err != nil { - t.Fatal("second open should not fail:", err) - } - s2.Close() -} - func TestBuildPacketDataRegionFromPayload(t *testing.T) { msg := &MQTTPacketMessage{Raw: "0102030405060708", Region: "PDX"} decoded := &DecodedPacket{ @@ -2806,88 +2646,10 @@ func TestBackfillPathJSONAsync(t *testing.T) { dbPath := filepath.Join(dir, "async_test.db") // Bootstrap schema manually so we can insert test data BEFORE OpenStore - db, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL&_busy_timeout=5000") - if err != nil { - t.Fatal(err) - } - // Create tables manually (minimal schema for this test) - _, err = db.Exec(` - CREATE TABLE _migrations (name TEXT PRIMARY KEY); - CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raw_hex TEXT NOT NULL, - hash TEXT NOT NULL UNIQUE, - first_seen TEXT NOT NULL, - route_type INTEGER, - payload_type INTEGER, - payload_version INTEGER, - decoded_json TEXT, - created_at TEXT DEFAULT (datetime('now')), - channel_hash TEXT - ); - CREATE TABLE observers ( - id TEXT PRIMARY KEY, - name TEXT, - iata TEXT, - last_seen TEXT, - first_seen TEXT, - packet_count INTEGER DEFAULT 0, - model TEXT, - firmware TEXT, - client_version TEXT, - radio TEXT, - battery_mv INTEGER, - uptime_secs INTEGER, - noise_floor REAL, - inactive INTEGER DEFAULT 0, - last_packet_at TEXT - ); - CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, - name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, - battery_mv INTEGER, temperature_c REAL - ); - CREATE TABLE inactive_nodes ( - public_key TEXT PRIMARY KEY, - name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, - battery_mv INTEGER, temperature_c REAL - ); - CREATE TABLE observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - transmission_id INTEGER NOT NULL REFERENCES transmissions(id), - observer_idx INTEGER, - direction TEXT, - snr REAL, rssi REAL, score INTEGER, - path_json TEXT, - timestamp INTEGER NOT NULL, - raw_hex TEXT - ); - CREATE UNIQUE INDEX idx_observations_dedup ON observations(transmission_id, observer_idx, COALESCE(path_json, '')); - CREATE INDEX idx_observations_transmission_id ON observations(transmission_id); - CREATE INDEX idx_observations_observer_idx ON observations(observer_idx); - CREATE INDEX idx_observations_timestamp ON observations(timestamp); - CREATE TABLE observer_metrics ( - observer_id TEXT NOT NULL, - timestamp TEXT NOT NULL, - noise_floor REAL, tx_air_secs INTEGER, rx_air_secs INTEGER, - recv_errors INTEGER, battery_mv INTEGER, - packets_sent INTEGER, packets_recv INTEGER, - PRIMARY KEY (observer_id, timestamp) - ); - CREATE TABLE dropped_packets ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - hash TEXT, raw_hex TEXT, reason TEXT NOT NULL, - observer_id TEXT, observer_name TEXT, - node_pubkey TEXT, node_name TEXT, - dropped_at DATETIME DEFAULT CURRENT_TIMESTAMP - ); - `) + db, err := pgutil.Open(testPostgresURL(t, dbPath), false) if err != nil { - t.Fatal("bootstrap schema:", err) + t.Fatal(err) } - // Mark all migrations as done EXCEPT the path_json backfill for _, m := range []string{ "advert_count_unique_v1", "noise_floor_real_v1", "node_telemetry_v1", @@ -2896,19 +2658,19 @@ func TestBackfillPathJSONAsync(t *testing.T) { "dropped_packets_v1", "observations_raw_hex_v1", "observers_last_packet_at_v1", "cleanup_legacy_null_hash_ts", } { - db.Exec(`INSERT INTO _migrations (name) VALUES (?)`, m) + db.Exec(`INSERT INTO _migrations (name) VALUES ($1)`, m) } // Insert a transmission + observations with NULL path_json and valid raw_hex // raw_hex "0102AABBCCDD0000" has 2-hop path decodable by packetpath rawHex := "41020304AABBCCDD05060708" - _, err = db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES (?, 'hash1', '2025-01-01T00:00:00Z', 4)`, rawHex) + _, err = db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES ($1, 'hash1', '2025-01-01T00:00:00Z', 4)`, rawHex) if err != nil { t.Fatal("insert tx:", err) } // Insert 100 observations needing backfill for i := 0; i < 100; i++ { - _, err = db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp, raw_hex, path_json) VALUES (1, ?, ?, ?, NULL)`, + _, err = db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp, raw_hex, path_json) VALUES (1, $1, $2, $3, NULL)`, i+1, 1700000000+i, rawHex) if err != nil { // dedup index might fire — use unique observer_idx @@ -2919,7 +2681,7 @@ func TestBackfillPathJSONAsync(t *testing.T) { // Now open store via OpenStore — this must return QUICKLY (non-blocking) start := time.Now() - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) elapsed := time.Since(start) if err != nil { t.Fatal("OpenStore:", err) @@ -2967,7 +2729,7 @@ func TestBackfillPathJSONAsync(t *testing.T) { func TestBackfillPathJSONAsyncMethodExists(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "method_test.db") - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } @@ -2995,73 +2757,10 @@ func TestBackfillPathJSONAsync_BracketRowsTerminate(t *testing.T) { // Bootstrap a minimal schema directly so we can seed pre-existing '[]' rows // before OpenStore runs. - db, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL&_busy_timeout=5000") - if err != nil { - t.Fatal(err) - } - _, err = db.Exec(` - CREATE TABLE _migrations (name TEXT PRIMARY KEY); - CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raw_hex TEXT NOT NULL, - hash TEXT NOT NULL UNIQUE, - first_seen TEXT NOT NULL, - route_type INTEGER, - payload_type INTEGER, - payload_version INTEGER, - decoded_json TEXT, - created_at TEXT DEFAULT (datetime('now')), - channel_hash TEXT - ); - CREATE TABLE observers ( - id TEXT PRIMARY KEY, name TEXT, iata TEXT, - last_seen TEXT, first_seen TEXT, packet_count INTEGER DEFAULT 0, - model TEXT, firmware TEXT, client_version TEXT, radio TEXT, - battery_mv INTEGER, uptime_secs INTEGER, noise_floor REAL, - inactive INTEGER DEFAULT 0, last_packet_at TEXT - ); - CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, - lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, - advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL - ); - CREATE TABLE inactive_nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, - lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, - advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL - ); - CREATE TABLE observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - transmission_id INTEGER NOT NULL REFERENCES transmissions(id), - observer_idx INTEGER, direction TEXT, - snr REAL, rssi REAL, score INTEGER, - path_json TEXT, - timestamp INTEGER NOT NULL, - raw_hex TEXT - ); - CREATE UNIQUE INDEX idx_observations_dedup ON observations(transmission_id, observer_idx, COALESCE(path_json, '')); - CREATE INDEX idx_observations_transmission_id ON observations(transmission_id); - CREATE INDEX idx_observations_observer_idx ON observations(observer_idx); - CREATE INDEX idx_observations_timestamp ON observations(timestamp); - CREATE TABLE observer_metrics ( - observer_id TEXT NOT NULL, timestamp TEXT NOT NULL, - noise_floor REAL, tx_air_secs INTEGER, rx_air_secs INTEGER, - recv_errors INTEGER, battery_mv INTEGER, - packets_sent INTEGER, packets_recv INTEGER, - PRIMARY KEY (observer_id, timestamp) - ); - CREATE TABLE dropped_packets ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - hash TEXT, raw_hex TEXT, reason TEXT NOT NULL, - observer_id TEXT, observer_name TEXT, - node_pubkey TEXT, node_name TEXT, - dropped_at DATETIME DEFAULT CURRENT_TIMESTAMP - ); - `) + db, err := pgutil.Open(testPostgresURL(t, dbPath), false) if err != nil { - t.Fatal("bootstrap schema:", err) + t.Fatal(err) } - // Mark all migrations done EXCEPT backfill_path_json_from_raw_hex_v1. for _, m := range []string{ "advert_count_unique_v1", "noise_floor_real_v1", "node_telemetry_v1", @@ -3070,20 +2769,20 @@ func TestBackfillPathJSONAsync_BracketRowsTerminate(t *testing.T) { "dropped_packets_v1", "observations_raw_hex_v1", "observers_last_packet_at_v1", "cleanup_legacy_null_hash_ts", } { - db.Exec(`INSERT INTO _migrations (name) VALUES (?)`, m) + db.Exec(`INSERT INTO _migrations (name) VALUES ($1)`, m) } // raw_hex producing ZERO hops via DecodePathFromRawHex: // DIRECT route (type=2), payload_type=2, version=0 → header 0x0A; path byte 0x00. // (See internal/packetpath/path_test.go: TestDecodePathFromRawHex_ZeroHops.) rawHex := "0A00DEADBEEF" - _, err = db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES (?, 'h_brackets', '2025-01-01T00:00:00Z', 2)`, rawHex) + _, err = db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES ($1, 'h_brackets', '2025-01-01T00:00:00Z', 2)`, rawHex) if err != nil { t.Fatal("insert tx:", err) } const seedCount = 100 for i := 0; i < seedCount; i++ { - _, err = db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp, raw_hex, path_json) VALUES (1, ?, ?, ?, '[]')`, + _, err = db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp, raw_hex, path_json) VALUES (1, $1, $2, $3, '[]')`, i+1, 1700000000+i, rawHex) if err != nil { t.Fatalf("insert obs %d: %v", i, err) @@ -3091,7 +2790,7 @@ func TestBackfillPathJSONAsync_BracketRowsTerminate(t *testing.T) { } db.Close() - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal("OpenStore:", err) } @@ -3133,14 +2832,14 @@ func TestSchemaMultibyteSupColumns(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } defer store.Close() for _, table := range []string{"nodes", "inactive_nodes"} { - rows, err := store.db.Query("PRAGMA table_info(" + table + ")") + rows, err := store.db.Query(`SELECT ordinal_position::int,column_name,data_type,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name=$1`, table) if err != nil { t.Fatalf("PRAGMA table_info(%s): %v", table, err) } @@ -3173,7 +2872,7 @@ func TestSchemaMultibyteSupColumns(t *testing.T) { // in the legacy _migrations marker table — so we just re-assert the // columns exist and the second OpenStore is a no-op. store.Close() - store2, err := OpenStore(dbPath) + store2, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore (second open): %v", err) } @@ -3189,7 +2888,7 @@ func TestSchemaMultibyteSupColumns(t *testing.T) { func TestUpdateNodeDefaultScope_EmptyScopeIsNoop(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -3243,7 +2942,7 @@ func TestInsertClientRxObservation(t *testing.T) { // a swap is detectable. var gotLat, gotLon, gotSNR, gotPosAccM float64 var gotRSSI int - if err := s.db.QueryRow(`SELECT lat, lon, snr, rssi, pos_acc_m FROM client_rx_observations WHERE pkt_hash = ?`, o.PktHash). + if err := s.db.QueryRow(`SELECT lat, lon, snr, rssi, pos_acc_m FROM client_rx_observations WHERE pkt_hash = $1`, o.PktHash). Scan(&gotLat, &gotLon, &gotSNR, &gotRSSI, &gotPosAccM); err != nil { t.Fatalf("read back geometry: %v", err) } @@ -3273,7 +2972,7 @@ func TestInsertClientRxObservation(t *testing.T) { t.Fatalf("second copy: ins=%v err=%v", ins, err) } var n int - s.db.QueryRow(`SELECT COUNT(*) FROM client_rx_observations WHERE pkt_hash = ?`, o.PktHash).Scan(&n) + s.db.QueryRow(`SELECT COUNT(*) FROM client_rx_observations WHERE pkt_hash = $1`, o.PktHash).Scan(&n) if n != 2 { t.Errorf("rows for pkt_hash = %d, want 2", n) } @@ -3331,7 +3030,7 @@ func TestInsertClientRfSample(t *testing.T) { SELECT lat, lon, pos_acc_m, stationary, uptime_secs, battery_mv, queue_len, errors, noise_floor, last_rssi, last_snr, tx_air_secs, rx_air_secs, recv, sent, flood_rx, direct_rx, flood_tx, direct_tx, recv_errors - FROM client_rf_samples WHERE sampled_at = ?`, o.SampledAt).Scan( + FROM client_rf_samples WHERE sampled_at = $1`, o.SampledAt).Scan( &gotLat, &gotLon, &gotPosAccM, &gotStationary, &gotUptimeSecs, &gotBatteryMV, &gotQueueLen, &gotErrors, &gotNoiseFloor, &gotLastRSSI, &gotLastSNR, &gotTxAirSecs, &gotRxAirSecs, &gotRecv, &gotSent, &gotFloodRx, &gotDirectRx, &gotFloodTx, &gotDirectTx, &gotRecvErrors); err != nil { @@ -3406,7 +3105,7 @@ func TestInsertClientRfSample(t *testing.T) { t.Fatalf("insert second sample: ins=%v err=%v", ins, err) } var gotRecvErrors2 *int64 - if err := s.db.QueryRow(`SELECT recv_errors FROM client_rf_samples WHERE sampled_at = ?`, o2.SampledAt). + if err := s.db.QueryRow(`SELECT recv_errors FROM client_rf_samples WHERE sampled_at = $1`, o2.SampledAt). Scan(&gotRecvErrors2); err != nil { t.Fatalf("read second sample: %v", err) } @@ -3423,16 +3122,18 @@ func TestInsertClientRfSample(t *testing.T) { // this is a regression guard. func TestPruneClientRfSamplesUsesIndex(t *testing.T) { s := newTestStore(t) - rows, err := s.db.Query(`EXPLAIN QUERY PLAN DELETE FROM client_rf_samples WHERE sampled_at < ?`, "2026-01-01T00:00:00Z") + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } + rows, err := s.db.Query(`EXPLAIN (COSTS OFF) DELETE FROM client_rf_samples WHERE sampled_at < $1`, "2026-01-01T00:00:00Z") if err != nil { t.Fatal(err) } defer rows.Close() plan := "" for rows.Next() { - var id, parent, notused int var detail string - if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + if err := rows.Scan(&detail); err != nil { t.Fatal(err) } plan += detail + "\n" @@ -3492,7 +3193,7 @@ func TestPruneOldClientRfSamples(t *testing.T) { t.Fatalf("expected 2 samples remaining (recent + boundary), got %d", remaining) } var boundarySurvived int - s.db.QueryRow(`SELECT COUNT(*) FROM client_rf_samples WHERE sampled_at = ?`, boundary).Scan(&boundarySurvived) + s.db.QueryRow(`SELECT COUNT(*) FROM client_rf_samples WHERE sampled_at = $1`, boundary).Scan(&boundarySurvived) if boundarySurvived != 1 { t.Fatalf("boundary row (500ms after cutoff instant) must survive; an RFC3339 (no-ms) cutoff would wrongly delete it because '.' < 'Z' lexicographically — got %d", boundarySurvived) } diff --git a/cmd/ingestor/db_writer_perf_test.go b/cmd/ingestor/db_writer_perf_test.go index 740663f92..4c1312a18 100644 --- a/cmd/ingestor/db_writer_perf_test.go +++ b/cmd/ingestor/db_writer_perf_test.go @@ -30,7 +30,7 @@ func TestWriterStarvationVisibleInPerf(t *testing.T) { // follower samples can't move p99 above 50s. ResetWriterStatsForTest() - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -46,7 +46,7 @@ func TestWriterStarvationVisibleInPerf(t *testing.T) { go func() { defer close(blockerDone) err := s.WriterTx("neighbor_builder", func(tx *sql.Tx) error { - if _, err := tx.Exec(`UPDATE nodes SET name = name WHERE 0`); err != nil { + if _, err := tx.Exec(`UPDATE nodes SET name = name WHERE false`); err != nil { return err } close(blockStarted) @@ -59,7 +59,11 @@ func TestWriterStarvationVisibleInPerf(t *testing.T) { }() // Wait for the blocker to be inside its transaction. - <-blockStarted + select { + case <-blockStarted: + case <-blockerDone: + t.Fatal("blocker failed before acquiring its transaction") + } // Small safety margin so the blocker is firmly holding the conn. time.Sleep(100 * time.Millisecond) @@ -74,7 +78,7 @@ func TestWriterStarvationVisibleInPerf(t *testing.T) { defer wg.Done() _, err := s.WriterExec( "mqtt_handler", - `INSERT OR IGNORE INTO _migrations (name) VALUES (?)`, + `INSERT INTO _migrations (name) VALUES ($1) ON CONFLICT DO NOTHING`, fmt.Sprintf("writer_starvation_test_%d", i), ) if err != nil { diff --git a/cmd/ingestor/decoder.go b/cmd/ingestor/decoder.go index 41aa3afc4..ae0c2f9a4 100644 --- a/cmd/ingestor/decoder.go +++ b/cmd/ingestor/decoder.go @@ -1068,55 +1068,7 @@ func DecodePacket(hexString string, channelKeys map[string]string, validateSigna // route-independent identifier for the same logical packet. For TRACE packets, // path_len is included in the hash to match firmware behavior. func ComputeContentHash(rawHex string) string { - buf, err := hex.DecodeString(rawHex) - if err != nil || len(buf) < 2 { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex - } - - headerByte := buf[0] - offset := 1 - if isTransportRoute(int(headerByte & 0x03)) { - offset += 4 - } - if offset >= len(buf) { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex - } - pathByte := buf[offset] - offset++ - hashSize := int((pathByte>>6)&0x3) + 1 - hashCount := int(pathByte & 0x3F) - pathBytes := hashSize * hashCount - - payloadStart := offset + pathBytes - if payloadStart > len(buf) { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex - } - - payload := buf[payloadStart:] - - // Hash payload-type byte only (bits 2-5 of header), not the full header. - // Firmware: SHA256(payload_type + [path_len for TRACE] + payload) - // Using the full header caused different hashes for the same logical packet - // when route type or version bits differed. See issue #786. - payloadType := (headerByte >> 2) & 0x0F - toHash := []byte{payloadType} - if int(payloadType) == PayloadTRACE { - // Firmware uses uint16_t path_len (2 bytes, little-endian) - toHash = append(toHash, pathByte, 0x00) - } - toHash = append(toHash, payload...) - - h := sha256.Sum256(toHash) - return hex.EncodeToString(h[:])[:16] + return packetpath.ContentHash(rawHex) } // PayloadJSON serializes the payload to JSON for DB storage. diff --git a/cmd/ingestor/from_pubkey_test.go b/cmd/ingestor/from_pubkey_test.go index 5fb614961..c0a47b862 100644 --- a/cmd/ingestor/from_pubkey_test.go +++ b/cmd/ingestor/from_pubkey_test.go @@ -10,7 +10,7 @@ import ( ) func TestInsertTransmission_FromPubkeyPopulatedForAdvert(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -34,14 +34,14 @@ func TestInsertTransmission_FromPubkeyPopulatedForAdvert(t *testing.T) { } var got sql.NullString - s.db.QueryRow("SELECT from_pubkey FROM transmissions WHERE hash = ?", data.Hash).Scan(&got) + s.db.QueryRow("SELECT from_pubkey FROM transmissions WHERE hash = $1", data.Hash).Scan(&got) if !got.Valid || got.String != pk { t.Fatalf("from_pubkey = %v (valid=%v), want %q", got.String, got.Valid, pk) } } func TestInsertTransmission_FromPubkeyNullForNonAdvert(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -64,7 +64,7 @@ func TestInsertTransmission_FromPubkeyNullForNonAdvert(t *testing.T) { } var got sql.NullString - s.db.QueryRow("SELECT from_pubkey FROM transmissions WHERE hash = ?", data.Hash).Scan(&got) + s.db.QueryRow("SELECT from_pubkey FROM transmissions WHERE hash = $1", data.Hash).Scan(&got) if got.Valid { t.Fatalf("from_pubkey for non-ADVERT must be NULL, got %q", got.String) } diff --git a/cmd/ingestor/go.mod b/cmd/ingestor/go.mod index a758d56c6..526200b38 100644 --- a/cmd/ingestor/go.mod +++ b/cmd/ingestor/go.mod @@ -1,6 +1,6 @@ module github.com/corescope/ingestor -go 1.22 +go 1.25.0 require ( github.com/eclipse/paho.mqtt.golang v1.5.0 @@ -30,21 +30,29 @@ replace github.com/meshcore-analyzer/dbschema => ../../internal/dbschema require ( github.com/gorilla/websocket v1.5.3 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.7.6 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + golang.org/x/crypto v0.37.0 // indirect golang.org/x/net v0.27.0 // indirect - golang.org/x/sync v0.7.0 // indirect + golang.org/x/sync v0.13.0 // indirect + golang.org/x/text v0.24.0 // indirect ) require github.com/meshcore-analyzer/prunequeue v0.0.0 replace github.com/meshcore-analyzer/prunequeue => ../../internal/prunequeue -require ( - github.com/mattn/go-sqlite3 v1.14.52 - github.com/meshcore-analyzer/mbcapqueue v0.0.0 -) +require github.com/meshcore-analyzer/mbcapqueue v0.0.0 replace github.com/meshcore-analyzer/mbcapqueue => ../../internal/mbcapqueue -require github.com/meshcore-analyzer/channel v0.0.0 +require ( + github.com/meshcore-analyzer/channel v0.0.0 + github.com/meshcore-analyzer/pgutil v0.0.0 +) replace github.com/meshcore-analyzer/channel => ../../internal/channel + +replace github.com/meshcore-analyzer/pgutil => ../../internal/pgutil diff --git a/cmd/ingestor/go.sum b/cmd/ingestor/go.sum index 554a48272..fca6ac2ea 100644 --- a/cmd/ingestor/go.sum +++ b/cmd/ingestor/go.sum @@ -1,10 +1,34 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/eclipse/paho.mqtt.golang v1.5.0 h1:EH+bUVJNgttidWFkLLVKaQPGmkTUfQQqjOsyvMGvD6o= github.com/eclipse/paho.mqtt.golang v1.5.0/go.mod h1:du/2qNQVqJf/Sqs4MEL77kR8QTqANF7XU7Fk0aOTAgk= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= -github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cmd/ingestor/ingest_buffer.go b/cmd/ingestor/ingest_buffer.go index 641801a3d..a762e6159 100644 --- a/cmd/ingestor/ingest_buffer.go +++ b/cmd/ingestor/ingest_buffer.go @@ -10,7 +10,7 @@ import ( // IngestBuffer decouples MQTT message receipt from DB writes (#1608). // // On boot the ingestor must subscribe to MQTT immediately, but the single -// SQLite writer (#1283) can be held for minutes by a startup migration +// serialized writer (#1283) can be held for minutes by a startup migration // (e.g. a large CREATE INDEX) or prune. Without buffering, every QoS-0 packet // received in that window is lost. IngestBuffer holds received work in a // bounded FIFO and a single consumer goroutine drains it once Ready() is diff --git a/cmd/ingestor/issue1283_test.go b/cmd/ingestor/issue1283_test.go index 4cca01641..96a04afac 100644 --- a/cmd/ingestor/issue1283_test.go +++ b/cmd/ingestor/issue1283_test.go @@ -1,12 +1,9 @@ package main import ( - "database/sql" "path/filepath" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // TestIngestorPruneOldPackets enforces #1283: the writer for @@ -16,7 +13,7 @@ import ( func TestIngestorPruneOldPackets(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "prune.db") - store, err := OpenStore(path) + store, err := openPostgresTestStore(t, path) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -27,7 +24,7 @@ func TestIngestorPruneOldPackets(t *testing.T) { for i, ts := range []string{old, old, new} { _, err := store.db.Exec( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) - VALUES (?, ?, ?, 0, 1, 1, '{}')`, + VALUES ($1, $2, $3, 0, 1, 1, '{}')`, "AA", "h"+string(rune('a'+i)), ts, ) if err != nil { @@ -52,47 +49,9 @@ func TestIngestorPruneOldPackets(t *testing.T) { } } -// TestIngestorVacuumOnStartupMigratesNONEtoINCREMENTAL exercises the -// scenario that originally broke in #1283: a fresh DB with -// auto_vacuum=NONE, vacuumOnStartup=true, no contention from a server -// process. The ingestor must complete the VACUUM and flip auto_vacuum to -// INCREMENTAL. Before the fix, the migration ran inside cmd/server and -// hit SQLITE_BUSY because the ingestor (sharing the container) was -// already writing. -func TestIngestorVacuumOnStartupMigratesNONEtoINCREMENTAL(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "vac.db") - - // Create a NONE-auto_vacuum DB (simulates an older deployment). - seed, err := sql.Open("sqlite3", path+"?_journal_mode=WAL") - if err != nil { - t.Fatal(err) - } - seed.SetMaxOpenConns(1) - if _, err := seed.Exec(`CREATE TABLE dummy(id INTEGER PRIMARY KEY)`); err != nil { - t.Fatal(err) - } - var before int - seed.QueryRow("PRAGMA auto_vacuum").Scan(&before) - if before != 0 { - t.Fatalf("precondition: auto_vacuum=%d, want 0", before) - } - seed.Close() - - store, err := OpenStore(path) - if err != nil { - t.Fatalf("OpenStore: %v", err) - } - defer store.Close() - - cfg := &Config{DB: &DBConfig{VacuumOnStartup: true}} - store.CheckAutoVacuum(cfg) - - var after int - if err := store.db.QueryRow("PRAGMA auto_vacuum").Scan(&after); err != nil { - t.Fatal(err) - } - if after != 2 { - t.Fatalf("expected auto_vacuum=2 after ingestor VACUUM, got %d", after) +func TestPostgresRuntimeCannotChangeTableVacuumPolicy(t *testing.T) { + s := newTestStore(t) + if _, err := s.db.Exec(`ALTER TABLE transmissions SET (autovacuum_enabled=false)`); err == nil { + t.Fatal("runtime writer changed owner-only vacuum policy") } } diff --git a/cmd/ingestor/issue1690_last_seen_test.go b/cmd/ingestor/issue1690_last_seen_test.go index 5b31917ec..6578f1f14 100644 --- a/cmd/ingestor/issue1690_last_seen_test.go +++ b/cmd/ingestor/issue1690_last_seen_test.go @@ -13,7 +13,7 @@ import ( ) func TestIssue1690_LastSeenUpdatedOnObservation(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -42,7 +42,7 @@ func TestIssue1690_LastSeenUpdatedOnObservation(t *testing.T) { // Sanity: confirm the seed last_seen is the 7d-ago time. var seededLastSeen int64 - if err := s.db.QueryRow(`SELECT COALESCE(last_seen, 0) FROM transmissions WHERE hash = ?`, hash).Scan(&seededLastSeen); err != nil { + if err := s.db.QueryRow(`SELECT COALESCE(last_seen, 0) FROM transmissions WHERE hash = $1`, hash).Scan(&seededLastSeen); err != nil { t.Fatalf("seed select last_seen: %v (column missing? post-fix must add it)", err) } weekAgoUnix, _ := time.Parse(time.RFC3339, weekAgo) @@ -71,7 +71,7 @@ func TestIssue1690_LastSeenUpdatedOnObservation(t *testing.T) { } var ls int64 - if err := s.db.QueryRow(`SELECT last_seen FROM transmissions WHERE hash = ?`, hash).Scan(&ls); err != nil { + if err := s.db.QueryRow(`SELECT last_seen FROM transmissions WHERE hash = $1`, hash).Scan(&ls); err != nil { t.Fatalf("post-insert select last_seen: %v", err) } // The post-fix writer must bump last_seen to at least the new observation's diff --git a/cmd/ingestor/issue1865_test.go b/cmd/ingestor/issue1865_test.go index 4bf3e241e..fff3ab600 100644 --- a/cmd/ingestor/issue1865_test.go +++ b/cmd/ingestor/issue1865_test.go @@ -14,10 +14,10 @@ import ( // seedNode inserts a node into both nodes and inactive_nodes (lowercase key). func seedNode(t *testing.T, store *Store, pubkey string) { t.Helper() - if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name) VALUES (?, ?)`, pubkey, "n_"+pubkey[:4]); err != nil { + if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name) VALUES ($1, $2)`, pubkey, "n_"+pubkey[:4]); err != nil { t.Fatalf("seed node %s: %v", pubkey, err) } - if _, err := store.db.Exec(`INSERT INTO inactive_nodes (public_key, name) VALUES (?, ?)`, pubkey, "n_"+pubkey[:4]); err != nil { + if _, err := store.db.Exec(`INSERT INTO inactive_nodes (public_key, name) VALUES ($1, $2)`, pubkey, "n_"+pubkey[:4]); err != nil { t.Fatalf("seed inactive node %s: %v", pubkey, err) } } @@ -26,7 +26,7 @@ func configuredScope(t *testing.T, store *Store, pubkey string) (sql.NullString, t.Helper() var sc, at sql.NullString if err := store.db.QueryRow( - `SELECT configured_scope, configured_scope_at FROM nodes WHERE public_key = ?`, pubkey, + `SELECT configured_scope, configured_scope_at FROM nodes WHERE public_key = $1`, pubkey, ).Scan(&sc, &at); err != nil { t.Fatalf("read configured_scope for %s: %v", pubkey, err) } @@ -35,7 +35,7 @@ func configuredScope(t *testing.T, store *Store, pubkey string) (sql.NullString, func openNeighborsStore(t *testing.T) *Store { t.Helper() - store, err := OpenStore(filepath.Join(t.TempDir(), "test.db")) + store, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), "test.db")) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -151,7 +151,7 @@ func TestUpdateNodeConfiguredScopeLastWriteWins(t *testing.T) { } // inactive_nodes mirrored. var inactive sql.NullString - if err := store.db.QueryRow(`SELECT configured_scope FROM inactive_nodes WHERE public_key = ?`, pk).Scan(&inactive); err != nil { + if err := store.db.QueryRow(`SELECT configured_scope FROM inactive_nodes WHERE public_key = $1`, pk).Scan(&inactive); err != nil { t.Fatal(err) } if inactive.String != "#de" { diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index d125601fe..cecf47e68 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -43,7 +43,13 @@ func main() { } configPath := flag.String("config", "config.json", "path to config file") + databaseURL := flag.String("database-url", "", "PostgreSQL telemetry writer URL") + stateDir := flag.String("state-dir", "", "local queue and statistics directory") + legacyDB := flag.String("db", "", "removed SQLite path; use the offline importer") flag.Parse() + if *legacyDB != "" { + log.Fatal("-db is no longer supported; import SQLite offline and configure -database-url") + } log.SetFlags(log.LstdFlags | log.Lmsgprefix) log.SetPrefix("[ingestor] ") @@ -74,12 +80,21 @@ func main() { sources := cfg.ResolvedSources() - store, err := OpenStoreWithInterval(cfg.DBPath, cfg.MetricsSampleInterval()) + if *databaseURL != "" { + cfg.DatabaseURL = *databaseURL + } + if *stateDir != "" { + cfg.StateDir = *stateDir + } + if cfg.DatabaseURL == "" { + log.Fatal("databaseURL or CORESCOPE_DATABASE_URL is required; migrate legacy SQLite files offline") + } + store, err := OpenStoreWithState(cfg.DatabaseURL, cfg.StateDir, cfg.MetricsSampleInterval()) if err != nil { log.Fatalf("db: %v", err) } defer store.Close() - log.Printf("SQLite opened: %s", cfg.DBPath) + log.Print("PostgreSQL telemetry writer connected") // Async backfill: path_json from raw_hex (#888) — must not block MQTT startup store.BackfillPathJSONAsync() @@ -94,7 +109,6 @@ func main() { go store.BackfillFromPubkey(5000, 100*time.Millisecond, nil) // Check auto_vacuum mode and optionally migrate (#919) - store.CheckAutoVacuum(cfg) channelKeys := loadChannelKeys(cfg, *configPath) if len(channelKeys) > 0 { @@ -103,9 +117,9 @@ func main() { log.Printf("No channel keys loaded — GRP_TXT packets will not be decrypted") } - keySet := newChannelKeySet(channelKeys, cfg.UsersDBPath(), cfg.ApprovedChannelsMax()) + keySet := newChannelKeySet(channelKeys, cfg.ApprovedChannelsURL(), cfg.ApprovedChannelsMax()) if cfg.ApprovedChannelsEnabled() { - logApprovedChannelsSource(cfg.UsersDBPath()) + logApprovedChannelsSource(cfg.ApprovedChannelsURL()) keySet.refresh() go func() { t := time.NewTicker(approvedChannelsRefresh) @@ -129,7 +143,7 @@ func main() { // Subscribe-early + buffer (#1608): the MQTT subscription is brought up // before startup maintenance so no packets are missed while the single - // SQLite writer is blocked (e.g. a large CREATE INDEX migration). Received + // serialized writer is busy with a bounded data backfill. Received // messages are buffered here and drained once Ready() is called below. ingestBuffer := NewIngestBuffer(cfg.IngestBufferSizeOrDefault()) ingestBuffer.Start() @@ -347,12 +361,8 @@ func main() { } } - vacuumPages := cfg.IncrementalVacuumPages() - store.RunIncrementalVacuum(vacuumPages) - // Gate open: the synchronous startup writes above cannot return until the - // single SQLite writer is free, which means any blocking async migration - // (e.g. the CREATE INDEX) has finished. WaitForAsyncMigrations() makes that + // single serialized writer is free. WaitForAsyncMigrations() makes that // explicit. Now drain everything the subscription buffered during startup. store.WaitForAsyncMigrations() ingestBuffer.Ready() @@ -374,7 +384,7 @@ func main() { go func() { for range retentionTicker.C { store.MoveStaleNodes(nodeDays) - store.RunIncrementalVacuum(vacuumPages) + } }() @@ -384,11 +394,11 @@ func main() { time.Sleep(90 * time.Second) // stagger after metrics prune store.RemoveStaleObservers(observerDays) store.PurgeStaleObservers(observerPurgeDays) - store.RunIncrementalVacuum(vacuumPages) + for range observerRetentionTicker.C { store.RemoveStaleObservers(observerDays) store.PurgeStaleObservers(observerPurgeDays) - store.RunIncrementalVacuum(vacuumPages) + } }() @@ -398,7 +408,7 @@ func main() { for range metricsRetentionTicker.C { store.PruneOldMetrics(metricsDays) store.PruneDroppedPackets(metricsDays) - store.RunIncrementalVacuum(vacuumPages) + } }() @@ -411,7 +421,7 @@ func main() { if n, err := store.PruneOldPackets(packetDays); err != nil { log.Printf("[prune] error: %v", err) } else if n > 0 { - store.RunIncrementalVacuum(vacuumPages) + } } }() @@ -431,28 +441,28 @@ func main() { if n, err := store.PruneOldClientReceptions(clientRxDays); err != nil { log.Printf("[prune] error: %v", err) } else if n > 0 { - store.RunIncrementalVacuum(vacuumPages) + } } if clientRxObsDays > 0 { if n, err := store.PruneOldClientRxObservations(clientRxObsDays); err != nil { log.Printf("[prune] client_rx_observations: %v", err) } else if n > 0 { - store.RunIncrementalVacuum(vacuumPages) + } } if clientRfDays > 0 { if n, err := store.PruneOldClientRfSamples(clientRfDays); err != nil { log.Printf("[prune] client_rf_samples: %v", err) } else if n > 0 { - store.RunIncrementalVacuum(vacuumPages) + } } if clientRegionsDays > 0 { if n, err := store.PruneOldClientDeclaredRegions(clientRegionsDays); err != nil { log.Printf("[prune] node_declared_regions: %v", err) } else if n > 0 { - store.RunIncrementalVacuum(vacuumPages) + } } } @@ -487,56 +497,7 @@ func main() { log.Printf("[regions] auto-derived region keys enabled: refreshing every %v, cap %d", interval, cfg.AutoRegionKeysMaxDerived()) } - // Hourly WAL checkpoint to prevent unbounded WAL growth. - // TRUNCATE resets the WAL file to zero bytes when all frames are flushed; - // if the server's read connection holds frames, remaining pages stay in the - // WAL until the next tick. Staggered 30s after startup to avoid competing - // with the initial burst of ingest writes. - walCheckpointTicker := time.NewTicker(1 * time.Hour) - go func() { - time.Sleep(30 * time.Second) - store.Checkpoint() - for range walCheckpointTicker.C { - store.Checkpoint() - } - }() - log.Printf("[db] WAL checkpoint scheduled every 1h") - - // Daily planner statistics refresh (#2058), in two parts. - // - // The routine refresh is staggered 2 minutes past startup for the same reason - // as the checkpoint above: it takes the write lock, and by then the initial - // ingest burst has passed, so it also sees the rows that burst added. - // - // The build in front of it deliberately does compete with that burst, because - // a database with no statistics at all has nothing better to offer the queries - // arriving in those 2 minutes. It only runs once per database; see - // Store.EnsurePlannerStats, which also carries what that costs. - // - // Bounded by analysis_limit either way, so neither grows with the file the way - // an unbounded ANALYZE does: 2.0s against 242.9s on a 9.4 GB database, both - // timed warm. Cold, on a first start, it is 3m43.9s. - { - analysisLimit := cfg.AnalysisLimit() - if analysisLimit < 0 { - log.Printf("[analyze] planner statistics refresh disabled (db.analysisLimit=%d)", analysisLimit) - } else { - analyzeTicker := time.NewTicker(24 * time.Hour) - go func() { - // Before the stagger, and only on a database that has never been - // analyzed: the stagger is a 2 minute window in which the first - // query would otherwise run on no statistics at all. A restart - // finds sqlite_stat1 already in the file and skips this. - store.EnsurePlannerStats(analysisLimit) - time.Sleep(2 * time.Minute) - store.RefreshPlannerStats(analysisLimit) - for range analyzeTicker.C { - store.RefreshPlannerStats(analysisLimit) - } - }() - log.Printf("[analyze] planner statistics refresh scheduled every 24h (analysis_limit=%d)", analysisLimit) - } - } + // PostgreSQL autovacuum, checkpoints and planner statistics are owned by the database service. // Daily neighbor_edges retention (#1287 — moved from cmd/server). { @@ -645,7 +606,6 @@ func main() { } statsTicker.Stop() pruneQueueTicker.Stop() - walCheckpointTicker.Stop() stopWatchdog() // A deploy is a SIGTERM, which is exactly the case that used to lose // the tally: save before the process goes away rather than leaving up diff --git a/cmd/ingestor/main_test.go b/cmd/ingestor/main_test.go index df57c7255..dfed98948 100644 --- a/cmd/ingestor/main_test.go +++ b/cmd/ingestor/main_test.go @@ -125,7 +125,7 @@ func newTestStore(t *testing.T) *Store { t.Helper() dir := t.TempDir() dbPath := dir + "/test.db" - s, err := OpenStore(dbPath) + s, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } @@ -1135,7 +1135,7 @@ func TestHandleMessageAdvert_EmptyScopeSkipsDefaultScopeUpdate(t *testing.T) { // Pre-seed the node with a non-empty default_scope so we can detect an // erroneous overwrite with "". - if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name, default_scope) VALUES (?, 'Node1', '#belgium')`, pubkey); err != nil { + if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name, default_scope) VALUES ($1, 'Node1', '#belgium')`, pubkey); err != nil { t.Fatalf("seed node: %v", err) } @@ -1147,7 +1147,7 @@ func TestHandleMessageAdvert_EmptyScopeSkipsDefaultScopeUpdate(t *testing.T) { handleMessage(store, "test", source, msg, nil, nil, &Config{}) var got sql.NullString - if err := store.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = ?`, pubkey).Scan(&got); err != nil { + if err := store.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = $1`, pubkey).Scan(&got); err != nil { t.Fatalf("read default_scope: %v", err) } if !got.Valid || got.String != "#belgium" { @@ -1188,7 +1188,7 @@ func TestHandleMessageAdvert_MatchedScopeUpdatesDefaultScope(t *testing.T) { code1 := fmt.Sprintf("%02X%02X", byte(code&0xFF), byte(code>>8)) rawHex := "10" + code1 + "000000" + advertBytes - if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name, default_scope) VALUES (?, 'Node1', '#old')`, pubkey); err != nil { + if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name, default_scope) VALUES ($1, 'Node1', '#old')`, pubkey); err != nil { t.Fatalf("seed node: %v", err) } @@ -1199,7 +1199,7 @@ func TestHandleMessageAdvert_MatchedScopeUpdatesDefaultScope(t *testing.T) { handleMessage(store, "test", source, msg, nil, regionKeySetFromKeys(map[string][]byte{"#de": regionKey}), &Config{}) var got sql.NullString - if err := store.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = ?`, pubkey).Scan(&got); err != nil { + if err := store.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = $1`, pubkey).Scan(&got); err != nil { t.Fatalf("read default_scope: %v", err) } if !got.Valid || got.String != "#de" { diff --git a/cmd/ingestor/maintenance.go b/cmd/ingestor/maintenance.go index 6ee0fe77d..a1afa76ae 100644 --- a/cmd/ingestor/maintenance.go +++ b/cmd/ingestor/maintenance.go @@ -34,18 +34,15 @@ const pruneBatchTransmissions = 250 // set: nothing modifies `transmissions` between them inside the transaction. // // The ORDER BY must be satisfiable from idx_transmissions_first_seen. That -// index carries the rowid as its tiebreaker, so "first_seen, id" is walked -// straight off it and the LIMIT stays deterministic even when timestamps tie. -// Ordering by id alone looks equivalent but makes SQLite abandon the index for -// a rowid SCAN. That is harmless while rows are being deleted — the oldest -// rows have the lowest rowids and match at once — but the batch that finds -// nothing, which is the steady state whenever nothing has aged out, walks the -// whole table under writerMu. TestPruneAgedTransmissionIDsUsesFirstSeenIndex -// pins the plan. -const pruneAgedTransmissionIDs = `SELECT id FROM transmissions WHERE first_seen < ? ORDER BY first_seen, id LIMIT ?` +// PostgreSQL index explicitly includes id as the tiebreaker (SQLite carried +// its rowid implicitly). This lets the LIMIT stop the ordered scan even when +// timestamps tie. Ordering by id alone could scan the entire table when no +// rows have aged out. TestPruneAgedTransmissionIDsHasOrderedIndex pins ordered +// index eligibility for both the batch query and its DELETE callers. +const pruneAgedTransmissionIDs = `SELECT id FROM transmissions WHERE first_seen < $1 ORDER BY first_seen, id LIMIT $2` -// The two statements of one prune batch. Child observations go first (no -// CASCADE in SQLite). +// The two statements of one prune batch. Child observations go first, retaining +// the bounded deletion behavior of the SQLite implementation. const ( pruneObservationsBatch = `DELETE FROM observations WHERE transmission_id IN (` + pruneAgedTransmissionIDs + `)` pruneTransmissionsBatch = `DELETE FROM transmissions WHERE id IN (` + pruneAgedTransmissionIDs + `)` @@ -117,13 +114,13 @@ func (s *Store) PruneOldClientReceptions(days int) (int64, error) { var n int64 err := s.WriterTx("prune_client_receptions", func(tx *sql.Tx) error { - res, err := tx.Exec(`DELETE FROM client_receptions WHERE rx_at < ?`, cutoff) + res, err := tx.Exec(`DELETE FROM client_receptions WHERE rx_at < $1`, cutoff) if err != nil { return fmt.Errorf("prune client_receptions: %w", err) } n, _ = res.RowsAffected() // Drop companion name rows not refreshed within the window. - if _, err := tx.Exec(`DELETE FROM client_observers WHERE last_seen < ?`, cutoff); err != nil { + if _, err := tx.Exec(`DELETE FROM client_observers WHERE last_seen < $1`, cutoff); err != nil { return fmt.Errorf("prune client_observers: %w", err) } return nil @@ -149,7 +146,7 @@ func (s *Store) PruneOldClientRxObservations(days int) (int64, error) { return 0, nil } cutoff := time.Now().UTC().AddDate(0, 0, -days).Format(rxTimeMillisLayout) - res, err := s.db.Exec(`DELETE FROM client_rx_observations WHERE rx_at < ?`, cutoff) + res, err := s.db.Exec(`DELETE FROM client_rx_observations WHERE rx_at < $1`, cutoff) if err != nil { return 0, fmt.Errorf("prune client_rx_observations: %w", err) } @@ -172,7 +169,7 @@ func (s *Store) PruneOldClientRfSamples(days int) (int64, error) { return 0, nil } cutoff := time.Now().UTC().AddDate(0, 0, -days).Format(rxTimeMillisLayout) - res, err := s.db.Exec(`DELETE FROM client_rf_samples WHERE sampled_at < ?`, cutoff) + res, err := s.db.Exec(`DELETE FROM client_rf_samples WHERE sampled_at < $1`, cutoff) if err != nil { return 0, fmt.Errorf("prune client_rf_samples: %w", err) } @@ -210,7 +207,7 @@ func (s *Store) PruneOldClientDeclaredRegions(days int) (int64, error) { // specific side of it. func (s *Store) pruneOldClientDeclaredRegionsAt(cutoffInstant time.Time) (int64, error) { cutoff := cutoffInstant.Format(rxTimeMillisLayout) - res, err := s.db.Exec(`DELETE FROM node_declared_regions WHERE observed_at < ?`, cutoff) + res, err := s.db.Exec(`DELETE FROM node_declared_regions WHERE observed_at < $1`, cutoff) if err != nil { return 0, fmt.Errorf("prune node_declared_regions: %w", err) } @@ -240,7 +237,7 @@ func (s *Store) PruneNeighborEdges(maxAgeDays int) (int64, error) { return 0, nil } cutoff := time.Now().UTC().Add(-time.Duration(maxAgeDays) * 24 * time.Hour).Format(time.RFC3339) - res, err := s.db.Exec("DELETE FROM neighbor_edges WHERE last_seen < ?", cutoff) + res, err := s.db.Exec("DELETE FROM neighbor_edges WHERE last_seen < $1", cutoff) if err != nil { return 0, fmt.Errorf("prune neighbor_edges: %w", err) } @@ -302,7 +299,7 @@ func (s *Store) BackfillFromPubkey(chunkSize int, yieldDuration time.Duration, p } log.Printf("[backfill] from_pubkey starting: %d ADVERT rows", total) - stmt, err := s.db.Prepare("UPDATE transmissions SET from_pubkey = ? WHERE id = ?") + stmt, err := s.db.Prepare("UPDATE transmissions SET from_pubkey = $1 WHERE id = $2") if err != nil { log.Printf("[backfill] from_pubkey prepare: %v", err) return @@ -312,7 +309,7 @@ func (s *Store) BackfillFromPubkey(chunkSize int, yieldDuration time.Duration, p var processed int64 for { rows, err := s.db.Query( - "SELECT id, decoded_json FROM transmissions WHERE from_pubkey IS NULL AND payload_type = 4 LIMIT ?", + "SELECT id, decoded_json FROM transmissions WHERE from_pubkey IS NULL AND payload_type = 4 LIMIT $1", chunkSize) if err != nil { log.Printf("[backfill] from_pubkey select: %v", err) @@ -336,7 +333,7 @@ func (s *Store) BackfillFromPubkey(chunkSize int, yieldDuration time.Duration, p break } - tx, err := s.db.Begin() + tx, err := beginWrite(s.db) if err != nil { log.Printf("[backfill] from_pubkey begin tx: %v", err) return diff --git a/cmd/ingestor/multibyte_persist.go b/cmd/ingestor/multibyte_persist.go index d9938da55..9d51cf498 100644 --- a/cmd/ingestor/multibyte_persist.go +++ b/cmd/ingestor/multibyte_persist.go @@ -76,7 +76,7 @@ func (s *Store) RunMultibyteCapPersist() (MultibyteCapPersistStats, error) { return stats, nil } - tx, err := s.db.Begin() + tx, err := beginWrite(s.db) if err != nil { return stats, err } @@ -85,19 +85,19 @@ func (s *Store) RunMultibyteCapPersist() (MultibyteCapPersistStats, error) { // inactive_nodes. The pre-#1386 implementation issued one UPDATE // against each table per entry — 50% guaranteed-empty. We now // look up the table once, then issue the matching UPDATE. - stmtN, err := tx.Prepare(`UPDATE nodes SET multibyte_sup=?, multibyte_evidence=? WHERE public_key=?`) + stmtN, err := tx.Prepare(`UPDATE nodes SET multibyte_sup=$1, multibyte_evidence=$2 WHERE public_key=$3`) if err != nil { return stats, err } defer stmtN.Close() - stmtI, err := tx.Prepare(`UPDATE inactive_nodes SET multibyte_sup=?, multibyte_evidence=? WHERE public_key=?`) + stmtI, err := tx.Prepare(`UPDATE inactive_nodes SET multibyte_sup=$1, multibyte_evidence=$2 WHERE public_key=$3`) if err != nil { return stats, err } defer stmtI.Close() // Membership probe: one indexed PK lookup. Cheap; avoids the // guaranteed-miss second UPDATE. - stmtProbe, err := tx.Prepare(`SELECT 1 FROM nodes WHERE public_key=? LIMIT 1`) + stmtProbe, err := tx.Prepare(`SELECT 1 FROM nodes WHERE public_key=$1 LIMIT 1`) if err != nil { return stats, err } @@ -187,24 +187,9 @@ func containsCI(s, sub string) bool { // RunMultibyteCapPersist on legacy DBs that pre-date the // internal/dbschema migration (#1386). func (s *Store) hasMultibyteSupColumns() bool { - rows, err := s.db.Query(`PRAGMA table_info(nodes)`) - if err != nil { - return false - } - defer rows.Close() - for rows.Next() { - var cid int - var name, ctype string - var notnull, pk int - var dflt interface{} - if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil { - return false - } - if name == "multibyte_sup" { - return true - } - } - return false + var found bool + _ = s.db.QueryRow(`SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='nodes' AND column_name='multibyte_sup')`).Scan(&found) + return found } // multibyteStatusToInt mirrors the mapping the server used before relocation. diff --git a/cmd/ingestor/multibyte_persist_helpers_test.go b/cmd/ingestor/multibyte_persist_helpers_test.go index 5e57fd8a5..d82501d37 100644 --- a/cmd/ingestor/multibyte_persist_helpers_test.go +++ b/cmd/ingestor/multibyte_persist_helpers_test.go @@ -57,22 +57,9 @@ func logContains(buf *syncBuffer, substr string) bool { // columnExists reports whether the named column exists on the table. func columnExists(t *testing.T, db *sql.DB, table, col string) bool { t.Helper() - rows, err := db.Query("PRAGMA table_info(" + table + ")") - if err != nil { - t.Fatalf("PRAGMA table_info(%s): %v", table, err) + var found bool + if err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name=$1 AND column_name=$2)`, table, col).Scan(&found); err != nil { + t.Fatal(err) } - defer rows.Close() - for rows.Next() { - var cid int - var name, ctype string - var notnull, pk int - var dfltValue sql.NullString - if err := rows.Scan(&cid, &name, &ctype, ¬null, &dfltValue, &pk); err != nil { - t.Fatalf("scan PRAGMA: %v", err) - } - if name == col { - return true - } - } - return false + return found } diff --git a/cmd/ingestor/multibyte_persist_test.go b/cmd/ingestor/multibyte_persist_test.go index 065a2381c..b270e6c4f 100644 --- a/cmd/ingestor/multibyte_persist_test.go +++ b/cmd/ingestor/multibyte_persist_test.go @@ -23,7 +23,7 @@ import ( func TestRunMultibyteCapPersist_AppliesSnapshot(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -105,7 +105,7 @@ func TestRunMultibyteCapPersist_AppliesSnapshot(t *testing.T) { func TestRunMultibyteCapPersist_NoSnapshot_NoOp(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -135,7 +135,7 @@ func TestRunMultibyteCapPersist_RoundTrip(t *testing.T) { dbPath := filepath.Join(dir, "test.db") // --- Phase 1: open store, seed, persist snapshot --- - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -172,7 +172,7 @@ func TestRunMultibyteCapPersist_RoundTrip(t *testing.T) { } // --- Phase 2: fresh Store, verify persisted state survived --- - store2, err := OpenStore(dbPath) + store2, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore (reopen): %v", err) } @@ -211,7 +211,7 @@ func TestRunMultibyteCapPersist_RoundTrip(t *testing.T) { func TestRunMultibyteCapPersist_MalformedSnapshot(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -257,7 +257,7 @@ func TestRunMultibyteCapPersist_MalformedSnapshot(t *testing.T) { func TestRunMultibyteCapPersist_MissingSchemaColumns(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -270,7 +270,7 @@ func TestRunMultibyteCapPersist_MissingSchemaColumns(t *testing.T) { `ALTER TABLE inactive_nodes DROP COLUMN multibyte_sup`, `ALTER TABLE inactive_nodes DROP COLUMN multibyte_evidence`, } { - if _, err := store.db.Exec(stmt); err != nil { + if _, err := testAdmin(t, store).Exec(stmt); err != nil { t.Fatalf("simulate legacy DB (%q): %v", stmt, err) } } @@ -315,7 +315,7 @@ func TestRunMultibyteCapPersist_MissingSchemaColumns(t *testing.T) { func TestRunMultibyteCapPersist_PreservesConfirmedOnUnknown(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } diff --git a/cmd/ingestor/neighbor_builder.go b/cmd/ingestor/neighbor_builder.go index 7a49ecc9a..8c77d6c6e 100644 --- a/cmd/ingestor/neighbor_builder.go +++ b/cmd/ingestor/neighbor_builder.go @@ -20,7 +20,7 @@ const NeighborEdgesBuilderInterval = 60 * time.Second // neighborBuilderMaxBatch caps how many observation rows a single // delta tick may process (#1339). With max_open_conns=1, an unbounded -// scan on a multi-million-row table holds the SQLite write lock for +// scan on a multi-million-row table holds the writer lock for // minutes and starves MQTT ingest. The cap keeps each tick bounded; // if a backlog accumulates, successive ticks drain it 50k rows at a // time without ever blocking ingest for long. @@ -197,9 +197,9 @@ func (s *Store) buildAndPersistNeighborEdges(trust *packetpath.TrustConfig) (int FROM observations o JOIN transmissions t ON t.id = o.transmission_id LEFT JOIN observers obs ON obs.rowid = o.observer_idx - WHERE o.timestamp > ? + WHERE o.timestamp > $1 ORDER BY o.timestamp - LIMIT ?`, watermarkEpoch, neighborBuilderMaxBatch) + LIMIT $2`, watermarkEpoch, neighborBuilderMaxBatch) if err != nil { return 0, fmt.Errorf("scan observations: %w", err) } @@ -254,36 +254,56 @@ func (s *Store) buildAndPersistNeighborEdges(trust *packetpath.TrustConfig) (int return 0, nil } - // Wrap the whole edge-persist tx under writer-perf instrumentation - // (#1340). Slow neighbor-builder ticks (the #1339 root cause) now - // show up on /api/perf under component=neighbor_builder. - var inserted int - err = s.WriterTx("neighbor_builder", func(tx *sql.Tx) error { - stmt, err := tx.Prepare(`INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) - VALUES (?, ?, 1, ?) - ON CONFLICT(node_a, node_b) DO UPDATE SET - count = count + 1, - last_seen = MAX(last_seen, excluded.last_seen)`) - if err != nil { - return fmt.Errorf("prepare: %w", err) + // Aggregate repeated pairs before taking the writer lock. Counts and the + // latest timestamp are additive; the working set is bounded by this scan's + // candidate slice. SQL batches cap parameters and server round trips. + type countedEdge struct { + a, b, ts string + count int + } + counts := make(map[[2]string]*countedEdge) + for _, e := range edges { + key := [2]string{e.a, e.b} + if current := counts[key]; current != nil { + current.count++ + if e.ts > current.ts { + current.ts = e.ts + } + } else { + counts[key] = &countedEdge{e.a, e.b, e.ts, 1} } - defer stmt.Close() - var firstErr error - for _, e := range edges { - if _, err := stmt.Exec(e.a, e.b, e.ts); err != nil && firstErr == nil { - firstErr = err + } + err = s.WriterTx("neighbor_builder", func(tx *sql.Tx) error { + const batchSize = 1000 + values := make([]string, 0, batchSize) + args := make([]any, 0, batchSize*4) + flush := func() error { + if len(values) == 0 { + return nil } + _, err := tx.Exec(`INSERT INTO neighbor_edges(node_a,node_b,count,last_seen) VALUES `+strings.Join(values, ",")+` + ON CONFLICT(node_a,node_b) DO UPDATE SET count=neighbor_edges.count+excluded.count, + last_seen=CASE WHEN neighbor_edges.last_seen IS NULL OR excluded.last_seen IS NULL THEN NULL ELSE GREATEST(neighbor_edges.last_seen,excluded.last_seen) END`, args...) + values = values[:0] + args = args[:0] + return err } - if firstErr != nil { - return fmt.Errorf("upsert: %w", firstErr) + for _, e := range counts { + n := len(args) + values = append(values, fmt.Sprintf("($%d,$%d,$%d,$%d)", n+1, n+2, n+3, n+4)) + args = append(args, e.a, e.b, e.count, e.ts) + if len(values) == batchSize { + if err := flush(); err != nil { + return err + } + } } - inserted = len(edges) - return nil + return flush() }) if err != nil { return 0, err } - return inserted, nil + return len(edges), nil } // canonEdge orders the pair so node_a <= node_b (matches the existing diff --git a/cmd/ingestor/neighbor_builder_delta_test.go b/cmd/ingestor/neighbor_builder_delta_test.go index 4e530953e..3d8742e20 100644 --- a/cmd/ingestor/neighbor_builder_delta_test.go +++ b/cmd/ingestor/neighbor_builder_delta_test.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "path/filepath" "testing" "time" @@ -29,27 +28,27 @@ func TestNeighborEdgesBuilderDeltaScan(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "delta.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } defer store.Close() if _, err := store.db.Exec( - `INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`, + `INSERT INTO nodes (public_key, name) VALUES ($1, $2), ($3, $4)`, "aaaaaaaaaa", "from-node", "bbbbbbbbbb", "first-hop", ); err != nil { t.Fatal(err) } if _, err := store.db.Exec( - `INSERT INTO observers (id, name) VALUES (?, ?)`, + `INSERT INTO observers (id, name) VALUES ($1, $2)`, "obs-1", "observer-1", ); err != nil { t.Fatal(err) } var obsRowid int64 - if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = ?`, "obs-1").Scan(&obsRowid); err != nil { + if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = $1`, "obs-1").Scan(&obsRowid); err != nil { t.Fatal(err) } @@ -58,34 +57,20 @@ func TestNeighborEdgesBuilderDeltaScan(t *testing.T) { const baselineStartTs int64 = 1735689600 // 2025-01-01 UTC baselineMaxTs := baselineStartTs + int64(baseline) - 1 - tx, err := store.db.Begin() - if err != nil { - t.Fatal(err) - } - txStmt, err := tx.Prepare(`INSERT INTO transmissions - (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, from_pubkey) - VALUES ('', ?, ?, 0, ?, 0, '{}', 'aaaaaaaaaa')`) - if err != nil { - t.Fatal(err) - } - obsStmt, err := tx.Prepare(`INSERT INTO observations - (transmission_id, observer_idx, path_json, timestamp) VALUES (?, ?, '["bb"]', ?)`) - if err != nil { - t.Fatal(err) - } - for i := 0; i < baseline; i++ { - res, err := txStmt.Exec(fmt.Sprintf("h%d", i), baselineStartTs+int64(i), payloadADVERT) + // Seed the same 100K logical rows in one transaction without per-row RPCs. + seed := func(prefix string, start int64, count int) { + t.Helper() + _, err := store.db.Exec(`WITH inserted AS ( + INSERT INTO transmissions(raw_hex,hash,first_seen,route_type,payload_type,payload_version,decoded_json,from_pubkey) + SELECT '',$1::text||n::text,to_char(to_timestamp($2::bigint+n) AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS"Z"'),0,$3,0,'{}','aaaaaaaaaa' + FROM generate_series(0,$4::integer-1) n RETURNING id,hash + ) INSERT INTO observations(transmission_id,observer_idx,path_json,timestamp) + SELECT id,$5,'["bb"]',$2::bigint+substring(hash,2)::bigint FROM inserted`, prefix, start, payloadADVERT, count, obsRowid) if err != nil { t.Fatal(err) } - txID, _ := res.LastInsertId() - if _, err := obsStmt.Exec(txID, obsRowid, baselineStartTs+int64(i)); err != nil { - t.Fatal(err) - } - } - if err := tx.Commit(); err != nil { - t.Fatal(err) } + seed("h", baselineStartTs, baseline) // Initial warm-up: drain to completion (StartNeighborEdgesBuilder // does the same — call directly so the test doesn't depend on the @@ -136,34 +121,7 @@ func TestNeighborEdgesBuilderDeltaScan(t *testing.T) { // than baselineMaxTs. const delta = 100 deltaStartTs := baselineMaxTs + 1 - tx2, err := store.db.Begin() - if err != nil { - t.Fatal(err) - } - txStmt2, err := tx2.Prepare(`INSERT INTO transmissions - (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, from_pubkey) - VALUES ('', ?, ?, 0, ?, 0, '{}', 'aaaaaaaaaa')`) - if err != nil { - t.Fatal(err) - } - obsStmt2, err := tx2.Prepare(`INSERT INTO observations - (transmission_id, observer_idx, path_json, timestamp) VALUES (?, ?, '["bb"]', ?)`) - if err != nil { - t.Fatal(err) - } - for i := 0; i < delta; i++ { - res, err := txStmt2.Exec(fmt.Sprintf("d%d", i), deltaStartTs+int64(i), payloadADVERT) - if err != nil { - t.Fatal(err) - } - txID, _ := res.LastInsertId() - if _, err := obsStmt2.Exec(txID, obsRowid, deltaStartTs+int64(i)); err != nil { - t.Fatal(err) - } - } - if err := tx2.Commit(); err != nil { - t.Fatal(err) - } + seed("d", deltaStartTs, delta) deltaStart := time.Now() n3, err := store.buildAndPersistNeighborEdges(trustAllPrefixes()) @@ -184,6 +142,8 @@ func TestNeighborEdgesBuilderDeltaScan(t *testing.T) { t.Fatalf("delta build of %d rows took %v; expected <500ms. (#1339)", delta, deltaDur) } + t.Logf("100K baseline: empty delta=%s; 100-row delta=%s", noopDur, deltaDur) + // Sanity: MAX(last_seen) advanced. var maxLastSeen2 string if err := store.db.QueryRow(`SELECT MAX(last_seen) FROM neighbor_edges`).Scan(&maxLastSeen2); err != nil { diff --git a/cmd/ingestor/neighbor_builder_test.go b/cmd/ingestor/neighbor_builder_test.go index 5cd777df9..c4dead36b 100644 --- a/cmd/ingestor/neighbor_builder_test.go +++ b/cmd/ingestor/neighbor_builder_test.go @@ -21,7 +21,7 @@ func TestNeighborEdgesBuilderUpsertsFromObservations(t *testing.T) { // Open via the ingestor's normal opener so applySchema and // dbschema.Apply both run (the builder requires neighbor_edges + // observers.iata etc.). - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -29,7 +29,7 @@ func TestNeighborEdgesBuilderUpsertsFromObservations(t *testing.T) { // Seed two nodes whose pubkey prefixes will be used as hops. if _, err := store.db.Exec( - `INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`, + `INSERT INTO nodes (public_key, name) VALUES ($1, $2), ($3, $4)`, "aaaaaaaaaa", "from-node", "bbbbbbbbbb", "first-hop", ); err != nil { @@ -38,31 +38,31 @@ func TestNeighborEdgesBuilderUpsertsFromObservations(t *testing.T) { // Seed one observer. if _, err := store.db.Exec( - `INSERT INTO observers (id, name) VALUES (?, ?)`, + `INSERT INTO observers (id, name) VALUES ($1, $2)`, "obs-1", "observer-1", ); err != nil { t.Fatal(err) } var obsRowid int64 - if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = ?`, "obs-1").Scan(&obsRowid); err != nil { + if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = $1`, "obs-1").Scan(&obsRowid); err != nil { t.Fatal(err) } // Insert one ADVERT transmission with from_pubkey = aaaaa… - res, err := store.db.Exec( + var txID int64 + err = store.db.QueryRow( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, from_pubkey) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING id`, "", "h1", "2026-01-01T00:00:00Z", 0, payloadADVERT, 0, "{}", "aaaaaaaaaa", - ) + ).Scan(&txID) if err != nil { t.Fatal(err) } - txID, _ := res.LastInsertId() // Insert one observation whose path[0] = "bb" (2-hex prefix unique // to bbbbb… in the nodes table). Expected edge: a↔b. if _, err := store.db.Exec( - `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (?, ?, ?, ?)`, + `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES ($1, $2, $3, $4)`, txID, obsRowid, `["bb"]`, int64(1735689600), ); err != nil { t.Fatal(err) @@ -77,7 +77,7 @@ func TestNeighborEdgesBuilderUpsertsFromObservations(t *testing.T) { } var got int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM neighbor_edges WHERE node_a = ? AND node_b = ?`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM neighbor_edges WHERE node_a = $1 AND node_b = $2`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { t.Fatal(err) } if got != 1 { @@ -97,14 +97,14 @@ func TestNeighborEdgesBuilderUpsertsFromAnonReqEphemeralPubKey(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "build.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } defer store.Close() if _, err := store.db.Exec( - `INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`, + `INSERT INTO nodes (public_key, name) VALUES ($1, $2), ($3, $4)`, "aaaaaaaaaa", "sender", "bbbbbbbbbb", "first-hop", ); err != nil { @@ -112,31 +112,31 @@ func TestNeighborEdgesBuilderUpsertsFromAnonReqEphemeralPubKey(t *testing.T) { } if _, err := store.db.Exec( - `INSERT INTO observers (id, name) VALUES (?, ?)`, + `INSERT INTO observers (id, name) VALUES ($1, $2)`, "obs-1", "observer-1", ); err != nil { t.Fatal(err) } var obsRowid int64 - if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = ?`, "obs-1").Scan(&obsRowid); err != nil { + if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = $1`, "obs-1").Scan(&obsRowid); err != nil { t.Fatal(err) } // ANON_REQ transmission: from_pubkey left NULL (as real ingest does — // only ADVERT populates it at write time), sender identity carried in // decoded_json.ephemeralPubKey instead. - res, err := store.db.Exec( + var txID int64 + err = store.db.QueryRow( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) - VALUES (?, ?, ?, ?, ?, ?, ?)`, + VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id`, "", "h2", "2026-01-01T00:00:00Z", 0, payloadAnonReq, 0, `{"ephemeralPubKey":"aaaaaaaaaa"}`, - ) + ).Scan(&txID) if err != nil { t.Fatal(err) } - txID, _ := res.LastInsertId() if _, err := store.db.Exec( - `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (?, ?, ?, ?)`, + `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES ($1, $2, $3, $4)`, txID, obsRowid, `["bb"]`, int64(1735689600), ); err != nil { t.Fatal(err) @@ -151,7 +151,7 @@ func TestNeighborEdgesBuilderUpsertsFromAnonReqEphemeralPubKey(t *testing.T) { } var got int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM neighbor_edges WHERE node_a = ? AND node_b = ?`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM neighbor_edges WHERE node_a = $1 AND node_b = $2`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { t.Fatal(err) } if got != 1 { @@ -169,14 +169,14 @@ func TestNeighborEdgesBuilderExcludesOtherNonAdvertTypes(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "build.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } defer store.Close() if _, err := store.db.Exec( - `INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`, + `INSERT INTO nodes (public_key, name) VALUES ($1, $2), ($3, $4)`, "aaaaaaaaaa", "sender", "bbbbbbbbbb", "first-hop", ); err != nil { @@ -184,29 +184,29 @@ func TestNeighborEdgesBuilderExcludesOtherNonAdvertTypes(t *testing.T) { } if _, err := store.db.Exec( - `INSERT INTO observers (id, name) VALUES (?, ?)`, + `INSERT INTO observers (id, name) VALUES ($1, $2)`, "obs-1", "observer-1", ); err != nil { t.Fatal(err) } var obsRowid int64 - if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = ?`, "obs-1").Scan(&obsRowid); err != nil { + if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = $1`, "obs-1").Scan(&obsRowid); err != nil { t.Fatal(err) } const payloadREQ = 2 - res, err := store.db.Exec( + var txID int64 + err = store.db.QueryRow( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, from_pubkey) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING id`, "", "h3", "2026-01-01T00:00:00Z", 0, payloadREQ, 0, "{}", "aaaaaaaaaa", - ) + ).Scan(&txID) if err != nil { t.Fatal(err) } - txID, _ := res.LastInsertId() if _, err := store.db.Exec( - `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (?, ?, ?, ?)`, + `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES ($1, $2, $3, $4)`, txID, obsRowid, `["bb"]`, int64(1735689600), ); err != nil { t.Fatal(err) @@ -217,7 +217,7 @@ func TestNeighborEdgesBuilderExcludesOtherNonAdvertTypes(t *testing.T) { } var got int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM neighbor_edges WHERE node_a = ? AND node_b = ?`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM neighbor_edges WHERE node_a = $1 AND node_b = $2`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { t.Fatal(err) } if got != 0 { @@ -240,30 +240,30 @@ func trustAllPrefixes() *packetpath.TrustConfig { func seedTrustFixture(t *testing.T, store *Store, hop string) { t.Helper() if _, err := store.db.Exec( - `INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`, + `INSERT INTO nodes (public_key, name) VALUES ($1, $2), ($3, $4)`, "aaaaaaaaaa", "from-node", "bbbbbbbbbb", "first-hop", ); err != nil { t.Fatal(err) } - if _, err := store.db.Exec(`INSERT INTO observers (id, name) VALUES (?, ?)`, "obs-1", "observer-1"); err != nil { + if _, err := store.db.Exec(`INSERT INTO observers (id, name) VALUES ($1, $2)`, "obs-1", "observer-1"); err != nil { t.Fatal(err) } var obsRowid int64 - if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = ?`, "obs-1").Scan(&obsRowid); err != nil { + if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = $1`, "obs-1").Scan(&obsRowid); err != nil { t.Fatal(err) } - res, err := store.db.Exec( + var txID int64 + err := store.db.QueryRow( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, from_pubkey) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING id`, "", "h1", "2026-01-01T00:00:00Z", 0, payloadADVERT, 0, "{}", "aaaaaaaaaa", - ) + ).Scan(&txID) if err != nil { t.Fatal(err) } - txID, _ := res.LastInsertId() if _, err := store.db.Exec( - `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (?, ?, ?, ?)`, + `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES ($1, $2, $3, $4)`, txID, obsRowid, `["`+hop+`"]`, int64(1735689600), ); err != nil { t.Fatal(err) @@ -279,7 +279,7 @@ func seedTrustFixture(t *testing.T, store *Store, hop string) { // one byte) a later-joining repeater sharing that byte turns today's // "unique" resolution into a wrong edge. func TestNeighborEdgesBuilderPathTrustExcludesOneByte(t *testing.T) { - store, err := OpenStore(filepath.Join(t.TempDir(), "trust1.db")) + store, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), "trust1.db")) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -297,7 +297,7 @@ func TestNeighborEdgesBuilderPathTrustExcludesOneByte(t *testing.T) { var got int if err := store.db.QueryRow( - `SELECT COUNT(*) FROM neighbor_edges WHERE node_a = ? AND node_b = ?`, + `SELECT COUNT(*) FROM neighbor_edges WHERE node_a = $1 AND node_b = $2`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { t.Fatal(err) } @@ -311,7 +311,7 @@ func TestNeighborEdgesBuilderPathTrustExcludesOneByte(t *testing.T) { // so the threshold narrows the evidence base rather than disabling the // builder. func TestNeighborEdgesBuilderPathTrustAllowsTwoByte(t *testing.T) { - store, err := OpenStore(filepath.Join(t.TempDir(), "trust2.db")) + store, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), "trust2.db")) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -324,7 +324,7 @@ func TestNeighborEdgesBuilderPathTrustAllowsTwoByte(t *testing.T) { var got int if err := store.db.QueryRow( - `SELECT COUNT(*) FROM neighbor_edges WHERE node_a = ? AND node_b = ?`, + `SELECT COUNT(*) FROM neighbor_edges WHERE node_a = $1 AND node_b = $2`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/observer_lastseen_1465_test.go b/cmd/ingestor/observer_lastseen_1465_test.go index dedd7ab76..9c3809b4f 100644 --- a/cmd/ingestor/observer_lastseen_1465_test.go +++ b/cmd/ingestor/observer_lastseen_1465_test.go @@ -31,7 +31,7 @@ func TestStatusMessage_ObserverLastSeen_AlwaysIngestTime_PastEnvelope_1465(t *te after := time.Now().Unix() var lastSeen string - if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = ?`, "obs-past").Scan(&lastSeen); err != nil { + if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = $1`, "obs-past").Scan(&lastSeen); err != nil { t.Fatalf("scan last_seen: %v", err) } ls, err := time.Parse(time.RFC3339, lastSeen) @@ -61,7 +61,7 @@ func TestStatusMessage_ObserverLastSeen_AlwaysIngestTime_FutureEnvelope_1465(t * after := time.Now().Unix() var lastSeen string - if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = ?`, "obs-future").Scan(&lastSeen); err != nil { + if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = $1`, "obs-future").Scan(&lastSeen); err != nil { t.Fatalf("scan last_seen: %v", err) } ls, err := time.Parse(time.RFC3339, lastSeen) @@ -94,7 +94,7 @@ func TestPacketMessage_ObserverLastSeen_AlwaysIngestTime_PastEnvelope_1465(t *te after := time.Now().Unix() var lastSeen string - if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = ?`, "obs-pkt").Scan(&lastSeen); err != nil { + if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = $1`, "obs-pkt").Scan(&lastSeen); err != nil { t.Fatalf("scan last_seen: %v", err) } ls, err := time.Parse(time.RFC3339, lastSeen) diff --git a/cmd/ingestor/observer_purge_test.go b/cmd/ingestor/observer_purge_test.go index 8a3f6a6a6..5646ba7a2 100644 --- a/cmd/ingestor/observer_purge_test.go +++ b/cmd/ingestor/observer_purge_test.go @@ -13,11 +13,11 @@ func staleObserver(t *testing.T, s *Store, id string) int64 { t.Fatal(err) } old := time.Now().UTC().AddDate(0, 0, -90).Format(time.RFC3339) - if _, err := s.db.Exec(`UPDATE observers SET last_seen = ?, inactive = 1 WHERE id = ?`, old, id); err != nil { + if _, err := s.db.Exec(`UPDATE observers SET last_seen = $1, inactive = 1 WHERE id = $2`, old, id); err != nil { t.Fatal(err) } var rowid int64 - if err := s.db.QueryRow(`SELECT rowid FROM observers WHERE id = ?`, id).Scan(&rowid); err != nil { + if err := s.db.QueryRow(`SELECT rowid FROM observers WHERE id = $1`, id).Scan(&rowid); err != nil { t.Fatal(err) } return rowid @@ -26,7 +26,7 @@ func staleObserver(t *testing.T, s *Store, id string) int64 { func observerExists(t *testing.T, s *Store, id string) bool { t.Helper() var n int - if err := s.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = ?`, id).Scan(&n); err != nil { + if err := s.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = $1`, id).Scan(&n); err != nil { t.Fatal(err) } return n > 0 @@ -56,13 +56,13 @@ func TestPurgeStaleObserversKeepsObserverWithObservations(t *testing.T) { rowid := staleObserver(t, store, "obs-referenced") if _, err := store.db.Exec( - `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, 'AA', 'h1', ?)`, + `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, 'AA', 'h1', $1)`, time.Now().UTC().Format(time.RFC3339), ); err != nil { t.Fatal(err) } if _, err := store.db.Exec( - `INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (1, ?, 0)`, rowid, + `INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (1, $1, 0)`, rowid, ); err != nil { t.Fatal(err) } @@ -95,7 +95,7 @@ func TestPurgeStaleObserversKeepsObserverWithMetrics(t *testing.T) { staleObserver(t, store, "obs-metrics") if _, err := store.db.Exec( - `INSERT INTO observer_metrics (observer_id, timestamp) VALUES ('obs-metrics', ?)`, + `INSERT INTO observer_metrics (observer_id, timestamp) VALUES ('obs-metrics', $1)`, time.Now().UTC().Format(time.RFC3339), ); err != nil { t.Fatal(err) @@ -143,7 +143,7 @@ func TestPurgeStaleObserversKeepsActiveObserver(t *testing.T) { t.Fatal(err) } old := time.Now().UTC().AddDate(0, 0, -90).Format(time.RFC3339) - if _, err := store.db.Exec(`UPDATE observers SET last_seen = ? WHERE id = ?`, old, "obs-active"); err != nil { + if _, err := store.db.Exec(`UPDATE observers SET last_seen = $1 WHERE id = $2`, old, "obs-active"); err != nil { t.Fatal(err) } @@ -166,7 +166,7 @@ func TestPurgeStaleObserversKeepsObserverInsideWindow(t *testing.T) { } recent := time.Now().UTC().AddDate(0, 0, -20).Format(time.RFC3339) if _, err := store.db.Exec( - `UPDATE observers SET last_seen = ?, inactive = 1 WHERE id = ?`, recent, "obs-recent", + `UPDATE observers SET last_seen = $1, inactive = 1 WHERE id = $2`, recent, "obs-recent", ); err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/observer_status_meta_test.go b/cmd/ingestor/observer_status_meta_test.go index a08f3db33..47a5ceeb8 100644 --- a/cmd/ingestor/observer_status_meta_test.go +++ b/cmd/ingestor/observer_status_meta_test.go @@ -53,7 +53,7 @@ func TestStatusMessageMetadataPersisted_Issue1044(t *testing.T) { } // Now drive the meta through UpsertObserver and verify the row. - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/postgres_config_test.go b/cmd/ingestor/postgres_config_test.go new file mode 100644 index 000000000..b03e867b8 --- /dev/null +++ b/cmd/ingestor/postgres_config_test.go @@ -0,0 +1,63 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestPostgresConfigurationPrecedence(t *testing.T) { + for _, tc := range []struct{ name, generic, writer, want string }{ + {"config", "", "", "postgres://configured@localhost/telemetry"}, + {"generic", "postgres://generic@localhost/telemetry", "", "postgres://generic@localhost/telemetry"}, + {"writer", "postgres://generic@localhost/telemetry", "postgres://writer@localhost/telemetry", "postgres://writer@localhost/telemetry"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("CORESCOPE_DATABASE_URL", tc.generic) + t.Setenv("CORESCOPE_WRITER_DATABASE_URL", tc.writer) + t.Setenv("CORESCOPE_STATE_DIR", "private-state") + t.Setenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", "postgres://channels@localhost/accounts") + path := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(path, []byte(`{"databaseURL":"postgres://configured@localhost/telemetry","dbPath":"legacy.db"}`), 0600); err != nil { + t.Fatal(err) + } + cfg, err := LoadConfig(path) + if err != nil { + t.Fatal(err) + } + if cfg.DatabaseURL != tc.want || cfg.StateDir != "private-state" || cfg.ApprovedChannelsURL() != "postgres://channels@localhost/accounts" { + t.Fatal("PostgreSQL overrides did not resolve correctly") + } + }) + } +} + +func TestOpenStoreRejectsLegacyPathsWithoutCreatingFiles(t *testing.T) { + path := filepath.Join(t.TempDir(), "missing", "database.db") + if s, err := OpenStore(path); err == nil { + s.Close() + t.Fatal("SQLite path accepted as DSN") + } + if _, err := os.Stat(filepath.Dir(path)); !os.IsNotExist(err) { + t.Fatal("legacy path created a directory") + } +} + +func TestOpenStoreConnectionErrorDoesNotLeakCredentials(t *testing.T) { + _, err := OpenStore("postgres://reader:do-not-log-this@localhost:1/test?sslmode=disable") + if err == nil { + t.Fatal("closed port unexpectedly connected") + } + if strings.Contains(err.Error(), "do-not-log-this") || strings.Contains(err.Error(), "postgres://") { + t.Fatal("connection error leaked credentials") + } +} + +func TestStatsDefaultUsesStateDirectory(t *testing.T) { + t.Setenv("CORESCOPE_INGESTOR_STATS", "") + t.Setenv("CORESCOPE_STATE_DIR", "private-state") + if got := statsFilePath(); got != filepath.Join("private-state", "ingestor-stats.json") { + t.Fatalf("stats path %q", got) + } +} diff --git a/cmd/ingestor/postgres_test.go b/cmd/ingestor/postgres_test.go new file mode 100644 index 000000000..7dc7a4354 --- /dev/null +++ b/cmd/ingestor/postgres_test.go @@ -0,0 +1,179 @@ +package main + +import ( + "context" + "database/sql" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +var testDatabases sync.Map + +func openPostgresTestStore(t testing.TB, key string) (*Store, error) { + return openPostgresTestStoreInterval(t, key, 300) +} +func openPostgresTestStoreInterval(t testing.TB, key string, interval int) (*Store, error) { + stateDir := filepath.Dir(key) + if strings.HasPrefix(key, "postgres") { + stateDir = t.TempDir() + } + return OpenStoreWithState(testPostgresURL(t, key), stateDir, interval) +} + +// A historical fixture path is only a test key. Runtime OpenStore accepts +// PostgreSQL URLs exclusively, with a real restricted writer credential. +func testPostgresURL(t testing.TB, key string) string { + t.Helper() + if strings.HasPrefix(key, "postgres://") || strings.HasPrefix(key, "postgresql://") { + return key + } + cacheKey := t.Name() + "|" + key + if v, ok := testDatabases.Load(cacheKey); ok { + return v.(string) + } + owner := pgtest.NewSchema(t) + db, err := pgutil.Open(owner, false) + if err != nil { + t.Fatal(err) + } + if err = dbschema.Apply(db, nil); err != nil { + db.Close() + t.Fatal(err) + } + u, _ := url.Parse(owner) + schema := u.Query().Get("search_path") + role := schema + "_writer" + for _, stmt := range []string{`CREATE ROLE ` + role + ` LOGIN PASSWORD '` + role + `'`, + `GRANT USAGE ON SCHEMA ` + schema + ` TO ` + role, + `GRANT SELECT,INSERT,UPDATE,DELETE ON ALL TABLES IN SCHEMA ` + schema + ` TO ` + role, + `GRANT USAGE,SELECT ON ALL SEQUENCES IN SCHEMA ` + schema + ` TO ` + role, + `REVOKE INSERT,UPDATE,DELETE ON ` + schema + `.corescope_schema FROM ` + role} { + if _, err = db.Exec(stmt); err != nil { + db.Close() + t.Fatal(err) + } + } + u.User = url.UserPassword(role, role) + dsn := u.String() + testDatabases.Store(cacheKey, dsn) + t.Cleanup(func() { + testDatabases.Delete(cacheKey) + defer db.Close() + if _, err := db.Exec(`DROP OWNED BY ` + role); err != nil { + t.Error(err) + } + if _, err := db.Exec(`DROP ROLE ` + role); err != nil { + t.Error(err) + } + }) + return dsn +} + +func testAdmin(t testing.TB, s *Store) *sql.DB { + t.Helper() + var schema string + if err := s.db.QueryRow(`SELECT current_schema()`).Scan(&schema); err != nil { + t.Fatal(err) + } + u, _ := url.Parse(os.Getenv("CORESCOPE_TEST_POSTGRES_URL")) + q := u.Query() + q.Set("search_path", schema) + u.RawQuery = q.Encode() + db, err := pgutil.Open(u.String(), false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + return db +} + +func TestPostgresPacketCommitOrdering(t *testing.T) { + s, err := OpenStore(testPostgresURL(t, "cursor")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + owner := testAdmin(t, s) + tx, err := beginWrite(owner) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + var earlier int64 + if err = tx.QueryRow(`INSERT INTO transmissions(raw_hex,hash,first_seen) VALUES('AA','delayed','2026-01-01T00:00:00Z') RETURNING id`).Scan(&earlier); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { + _, e := s.InsertTransmission(&PacketData{Hash: "following", RawHex: "BB", Timestamp: "2026-01-01T00:00:01Z"}) + done <- e + }() + select { + case err := <-done: + t.Fatalf("later ingest passed unfinished transaction: %v", err) + case <-time.After(75 * time.Millisecond): + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("writer did not resume") + } + var later int64 + if err = s.db.QueryRow(`SELECT id FROM transmissions WHERE hash='following'`).Scan(&later); err != nil { + t.Fatal(err) + } + if later <= earlier { + t.Fatalf("cursor order %d then %d", earlier, later) + } +} + +func TestPostgresRuntimeCannotMigrate(t *testing.T) { + s, err := OpenStore(testPostgresURL(t, "roles")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + for _, q := range []string{`CREATE TABLE forbidden(id int)`, `UPDATE corescope_schema SET ready=false`, `DROP TABLE transmissions`} { + if _, err = s.db.Exec(q); err == nil { + t.Errorf("runtime role allowed %s", q) + } + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err = s.db.PingContext(ctx); err != nil { + t.Fatal(err) + } +} + +func TestPostgresNaiveClockUpsert(t *testing.T) { + s := newTestStore(t) + now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) + for i, at := range []time.Time{now, now.Add(time.Hour), now.Add(48 * time.Hour)} { + if err := s.RecordNaiveSkew("clock-observer", -90, at); err != nil { + t.Fatal(err) + } + var count, skew int + if err := s.db.QueryRow(`SELECT clock_skew_count_24h,clock_skew_seconds FROM observers WHERE id='clock-observer'`).Scan(&count, &skew); err != nil { + t.Fatal(err) + } + want := []int{1, 2, 1}[i] + if count != want || skew != -90 { + t.Fatalf("event %d: count=%d skew=%d", i, count, skew) + } + } +} diff --git a/cmd/ingestor/pragma_parity_test.go b/cmd/ingestor/pragma_parity_test.go index e90e4d97a..29f11beef 100644 --- a/cmd/ingestor/pragma_parity_test.go +++ b/cmd/ingestor/pragma_parity_test.go @@ -1,55 +1,20 @@ package main -import ( - "path/filepath" - "testing" -) +import "testing" -// TestOpenStorePragmas pins the pragmas the writer connection runs with. -// -// These were silently at risk during the modernc.org/sqlite → -// github.com/mattn/go-sqlite3 migration in two different ways: -// -// - The DSN dialect changed. modernc understood only `_pragma=name(value)`; -// mattn understands only `_`-prefixed parameters and ignores the other -// form outright. A driver-only rename would have dropped every pragma on -// the floor with no error anywhere. -// -// - mattn runs `PRAGMA synchronous = NORMAL` unconditionally, defaulting the -// mode itself to NORMAL rather than leaving SQLite's own FULL in place. So -// the writer would have quietly moved from FULL (2) to NORMAL (1), -// weakening durability under power loss. `_synchronous=FULL` in the DSN is -// what holds it — this test is what notices if it is ever dropped. -// -// Read through the store's own connection on purpose: a separate connection, or -// the log line in OpenStoreWithInterval, would prove nothing about what the -// writer is actually using. -func TestOpenStorePragmas(t *testing.T) { - store, err := OpenStore(filepath.Join(t.TempDir(), "pragma.db")) - if err != nil { - t.Fatalf("OpenStore: %v", err) - } - defer store.db.Close() - - for _, want := range []struct { - pragma string - value string - why string - }{ - {"journal_mode", "wal", "concurrent reader (cmd/server) requires WAL"}, - {"synchronous", "2", "FULL; mattn defaults to 1 (NORMAL) unless the DSN says otherwise"}, - {"auto_vacuum", "2", "INCREMENTAL; maintenance.go drives incremental_vacuum"}, - {"foreign_keys", "1", "schema relies on FK enforcement"}, - {"busy_timeout", "5000", "writer serialises behind the reader"}, - {"cache_size", "-2000", "2 MiB of C-allocated page cache, pinned: it sits outside GOMEMLIMIT"}, - } { +// Durable settings are measured through the runtime connection, not logs. +func TestOpenStoreDurability(t *testing.T) { + s := newTestStore(t) + for _, setting := range []string{"fsync", "full_page_writes", "synchronous_commit"} { var got string - if err := store.db.QueryRow("PRAGMA " + want.pragma).Scan(&got); err != nil { - t.Errorf("PRAGMA %s: %v", want.pragma, err) - continue + if err := s.db.QueryRow("SHOW " + setting).Scan(&got); err != nil { + t.Fatal(err) } - if got != want.value { - t.Errorf("PRAGMA %s = %q, want %q (%s)", want.pragma, got, want.value, want.why) + if got != "on" { + t.Errorf("%s=%s; require on", setting, got) } } + if _, err := s.db.Exec(`INSERT INTO observations(transmission_id,timestamp) VALUES(999999,1)`); err == nil { + t.Fatal("foreign key not enforced") + } } diff --git a/cmd/ingestor/prune_chunked_test.go b/cmd/ingestor/prune_chunked_test.go index a3be3261a..567f6396c 100644 --- a/cmd/ingestor/prune_chunked_test.go +++ b/cmd/ingestor/prune_chunked_test.go @@ -6,8 +6,6 @@ import ( "strings" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // seedAgedTransmissions inserts n transmissions stamped `age` days in the @@ -18,22 +16,22 @@ func seedAgedTransmissions(t *testing.T, store *Store, n, obsPerTx, ageDays int) t.Helper() ts := time.Now().UTC().AddDate(0, 0, -ageDays).Format(time.RFC3339) for i := 0; i < n; i++ { - res, err := store.db.Exec( + var txID int64 + err := store.db.QueryRow( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) - VALUES (?, ?, ?, 0, 1, 1, '{}')`, + VALUES ($1, $2, $3, 0, 1, 1, '{}') RETURNING id`, "AA", fmt.Sprintf("h%d-%d", ageDays, i), ts, - ) + ).Scan(&txID) if err != nil { t.Fatalf("seed tx %d: %v", i, err) } - txID, err := res.LastInsertId() if err != nil { t.Fatalf("seed tx %d LastInsertId: %v", i, err) } for j := 0; j < obsPerTx; j++ { if _, err := store.db.Exec( `INSERT INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp) - VALUES (?, ?, 'rx', 1.0, -100, 0, '[]', ?)`, + VALUES ($1, $2, 'rx', 1.0, -100, 0, '[]', $3)`, txID, j, time.Now().Unix(), ); err != nil { t.Fatalf("seed obs %d/%d: %v", i, j, err) @@ -53,7 +51,7 @@ func countRows(t *testing.T, store *Store, table string) int { func openPruneStore(t *testing.T, name string) *Store { t.Helper() - store, err := OpenStore(filepath.Join(t.TempDir(), name)) + store, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), name)) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -212,7 +210,7 @@ func TestPruneOldPacketsDisabledTakesNoWriterLock(t *testing.T) { // steady-state cost when nothing has aged out yet: a single empty batch, then // the loop exits. That batch is only cheap because the subquery is walked off // idx_transmissions_first_seen — see -// TestPruneAgedTransmissionIDsUsesFirstSeenIndex. +// TestPruneAgedTransmissionIDsHasOrderedIndex. func TestPruneOldPacketsNothingToDeleteRunsOneEmptyBatch(t *testing.T) { store := openPruneStore(t, "prune-noop.db") seedAgedTransmissions(t, store, 9, 2, 0) @@ -234,7 +232,7 @@ func TestPruneOldPacketsNothingToDeleteRunsOneEmptyBatch(t *testing.T) { } } -// TestPruneAgedTransmissionIDsUsesFirstSeenIndex pins the query plan of the +// TestPruneAgedTransmissionIDsHasOrderedIndex pins the query plan of the // batch subquery and of both statements that embed it. // // Ordering the batch by id instead of first_seen makes SQLite drop @@ -242,7 +240,7 @@ func TestPruneOldPacketsNothingToDeleteRunsOneEmptyBatch(t *testing.T) { // the terminating, nothing-left batch from ~10µs to ~73ms — once per statement, // under writerMu, in the state an instance is in whenever nothing has aged out. // Transaction counts cannot see that regression, so the plan is the assertion. -func TestPruneAgedTransmissionIDsUsesFirstSeenIndex(t *testing.T) { +func TestPruneAgedTransmissionIDsHasOrderedIndex(t *testing.T) { store := openPruneStore(t, "prune-plan.db") seedAgedTransmissions(t, store, 20, 2, 10) @@ -252,15 +250,17 @@ func TestPruneAgedTransmissionIDsUsesFirstSeenIndex(t *testing.T) { "observations delete": pruneObservationsBatch, "transmissions delete": pruneTransmissionsBatch, } { - rows, err := store.db.Query("EXPLAIN QUERY PLAN "+q, cutoff, pruneBatchTransmissions) + if _, err := store.db.Exec(`SET enable_seqscan=off; SET enable_bitmapscan=off`); err != nil { + t.Fatal(err) + } + rows, err := store.db.Query("EXPLAIN (COSTS OFF) "+q, cutoff, pruneBatchTransmissions) if err != nil { - t.Fatalf("%s: EXPLAIN QUERY PLAN: %v", name, err) + t.Fatalf("%s: EXPLAIN (COSTS OFF): %v", name, err) } var steps []string for rows.Next() { - var id, parent, notused int var detail string - if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + if err := rows.Scan(&detail); err != nil { rows.Close() t.Fatalf("%s: scan plan row: %v", name, err) } @@ -276,11 +276,11 @@ func TestPruneAgedTransmissionIDsUsesFirstSeenIndex(t *testing.T) { if !strings.Contains(plan, "idx_transmissions_first_seen") { t.Errorf("%s: plan does not use idx_transmissions_first_seen: %s", name, plan) } - if strings.Contains(plan, "SCAN transmissions") { + if strings.Contains(plan, "Seq Scan on transmissions") { t.Errorf("%s: plan scans transmissions, so the empty terminating batch walks the whole table under writerMu: %s", name, plan) } - if strings.Contains(plan, "TEMP B-TREE") { - t.Errorf("%s: plan sorts in a temp b-tree instead of walking the index in order: %s", name, plan) + if strings.Contains(plan, "Sort Key:") { + t.Errorf("%s: ordered-index eligibility query unexpectedly requires a sort: %s", name, plan) } } } diff --git a/cmd/ingestor/prune_geofilter.go b/cmd/ingestor/prune_geofilter.go index 38f75857f..6464c044f 100644 --- a/cmd/ingestor/prune_geofilter.go +++ b/cmd/ingestor/prune_geofilter.go @@ -1,7 +1,7 @@ // Package main: ingestor-side processor for prune-request marker files // written by the read-only server (see internal/prunequeue). // -// The server cannot DELETE because it opens SQLite mode=ro (#1283/#1289). +// The server cannot DELETE because it uses a restricted PostgreSQL reader (#1283/#1289). // Instead, the server writes request-.json under /prune-requests/ // and the ingestor consumes it here. package main @@ -22,7 +22,7 @@ func (s *Store) DeleteNodesByPubkeys(pubkeys []string) (int64, error) { if len(pubkeys) == 0 { return 0, nil } - // Chunk to keep statements under SQLite's variable limit (default 999). + // Bound each delete batch and the number of bind parameters. const chunk = 500 var total int64 for start := 0; start < len(pubkeys); start += chunk { @@ -31,8 +31,11 @@ func (s *Store) DeleteNodesByPubkeys(pubkeys []string) (int64, error) { end = len(pubkeys) } batch := pubkeys[start:end] - placeholders := strings.Repeat("?,", len(batch)) - placeholders = placeholders[:len(placeholders)-1] + slots := make([]string, len(batch)) + for i := range slots { + slots[i] = fmt.Sprintf("$%d", i+1) + } + placeholders := strings.Join(slots, ",") args := make([]interface{}, len(batch)) for i, pk := range batch { args[i] = pk @@ -59,7 +62,7 @@ func (s *Store) DeleteNodesByPubkeys(pubkeys []string) (int64, error) { } // RunPendingPruneRequests scans the prune-requests/ directory next to the -// SQLite database and processes any request-.json markers written by +// state directory and processes any request-.json markers written by // the server. Each request is honored verbatim — the server is responsible // for the TOCTOU snapshot (only pubkeys that were still outside the // geofilter at confirm time). After running DELETE, the ingestor writes diff --git a/cmd/ingestor/prune_geofilter_test.go b/cmd/ingestor/prune_geofilter_test.go index f12654c89..def2e353c 100644 --- a/cmd/ingestor/prune_geofilter_test.go +++ b/cmd/ingestor/prune_geofilter_test.go @@ -12,7 +12,7 @@ func TestRunPendingPruneRequests(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -67,7 +67,7 @@ func TestRunPendingPruneRequests(t *testing.T) { func TestRunPendingPruneRequests_EmptyQueueIsNoop(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } diff --git a/cmd/ingestor/refresh_planner_stats_test.go b/cmd/ingestor/refresh_planner_stats_test.go index e7f0aee21..a83c2fb10 100644 --- a/cmd/ingestor/refresh_planner_stats_test.go +++ b/cmd/ingestor/refresh_planner_stats_test.go @@ -1,102 +1,12 @@ package main import ( - "bytes" - "log" + "github.com/meshcore-analyzer/dbconfig" "os" "path/filepath" - "strings" "testing" - - "github.com/meshcore-analyzer/dbconfig" ) -// #2058: the planner has no cardinality statistics because ANALYZE has never -// run, so it picks a plain index over the partial index built for the query. -// These pin the three things that make the refresh work at all: the statement is -// one that actually writes statistics, the pragma reaches the connection, and a -// negative limit leaves the database untouched. - -func hasStat1(t *testing.T, s *Store) bool { - t.Helper() - var n int - if err := s.db.QueryRow( - `SELECT count(*) FROM sqlite_master WHERE type='table' AND name='sqlite_stat1'`).Scan(&n); err != nil { - t.Fatalf("query sqlite_master: %v", err) - } - return n > 0 -} - -// This is the guard against going back to PRAGMA optimize, which is what the -// first draft of this change used. Measured against the 9.4 GB staging database: -// optimize analyzes only tables the calling connection has itself queried during -// the session, so from a maintenance call it writes nothing and sqlite_stat1 -// never appears. A fresh store here has queried nothing either, so this test -// fails on that mistake instead of passing on a no-op. -func TestRefreshPlannerStatsWritesStatistics_Issue2058(t *testing.T) { - s := newTestStore(t) - defer s.Close() - - if hasStat1(t, s) { - t.Fatal("a fresh store already carries sqlite_stat1, so this test cannot tell whether the refresh did anything") - } - - if !s.RefreshPlannerStats(10000) { - t.Fatal("RefreshPlannerStats reported no refresh") - } - - if !hasStat1(t, s) { - t.Error("sqlite_stat1 was not created, so the planner still has no statistics") - } -} - -func TestRefreshPlannerStatsAppliesTheLimit_Issue2058(t *testing.T) { - s := newTestStore(t) - defer s.Close() - - s.RefreshPlannerStats(250) - - // analysis_limit is per connection. The store runs SetMaxOpenConns(1) - // (db.go:142), which is the only reason setting it through Exec is sound - // here: on a multi-connection pool the pragma could land on a connection - // the ANALYZE never uses, and the limit would silently not apply. - var limit int - if err := s.db.QueryRow("PRAGMA analysis_limit").Scan(&limit); err != nil { - t.Fatalf("read back analysis_limit: %v", err) - } - if limit != 250 { - t.Errorf("analysis_limit did not reach the connection: want 250, got %d", limit) - } -} - -func TestRefreshPlannerStatsNegativeLimitIsANoop_Issue2058(t *testing.T) { - s := newTestStore(t) - defer s.Close() - - if s.RefreshPlannerStats(-1) { - t.Error("a negative limit must not report a refresh") - } - if hasStat1(t, s) { - t.Error("a negative limit still built sqlite_stat1; the refresh is not actually disabled") - } -} - -func TestRefreshPlannerStatsIsRepeatable_Issue2058(t *testing.T) { - s := newTestStore(t) - defer s.Close() - - // The ticker calls this every 24h for the life of the process. A second - // call must not error or undo the first, which is the part a single-call - // test would not notice. - s.RefreshPlannerStats(10000) - if !s.RefreshPlannerStats(10000) { - t.Fatal("the second refresh reported failure") - } - if !hasStat1(t, s) { - t.Error("sqlite_stat1 disappeared across two refreshes") - } -} - func TestAnalysisLimitConfigDefault_Issue2058(t *testing.T) { cases := []struct { name string @@ -148,120 +58,26 @@ func TestAnalysisLimitSurvivesTheConfigFile_Issue2058(t *testing.T) { } } -// EnsurePlannerStats closes the 2 minute window the startup stagger opens, and -// must do so exactly once per database rather than once per restart. - -func TestEnsurePlannerStatsBuildsWhenAbsent_Issue2058(t *testing.T) { +func TestPostgresOwnerAnalyzesTelemetry(t *testing.T) { s := newTestStore(t) - defer s.Close() - - if !s.EnsurePlannerStats(10000) { - t.Fatal("EnsurePlannerStats did not build statistics on a database that has none") - } - if !hasStat1(t, s) { - t.Error("sqlite_stat1 absent after EnsurePlannerStats") - } -} - -func TestEnsurePlannerStatsSkipsWhenPresent_Issue2058(t *testing.T) { - s := newTestStore(t) - defer s.Close() - - s.RefreshPlannerStats(10000) - // The point of the skip: on every restart after the first this must cost one - // sqlite_master query, not an ANALYZE. If it ever returns true here it is - // running the 2s refresh on every boot. - if s.EnsurePlannerStats(10000) { - t.Error("EnsurePlannerStats rebuilt statistics that were already there") - } -} - -func TestEnsurePlannerStatsRespectsDisabled_Issue2058(t *testing.T) { - s := newTestStore(t) - defer s.Close() - - if s.EnsurePlannerStats(-1) { - t.Error("a negative limit must not build statistics") - } - if hasStat1(t, s) { - t.Error("sqlite_stat1 built despite the refresh being disabled") - } -} - -// The whole design rests on this: statistics live in the file, so the boot-time -// build is a one-off per database. If a future change ever wrote them somewhere -// per-process, EnsurePlannerStats would silently run a full ANALYZE on every -// restart and this test is what says so. -func TestPlannerStatsSurviveReopen_Issue2058(t *testing.T) { - dir := t.TempDir() - dbPath := dir + "/reopen.db" - - first, err := OpenStore(dbPath) - if err != nil { + owner := testAdmin(t, s) + if _, err := owner.Exec(`INSERT INTO transmissions(raw_hex,hash,first_seen) SELECT 'aa', n::text,'2026-01-01T00:00:00Z' FROM generate_series(1,1000) n`); err != nil { t.Fatal(err) } - first.WaitForAsyncMigrations() - if !first.EnsurePlannerStats(10000) { - t.Fatal("first store did not build statistics") - } - first.Close() - - second, err := OpenStore(dbPath) - if err != nil { + if _, err := owner.Exec(`ANALYZE transmissions`); err != nil { t.Fatal(err) } - defer second.Close() - second.WaitForAsyncMigrations() - - if !hasStat1(t, second) { - t.Fatal("statistics did not survive reopening the database") - } - if second.EnsurePlannerStats(10000) { - t.Error("the reopened store rebuilt statistics, so a restart would pay for an ANALYZE it does not need") - } -} - -// An operator watching a first deploy sees ingest stop for minutes. The warning -// is what tells them it is an ANALYZE and not a hang, so it is worth pinning: -// measured on staging, the build held the write connection 3m43.9s and the -// observations table took zero rows for four minutes. -func TestEnsurePlannerStatsWarnsBeforeBuilding_Issue2058(t *testing.T) { - s := newTestStore(t) - defer s.Close() - - var buf bytes.Buffer - orig := log.Writer() - log.SetOutput(&buf) - defer log.SetOutput(orig) - - s.EnsurePlannerStats(10000) - - out := buf.String() - for _, want := range []string{"no planner statistics", "write connection", "Once per database"} { - if !strings.Contains(out, want) { - t.Errorf("the first-build warning does not mention %q; an operator seeing ingest stall gets no explanation.\ngot: %s", want, out) - } + var estimate float64 + if err := owner.QueryRow(`SELECT reltuples FROM pg_class WHERE oid='transmissions'::regclass`).Scan(&estimate); err != nil { + t.Fatal(err) } - if i, j := strings.Index(out, "no planner statistics"), strings.Index(out, "statistics built in"); i == -1 || j == -1 || i > j { - t.Errorf("the warning must come before the completion line, so it is visible while the write path is held; got: %s", out) + if estimate != 1000 { + t.Fatalf("planner row estimate=%v, want 1000", estimate) } } - -// The counterpart: a restart must not log the warning, or every boot looks like -// it is about to stall. -func TestEnsurePlannerStatsIsQuietWhenPresent_Issue2058(t *testing.T) { +func TestPostgresRuntimeCannotAlterPlannerTargets(t *testing.T) { s := newTestStore(t) - defer s.Close() - s.RefreshPlannerStats(10000) - - var buf bytes.Buffer - orig := log.Writer() - log.SetOutput(&buf) - defer log.SetOutput(orig) - - s.EnsurePlannerStats(10000) - - if out := buf.String(); strings.Contains(out, "no planner statistics") { - t.Errorf("warned about building on a database that already has statistics: %s", out) + if _, err := s.db.Exec(`ALTER TABLE transmissions ALTER COLUMN payload_type SET STATISTICS 1000`); err == nil { + t.Fatal("runtime role changed owner planner policy") } } diff --git a/cmd/ingestor/region_keys.go b/cmd/ingestor/region_keys.go index 298aba930..376affeed 100644 --- a/cmd/ingestor/region_keys.go +++ b/cmd/ingestor/region_keys.go @@ -74,7 +74,7 @@ func regionNameAcceptable(name string) (string, bool) { // (#1865). Always present; the column is part of the schema. // - node_declared_regions, an optional table a deployment may fill by other // means. Absent on a stock install, so its absence is not an error: the -// probe below asks sqlite_master first rather than letting "no such table" +// probe below asks information_schema first rather than letting "no such table" // abort a refresh that the first source could still answer. // // A node counts once per name however many times it declares it, so the @@ -129,7 +129,7 @@ func (s *Store) declaredRegionSources() ([]declaredRegionStat, error) { rows.Close() var present string - if err := s.db.QueryRow(`SELECT name FROM sqlite_master WHERE type='table' AND name='node_declared_regions'`).Scan(&present); err == nil && present != "" { + if err := s.db.QueryRow(`SELECT table_name FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='node_declared_regions'`).Scan(&present); err == nil && present != "" { ndr, err := s.db.Query(` WITH ranked AS ( SELECT target, observed_at, regions_csv, diff --git a/cmd/ingestor/region_keys_test.go b/cmd/ingestor/region_keys_test.go index 209e47b68..927c37050 100644 --- a/cmd/ingestor/region_keys_test.go +++ b/cmd/ingestor/region_keys_test.go @@ -397,22 +397,9 @@ func TestDeclaredRegionSourcesIgnoresWildcard(t *testing.T) { // wants the optional source has to bring it. func insertDeclaredRegionsRow(t *testing.T, s *Store, target, observedAt, regionsCSV string) { t.Helper() - if _, err := s.db.Exec(` - CREATE TABLE IF NOT EXISTS node_declared_regions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - target TEXT NOT NULL, - rx_pubkey TEXT NOT NULL, - observed_at TEXT NOT NULL, - ingested_at TEXT NOT NULL, - regions_csv TEXT NOT NULL, - truncated INTEGER NOT NULL DEFAULT 0, - UNIQUE(target, rx_pubkey, observed_at) - )`); err != nil { - t.Fatal(err) - } if _, err := s.db.Exec( `INSERT INTO node_declared_regions (target, rx_pubkey, observed_at, ingested_at, regions_csv, truncated) - VALUES (?, 'rx', ?, ?, ?, 0)`, target, observedAt, observedAt, regionsCSV); err != nil { + VALUES ($1, 'rx', $2, $3, $4, 0)`, target, observedAt, observedAt, regionsCSV); err != nil { t.Fatal(err) } } @@ -639,7 +626,7 @@ func TestDeclaredRegionSourcesMergesBothSources(t *testing.T) { func seedNodeWithConfiguredScope(t *testing.T, s *Store, pubkey, scopes, at string) { t.Helper() if _, err := s.db.Exec( - `INSERT INTO nodes (public_key, configured_scope, configured_scope_at) VALUES (?, ?, ?)`, + `INSERT INTO nodes (public_key, configured_scope, configured_scope_at) VALUES ($1, $2, $3)`, pubkey, scopes, at); err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/regions_gate_test.go b/cmd/ingestor/regions_gate_test.go index a9dde1b3b..b2e4fc10c 100644 --- a/cmd/ingestor/regions_gate_test.go +++ b/cmd/ingestor/regions_gate_test.go @@ -67,7 +67,7 @@ func TestClientRegionsGateOffDoesNotFallThroughToObserverPath(t *testing.T) { t.Fatalf("feature OFF: expected 0 node_declared_regions rows, got %d", n) } var observerRows int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = ?`, testCompanionPK).Scan(&observerRows); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = $1`, testCompanionPK).Scan(&observerRows); err != nil { t.Fatal(err) } if observerRows != 0 { diff --git a/cmd/ingestor/relay_touch_test.go b/cmd/ingestor/relay_touch_test.go index 53fa23480..7bbf8869f 100644 --- a/cmd/ingestor/relay_touch_test.go +++ b/cmd/ingestor/relay_touch_test.go @@ -29,7 +29,7 @@ func seedRelayNode(t *testing.T, s *Store, pubkey, name, lastSeen string) { func nodeLastSeen(t *testing.T, s *Store, pubkey string) string { t.Helper() var ls string - if err := s.db.QueryRow(`SELECT COALESCE(last_seen,'') FROM nodes WHERE public_key=?`, pubkey).Scan(&ls); err != nil { + if err := s.db.QueryRow(`SELECT COALESCE(last_seen,'') FROM nodes WHERE public_key=$1`, pubkey).Scan(&ls); err != nil { t.Fatalf("read last_seen for %s: %v", pubkey, err) } return ls diff --git a/cmd/ingestor/resolved_path_test.go b/cmd/ingestor/resolved_path_test.go index 308868506..cbc07da79 100644 --- a/cmd/ingestor/resolved_path_test.go +++ b/cmd/ingestor/resolved_path_test.go @@ -95,7 +95,7 @@ func TestInsertTransmissionWritesResolvedPath(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "ingest.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -103,7 +103,7 @@ func TestInsertTransmissionWritesResolvedPath(t *testing.T) { // Seed nodes with unique 1-byte prefixes. if _, err := store.db.Exec( - `INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`, + `INSERT INTO nodes (public_key, name) VALUES ($1, $2), ($3, $4)`, "aaaaaaaaaa", "from-node", "bbbbbbbbbb", "first-hop", ); err != nil { @@ -138,7 +138,7 @@ func TestInsertTransmissionWritesResolvedPath(t *testing.T) { var rp sql.NullString if err := store.db.QueryRow( - `SELECT resolved_path FROM observations WHERE transmission_id = (SELECT id FROM transmissions WHERE hash = ?)`, + `SELECT resolved_path FROM observations WHERE transmission_id = (SELECT id FROM transmissions WHERE hash = $1)`, "h-1547", ).Scan(&rp); err != nil { t.Fatalf("query: %v", err) @@ -374,14 +374,14 @@ func TestInsertTransmissionDoesNotClobberResolvedPathOnAllNil(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "ingest.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } defer store.Close() if _, err := store.db.Exec( - `INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`, + `INSERT INTO nodes (public_key, name) VALUES ($1, $2), ($3, $4)`, "aaaaaaaaaa", "from-node", "bbbbbbbbbb", "first-hop", ); err != nil { @@ -412,7 +412,7 @@ func TestInsertTransmissionDoesNotClobberResolvedPathOnAllNil(t *testing.T) { // Sanity: first write populated resolved_path. var first sql.NullString if err := store.db.QueryRow( - `SELECT resolved_path FROM observations WHERE transmission_id = (SELECT id FROM transmissions WHERE hash = ?)`, + `SELECT resolved_path FROM observations WHERE transmission_id = (SELECT id FROM transmissions WHERE hash = $1)`, "h-clobber", ).Scan(&first); err != nil { t.Fatalf("first query: %v", err) @@ -432,7 +432,7 @@ func TestInsertTransmissionDoesNotClobberResolvedPathOnAllNil(t *testing.T) { var after sql.NullString if err := store.db.QueryRow( - `SELECT resolved_path FROM observations WHERE transmission_id = (SELECT id FROM transmissions WHERE hash = ?)`, + `SELECT resolved_path FROM observations WHERE transmission_id = (SELECT id FROM transmissions WHERE hash = $1)`, "h-clobber", ).Scan(&after); err != nil { t.Fatalf("post-reingest query: %v", err) diff --git a/cmd/ingestor/retained_status_test.go b/cmd/ingestor/retained_status_test.go index 64a7ffc47..5ad07a574 100644 --- a/cmd/ingestor/retained_status_test.go +++ b/cmd/ingestor/retained_status_test.go @@ -26,7 +26,7 @@ func seedObserver(t *testing.T, s *Store, id string, ageDays int) string { t.Fatal(err) } old := time.Now().UTC().AddDate(0, 0, -ageDays).Format(time.RFC3339) - if _, err := s.db.Exec(`UPDATE observers SET last_seen = ? WHERE id = ?`, old, id); err != nil { + if _, err := s.db.Exec(`UPDATE observers SET last_seen = $1 WHERE id = $2`, old, id); err != nil { t.Fatal(err) } return old @@ -35,7 +35,7 @@ func seedObserver(t *testing.T, s *Store, id string, ageDays int) string { func observerLastSeen(t *testing.T, s *Store, id string) string { t.Helper() var got string - if err := s.db.QueryRow(`SELECT last_seen FROM observers WHERE id = ?`, id).Scan(&got); err != nil { + if err := s.db.QueryRow(`SELECT last_seen FROM observers WHERE id = $1`, id).Scan(&got); err != nil { t.Fatal(err) } return got @@ -57,7 +57,7 @@ func TestRetainedStatusDoesNotAdvanceLastSeen(t *testing.T) { func TestRetainedStatusDoesNotReactivateInactiveObserver(t *testing.T) { store := newTestStore(t) seedObserver(t, store, "obs-retained", 60) - if _, err := store.db.Exec(`UPDATE observers SET inactive = 1 WHERE id = ?`, "obs-retained"); err != nil { + if _, err := store.db.Exec(`UPDATE observers SET inactive = 1 WHERE id = $1`, "obs-retained"); err != nil { t.Fatal(err) } @@ -66,7 +66,7 @@ func TestRetainedStatusDoesNotReactivateInactiveObserver(t *testing.T) { nil, nil, &Config{}) var inactive int - if err := store.db.QueryRow(`SELECT inactive FROM observers WHERE id = ?`, "obs-retained").Scan(&inactive); err != nil { + if err := store.db.QueryRow(`SELECT inactive FROM observers WHERE id = $1`, "obs-retained").Scan(&inactive); err != nil { t.Fatal(err) } if inactive != 1 { @@ -85,7 +85,7 @@ func TestRetainedStatusDoesNotCreateUnknownObserver(t *testing.T) { nil, nil, &Config{}) var count int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = ?`, "obs-never-seen").Scan(&count); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = $1`, "obs-never-seen").Scan(&count); err != nil { t.Fatal(err) } if count != 0 { @@ -105,7 +105,7 @@ func TestRetainedStatusDoesNotInsertMetricsSample(t *testing.T) { var count int if err := store.db.QueryRow( - `SELECT COUNT(*) FROM observer_metrics WHERE observer_id = ?`, "obs-retained", + `SELECT COUNT(*) FROM observer_metrics WHERE observer_id = $1`, "obs-retained", ).Scan(&count); err != nil { t.Fatal(err) } @@ -127,7 +127,7 @@ func TestRetainedStatusStillUpdatesMetadata(t *testing.T) { var firmware string var noiseFloor float64 if err := store.db.QueryRow( - `SELECT firmware, noise_floor FROM observers WHERE id = ?`, "obs-retained", + `SELECT firmware, noise_floor FROM observers WHERE id = $1`, "obs-retained", ).Scan(&firmware, &noiseFloor); err != nil { t.Fatal(err) } @@ -162,7 +162,7 @@ func TestLiveStatusStillCreatesUnknownObserver(t *testing.T) { nil, nil, &Config{}) var count int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = ?`, "obs-fresh").Scan(&count); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = $1`, "obs-fresh").Scan(&count); err != nil { t.Fatal(err) } if count != 1 { diff --git a/cmd/ingestor/rf_gate_test.go b/cmd/ingestor/rf_gate_test.go index 4b6f8dee7..8f05afe6f 100644 --- a/cmd/ingestor/rf_gate_test.go +++ b/cmd/ingestor/rf_gate_test.go @@ -76,7 +76,7 @@ func TestClientRfSamplesGateOffDoesNotFallThroughToObserverPath(t *testing.T) { t.Fatalf("feature OFF: expected 0 client_rf_samples rows, got %d", n) } var observerRows int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = ?`, testCompanionPK).Scan(&observerRows); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = $1`, testCompanionPK).Scan(&observerRows); err != nil { t.Fatal(err) } if observerRows != 0 { diff --git a/cmd/ingestor/scope_match_tally.go b/cmd/ingestor/scope_match_tally.go index 378b563f7..cb3c2702a 100644 --- a/cmd/ingestor/scope_match_tally.go +++ b/cmd/ingestor/scope_match_tally.go @@ -20,20 +20,7 @@ import ( // scope_match_totals is one row. The ingestor reads it into the counters at // startup and writes them back periodically and on shutdown, which makes the // tally cumulative since `since_unix` instead of since boot. It is deliberately -// NOT in internal/dbschema: the server neither reads this table nor detects it, -// and putting it in AssertReady would make an old DB without the table fail the -// server's startup check for data the server never looks at. - -const scopeMatchTotalsSchema = ` - CREATE TABLE IF NOT EXISTS scope_match_totals ( - id INTEGER PRIMARY KEY CHECK (id = 1), - since_unix INTEGER NOT NULL, - unique_matches INTEGER NOT NULL, - explicit_over_derived INTEGER NOT NULL, - ambiguous INTEGER NOT NULL, - none_matches INTEGER NOT NULL, - updated_unix INTEGER NOT NULL - )` +// included in the canonical telemetry schema owned by the migration command. // LoadScopeMatchTotals seeds the in-process counters from the stored row so // counting continues where the previous process stopped. On a DB that has no @@ -68,7 +55,7 @@ func (s *Store) SaveScopeMatchTotals() error { _, err := s.db.Exec(` INSERT INTO scope_match_totals (id, since_unix, unique_matches, explicit_over_derived, ambiguous, none_matches, updated_unix) - VALUES (1, ?, ?, ?, ?, ?, ?) + VALUES (1, $1, $2, $3, $4, $5, $6) ON CONFLICT(id) DO UPDATE SET since_unix = excluded.since_unix, unique_matches = excluded.unique_matches, diff --git a/cmd/ingestor/scope_match_tally_test.go b/cmd/ingestor/scope_match_tally_test.go index e62292da1..3332797dd 100644 --- a/cmd/ingestor/scope_match_tally_test.go +++ b/cmd/ingestor/scope_match_tally_test.go @@ -37,7 +37,7 @@ func TestScopeMatchTallySurvivesReopen(t *testing.T) { resetScopeMatchCounters(t) dbPath := t.TempDir() + "/tally.db" - s, err := OpenStore(dbPath) + s, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } @@ -58,7 +58,7 @@ func TestScopeMatchTallySurvivesReopen(t *testing.T) { scopeMatchCounters.none.Store(0) scopeMatchCounters.sinceUnix.Store(0) - s2, err := OpenStore(dbPath) + s2, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } @@ -89,7 +89,7 @@ func TestScopeMatchTallyAnchorsSinceOnFirstOpen(t *testing.T) { resetScopeMatchCounters(t) before := time.Now().Unix() - s, err := OpenStore(t.TempDir() + "/tally.db") + s, err := openPostgresTestStore(t, t.TempDir()+"/tally.db") if err != nil { t.Fatal(err) } @@ -115,7 +115,7 @@ func TestScopeMatchTallyContinuesAfterRestore(t *testing.T) { resetScopeMatchCounters(t) dbPath := t.TempDir() + "/tally.db" - s, err := OpenStore(dbPath) + s, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } @@ -126,7 +126,7 @@ func TestScopeMatchTallyContinuesAfterRestore(t *testing.T) { } s.Close() - s2, err := OpenStore(dbPath) + s2, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } @@ -154,7 +154,7 @@ func TestScopeMatchTallyContinuesAfterRestore(t *testing.T) { func TestScopeMatchTallySaveIsIdempotent(t *testing.T) { resetScopeMatchCounters(t) - s, err := OpenStore(t.TempDir() + "/tally.db") + s, err := openPostgresTestStore(t, t.TempDir()+"/tally.db") if err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/sig_validate_ingest_test.go b/cmd/ingestor/sig_validate_ingest_test.go index ab09e1fb3..75d15196c 100644 --- a/cmd/ingestor/sig_validate_ingest_test.go +++ b/cmd/ingestor/sig_validate_ingest_test.go @@ -47,7 +47,7 @@ func makeAppdata(name string) []byte { func TestSigValidation_ValidAdvertStored(t *testing.T) { dbPath := t.TempDir() + "/test.db" - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } @@ -73,7 +73,7 @@ func TestSigValidation_ValidAdvertStored(t *testing.T) { func TestSigValidation_TamperedSignatureDropped(t *testing.T) { dbPath := t.TempDir() + "/test.db" - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } @@ -138,7 +138,7 @@ func TestSigValidation_TamperedSignatureDropped(t *testing.T) { func TestSigValidation_TruncatedAppdataDropped(t *testing.T) { dbPath := t.TempDir() + "/test.db" - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } @@ -168,7 +168,7 @@ func TestSigValidation_TruncatedAppdataDropped(t *testing.T) { func TestSigValidation_DisabledByConfig(t *testing.T) { dbPath := t.TempDir() + "/test.db" - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } @@ -204,7 +204,7 @@ func TestSigValidation_DisabledByConfig(t *testing.T) { func TestSigValidation_DropCounterIncrements(t *testing.T) { dbPath := t.TempDir() + "/test.db" - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } @@ -236,7 +236,7 @@ func TestSigValidation_DropCounterIncrements(t *testing.T) { func TestSigValidation_LogContainsFields(t *testing.T) { // This test verifies the dropped_packets row has all required fields dbPath := t.TempDir() + "/test.db" - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } @@ -289,15 +289,15 @@ func TestSigValidation_LogContainsFields(t *testing.T) { func TestPruneDroppedPackets(t *testing.T) { dbPath := t.TempDir() + "/test.db" - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } defer store.Close() // Insert an old dropped packet - store.db.Exec(`INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('old', 'test', datetime('now', '-60 days'))`) - store.db.Exec(`INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('new', 'test', datetime('now'))`) + store.db.Exec(`INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('old', 'test', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC' - interval '60 days','YYYY-MM-DD HH24:MI:SS'))`) + store.db.Exec(`INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('new', 'test', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD HH24:MI:SS'))`) n, err := store.PruneDroppedPackets(30) if err != nil { diff --git a/cmd/ingestor/stats_file.go b/cmd/ingestor/stats_file.go index 449156108..67375ffb1 100644 --- a/cmd/ingestor/stats_file.go +++ b/cmd/ingestor/stats_file.go @@ -6,6 +6,7 @@ import ( "encoding/json" "log" "os" + "path/filepath" "sync" "time" @@ -44,7 +45,7 @@ type IngestorStatsSnapshot struct { // the server's /api/perf/io endpoint under .ingestor (#1120 — "Both // ingestor and server"). Optional; absent on non-Linux hosts. ProcIO *PerfIOSample `json:"procIO,omitempty"` - // WriterPerf is the per-component SQLite writer-lock latency + // WriterPerf is the per-component serialized writer-lock latency // snapshot (#1340) — wait_ms / hold_ms / contention_total tagged // by component (neighbor_builder, mqtt_handler, prune_packets, // prune_observers, prune_metrics, vacuum). Surfaced by the server @@ -106,15 +107,17 @@ type SourceLivenessSnapshot struct { // statsFilePath returns the writable path the ingestor will publish stats to. // Override via env CORESCOPE_INGESTOR_STATS for tests / non-default deploys. // -// SECURITY: the default lives in /tmp which is world-writable. The writer uses -// O_NOFOLLOW + 0o600 so a pre-planted symlink cannot be used to clobber an -// arbitrary file via this path. Operators who want stronger guarantees should -// point CORESCOPE_INGESTOR_STATS at a private directory (e.g. /var/lib/corescope/). +// The default lives in the private state directory. The writer retains +// O_NOFOLLOW and private permissions for an explicitly overridden path. func statsFilePath() string { if p := os.Getenv("CORESCOPE_INGESTOR_STATS"); p != "" { return p } - return "/tmp/corescope-ingestor-stats.json" + stateDir := os.Getenv("CORESCOPE_STATE_DIR") + if stateDir == "" { + stateDir = "data" + } + return filepath.Join(stateDir, "ingestor-stats.json") } // writeStatsAtomic writes b to path via a tmp-then-rename, refusing to follow @@ -258,6 +261,9 @@ func StartStatsFileWriter(s *Store, interval time.Duration) (stop func()) { t := time.NewTicker(interval) defer t.Stop() path := statsFilePath() + if os.Getenv("CORESCOPE_INGESTOR_STATS") == "" && s.stateDir != "" { + path = filepath.Join(s.stateDir, "ingestor-stats.json") + } // Track previous procIO sample so we can compute per-second deltas // across ticks (#1120 follow-up: ingestor /proc/self/io exposure). prevIO := readProcSelfIOFn() diff --git a/cmd/ingestor/stats_file_test.go b/cmd/ingestor/stats_file_test.go index e96d56bd1..431e104e6 100644 --- a/cmd/ingestor/stats_file_test.go +++ b/cmd/ingestor/stats_file_test.go @@ -2,8 +2,11 @@ package main import ( "encoding/json" + "errors" "os" "path/filepath" + "runtime" + "syscall" "testing" "time" ) @@ -50,7 +53,7 @@ func TestStatsFileWriter_PublishesProcIO(t *testing.T) { statsPath := filepath.Join(dir, "ingestor-stats.json") t.Setenv("CORESCOPE_INGESTOR_STATS", statsPath) - store, err := OpenStore(filepath.Join(dir, "test.db")) + store, err := openPostgresTestStore(t, filepath.Join(dir, "test.db")) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -129,6 +132,9 @@ func TestWriteStatsAtomic_SymlinkAtDestIsReplaced(t *testing.T) { // Pre-plant a symlink at the destination path. path := filepath.Join(dir, "stats.json") if err := os.Symlink(target, path); err != nil { + if runtime.GOOS == "windows" && errors.Is(err, syscall.Errno(1314)) { + t.Skip("Windows account cannot create symlinks; this security fixture must run on Linux CI") + } t.Fatalf("symlink: %v", err) } diff --git a/cmd/ingestor/stats_file_timestamp_test.go b/cmd/ingestor/stats_file_timestamp_test.go index fd3a66597..abae95de8 100644 --- a/cmd/ingestor/stats_file_timestamp_test.go +++ b/cmd/ingestor/stats_file_timestamp_test.go @@ -38,7 +38,7 @@ func TestStatsFileWriter_SampledAtMatchesProcIOSampledAt(t *testing.T) { statsPath := filepath.Join(dir, "ingestor-stats.json") t.Setenv("CORESCOPE_INGESTOR_STATS", statsPath) - store, err := OpenStore(filepath.Join(dir, "test.db")) + store, err := openPostgresTestStore(t, filepath.Join(dir, "test.db")) if err != nil { t.Fatalf("OpenStore: %v", err) } diff --git a/cmd/ingestor/status_liveness_test.go b/cmd/ingestor/status_liveness_test.go index 6f96b07d1..a72b5aff2 100644 --- a/cmd/ingestor/status_liveness_test.go +++ b/cmd/ingestor/status_liveness_test.go @@ -55,7 +55,7 @@ func TestStaleStatusPayloadDoesNotCreateObserver(t *testing.T) { nil, nil, &Config{}) var count int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = ?`, "obs-purged").Scan(&count); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = $1`, "obs-purged").Scan(&count); err != nil { t.Fatal(err) } if count != 0 { @@ -82,7 +82,7 @@ func TestOfflineStatusIsNotLiveness(t *testing.T) { func TestStaleStatusPayloadDoesNotReactivateInactiveObserver(t *testing.T) { store := newTestStore(t) seedObserver(t, store, "obs-zombie", 60) - if _, err := store.db.Exec(`UPDATE observers SET inactive = 1 WHERE id = ?`, "obs-zombie"); err != nil { + if _, err := store.db.Exec(`UPDATE observers SET inactive = 1 WHERE id = $1`, "obs-zombie"); err != nil { t.Fatal(err) } @@ -92,7 +92,7 @@ func TestStaleStatusPayloadDoesNotReactivateInactiveObserver(t *testing.T) { nil, nil, &Config{}) var inactive int - if err := store.db.QueryRow(`SELECT inactive FROM observers WHERE id = ?`, "obs-zombie").Scan(&inactive); err != nil { + if err := store.db.QueryRow(`SELECT inactive FROM observers WHERE id = $1`, "obs-zombie").Scan(&inactive); err != nil { t.Fatal(err) } if inactive != 1 { @@ -172,7 +172,7 @@ func TestStaleStatusPayloadStillUpdatesMetadata(t *testing.T) { nil, nil, &Config{}) var firmware string - if err := store.db.QueryRow(`SELECT firmware FROM observers WHERE id = ?`, "obs-zombie").Scan(&firmware); err != nil { + if err := store.db.QueryRow(`SELECT firmware FROM observers WHERE id = $1`, "obs-zombie").Scan(&firmware); err != nil { t.Fatal(err) } if firmware != "v1.14.1" { @@ -193,7 +193,7 @@ func TestStaleStatusPayloadDoesNotInsertMetricsSample(t *testing.T) { var count int if err := store.db.QueryRow( - `SELECT COUNT(*) FROM observer_metrics WHERE observer_id = ?`, "obs-zombie", + `SELECT COUNT(*) FROM observer_metrics WHERE observer_id = $1`, "obs-zombie", ).Scan(&count); err != nil { t.Fatal(err) } diff --git a/cmd/migrate/cli_test.go b/cmd/migrate/cli_test.go new file mode 100644 index 000000000..6bf0e0d9c --- /dev/null +++ b/cmd/migrate/cli_test.go @@ -0,0 +1,106 @@ +package main + +import ( + "bytes" + "context" + "database/sql" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +func TestCheckReadyDoesNotBootstrap(t *testing.T) { + dsn := pgtest.NewSchema(t) + t.Setenv("CORESCOPE_DATABASE_URL", dsn) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") + var output bytes.Buffer + if err := runCommand(context.Background(), []string{"-check-ready"}, &output); err == nil { + t.Fatal("empty schema reported ready") + } + var n int + if err := openImportDB(t, dsn).QueryRow(`SELECT count(*) FROM information_schema.tables WHERE table_schema=current_schema()`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 0 { + t.Fatal("readiness check created tables") + } + if err := runCommand(context.Background(), nil, &output); err != nil { + t.Fatal(err) + } + if err := runCommand(context.Background(), []string{"-check-ready"}, &output); err != nil { + t.Fatal(err) + } + if strings.Contains(output.String(), dsn) { + t.Fatal("CLI output leaked a database URL") + } +} + +func TestCLIImportRequiresOfflineAndDoesNotFinalizeOnAccountFailure(t *testing.T) { + source := telemetrySource(t) + telemetry, accounts := pgtest.NewDatabase(t), pgtest.NewDatabase(t) + t.Setenv("CORESCOPE_DATABASE_URL", telemetry) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", accounts) + var output bytes.Buffer + args := []string{"-from-sqlite", source, "-state-dir", t.TempDir()} + if err := runCommand(context.Background(), args, &output); err == nil { + t.Fatal("import accepted without explicit offline confirmation") + } + args = append(args, "-offline", "-users-from-sqlite", "missing-account-source.db") + if err := runCommand(context.Background(), args, &output); err == nil { + t.Fatal("missing account source was ignored") + } + if err := dbschema.AssertReady(openImportDB(t, telemetry)); err == nil { + t.Fatal("telemetry became ready despite failed account import") + } +} + +func TestCheckImportMarkerBindsExactSource(t *testing.T) { + dsn, source := pgtest.NewSchema(t), telemetrySource(t) + t.Setenv("CORESCOPE_DATABASE_URL", dsn) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") + var output bytes.Buffer + args := []string{"-check-import-kind=telemetry", "-from-sqlite", source} + if err := runCommand(context.Background(), args, &output); err == nil { + t.Fatal("empty database passed imported-source check") + } + if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry"}); err != nil { + t.Fatal(err) + } + if err := runCommand(context.Background(), args, &output); err == nil { + t.Fatal("unfinalized import passed bootstrap guard") + } + if err := finalizeImport(context.Background(), dsn, "telemetry"); err != nil { + t.Fatal(err) + } + if err := runCommand(context.Background(), args, &output); err != nil { + t.Fatal(err) + } + args[len(args)-1] = accountSource(t, 1) + if err := runCommand(context.Background(), args, &output); err == nil { + t.Fatal("different legacy source passed bootstrap guard") + } +} + +func TestWALModeCleanShutdownImportAndGuard(t *testing.T) { + source := telemetrySource(t) + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`PRAGMA journal_mode=WAL`); err != nil { + t.Fatal(err) + } + db.Close() + dsn := pgtest.NewSchema(t) + t.Setenv("CORESCOPE_DATABASE_URL", dsn) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") + var output bytes.Buffer + if err := runCommand(context.Background(), []string{"-offline", "-from-sqlite", source, "-state-dir", t.TempDir()}, &output); err != nil { + t.Fatal(err) + } + if err := runCommand(context.Background(), []string{"-check-import-kind=telemetry", "-from-sqlite", source}, &output); err != nil { + t.Fatal(err) + } +} diff --git a/cmd/migrate/generated_columns_test.go b/cmd/migrate/generated_columns_test.go new file mode 100644 index 000000000..a879f2c03 --- /dev/null +++ b/cmd/migrate/generated_columns_test.go @@ -0,0 +1,82 @@ +package main + +import ( + "context" + "database/sql" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/pgutil/pgtest" + "github.com/meshcore-analyzer/users" +) + +func TestImportRejectsGeneratedAccountColumnsBeforeNormalization(t *testing.T) { + for _, storage := range []string{"STORED", "VIRTUAL"} { + for _, column := range []string{"unsupported_note", "last_login_at"} { + t.Run(storage+"/"+column, func(t *testing.T) { + source := filepath.Join(t.TempDir(), "users.db") + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`CREATE TABLE schema_version(version INTEGER NOT NULL); INSERT INTO schema_version VALUES(1)`); err != nil { + t.Fatal(err) + } + for _, stmt := range users.LegacyMigrations()[0] { + if strings.HasPrefix(stmt, "CREATE TABLE users (") { + if column == "last_login_at" { + stmt = strings.Replace(stmt, "last_login_at INTEGER,", "last_login_at INTEGER GENERATED ALWAYS AS (created_at + 1) "+storage+",", 1) + } else { + stmt = strings.TrimSuffix(strings.TrimSpace(stmt), ")") + ", unsupported_note TEXT GENERATED ALWAYS AS (display_name || ':retained') " + storage + ")" + } + } + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } + } + if _, err := db.Exec(`INSERT INTO users(id,email,display_name,password_hash,role,status,created_at) VALUES(42,'synthetic@example.invalid','synthetic','hash','user','active',1)`); err != nil { + t.Fatal(err) + } + var value string + if err := db.QueryRow(`SELECT ` + quote(column) + ` FROM users WHERE id=42`).Scan(&value); err != nil || value == "" { + t.Fatal("generated fixture value missing", err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + dsn, state := pgtest.NewSchema(t), t.TempDir() + report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: state, Kind: "accounts"}) + if err == nil || !strings.Contains(err.Error(), "generated or hidden") { + t.Errorf("generated source must be explicitly refused before normalization: verified=%v error=%v", report.Verified, err) + } + target := openImportDB(t, dsn) + var tables int + if err := target.QueryRow(`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema()`).Scan(&tables); err != nil || tables != 0 { + t.Errorf("unsupported source changed the target: tables=%d error=%v", tables, err) + } + if err := users.AssertReady(target); err == nil { + t.Error("unsupported source became runtime-ready") + } + normalized, err := sql.Open("sqlite3", sqliteURL(filepath.Join(state, "accounts", "normalized.sqlite"), "ro")) + if err != nil { + t.Fatal(err) + } + defer normalized.Close() + var version int + if err := normalized.QueryRow(`SELECT version FROM schema_version`).Scan(&version); err != nil || version != 1 { + t.Errorf("unsupported source reached account normalization: version=%d error=%v", version, err) + } + after, err := sourceFingerprint(source) + if err != nil || before != after { + t.Fatal("original source changed", err) + } + }) + } + } +} diff --git a/cmd/migrate/go.mod b/cmd/migrate/go.mod index 1d24e0a43..563d410ac 100644 --- a/cmd/migrate/go.mod +++ b/cmd/migrate/go.mod @@ -1,10 +1,30 @@ module github.com/corescope/migrate -go 1.22 +go 1.25.0 require ( + github.com/jackc/pgx/v5 v5.7.6 github.com/mattn/go-sqlite3 v1.14.52 github.com/meshcore-analyzer/dbschema v0.0.0 + github.com/meshcore-analyzer/pgutil v0.0.0 + github.com/meshcore-analyzer/users v0.0.0 +) + +require ( + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/meshcore-analyzer/packetpath v0.0.0 + golang.org/x/crypto v0.37.0 // indirect + golang.org/x/sync v0.13.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/text v0.24.0 // indirect ) replace github.com/meshcore-analyzer/dbschema => ../../internal/dbschema + +replace github.com/meshcore-analyzer/users => ../../internal/users + +replace github.com/meshcore-analyzer/pgutil => ../../internal/pgutil + +replace github.com/meshcore-analyzer/packetpath => ../../internal/packetpath diff --git a/cmd/migrate/go.sum b/cmd/migrate/go.sum index 75bf6eb53..4915dde04 100644 --- a/cmd/migrate/go.sum +++ b/cmd/migrate/go.sum @@ -1,2 +1,32 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cmd/migrate/import.go b/cmd/migrate/import.go new file mode 100644 index 000000000..eaac3c0ee --- /dev/null +++ b/cmd/migrate/import.go @@ -0,0 +1,782 @@ +package main + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "database/sql" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "hash" + "math" + "os" + "path/filepath" + "strings" + "unicode/utf8" + + "github.com/jackc/pgx/v5" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" +) + +type importOptions struct { + Source, DatabaseURL, StateDir, Kind string + BatchSize int + Resume bool + afterBatch func() error + targetDigest string +} +type tableReport struct { + Table string `json:"table"` + Rows int64 `json:"rows"` + SHA256 string `json:"sha256"` +} +type importReport struct { + Kind string `json:"kind"` + Verified bool `json:"verified"` + Tables []tableReport `json:"tables"` +} +type sourceManifest struct { + Version int `json:"version"` + RunID string `json:"run_id"` + Kind string `json:"kind"` + SourceDigest string `json:"source_digest"` + RawDigest string `json:"raw_digest"` + NormalizedDigest string `json:"normalized_digest"` + TargetDigest string `json:"target_digest"` +} + +const importerLock int64 = 7289041410 + +func fingerprint(ctx context.Context, path string) (string, error) { + fp, err := sourceFingerprintContext(ctx, path) + if err != nil { + return "", err + } + return fmt.Sprintf("%x:%x", fp[0], fp[1]), nil +} + +func prepareSource(ctx context.Context, o importOptions, tables []importTable) (sourceManifest, string, error) { + var manifest sourceManifest + if o.Source == "" || o.StateDir == "" { + return manifest, "", errors.New("source and state directory are required for import") + } + digest, err := fingerprint(ctx, o.Source) + if err != nil { + return manifest, "", err + } + dir := filepath.Join(o.StateDir, o.Kind) + raw := filepath.Join(dir, "recovery.sqlite") + normalized := filepath.Join(dir, "normalized.sqlite") + path := filepath.Join(dir, "manifest.json") + data, err := os.ReadFile(path) + if err == nil { + if !o.Resume { + return manifest, "", errors.New("migration state already exists; use -resume with the same source and destination") + } + if err := json.Unmarshal(data, &manifest); err != nil { + return manifest, "", errors.New("invalid migration state manifest") + } + if manifest.Version != 1 || manifest.Kind != o.Kind || manifest.SourceDigest != digest || manifest.TargetDigest != o.targetDigest { + return manifest, "", errors.New("migration resume source or destination does not match the immutable manifest") + } + for p, want := range map[string]string{raw: manifest.RawDigest, normalized: manifest.NormalizedDigest} { + got, err := fingerprint(ctx, p) + if err != nil { + return manifest, "", err + } + if got != want { + return manifest, "", errors.New("migration recovery or normalized snapshot changed; resume refused") + } + } + return manifest, normalized, nil + } + if !errors.Is(err, os.ErrNotExist) { + return manifest, "", err + } + if o.Resume { + return manifest, "", errors.New("no complete migration state manifest exists to resume") + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return manifest, "", err + } + if err := snapshotSource(ctx, o.Source, raw); err != nil { + return manifest, "", err + } + if err := snapshotSource(ctx, raw, normalized); err != nil { + return manifest, "", err + } + db, err := sql.Open(importSQLiteDriver, sqliteURL(normalized, "rw")) + if err != nil { + return manifest, "", err + } + db.SetMaxOpenConns(1) + if _, err = db.ExecContext(ctx, `PRAGMA trusted_schema=OFF`); err == nil { + err = normalizeSource(db, o.Kind, raw, tables) + } + closeErr := db.Close() + if err != nil { + return manifest, "", err + } + if closeErr != nil { + return manifest, "", closeErr + } + manifest = sourceManifest{Version: 1, Kind: o.Kind, SourceDigest: digest, TargetDigest: o.targetDigest} + var random [16]byte + if _, err := rand.Read(random[:]); err != nil { + return manifest, "", err + } + manifest.RunID = hex.EncodeToString(random[:]) + if manifest.RawDigest, err = fingerprint(ctx, raw); err != nil { + return manifest, "", err + } + if manifest.NormalizedDigest, err = fingerprint(ctx, normalized); err != nil { + return manifest, "", err + } + after, err := fingerprint(ctx, o.Source) + if err != nil { + return manifest, "", err + } + if after != digest { + return manifest, "", errors.New("SQLite source changed during preparation; keep every instance writer stopped") + } + data, err = json.MarshalIndent(manifest, "", " ") + if err != nil { + return manifest, "", err + } + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return manifest, "", err + } + _, writeErr := f.Write(data) + syncErr := f.Sync() + closeErr = f.Close() + if err := errors.Join(writeErr, syncErr, closeErr); err != nil { + return manifest, "", err + } + return manifest, normalized, nil +} + +func importSQLite(ctx context.Context, o importOptions) (importReport, error) { + report := importReport{Kind: o.Kind} + if err := ctx.Err(); err != nil { + return report, err + } + if o.BatchSize == 0 { + o.BatchSize = 1000 + } + if o.BatchSize < 1 || o.BatchSize > 10000 { + return report, errors.New("import batch size must be between 1 and 10000") + } + tables, err := layouts(o.Kind) + if err != nil { + return report, err + } + if _, err := pgutil.ParseConfig(o.DatabaseURL); err != nil { + return report, err + } + owner, err := pgutil.Open(o.DatabaseURL, false) + if err != nil { + return report, err + } + defer owner.Close() + owner.SetMaxOpenConns(1) + var database, schema, address string + var databaseOID, schemaOID int64 + var port int + if err := owner.QueryRowContext(ctx, `SELECT current_database(),current_schema(), + (SELECT oid::bigint FROM pg_database WHERE datname=current_database()), + (SELECT oid::bigint FROM pg_namespace WHERE nspname=current_schema()), + COALESCE(inet_server_addr()::text,'local'),COALESCE(inet_server_port(),0)`).Scan(&database, &schema, &databaseOID, &schemaOID, &address, &port); err != nil { + return report, err + } + identity, _ := json.Marshal([]any{database, schema, databaseOID, schemaOID, address, port}) + identityHash := sha256.Sum256(identity) + o.targetDigest = hex.EncodeToString(identityHash[:]) + config, err := pgutil.ParseConfig(o.DatabaseURL) + if err != nil { + return report, err + } + conn, err := pgx.ConnectConfig(ctx, config) + if err != nil { + return report, errors.New("connect to PostgreSQL import destination failed") + } + defer conn.Close(context.Background()) + // One importer owns this database until completion/cancellation. Session lock + // works across committed COPY batches; closing the connection always releases it. + if err := lockImport(ctx, conn); err != nil { + return report, err + } + manifest, path, err := prepareSource(ctx, o, tables) + if err != nil { + return report, err + } + source, err := sql.Open(importSQLiteDriver, sqliteURL(path, "ro")) + if err != nil { + return report, err + } + defer source.Close() + source.SetMaxOpenConns(1) + if _, err := source.ExecContext(ctx, `PRAGMA trusted_schema=OFF`); err != nil { + return report, err + } + + var exists bool + if err := conn.QueryRow(ctx, `SELECT to_regclass('corescope_import') IS NOT NULL`).Scan(&exists); err != nil { + return report, err + } + if !exists { + var occupied int + if err := conn.QueryRow(ctx, `SELECT count(*) FROM information_schema.tables WHERE table_schema=current_schema()`).Scan(&occupied); err != nil { + return report, err + } + if occupied != 0 { + return report, errors.New("import destination is not empty; existing data was not changed") + } + tx, err := owner.BeginTx(ctx, nil) + if err != nil { + return report, err + } + defer tx.Rollback() + if o.Kind == "telemetry" { + err = dbschema.ApplyForImportTx(tx, nil) + } else { + err = users.ApplyForImportTx(tx) + } + if err != nil { + return report, err + } + if _, err := tx.ExecContext(ctx, `CREATE TABLE corescope_import (id INTEGER PRIMARY KEY CHECK(id=1),run_id TEXT NOT NULL,kind TEXT NOT NULL,source_digest TEXT NOT NULL,normalized_digest TEXT NOT NULL,verified BOOLEAN NOT NULL DEFAULT false,report TEXT NOT NULL DEFAULT '') + ;CREATE TABLE corescope_import_progress (table_name TEXT PRIMARY KEY,rows_copied BIGINT NOT NULL,sha256 TEXT NOT NULL,complete BOOLEAN NOT NULL)`); err != nil { + return report, err + } + if _, err := tx.ExecContext(ctx, `INSERT INTO corescope_import(id,run_id,kind,source_digest,normalized_digest) VALUES(1,$1,$2,$3,$4)`, manifest.RunID, o.Kind, manifest.SourceDigest, manifest.NormalizedDigest); err != nil { + return report, err + } + if err := tx.Commit(); err != nil { + return report, err + } + } else if !o.Resume { + return report, errors.New("destination has an import in progress; use -resume") + } + var runID, kind, sourceDigest, normalizedDigest string + if err := conn.QueryRow(ctx, `SELECT run_id,kind,source_digest,normalized_digest FROM corescope_import WHERE id=1`).Scan(&runID, &kind, &sourceDigest, &normalizedDigest); err != nil { + return report, err + } + if runID != manifest.RunID || kind != o.Kind || sourceDigest != manifest.SourceDigest || normalizedDigest != manifest.NormalizedDigest { + return report, errors.New("destination import identity does not match this source/state directory") + } + var version int + var ready bool + if err := conn.QueryRow(ctx, `SELECT version,ready FROM corescope_schema WHERE kind=$1`, o.Kind).Scan(&version, &ready); err != nil { + return report, err + } + if version != 1 { + return report, errors.New("unsupported PostgreSQL schema version for import") + } + if ready { + return report, errors.New("import destination is already live; do not resume after cutover") + } + for _, table := range tables { + got, err := copyTable(ctx, source, owner, conn, table, o) + if err != nil { + return report, fmt.Errorf("import %s: %w", table.Name, err) + } + report.Tables = append(report.Tables, got) + } + if err := reseed(ctx, source, conn, tables, o.Kind); err != nil { + return report, err + } + // COPY does not populate planner statistics. Prepare only this store's + // imported tables before declaring verification complete; the cost belongs + // to migration timing, never to the steady-state ingestion benchmark. + for _, table := range tables { + if _, err := conn.Exec(ctx, `ANALYZE `+quote(table.Name)); err != nil { + return report, fmt.Errorf("analyze imported %s: %w", table.Name, err) + } + } + // Source stays frozen through verification, not merely while snapshotting. + current, err := fingerprint(ctx, o.Source) + if err != nil { + return report, err + } + if current != manifest.SourceDigest { + return report, errors.New("original SQLite source changed during import; target remains unready") + } + normalized, err := fingerprint(ctx, path) + if err != nil { + return report, err + } + if normalized != manifest.NormalizedDigest { + return report, errors.New("normalized snapshot changed during import") + } + report.Verified = true + data, err := json.Marshal(report) + if err != nil { + return report, err + } + if _, err := conn.Exec(ctx, `UPDATE corescope_import SET verified=true,report=$1 WHERE id=1`, string(data)); err != nil { + return report, err + } + return report, nil +} + +func columnNames(table importTable) ([]string, []string) { + names := make([]string, len(table.Columns)) + quoted := make([]string, len(names)) + for i, col := range table.Columns { + names[i] = col.Name + quoted[i] = quote(col.Name) + } + return names, quoted +} + +func orderBy(db *sql.DB, table importTable, postgres bool) (string, error) { + // Sort by logical primary key with identical byte collation in both engines. + rows, err := db.Query(`SELECT a.attname FROM pg_index i JOIN pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=ANY(i.indkey) + WHERE i.indrelid=to_regclass($1) AND i.indisprimary ORDER BY array_position(i.indkey,a.attnum)`, table.Name) + if err != nil { + return "", err + } + defer rows.Close() + var keys []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return "", err + } + keys = append(keys, name) + } + if err := rows.Err(); err != nil { + return "", err + } + if len(keys) == 0 { + return "", errors.New("destination table lacks a stable primary key") + } + var out []string + for _, name := range keys { + colType := "" + for _, col := range table.Columns { + if col.Name == name { + colType = col.Type + } + } + expr := quote(name) + if table.Name == "mail_events" && name == "id" && !postgres { + expr = "rowid" + } + if colType == "TEXT" { + if postgres { + expr += ` COLLATE "C"` + } else { + expr += ` COLLATE BINARY` + } + } + if postgres { + expr += " NULLS FIRST" + } + out = append(out, expr) + } + return strings.Join(out, ","), nil +} + +func canonical(value any, kind string) (any, error) { + if value == nil { + return nil, nil + } + switch kind { + case "TEXT": + var s string + switch v := value.(type) { + case string: + s = v + case []byte: + s = string(v) + default: + return nil, errors.New("non-text value in a text column") + } + if !utf8.ValidString(s) || strings.ContainsRune(s, 0) { + return nil, errors.New("text contains invalid UTF-8 or NUL, which PostgreSQL cannot preserve") + } + return s, nil + case "BIGINT": + switch v := value.(type) { + case int64: + return v, nil + case int32: + return int64(v), nil + case int: + return int64(v), nil + case float64: + if math.Trunc(v) == v && v >= math.MinInt64 && v < 9223372036854775808.0 { + return int64(v), nil + } + } + return nil, errors.New("non-integral or out-of-range value in an integer column") + case "DOUBLE PRECISION": + switch v := value.(type) { + case float64: + return v, nil + case int64: + if int64(float64(v)) != v { + return nil, errors.New("integer cannot be represented exactly by the destination floating-point column") + } + return float64(v), nil + case int32: + return float64(v), nil + } + return nil, errors.New("non-numeric value in a floating-point column") + } + return nil, fmt.Errorf("unsupported destination column type %s", kind) +} + +func hashRow(h hash.Hash, values []any) { + for _, value := range values { + switch v := value.(type) { + case nil: + h.Write([]byte{0}) + case int64: + h.Write([]byte{1}) + var b [8]byte + binary.BigEndian.PutUint64(b[:], uint64(v)) + h.Write(b[:]) + case float64: + h.Write([]byte{2}) + var b [8]byte + binary.BigEndian.PutUint64(b[:], math.Float64bits(v)) + h.Write(b[:]) + case string: + h.Write([]byte{3}) + var b [8]byte + binary.BigEndian.PutUint64(b[:], uint64(len(v))) + h.Write(b[:]) + h.Write([]byte(v)) + } + } + h.Write([]byte{255}) +} + +func digestTarget(ctx context.Context, db *sql.DB, table importTable) (tableReport, error) { + report := tableReport{Table: table.Name} + _, quoted := columnNames(table) + order, err := orderBy(db, table, true) + if err != nil { + return report, err + } + rows, err := db.QueryContext(ctx, `SELECT `+strings.Join(quoted, ",")+` FROM `+quote(table.Name)+` ORDER BY `+order) + if err != nil { + return report, err + } + defer rows.Close() + h := sha256.New() + for rows.Next() { + values := make([]any, len(quoted)) + dest := make([]any, len(quoted)) + for i := range dest { + dest[i] = &values[i] + } + if err := rows.Scan(dest...); err != nil { + return report, err + } + for i, col := range table.Columns { + values[i], err = canonical(values[i], col.Type) + if err != nil { + return report, err + } + } + hashRow(h, values) + report.Rows++ + } + report.SHA256 = hex.EncodeToString(h.Sum(nil)) + return report, rows.Err() +} + +func copyTable(ctx context.Context, source, owner *sql.DB, conn *pgx.Conn, table importTable, o importOptions) (tableReport, error) { + report := tableReport{Table: table.Name} + var copied int64 + var storedDigest string + var complete bool + err := conn.QueryRow(ctx, `SELECT rows_copied,sha256,complete FROM corescope_import_progress WHERE table_name=$1`, table.Name).Scan(&copied, &storedDigest, &complete) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return report, err + } + existing, err := digestTarget(ctx, owner, table) + if err != nil { + return report, err + } + emptyHash := sha256.Sum256(nil) + if storedDigest == "" { + storedDigest = hex.EncodeToString(emptyHash[:]) + } + if existing.Rows != copied || existing.SHA256 != storedDigest { + return report, errors.New("destination rows differ from committed import progress; resume refused") + } + var exists int + if err := source.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_master WHERE type='table' AND name=?`, table.Name).Scan(&exists); err != nil { + return report, err + } + h := sha256.New() + names, quoted := columnNames(table) + var batch [][]any + var batchBytes int + flush := func(done bool) error { + if len(batch) == 0 && !done { + return nil + } + tx, err := conn.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + if len(batch) > 0 { + if n, err := tx.CopyFrom(ctx, pgx.Identifier{table.Name}, names, pgx.CopyFromRows(batch)); err != nil { + return err + } else if n != int64(len(batch)) { + return errors.New("COPY row count mismatch") + } + } + digest := hex.EncodeToString(h.Sum(nil)) + if _, err := tx.Exec(ctx, `INSERT INTO corescope_import_progress(table_name,rows_copied,sha256,complete) VALUES($1,$2,$3,$4) + ON CONFLICT(table_name) DO UPDATE SET rows_copied=excluded.rows_copied,sha256=excluded.sha256,complete=excluded.complete`, table.Name, report.Rows, digest, done); err != nil { + return err + } + if err := tx.Commit(ctx); err != nil { + return err + } + wrote := len(batch) > 0 + batch = nil + batchBytes = 0 + if wrote && o.afterBatch != nil { + return o.afterBatch() + } + return nil + } + if exists != 0 { + order, err := orderBy(owner, table, false) + if err != nil { + return report, err + } + expressions := append([]string(nil), quoted...) + if o.Kind == "accounts" && table.Name == "mail_events" { + expressions[0] = "rowid" + } + rows, err := source.QueryContext(ctx, `SELECT `+strings.Join(expressions, ",")+` FROM `+quote(table.Name)+` ORDER BY `+order) + if err != nil { + return report, err + } + defer rows.Close() + for rows.Next() { + values := make([]any, len(names)) + dest := make([]any, len(names)) + for i := range dest { + dest[i] = &values[i] + } + if err := rows.Scan(dest...); err != nil { + return report, err + } + rowBytes := 0 + for i, col := range table.Columns { + values[i], err = canonical(values[i], col.Type) + if err != nil { + return report, fmt.Errorf("column %s: %w", col.Name, err) + } + if s, ok := values[i].(string); ok { + rowBytes += len(s) + } else { + rowBytes += 8 + } + } + if rowBytes > maxImportRowBytes { + return report, errors.New("a source row exceeds the 64 MiB migration safety limit") + } + hashRow(h, values) + report.Rows++ + if report.Rows <= copied { + if report.Rows == copied && hex.EncodeToString(h.Sum(nil)) != storedDigest { + return report, errors.New("source prefix differs from import progress") + } + continue + } + if complete { + return report, errors.New("completed source table contains unexpected rows") + } + batch = append(batch, values) + batchBytes += rowBytes + if len(batch) >= o.BatchSize || batchBytes >= 8*1024*1024 { + if err := flush(false); err != nil { + return report, err + } + } + } + if err := rows.Err(); err != nil { + return report, err + } + rows.Close() + } + if report.Rows < copied { + return report, errors.New("source table shorter than committed progress") + } + if err := flush(true); err != nil { + return report, err + } + report.SHA256 = hex.EncodeToString(h.Sum(nil)) + final, err := digestTarget(ctx, owner, table) + if err != nil { + return report, err + } + if final != report { + return report, errors.New("row count or logical digest verification failed") + } + return report, nil +} + +func reseed(ctx context.Context, source *sql.DB, conn *pgx.Conn, tables []importTable, kind string) error { + highWater := map[string]int64{} + var hasSequence int + if err := source.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name='sqlite_sequence'`).Scan(&hasSequence); err != nil { + return err + } + if hasSequence > 0 { + rows, err := source.Query(`SELECT name,seq FROM sqlite_sequence`) + if err != nil { + return err + } + for rows.Next() { + var name string + var n int64 + if err := rows.Scan(&name, &n); err != nil { + rows.Close() + return err + } + highWater[name] = n + } + if err := rows.Err(); err != nil { + rows.Close() + return err + } + rows.Close() + } + references := map[string][][2]string{} + if kind == "telemetry" { + references["observers"] = [][2]string{{"observations", "observer_idx"}} + references["transmissions"] = [][2]string{{"observations", "transmission_id"}, {"advert_route_evidence", "tx_id"}, {"advert_evidence_backfill", "tx_cursor"}} + references["observations"] = [][2]string{{"advert_evidence_backfill", "obs_cursor"}} + } else { + references["users"] = [][2]string{{"audit_log", "actor_user_id"}, {"audit_log", "target_user_id"}, {"users", "activated_by"}, {"sessions", "user_id"}, {"tokens", "user_id"}, {"mail_log", "user_id"}, {"proposals", "proposer_id"}, {"proposals", "reviewer_id"}} + references["mail_log"] = [][2]string{{"mail_events", "mail_id"}} + } + for _, table := range tables { + for _, col := range table.Columns { + if !col.Identity { + continue + } + high := highWater[table.Name] + var n int64 + if err := conn.QueryRow(ctx, `SELECT COALESCE(MAX(`+quote(col.Name)+`),0) FROM `+quote(table.Name)).Scan(&n); err != nil { + return err + } + if n > high { + high = n + } + for _, ref := range references[table.Name] { + if err := conn.QueryRow(ctx, `SELECT COALESCE(MAX(`+quote(ref[1])+`),0) FROM `+quote(ref[0])).Scan(&n); err != nil { + return err + } + if n > high { + high = n + } + } + called := high > 0 + if !called { + high = 1 + } + if _, err := conn.Exec(ctx, `SELECT setval(pg_get_serial_sequence($1,$2),$3,$4)`, table.Name, col.Name, high, called); err != nil { + return err + } + } + } + return nil +} + +func finalizeImport(ctx context.Context, databaseURL, kind string) error { + tables, err := layouts(kind) + if err != nil { + return err + } + db, err := pgutil.Open(databaseURL, false) + if err != nil { + return err + } + defer db.Close() + config, err := pgutil.ParseConfig(databaseURL) + if err != nil { + return err + } + conn, err := pgx.ConnectConfig(ctx, config) + if err != nil { + return errors.New("connect to PostgreSQL finalization destination failed") + } + defer conn.Close(context.Background()) + if err := lockImport(ctx, conn); err != nil { + return err + } + var verified bool + var recordedKind, reportText string + if err := conn.QueryRow(ctx, `SELECT kind,verified,report FROM corescope_import WHERE id=1`).Scan(&recordedKind, &verified, &reportText); err != nil { + return err + } + if !verified || recordedKind != kind { + return errors.New("cannot finalize an unverified or mismatched import") + } + var report importReport + if err := json.Unmarshal([]byte(reportText), &report); err != nil { + return err + } + if !report.Verified || report.Kind != kind || len(report.Tables) != len(tables) { + return errors.New("incomplete verification manifest") + } + tx, err := conn.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + var lockTables []string + for _, table := range tables { + lockTables = append(lockTables, quote(table.Name)) + } + if _, err := tx.Exec(ctx, `LOCK TABLE `+strings.Join(lockTables, ",")+` IN SHARE MODE`); err != nil { + return err + } + for i, table := range tables { + got, err := digestTarget(ctx, db, table) + if err != nil { + return err + } + if got != report.Tables[i] { + return fmt.Errorf("final verification failed for %s; target remains unready", table.Name) + } + } + result, err := tx.Exec(ctx, `UPDATE corescope_schema SET ready=true WHERE kind=$1 AND version=1`, kind) + if err != nil { + return err + } + if result.RowsAffected() != 1 { + return errors.New("missing or unsupported readiness marker") + } + return tx.Commit(ctx) +} + +func lockImport(ctx context.Context, conn *pgx.Conn) error { + var locked bool + if err := conn.QueryRow(ctx, `SELECT pg_try_advisory_lock(hashtextextended(current_schema(),$1))`, importerLock).Scan(&locked); err != nil { + return err + } + if !locked { + return errors.New("another importer owns this destination; wait for it to finish before retrying") + } + return nil +} diff --git a/cmd/migrate/import_safety_test.go b/cmd/migrate/import_safety_test.go new file mode 100644 index 000000000..c8d8f823c --- /dev/null +++ b/cmd/migrate/import_safety_test.go @@ -0,0 +1,294 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +func TestResumeRefusesDifferentDestination(t *testing.T) { + o := importOptions{Source: telemetrySource(t), DatabaseURL: pgtest.NewSchema(t), StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 1, afterBatch: func() error { return context.Canceled }} + if _, err := importSQLite(context.Background(), o); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + o.Resume = true + o.afterBatch = nil + o.DatabaseURL = pgtest.NewSchema(t) + if _, err := importSQLite(context.Background(), o); err == nil { + t.Fatal("resume accepted a different PostgreSQL destination") + } +} + +func TestResumeAndFinalizationRejectTargetCorruption(t *testing.T) { + for _, partial := range []bool{false, true} { + t.Run(map[bool]string{false: "finalization", true: "resume"}[partial], func(t *testing.T) { + o := importOptions{Source: telemetrySource(t), DatabaseURL: pgtest.NewSchema(t), StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 1} + if partial { + o.afterBatch = func() error { return context.Canceled } + } + _, err := importSQLite(context.Background(), o) + if partial { + if !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + } else if err != nil { + t.Fatal(err) + } + db := openImportDB(t, o.DatabaseURL) + if _, err := db.Exec(`UPDATE observers SET name='tampered'`); err != nil { + t.Fatal(err) + } + if partial { + o.Resume = true + o.afterBatch = nil + _, err = importSQLite(context.Background(), o) + } else { + err = finalizeImport(context.Background(), o.DatabaseURL, o.Kind) + } + if err == nil { + t.Fatal("corrupted target was accepted") + } + if err := dbschema.AssertReady(db); err == nil { + t.Fatal("corrupted target became runtime-ready") + } + }) + } +} + +func TestImportPreservesRawCaseJSONAndFractionalScore(t *testing.T) { + source := telemetrySource(t) + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + for _, stmt := range []string{`INSERT INTO nodes(public_key,name,advert_count) VALUES('AbCd','MiXeD',123)`, `UPDATE observations SET path_json='{malformed',score=0.75 WHERE id=90`, `UPDATE transmissions SET channel_hash='public',payload_type=5 WHERE id=80`} { + if _, err := db.Exec(stmt); err != nil { + db.Close() + t.Fatal(err) + } + } + db.Close() + dsn := pgtest.NewSchema(t) + if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry"}); err != nil { + t.Fatal(err) + } + target := openImportDB(t, dsn) + var key, jsonText, channel string + var count int + var score float64 + if err := target.QueryRow(`SELECT public_key,advert_count FROM nodes`).Scan(&key, &count); err != nil || key != "AbCd" || count != 123 { + t.Fatal("raw key/counter changed", err) + } + if err := target.QueryRow(`SELECT path_json,score FROM observations WHERE id=90`).Scan(&jsonText, &score); err != nil || jsonText != "{malformed" || score != 0.75 { + t.Fatal("raw JSON or fractional score changed", err) + } + if err := target.QueryRow(`SELECT channel_hash FROM transmissions WHERE id=80`).Scan(&channel); err != nil || channel != "public" { + t.Fatal("raw channel hash changed", err) + } +} + +func TestImportRejectsUnexpectedViewsTriggersAndFutureAccounts(t *testing.T) { + for _, statement := range []string{`CREATE VIEW unexpected AS SELECT * FROM users`, `CREATE TRIGGER surprise AFTER INSERT ON users BEGIN DELETE FROM tokens; END`, `UPDATE schema_version SET version=999`} { + source := accountSource(t, 5) + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(statement); err != nil { + t.Fatal(err) + } + db.Close() + dsn := pgtest.NewSchema(t) + _, err = importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "accounts"}) + if err == nil { + t.Fatal("unsupported source accepted") + } + var n int + if err := openImportDB(t, dsn).QueryRow(`SELECT count(*) FROM information_schema.tables WHERE table_schema=current_schema()`).Scan(&n); err != nil || n != 0 { + t.Fatal("unsupported source changed target", err) + } + } +} + +func TestAccountsReseedDeletedHistoricalReferences(t *testing.T) { + source := accountSource(t, 5) + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO audit_log(at,actor_user_id,target_user_id,action) VALUES(1,700000,900000,'deleted'); UPDATE users SET activated_by=1000000`); err != nil { + t.Fatal(err) + } + db.Close() + dsn := pgtest.NewSchema(t) + if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "accounts"}); err != nil { + t.Fatal(err) + } + var id int64 + if err := openImportDB(t, dsn).QueryRow(`INSERT INTO users(email,display_name,password_hash,created_at) VALUES('next@example.invalid','next','hash',1) RETURNING id`).Scan(&id); err != nil || id <= 1000000 { + t.Fatalf("historical account identity reused: %d,%v", id, err) + } +} + +func TestSourceFingerprintCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := sourceFingerprintContext(ctx, telemetrySource(t)) + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled source hashing: %v", err) + } +} + +func TestImportRefusesLegacyV2WithUpgradeGuidance(t *testing.T) { + source := telemetrySource(t) + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`ALTER TABLE observations RENAME COLUMN observer_idx TO observer_id`); err != nil { + t.Fatal(err) + } + db.Close() + _, err = importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: pgtest.NewSchema(t), StateDir: t.TempDir(), Kind: "telemetry"}) + if err == nil || !strings.Contains(err.Error(), "upgrade") { + t.Fatal("v2 refusal lacks explicit upgrade guidance") + } +} + +func TestAccountVersionMatrixAndMissingDeclaredTable(t *testing.T) { + for version := 0; version <= 5; version++ { + t.Run(fmt.Sprint(version), func(t *testing.T) { + var source string + if version > 0 { + source = accountSource(t, version) + } else { + source = filepath.Join(t.TempDir(), "empty.db") + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`CREATE TABLE schema_version(version INTEGER NOT NULL);INSERT INTO schema_version VALUES(0)`); err != nil { + t.Fatal(err) + } + db.Close() + } + report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: pgtest.NewSchema(t), StateDir: t.TempDir(), Kind: "accounts"}) + if err != nil || !report.Verified { + t.Fatalf("account version %d: %v", version, err) + } + }) + } + t.Run("missing-table", func(t *testing.T) { + source := accountSource(t, 5) + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`DROP TABLE mail_events`); err != nil { + t.Fatal(err) + } + db.Close() + if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: pgtest.NewSchema(t), StateDir: t.TempDir(), Kind: "accounts"}); err == nil { + t.Fatal("incomplete declared account schema was accepted") + } + }) +} + +func TestNormalizeDoesNotRewriteObservationRows(t *testing.T) { + raw := telemetrySource(t) + db, err := sql.Open("sqlite3", raw) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<1000) + INSERT INTO observations(id,transmission_id,observer_idx,timestamp) SELECT x+1000,80,NULL,x FROM n`); err != nil { + t.Fatal(err) + } + db.Close() + working := filepath.Join(t.TempDir(), "working.sqlite") + if err := snapshotSource(context.Background(), raw, working); err != nil { + t.Fatal(err) + } + db, err = sql.Open("sqlite3", sqliteURL(working, "rw")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + db.SetMaxOpenConns(1) + tables, err := layouts("telemetry") + if err != nil { + t.Fatal(err) + } + var before, after int64 + if err := db.QueryRow(`SELECT total_changes()`).Scan(&before); err != nil { + t.Fatal(err) + } + if err := normalizeSource(db, "telemetry", raw, tables); err != nil { + t.Fatal(err) + } + if err := db.QueryRow(`SELECT total_changes()`).Scan(&after); err != nil { + t.Fatal(err) + } + if after-before >= 50 { + t.Fatalf("normalization rewrote unchanged bulk rows: %d changes for 1003 observations", after-before) + } + t.Logf("normalization wrote %d rows for 1003 unchanged observations", after-before) +} + +func TestImportPopulatesPlannerStatsBeforeReadiness(t *testing.T) { + dsn := pgtest.NewSchema(t) + if _, err := importSQLite(context.Background(), importOptions{Source: telemetrySource(t), DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry"}); err != nil { + t.Fatal(err) + } + db := openImportDB(t, dsn) + var estimate float64 + if err := db.QueryRow(`SELECT reltuples FROM pg_class WHERE oid=to_regclass('observations')`).Scan(&estimate); err != nil { + t.Fatal(err) + } + if estimate != 3 { + t.Fatalf("planner estimate before readiness=%v; want 3 imported observations", estimate) + } + if err := dbschema.AssertReady(db); err == nil { + t.Fatal("statistics preparation opened readiness before finalization") + } +} + +func TestReservedNameDoesNotHideSourceTrigger(t *testing.T) { + source := accountSource(t, 5) + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + for _, stmt := range []string{ + `CREATE TRIGGER hidden_source AFTER INSERT ON users BEGIN DELETE FROM tokens; END`, + `PRAGMA writable_schema=ON`, + `UPDATE sqlite_master SET name='sqlite_hidden_source',sql=replace(sql,'hidden_source','sqlite_hidden_source') WHERE type='trigger' AND name='hidden_source'`, + `PRAGMA writable_schema=OFF`, + } { + if _, err := db.Exec(stmt); err != nil { + db.Close() + t.Fatal(err) + } + } + db.Close() + db, err = sql.Open("sqlite3", sqliteURL(source, "ro")) + if err != nil { + t.Fatal(err) + } + var count int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='trigger' AND name='sqlite_hidden_source'`).Scan(&count); err != nil || count != 1 { + db.Close() + t.Fatalf("tampered schema fixture: count=%d err=%v", count, err) + } + db.Close() + if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: pgtest.NewSchema(t), StateDir: t.TempDir(), Kind: "accounts"}); err == nil { + t.Fatal("reserved-name trigger bypassed source validation") + } +} diff --git a/cmd/migrate/import_test.go b/cmd/migrate/import_test.go new file mode 100644 index 000000000..de0905001 --- /dev/null +++ b/cmd/migrate/import_test.go @@ -0,0 +1,256 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" + "github.com/meshcore-analyzer/users" +) + +func telemetrySource(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "source.db") + db, err := sql.Open("sqlite3", path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := legacy.ApplyBase(db); err != nil { + t.Fatal(err) + } + if err := legacy.Apply(db, t.Logf); err != nil { + t.Fatal(err) + } + for _, query := range []string{ + `INSERT INTO observers(rowid,id,name) VALUES(0,'observer-zero',NULL),(50,'observer-gap','')`, + `INSERT INTO transmissions(id,raw_hex,hash,first_seen,last_seen) VALUES(0,'00','zero','2026-01-01 00:00:00',0),(80,'00','gap','2026-01-02T00:00:00Z',1),(99999,'00','deleted','2026-01-01',0)`, + `DELETE FROM transmissions WHERE id=99999`, + `INSERT INTO observations(id,transmission_id,observer_idx,path_json,timestamp) VALUES(0,0,0,NULL,0),(90,80,999,'',1),(91,80,NULL,'[]',2)`, + } { + if _, err := db.Exec(query); err != nil { + t.Fatal(err) + } + } + return path +} + +func openImportDB(t *testing.T, dsn string) *sql.DB { + t.Helper() + db, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + return db +} + +func TestImportTelemetryPreservesIDsNullsAndHighWater(t *testing.T) { + source := telemetrySource(t) + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + dsn := pgtest.NewSchema(t) + report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 2}) + if err != nil { + t.Fatalf("import valid telemetry: %v", err) + } + if !report.Verified || len(report.Tables) != len(dbschema.Tables) { + t.Fatalf("missing verification: %+v", report) + } + db := openImportDB(t, dsn) + if err := dbschema.AssertReady(db); err == nil { + t.Fatal("import accepted before explicit cutover finalization") + } + if err := finalizeImport(context.Background(), dsn, "telemetry"); err != nil { + t.Fatal(err) + } + if err := dbschema.AssertReady(db); err != nil { + t.Fatal(err) + } + var n int + if err := db.QueryRow(`SELECT count(*) FROM observations WHERE (id=0 AND observer_idx=0 AND path_json IS NULL) OR (id=90 AND observer_idx=999 AND path_json='') OR (id=91 AND observer_idx IS NULL)`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 3 { + t.Fatalf("lost observer identity/null/orphan links: %d", n) + } + var id int64 + if err := db.QueryRow(`INSERT INTO observers(id) VALUES('next-observer') RETURNING rowid`).Scan(&id); err != nil { + t.Fatal(err) + } + if id <= 999 { + t.Fatalf("observer identity reused historical reference: %d", id) + } + if err := db.QueryRow(`INSERT INTO transmissions(raw_hex,hash,first_seen) VALUES('00','next','2026-01-03') RETURNING id`).Scan(&id); err != nil { + t.Fatal(err) + } + if id <= 99999 { + t.Fatalf("transmission identity ignored deleted high water: %d", id) + } + after, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + if before != after { + t.Fatal("migration changed original SQLite source") + } +} + +func TestImportInterruptResumeAndWrongSource(t *testing.T) { + source := telemetrySource(t) + dsn, state := pgtest.NewSchema(t), t.TempDir() + options := importOptions{Source: source, DatabaseURL: dsn, StateDir: state, Kind: "telemetry", BatchSize: 1, afterBatch: func() error { return context.Canceled }} + if _, err := importSQLite(context.Background(), options); !errors.Is(err, context.Canceled) { + t.Fatalf("expected cancellation after committed batch, got %v", err) + } + db := openImportDB(t, dsn) + if err := dbschema.AssertReady(db); err == nil { + t.Fatal("partially imported database accepted") + } + options.afterBatch = nil + if _, err := importSQLite(context.Background(), options); err == nil { + t.Fatal("partial import resumed without explicit resume") + } + options.Resume = true + options.Source = telemetrySource(t) + changed, err := sql.Open("sqlite3", options.Source) + if err != nil { + t.Fatal(err) + } + if _, err := changed.Exec(`UPDATE transmissions SET raw_hex='ff' WHERE id=0`); err != nil { + t.Fatal(err) + } + changed.Close() + if _, err := importSQLite(context.Background(), options); err == nil { + t.Fatal("resumed with a different source") + } + options.Source = source + report, err := importSQLite(context.Background(), options) + if err != nil { + t.Fatal(err) + } + if !report.Verified { + t.Fatal("resumed import unverified") + } + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM observations`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 3 { + t.Fatalf("resume duplicated/lost observations: %d", n) + } + if _, err := importSQLite(context.Background(), options); err != nil { + t.Fatalf("verified retry must be idempotent: %v", err) + } +} + +func TestImportRefusesUnknownSourceAndOccupiedTarget(t *testing.T) { + for _, unknown := range []bool{false, true} { + source, dsn := telemetrySource(t), pgtest.NewSchema(t) + if unknown { + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`ALTER TABLE transmissions ADD COLUMN future_secret TEXT`); err != nil { + t.Fatal(err) + } + db.Close() + } else { + db := openImportDB(t, dsn) + if _, err := db.Exec(`CREATE TABLE keep_me(value TEXT); INSERT INTO keep_me VALUES('retained')`); err != nil { + t.Fatal(err) + } + } + if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry"}); err == nil { + t.Fatal("unsupported source/occupied destination accepted") + } + if !unknown { + var value string + if err := openImportDB(t, dsn).QueryRow(`SELECT value FROM keep_me`).Scan(&value); err != nil { + t.Fatal(err) + } + if value != "retained" { + t.Fatal("existing target changed") + } + } + } +} + +func accountSource(t *testing.T, version int) string { + t.Helper() + path := filepath.Join(t.TempDir(), "users.db") + db, err := sql.Open("sqlite3", path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`CREATE TABLE schema_version(version INTEGER NOT NULL)`); err != nil { + t.Fatal(err) + } + for _, migration := range users.LegacyMigrations()[:version] { + for _, stmt := range migration { + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } + } + } + if _, err := db.Exec(`INSERT INTO schema_version VALUES(?)`, version); err != nil { + t.Fatal(err) + } + for _, stmt := range []string{ + `INSERT INTO users(id,email,display_name,password_hash,role,status,created_at) VALUES(0,'user@example.invalid','test','preserved-hash','user','active',1)`, + `INSERT INTO sessions(id,token_hash,user_id,csrf_token,created_at,expires_at,last_seen_at) VALUES(90,'preserved-token',0,'preserved-csrf',1,9999999999,2)`, + `INSERT INTO tokens(token_hash,user_id,purpose,expires_at,used_at) VALUES('one-use',0,'reset',9999999999,NULL)`, + `INSERT INTO mail_log(id,to_email,purpose,sent_at,last_event_at) VALUES(4,'user@example.invalid','activate',1,1)`, + `INSERT INTO mail_events(rowid,mail_id,event,at) VALUES(900,4,'delivered',1)`, + } { + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } + } + return path +} + +func TestImportLegacyAccountsPreservesAuthenticationAndEventOrder(t *testing.T) { + source, dsn := accountSource(t, 1), pgtest.NewSchema(t) + report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "accounts", BatchSize: 1}) + if err != nil { + t.Fatal(err) + } + if !report.Verified { + t.Fatal("accounts not verified") + } + db := openImportDB(t, dsn) + if err := users.AssertReady(db); err == nil { + t.Fatal("accounts accepted before finalization") + } + if err := finalizeImport(context.Background(), dsn, "accounts"); err != nil { + t.Fatal(err) + } + if err := users.AssertReady(db); err != nil { + t.Fatal(err) + } + var value string + if err := db.QueryRow(`SELECT password_hash FROM users WHERE id=0`).Scan(&value); err != nil || value != "preserved-hash" { + t.Fatalf("password hash changed: %v", err) + } + if err := db.QueryRow(`SELECT token_hash FROM sessions WHERE id=90 AND user_id=0`).Scan(&value); err != nil || value != "preserved-token" { + t.Fatalf("session changed: %v", err) + } + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM tokens WHERE token_hash='one-use' AND used_at IS NULL`).Scan(&n); err != nil || n != 1 { + t.Fatalf("one-time token changed: %v", err) + } + if err := db.QueryRow(`SELECT id FROM mail_events WHERE mail_id=4`).Scan(&n); err != nil || n != 900 { + t.Fatalf("mail event identity changed: %d %v", n, err) + } +} diff --git a/cmd/migrate/layout.go b/cmd/migrate/layout.go new file mode 100644 index 000000000..9e0b566ba --- /dev/null +++ b/cmd/migrate/layout.go @@ -0,0 +1,329 @@ +package main + +import ( + "database/sql" + "errors" + "fmt" + "strings" + + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" + "github.com/meshcore-analyzer/users" +) + +type importColumn struct { + Name string + Type string + Identity bool +} +type importTable struct { + Name string + Columns []importColumn +} + +func quote(name string) string { return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` } + +func layouts(kind string) ([]importTable, error) { + var out []importTable + if kind == "telemetry" { + for _, table := range dbschema.Tables { + t := importTable{Name: table.Name} + for _, col := range table.Columns { + t.Columns = append(t.Columns, importColumn{col.Name, col.Type, col.Identity}) + } + out = append(out, t) + } + return out, nil + } + if kind != "accounts" { + return nil, errors.New("import kind must be telemetry or accounts") + } + // SQLite is used only by this offline importer. Derive the legacy account + // layout from the immutable migration metadata instead of duplicating it. + db, err := sql.Open("sqlite3", ":memory:") + if err != nil { + return nil, err + } + defer db.Close() + db.SetMaxOpenConns(1) + for _, migration := range users.LegacyMigrations() { + for _, stmt := range migration { + if _, err := db.Exec(stmt); err != nil { + return nil, err + } + } + } + identities := map[string]bool{"users": true, "sessions": true, "audit_log": true, "mail_log": true, "proposals": true, "mail_events": true} + for _, name := range users.Tables() { + t := importTable{Name: name} + if name == "mail_events" { + t.Columns = append(t.Columns, importColumn{"id", "BIGINT", true}) + } + cols, err := sqliteColumns(db, name) + if err != nil { + return nil, err + } + for _, col := range cols { + kind := "TEXT" + if strings.Contains(strings.ToUpper(col.Type), "INT") { + kind = "BIGINT" + } + t.Columns = append(t.Columns, importColumn{col.Name, kind, col.Name == "id" && identities[name]}) + } + out = append(out, t) + } + return out, nil +} + +func sqliteColumns(db *sql.DB, table string) ([]importColumn, error) { + // table_info omits generated and hidden columns, which must be refused even + // when their names match an ordinary column in a supported legacy layout. + rows, err := db.Query(`PRAGMA table_xinfo(` + quote(table) + `)`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []importColumn + for rows.Next() { + var cid, notnull, pk, hidden int + var name, kind string + var def sql.NullString + if err := rows.Scan(&cid, &name, &kind, ¬null, &def, &pk, &hidden); err != nil { + return nil, err + } + if hidden != 0 { + return nil, fmt.Errorf("unsupported generated or hidden SQLite column %s.%s", table, name) + } + out = append(out, importColumn{Name: name, Type: kind}) + } + return out, rows.Err() +} + +func validateSource(db *sql.DB, kind string, tables []importTable) error { + allowed := map[string]map[string]bool{} + for _, table := range tables { + cols := map[string]bool{} + for _, col := range table.Columns { + cols[col.Name] = true + } + if kind == "accounts" && table.Name == "mail_events" { + delete(cols, "id") + } + allowed[table.Name] = cols + } + if kind == "accounts" { + allowed["schema_version"] = map[string]bool{"version": true} + } + rows, err := db.Query(`SELECT name,type,COALESCE(sql,'') FROM sqlite_master WHERE type IN ('table','view','trigger')`) + if err != nil { + return err + } + type object struct{ name, kind, sql string } + internalTables := map[string]bool{"sqlite_sequence": true, "sqlite_stat1": true, "sqlite_stat2": true, "sqlite_stat3": true, "sqlite_stat4": true} + var objects []object + for rows.Next() { + var o object + if err := rows.Scan(&o.name, &o.kind, &o.sql); err != nil { + rows.Close() + return err + } + ordinary := strings.HasPrefix(strings.ToUpper(strings.Join(strings.Fields(o.sql), " ")), "CREATE TABLE ") + if o.kind == "table" && internalTables[o.name] && ordinary { + continue + } + objects = append(objects, o) + if len(objects) > len(allowed)+1 { + rows.Close() + return errors.New("source has unexpected schema objects") + } + } + if err := rows.Err(); err != nil { + rows.Close() + return err + } + rows.Close() + actual := map[string]map[string]bool{} + for _, o := range objects { + if o.kind == "table" && !strings.HasPrefix(strings.ToUpper(strings.Join(strings.Fields(o.sql), " ")), "CREATE TABLE ") { + return fmt.Errorf("unsupported nonordinary source table %q", o.name) + } + if o.kind == "view" && kind == "telemetry" && o.name == "packets_v" { + continue + } + cols, known := allowed[o.name] + if o.kind != "table" || !known { + return fmt.Errorf("unsupported SQLite %s %q; no source data was discarded", o.kind, o.name) + } + have, err := sqliteColumns(db, o.name) + if err != nil { + return err + } + actual[o.name] = map[string]bool{} + for _, col := range have { + actual[o.name][col.Name] = true + if !cols[col.Name] { + if kind == "telemetry" && o.name == "observations" && col.Name == "observer_id" { + return errors.New("legacy v2 observer_id layout requires an offline upgrade with the last SQLite release before importing") + } + return fmt.Errorf("unsupported SQLite column %s.%s", o.name, col.Name) + } + } + } + if kind == "telemetry" { + return legacy.CheckLegacySource(db) + } + var n, version int + if err := db.QueryRow(`SELECT COUNT(*),COALESCE(MAX(version),0) FROM schema_version`).Scan(&n, &version); err != nil { + return errors.New("account source must contain schema_version (supported versions 0 through 5)") + } + if n != 1 || version < 0 || version > users.CurrentSchemaVersion { + return fmt.Errorf("unsupported account schema version %d (supported 0 through %d)", version, users.CurrentSchemaVersion) + } + // A declared version must contain its complete historical shape. Otherwise + // treating an absent table as empty could conceal a damaged account source. + expected, err := sql.Open("sqlite3", ":memory:") + if err != nil { + return err + } + defer expected.Close() + expected.SetMaxOpenConns(1) + for _, migration := range users.LegacyMigrations()[:version] { + for _, stmt := range migration { + if _, err := expected.Exec(stmt); err != nil { + return err + } + } + } + for _, table := range tables { + columns, err := sqliteColumns(expected, table.Name) + if err != nil { + return err + } + if len(columns) == 0 { + if _, exists := actual[table.Name]; exists { + return fmt.Errorf("account table %s is newer than its declared schema version", table.Name) + } + continue + } + for _, col := range columns { + if !actual[table.Name][col.Name] { + return fmt.Errorf("account schema version %d is missing %s.%s", version, table.Name, col.Name) + } + } + } + var integrity string + if err := db.QueryRow(`PRAGMA quick_check`).Scan(&integrity); err != nil { + return err + } + if integrity != "ok" { + return errors.New("account source failed SQLite integrity check") + } + return nil +} + +func normalizeSource(db *sql.DB, kind, rawPath string, tables []importTable) error { + if err := validateSource(db, kind, tables); err != nil { + return err + } + if kind == "accounts" { + var version int + if err := db.QueryRow(`SELECT version FROM schema_version`).Scan(&version); err != nil { + return err + } + for i, migration := range users.LegacyMigrations()[version:] { + tx, err := db.Begin() + if err != nil { + return err + } + for _, stmt := range migration { + if _, err := tx.Exec(stmt); err != nil { + tx.Rollback() + return err + } + } + if _, err := tx.Exec(`UPDATE schema_version SET version=?`, version+i+1); err != nil { + tx.Rollback() + return err + } + if err := tx.Commit(); err != nil { + return err + } + } + return nil + } + if err := legacy.Normalize(db, nil); err != nil { + return err + } + // Mutation inventory of legacy.Normalize: only these columns can change + // existing rows. Duplicate/invalid-row deletion paths were refused above. + // Restore changed tuples by rowid, without rewriting the observation corpus. + if _, err := db.Exec(`ATTACH DATABASE ? AS recovery`, sqliteURL(rawPath, "ro")); err != nil { + return err + } + defer db.Exec(`DETACH DATABASE recovery`) + mutated := map[string]map[string]bool{ + "nodes": {"public_key": true, "advert_count": true}, + "observers": {"noise_floor": true, "last_packet_at": true}, + "transmissions": {"channel_hash": true}, + } + for _, table := range tables { + selected, changes := mutated[table.Name] + if !changes { + continue + } + var exists int + if err := db.QueryRow(`SELECT count(*) FROM recovery.sqlite_master WHERE type='table' AND name=?`, table.Name).Scan(&exists); err != nil { + return err + } + if exists == 0 { + continue + } + rows, err := db.Query(`PRAGMA recovery.table_info(` + quote(table.Name) + `)`) + if err != nil { + return err + } + var columns []string + for rows.Next() { + var cid, nn, pk int + var name, kind string + var def sql.NullString + if err := rows.Scan(&cid, &name, &kind, &nn, &def, &pk); err != nil { + rows.Close() + return err + } + if selected[name] { + columns = append(columns, name) + } + } + if err := rows.Err(); err != nil { + rows.Close() + return err + } + rows.Close() + var sourceCount, targetCount int64 + if err := db.QueryRow(`SELECT count(*) FROM recovery.` + quote(table.Name)).Scan(&sourceCount); err != nil { + return err + } + if err := db.QueryRow(`SELECT count(*) FROM main.` + quote(table.Name)).Scan(&targetCount); err != nil { + return err + } + if sourceCount != targetCount { + return fmt.Errorf("normalization changed row count in %s; source requires an explicit repair", table.Name) + } + if len(columns) == 0 { + continue + } + var targets, originals, differences []string + for _, name := range columns { + targets = append(targets, quote(name)) + originals = append(originals, "old."+quote(name)) + differences = append(differences, "current."+quote(name)+" IS NOT old."+quote(name)) + } + restore := `UPDATE main.` + quote(table.Name) + ` AS current SET (` + strings.Join(targets, ",") + `)=(SELECT ` + strings.Join(originals, ",") + ` FROM recovery.` + quote(table.Name) + ` old WHERE old.rowid=current.rowid) + WHERE EXISTS(SELECT 1 FROM recovery.` + quote(table.Name) + ` old WHERE old.rowid=current.rowid AND (` + strings.Join(differences, " OR ") + `))` + if _, err := db.Exec(restore); err != nil { + return fmt.Errorf("preserve original %s values: %w", table.Name, err) + } + } + return nil +} diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go index d6f7eae09..7778cb18e 100644 --- a/cmd/migrate/main.go +++ b/cmd/migrate/main.go @@ -1,58 +1,232 @@ -// Command migrate runs all dbschema migrations against a SQLite -// CoreScope database and exits. Used by CI / one-shot tooling to bring -// an unmigrated fixture (or a fresh DB) up to the schema shape the -// read-only server (cmd/server) requires via dbschema.AssertReady. -// -// In production the ingestor (cmd/ingestor) runs dbschema.Apply at -// startup before subscribing to MQTT — this binary exists so CI's E2E -// job can migrate the e2e-fixture.db without booting the full ingestor -// (which needs MQTT brokers). -// -// Usage: -// -// migrate -db path/to/file.db +// Command migrate bootstraps PostgreSQL or imports immutable offline SQLite +// recovery snapshots. Runtime services never apply DDL themselves. package main import ( - "database/sql" + "context" + "encoding/json" + "errors" "flag" - "log" + "fmt" + "io" + "os" + "os/signal" + "syscall" - _ "github.com/mattn/go-sqlite3" "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" ) func main() { - dbPath := flag.String("db", "", "path to SQLite database to migrate (required)") - flag.Parse() - - if *dbPath == "" { - log.Fatalf("[migrate] -db is required") + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + if err := runCommand(ctx, os.Args[1:], os.Stdout); err != nil { + fmt.Fprintln(os.Stderr, "[migrate]", err) + os.Exit(1) } +} - log.SetFlags(log.LstdFlags | log.Lmsgprefix) - log.SetPrefix("[migrate] ") +func runCommand(ctx context.Context, args []string, out io.Writer) error { + flags := flag.NewFlagSet("migrate", flag.ContinueOnError) + flags.SetOutput(io.Discard) + databaseURL := flags.String("database-url", "", "PostgreSQL telemetry owner URL (prefer CORESCOPE_DATABASE_URL)") + usersURL := flags.String("users-database-url", "", "PostgreSQL accounts owner URL (prefer CORESCOPE_USERS_DATABASE_URL)") + source := flags.String("from-sqlite", "", "offline telemetry SQLite source") + accountSource := flags.String("users-from-sqlite", "", "offline account SQLite source") + stateDir := flags.String("state-dir", "state/migration", "private migration snapshots and resume manifests") + offline := flags.Bool("offline", false, "confirm telemetry and account writers are stopped") + resume := flags.Bool("resume", false, "resume the same source/state/destination import") + checkReady := flags.Bool("check-ready", false, "check readiness without changing any schema or data") + checkImportKind := flags.String("check-import-kind", "", "read-only completed-import guard: telemetry or accounts, with its SQLite source path") + if err := flags.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + flags.SetOutput(out) + flags.PrintDefaults() + return nil + } + return errors.New("invalid migration arguments; use -help") + } + if flags.NArg() != 0 { + return errors.New("unexpected positional migration argument; use -help") + } + if *databaseURL == "" { + *databaseURL = os.Getenv("CORESCOPE_DATABASE_URL") + } + if *usersURL == "" { + *usersURL = os.Getenv("CORESCOPE_USERS_DATABASE_URL") + } + if *databaseURL == "" && *usersURL == "" { + return errors.New("provide CORESCOPE_DATABASE_URL and/or CORESCOPE_USERS_DATABASE_URL") + } + if *databaseURL != "" && *usersURL != "" { + same, err := pgutil.SameDatabase(ctx, *databaseURL, *usersURL) + if err != nil { + return err + } + if same { + return errors.New("telemetry and accounts require separate PostgreSQL databases") + } + } + type store struct{ kind, dsn, source string } + stores := []store{{"telemetry", *databaseURL, *source}, {"accounts", *usersURL, *accountSource}} + if *checkImportKind != "" { + if *checkReady || *resume { + return errors.New("-check-import-kind cannot be combined with -check-ready or -resume") + } + for _, s := range stores { + if s.kind == *checkImportKind { + if s.dsn == "" || s.source == "" { + return errors.New("completed-import check requires the matching PostgreSQL URL and SQLite source path") + } + if err := checkImportedSource(ctx, s.dsn, s.kind, s.source); err != nil { + return err + } + fmt.Fprintf(out, "%s verified import matches the retained SQLite source\n", s.kind) + return nil + } + } + return errors.New("-check-import-kind must be telemetry or accounts") + } + importing := *source != "" || *accountSource != "" + if importing && !*offline { + return errors.New("SQLite import requires -offline after stopping all telemetry and account writers") + } + if *checkReady && (importing || *resume) { + return errors.New("-check-ready cannot be combined with import or resume") + } + if *resume && !importing { + return errors.New("-resume requires a SQLite source") + } + var imported []store + for _, s := range stores { + if s.dsn == "" { + if s.source != "" { + return fmt.Errorf("%s source requires its PostgreSQL owner URL", s.kind) + } + continue + } + if *checkReady { + db, err := pgutil.Open(s.dsn, false) + if err != nil { + return err + } + if s.kind == "telemetry" { + err = dbschema.AssertReady(db) + } else { + err = users.AssertReady(db) + } + db.Close() + if err != nil { + return err + } + fmt.Fprintf(out, "%s PostgreSQL schema is ready\n", s.kind) + continue + } + if s.source != "" { + report, err := importSQLite(ctx, importOptions{Source: s.source, DatabaseURL: s.dsn, StateDir: *stateDir, Kind: s.kind, Resume: *resume}) + if err != nil { + return err + } + if !report.Verified { + return errors.New("import returned without complete verification") + } + if err := json.NewEncoder(out).Encode(report); err != nil { + return err + } + imported = append(imported, s) + } + } + if *checkReady { + return nil + } + // Empty/absent optional stores are initialized only after every requested + // SQLite import has verified, so an account failure cannot cut over telemetry. + for _, s := range stores { + if s.dsn == "" || s.source != "" { + continue + } + db, err := pgutil.Open(s.dsn, false) + if err != nil { + return err + } + if s.kind == "telemetry" { + err = dbschema.Apply(db, nil) + } else { + err = users.Apply(db) + } + db.Close() + if err != nil { + return err + } + fmt.Fprintf(out, "%s PostgreSQL schema initialized\n", s.kind) + } + for i, s := range imported { + if err := finalizeImport(ctx, s.dsn, s.kind); err != nil { + // There is no cross-database transaction. Writers stay offline throughout + // cutover; close any earlier readiness gate if a later finalization fails. + for _, prior := range imported[:i] { + db, openErr := pgutil.Open(prior.dsn, false) + if openErr == nil { + _, resetErr := db.Exec(`UPDATE corescope_schema SET ready=false WHERE kind=$1`, prior.kind) + db.Close() + if resetErr != nil { + return errors.Join(err, errors.New("an earlier readiness gate could not be reset; keep services stopped")) + } + } else { + return errors.Join(err, errors.New("an earlier readiness gate could not be reset; keep services stopped")) + } + } + return err + } + } + for _, s := range imported { + fmt.Fprintf(out, "%s import verified and finalized; keep the recovery snapshot for rollback\n", s.kind) + } + return nil +} - // Same DSN as the ingestor. A bare path here would mean synchronous=NORMAL - // under mattn, quietly weakening durability for a process that writes — and - // this one runs dbschema.Apply, which now also deletes duplicate rows. - db, err := sql.Open("sqlite3", dbschema.WriterDSN(*dbPath)) +func checkImportedSource(ctx context.Context, databaseURL, kind, source string) error { + db, err := pgutil.Open(databaseURL, true) if err != nil { - log.Fatalf("open %s: %v", *dbPath, err) + return err } defer db.Close() - - if err := db.Ping(); err != nil { - log.Fatalf("ping %s: %v", *dbPath, err) + if kind == "telemetry" { + err = dbschema.AssertReady(db) + } else { + err = users.AssertReady(db) } - - if err := dbschema.Apply(db, log.Printf); err != nil { - log.Fatalf("dbschema.Apply: %v", err) + if err != nil { + return err } - - if err := dbschema.AssertReady(db); err != nil { - log.Fatalf("dbschema.AssertReady after Apply: %v (this is a bug — Apply did not produce a ready schema)", err) + var runID, recordedKind, digest, normalized, reportText string + var verified bool + if err := db.QueryRowContext(ctx, `SELECT run_id,kind,source_digest,normalized_digest,verified,report FROM corescope_import WHERE id=1`).Scan(&runID, &recordedKind, &digest, &normalized, &verified, &reportText); err != nil { + return errors.New("PostgreSQL has no completed SQLite import; run the explicit offline upgrade before starting services") } - - log.Printf("OK: %s is migrated and ready", *dbPath) + var report importReport + if !verified || recordedKind != kind || runID == "" || normalized == "" || json.Unmarshal([]byte(reportText), &report) != nil || !report.Verified || report.Kind != kind { + return errors.New("PostgreSQL import verification marker is incomplete or belongs to another store") + } + tables, err := layouts(kind) + if err != nil { + return err + } + if len(report.Tables) != len(tables) { + return errors.New("PostgreSQL import report is incomplete") + } + for i, table := range tables { + if report.Tables[i].Table != table.Name || report.Tables[i].Rows < 0 || len(report.Tables[i].SHA256) != 64 { + return errors.New("PostgreSQL import report is invalid") + } + } + actual, err := fingerprint(ctx, source) + if err != nil { + return err + } + if actual != digest { + return errors.New("retained SQLite source does not match this PostgreSQL import; keep services stopped") + } + return nil } diff --git a/cmd/migrate/main_test.go b/cmd/migrate/main_test.go index b32b5c9b4..19dfec26a 100644 --- a/cmd/migrate/main_test.go +++ b/cmd/migrate/main_test.go @@ -6,14 +6,21 @@ package main import ( + "context" "database/sql" + "fmt" "io" "os" + "os/exec" "path/filepath" + "strings" "testing" _ "github.com/mattn/go-sqlite3" - "github.com/meshcore-analyzer/dbschema" + postgresSchema "github.com/meshcore-analyzer/dbschema" + dbschema "github.com/meshcore-analyzer/dbschema/legacy" + "github.com/meshcore-analyzer/packetpath" + "github.com/meshcore-analyzer/pgutil/pgtest" ) // fixtureCandidates lists possible locations of the committed e2e @@ -31,10 +38,152 @@ func locateFixture(t *testing.T) string { return abs } } - t.Skipf("e2e fixture not found (looked in: %v)", fixtureCandidates) + t.Fatalf("committed e2e fixture not found (looked in: %v)", fixtureCandidates) return "" } +func TestCommittedFixtureRequiresExplicitDuplicateRepair(t *testing.T) { + source := locateFixture(t) + _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: pgtest.NewSchema(t), StateDir: t.TempDir(), Kind: "telemetry"}) + if err == nil || !strings.Contains(err.Error(), "duplicate observation identity") { + t.Fatalf("raw legacy fixture must require explicit duplicate repair: %v", err) + } +} + +func TestPostgresImportCommittedFixture(t *testing.T) { + original := locateFixture(t) + before, err := sourceFingerprint(original) + if err != nil { + t.Fatal(err) + } + source := filepath.Join(t.TempDir(), "reviewed-fixture.sqlite") + if err := snapshotSource(context.Background(), original, source); err != nil { + t.Fatal(err) + } + legacyDB, err := sql.Open("sqlite3", sqliteURL(source, "rw")) + if err != nil { + t.Fatal(err) + } + // This historical fixture has one duplicate identity group. Its documented + // legacy repair is explicit and confined to this disposable copy; the runtime + // importer itself rejects duplicates and never silently collapses records. + if err := dbschema.Apply(legacyDB, t.Logf); err != nil { + legacyDB.Close() + t.Fatal(err) + } + var wantTransmissions, wantObservations int + if err := legacyDB.QueryRow(`SELECT count(*) FROM transmissions`).Scan(&wantTransmissions); err != nil { + t.Fatal(err) + } + if err := legacyDB.QueryRow(`SELECT count(*) FROM observations`).Scan(&wantObservations); err != nil { + t.Fatal(err) + } + legacyDB.Close() + dsn := pgtest.NewSchema(t) + report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 64}) + if err != nil { + t.Fatal(err) + } + if !report.Verified { + t.Fatal("committed fixture import was not verified") + } + if err := finalizeImport(context.Background(), dsn, "telemetry"); err != nil { + t.Fatal(err) + } + db := openImportDB(t, dsn) + if err := postgresSchema.AssertReady(db); err != nil { + t.Fatal(err) + } + var transmissions, observations int + if err := db.QueryRow(`SELECT count(*) FROM transmissions`).Scan(&transmissions); err != nil { + t.Fatal(err) + } + if err := db.QueryRow(`SELECT count(*) FROM observations`).Scan(&observations); err != nil { + t.Fatal(err) + } + if transmissions != wantTransmissions || observations != wantObservations || transmissions == 0 { + t.Fatalf("reviewed fixture changed: %d/%d transmissions, %d/%d observations", transmissions, wantTransmissions, observations, wantObservations) + } + after, err := sourceFingerprint(original) + if err != nil { + t.Fatal(err) + } + if after != before { + t.Fatal("committed fixture changed during import") + } +} + +func TestFixturePreparationToolKeepsOriginalAndRefusesOverwrite(t *testing.T) { + source := locateFixture(t) + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + destination := filepath.Join(t.TempDir(), "prepared.sqlite") + args := []string{"run", "../../scripts/prepare-postgres-fixture.go", "-source", source, "-destination", destination} + output, err := exec.Command("go", args...).CombinedOutput() + if err != nil { + t.Fatalf("fixture tool: %v: %s", err, output) + } + if !strings.Contains(string(output), "observations:") || !strings.Contains(string(output), "original fixture unchanged") { + t.Fatal("fixture tool did not report the repair counts") + } + if output, err := exec.Command("go", "run", "../../scripts/migrate-fixture-hashes.go", destination).CombinedOutput(); err != nil { + t.Fatalf("prepared fixture hash normalization: %v: %s", err, output) + } + db, err := sql.Open("sqlite3", sqliteURL(destination, "ro")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := dbschema.CheckLegacySource(db); err != nil { + t.Fatal(err) + } + if err := exec.Command("go", args...).Run(); err == nil { + t.Fatal("fixture tool overwrote an existing destination") + } + after, err := sourceFingerprint(source) + if err != nil || after != before { + t.Fatal("fixture tool changed original source", err) + } +} + +func TestFixtureHashToolUsesSharedRuntimeIdentity(t *testing.T) { + file := filepath.Join(t.TempDir(), "hashes.sqlite") + db, err := sql.Open("sqlite3", file) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`CREATE TABLE transmissions(id INTEGER PRIMARY KEY,raw_hex TEXT,hash TEXT UNIQUE)`); err != nil { + t.Fatal(err) + } + raws := []string{"14010203040001020304", "150001020305", "160001020306", "17010203040001020307", "250001020304", "1d0001020304"} + for i, raw := range raws { + if _, err := db.Exec(`INSERT INTO transmissions VALUES(?,?,?)`, i, raw, fmt.Sprint("old-", i)); err != nil { + t.Fatal(err) + } + } + db.Close() + output, err := exec.Command("go", "run", "../../scripts/migrate-fixture-hashes.go", file).CombinedOutput() + if err != nil { + t.Fatalf("hash fixture: %v: %s", err, output) + } + db, err = sql.Open("sqlite3", sqliteURL(file, "ro")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + for i, raw := range raws { + var got string + if err := db.QueryRow(`SELECT hash FROM transmissions WHERE id=?`, i).Scan(&got); err != nil { + t.Fatal(err) + } + if want := packetpath.ContentHash(raw); got != want { + t.Fatalf("fixture route/type %s hash=%s, want runtime %s", raw, got, want) + } + } +} + func copyFile(t *testing.T, src, dst string) { t.Helper() in, err := os.Open(src) diff --git a/cmd/migrate/source.go b/cmd/migrate/source.go new file mode 100644 index 000000000..c4668e519 --- /dev/null +++ b/cmd/migrate/source.go @@ -0,0 +1,220 @@ +package main + +import ( + "context" + "crypto/sha256" + "database/sql" + "errors" + "fmt" + "io" + "net/url" + "os" + "path/filepath" + "runtime" + "time" + + "github.com/mattn/go-sqlite3" +) + +const importSQLiteDriver = "corescope_sqlite_import" +const maxImportRowBytes = 64 * 1024 * 1024 + +func init() { + // Apply the bound before any SQLite row reaches the Go driver. A check + // after Scan would already have allocated an arbitrarily large value. + sql.Register(importSQLiteDriver, &sqlite3.SQLiteDriver{ConnectHook: func(conn *sqlite3.SQLiteConn) error { + conn.SetLimit(sqlite3.SQLITE_LIMIT_LENGTH, maxImportRowBytes) + return nil + }}) +} + +// snapshotSource creates an offline recovery copy without modifying the input. +// The backup API preserves hidden rowids used by observer links and mail event +// ordering. VACUUM INTO may renumber them and is not a safe migration snapshot. +func snapshotSource(ctx context.Context, source, destination string) (err error) { + if err := ctx.Err(); err != nil { + return err + } + sourceDB, err := sql.Open(importSQLiteDriver, sqliteURL(source, "ro")) + if err != nil { + return err + } + defer sourceDB.Close() + sourceDB.SetMaxOpenConns(1) + if _, err := sourceDB.ExecContext(ctx, `PRAGMA trusted_schema=OFF`); err != nil { + return err + } + var schemaVersion int + if err := sourceDB.QueryRowContext(ctx, `PRAGMA schema_version`).Scan(&schemaVersion); err != nil { + return fmt.Errorf("read SQLite source: %w", err) + } + before, err := sourceFingerprintContext(ctx, source) + if err != nil { + return err + } + reserved, err := os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return fmt.Errorf("create new recovery snapshot: %w", err) + } + if err := reserved.Close(); err != nil { + _ = os.Remove(destination) + return err + } + defer func() { + if err != nil { + for _, suffix := range []string{"", "-wal", "-shm", "-journal"} { + _ = os.Remove(destination + suffix) + } + } + }() + destinationDB, err := sql.Open(importSQLiteDriver, sqliteURL(destination, "rw")) + if err != nil { + return err + } + defer destinationDB.Close() + destinationDB.SetMaxOpenConns(1) + src, err := sourceDB.Conn(ctx) + if err != nil { + return err + } + defer src.Close() + dst, err := destinationDB.Conn(ctx) + if err != nil { + return err + } + defer dst.Close() + err = src.Raw(func(sourceDriver any) error { + return dst.Raw(func(destinationDriver any) (err error) { + backup, err := destinationDriver.(*sqlite3.SQLiteConn).Backup("main", sourceDriver.(*sqlite3.SQLiteConn), "main") + if err != nil { + return err + } + defer func() { + if finishErr := backup.Finish(); err == nil { + err = finishErr + } + }() + remaining, progressAt := -1, time.Now() + for { + if err := ctx.Err(); err != nil { + return err + } + done, err := backup.Step(256) + if err != nil { + return err + } + if done { + return nil + } + if current := backup.Remaining(); current != remaining { + remaining, progressAt = current, time.Now() + continue + } + if time.Since(progressAt) > 10*time.Second { + return errors.New("SQLite snapshot made no progress; stop all instance writers before retrying") + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(10 * time.Millisecond): + } + } + }) + }) + if err != nil { + return fmt.Errorf("copy SQLite recovery snapshot: %w", err) + } + var check string + if err := dst.QueryRowContext(ctx, `PRAGMA quick_check`).Scan(&check); err != nil { + return err + } + if check != "ok" { + return errors.New("SQLite recovery snapshot failed integrity verification") + } + after, err := sourceFingerprintContext(ctx, source) + if err != nil { + return err + } + if after != before { + return errors.New("SQLite source changed during snapshot; stop all instance writers before retrying") + } + if err := dst.Close(); err != nil { + return err + } + if err := destinationDB.Close(); err != nil { + return err + } + return nil +} + +func sqliteURL(path, mode string) string { + abs, err := filepath.Abs(path) + if err != nil { + abs = path + } + abs = filepath.ToSlash(abs) + if runtime.GOOS == "windows" { + abs = "/" + abs + } + u := url.URL{Scheme: "file", Path: abs} + q := url.Values{"mode": {mode}, "_busy_timeout": {"5000"}} + u.RawQuery = q.Encode() + return u.String() +} + +// Hash the main file and committed-write log without loading either into RAM. +// Shared-memory coordination files are not application data. +func sourceFingerprint(path string) ([2][32]byte, error) { + return sourceFingerprintContext(context.Background(), path) +} + +type contextReader struct { + ctx context.Context + r io.Reader +} + +func (r contextReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.r.Read(p) +} + +func sourceFingerprintContext(ctx context.Context, path string) ([2][32]byte, error) { + var result [2][32]byte + if err := ctx.Err(); err != nil { + return result, err + } + for i, suffix := range []string{"", "-wal"} { + f, err := os.Open(path + suffix) + if i == 1 && errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return result, err + } + if i == 1 { + info, err := f.Stat() + if err != nil { + f.Close() + return result, err + } + // Read-only WAL-mode opens can create/remove a zero-byte sidecar. + // Every nonempty byte, including headers, still affects identity. + if info.Size() == 0 { + if err := f.Close(); err != nil { + return result, err + } + continue + } + } + h := sha256.New() + _, copyErr := io.CopyBuffer(h, contextReader{ctx, f}, make([]byte, 128*1024)) + closeErr := f.Close() + if err := errors.Join(copyErr, closeErr); err != nil { + return result, err + } + copy(result[i][:], h.Sum(nil)) + } + return result, nil +} diff --git a/cmd/migrate/source_test.go b/cmd/migrate/source_test.go new file mode 100644 index 000000000..32f790946 --- /dev/null +++ b/cmd/migrate/source_test.go @@ -0,0 +1,188 @@ +package main + +import ( + "context" + "crypto/sha256" + "database/sql" + "os" + "path/filepath" + "testing" + + "github.com/mattn/go-sqlite3" +) + +func TestSnapshotSourceIncludesWALWithoutChangingSource(t *testing.T) { + dir := t.TempDir() + source := filepath.Join(dir, "source.db") + destination := filepath.Join(dir, "recovery.db") + db, err := sql.Open("sqlite3", source+"?_journal_mode=WAL") + if err != nil { + t.Fatal(err) + } + defer db.Close() + db.SetMaxOpenConns(1) + for _, statement := range []string{ + `PRAGMA wal_autocheckpoint=0`, + `CREATE TABLE records (id INTEGER PRIMARY KEY, value TEXT)`, + `INSERT INTO records VALUES (0, NULL), (90, '')`, + `CREATE TABLE identities (key TEXT PRIMARY KEY)`, + `INSERT INTO identities(rowid,key) VALUES (0,'zero'),(900,'gap')`, + } { + if _, err := db.Exec(statement); err != nil { + t.Fatal(err) + } + } + before := make(map[string][32]byte) + for _, path := range []string{source, source + "-wal"} { + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + before[path] = sha256.Sum256(content) + } + if err := snapshotSource(context.Background(), source, destination); err != nil { + t.Fatalf("WAL-aware recovery snapshot: %v", err) + } + for path, expected := range before { + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if sha256.Sum256(content) != expected { + t.Fatalf("source was modified: %s", filepath.Base(path)) + } + } + copyDB, err := sql.Open("sqlite3", destination) + if err != nil { + t.Fatal(err) + } + defer copyDB.Close() + var count int + if err := copyDB.QueryRow(`SELECT COUNT(*) FROM records WHERE (id=0 AND value IS NULL) OR (id=90 AND value='')`).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 2 { + t.Fatalf("snapshot lost committed WAL rows or null/empty values: got %d", count) + } + if err := copyDB.QueryRow(`SELECT COUNT(*) FROM identities WHERE (rowid=0 AND key='zero') OR (rowid=900 AND key='gap')`).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 2 { + t.Fatal("snapshot changed hidden rowids") + } +} + +func TestSnapshotSourceDoesNotOverwriteRecoveryOrOriginal(t *testing.T) { + dir := t.TempDir() + source := filepath.Join(dir, "source.db") + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`CREATE TABLE records(id INTEGER)`); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + for _, destination := range []string{source, filepath.Join(dir, "existing.db")} { + if destination != source { + if err := os.WriteFile(destination, []byte("existing recovery"), 0o600); err != nil { + t.Fatal(err) + } + } + before, err := os.ReadFile(destination) + if err != nil { + t.Fatal(err) + } + if err := snapshotSource(context.Background(), source, destination); err == nil { + t.Fatal("overwriting an existing file was accepted") + } + after, err := os.ReadFile(destination) + if err != nil { + t.Fatal(err) + } + if sha256.Sum256(before) != sha256.Sum256(after) { + t.Fatal("existing file was changed") + } + } +} + +func TestSnapshotSourceFailureLeavesNoPartialCopy(t *testing.T) { + for _, cancelled := range []bool{false, true} { + dir := t.TempDir() + source, destination := filepath.Join(dir, "source.db"), filepath.Join(dir, "recovery.db") + if err := os.WriteFile(source, []byte("not a SQLite database"), 0o600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + if cancelled { + cancel() + } + if err := snapshotSource(ctx, source, destination); err == nil { + t.Fatal("invalid/cancelled snapshot accepted") + } + cancel() + if _, err := os.Stat(destination); !os.IsNotExist(err) { + t.Fatalf("partial output retained: %v", err) + } + } +} + +func TestSourceFingerprintEmptyWALMatchesAbsentWAL(t *testing.T) { + source := filepath.Join(t.TempDir(), "source.db") + if err := os.WriteFile(source, []byte("stable database bytes"), 0o600); err != nil { + t.Fatal(err) + } + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(source+"-wal", nil, 0o600); err != nil { + t.Fatal(err) + } + empty, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + if empty != before { + t.Fatal("an empty WAL changed the source identity") + } + if err := os.WriteFile(source+"-wal", []byte("nonempty WAL bytes must be observed"), 0o600); err != nil { + t.Fatal(err) + } + nonempty, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + if nonempty == before { + t.Fatal("nonempty WAL bytes were ignored") + } +} + +func TestSQLiteSourceLengthLimitPrecedesScan(t *testing.T) { + db, err := sql.Open(importSQLiteDriver, ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + conn, err := db.Conn(context.Background()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if err := conn.Raw(func(driver any) error { + sqlite := driver.(*sqlite3.SQLiteConn) + if got := sqlite.GetLimit(sqlite3.SQLITE_LIMIT_LENGTH); got != maxImportRowBytes { + t.Fatalf("native SQLite length limit=%d", got) + } + sqlite.SetLimit(sqlite3.SQLITE_LIMIT_LENGTH, 1024) + return nil + }); err != nil { + t.Fatal(err) + } + var data []byte + if err := conn.QueryRowContext(context.Background(), `SELECT zeroblob(2048)`).Scan(&data); err == nil { + t.Fatal("native length limit did not reject a large result before Scan") + } +} diff --git a/cmd/server/account_test.go b/cmd/server/account_test.go index 554bb52e0..837e7ba38 100644 --- a/cmd/server/account_test.go +++ b/cmd/server/account_test.go @@ -135,8 +135,7 @@ func TestAuthResetTokenStoreFailureEndsSessionsFirst(t *testing.T) { c := f.registerAndActivate(t, "pia@example.org", "Pia", pw) thief := f.login(t, "pia@example.org", pw) _, reset := pendingLinks(t, f, c, "pia@example.org", "attacker@example.org") - f.execDB(t, `CREATE TRIGGER down BEFORE UPDATE ON tokens WHEN OLD.purpose = 'email_change' BEGIN - SELECT RAISE(ABORT, 'token store down'); END`) + f.execDB(t, `CREATE FUNCTION down() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'token store down'; END $$; CREATE TRIGGER down BEFORE UPDATE ON tokens FOR EACH ROW WHEN (OLD.purpose='email_change') EXECUTE FUNCTION down()`) expectStatus(t, f.do("POST", "/api/auth/reset", resetRequest{Token: reset, Password: "a brand new secret"}), 500) expectStatus(t, f.do("GET", "/api/auth/me", nil, as(thief)), 401) expectStatus(t, f.do("GET", "/api/auth/me", nil, as(c)), 401) diff --git a/cmd/server/advert_evidence_scale_test.go b/cmd/server/advert_evidence_scale_test.go index b569d30b6..eed081e8d 100644 --- a/cmd/server/advert_evidence_scale_test.go +++ b/cmd/server/advert_evidence_scale_test.go @@ -18,7 +18,7 @@ func TestAdvertEvidenceCatchupScale(t *testing.T) { if path == "" { t.Fatal("ADVERT_SCALE_DB must name the completed ingestor scale fixture") } - db, err := OpenDB(path) + db, err := openFixtureReader(t, path) if err != nil { t.Fatal(err) } diff --git a/cmd/server/advert_route_evidence.go b/cmd/server/advert_route_evidence.go index fc8336cbc..15bfe041a 100644 --- a/cmd/server/advert_route_evidence.go +++ b/cmd/server/advert_route_evidence.go @@ -16,7 +16,7 @@ func (db *DB) advertEvidencePresent() bool { return true } var exists int - if err := db.conn.QueryRow(`SELECT 1 FROM sqlite_master WHERE type='table' AND name='advert_route_evidence'`).Scan(&exists); err != nil { + if err := db.conn.QueryRow(`SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='advert_route_evidence'`).Scan(&exists); err != nil { return false } db.advertEvidenceTable.Store(true) @@ -39,7 +39,7 @@ func (db *DB) advertEvidenceForIDs(ids []int) (map[int]uint8, error) { for i, id := range ids[start:end] { args[i] = id } - query := `SELECT tx_id,SUM(bit) FROM advert_route_evidence WHERE tx_id IN (` + strings.TrimSuffix(strings.Repeat("?,", len(args)), ",") + `) GROUP BY tx_id` + query := `SELECT tx_id,SUM(bit) FROM advert_route_evidence WHERE tx_id IN (` + sqlPlaceholders(len(args)) + `) GROUP BY tx_id` if db.advertEvidenceReadHook != nil { db.advertEvidenceReadHook() } @@ -121,7 +121,7 @@ func (s *PacketStore) pollAdvertEvidence(limit int) error { if limit <= 0 || limit > 500 { limit = 500 } - rows, err := s.db.conn.Query(`SELECT id,tx_id,bit FROM advert_route_evidence WHERE id>? ORDER BY id LIMIT ?`, s.advertEvidenceCursor, limit) + rows, err := s.db.conn.Query(`SELECT id,tx_id,bit FROM advert_route_evidence WHERE id>$1 ORDER BY id LIMIT $2`, s.advertEvidenceCursor, limit) if err != nil { return err } diff --git a/cmd/server/advert_route_evidence_test.go b/cmd/server/advert_route_evidence_test.go index 3fc57e06b..983ae96d5 100644 --- a/cmd/server/advert_route_evidence_test.go +++ b/cmd/server/advert_route_evidence_test.go @@ -15,29 +15,32 @@ import ( func advertEvidenceFixture(t *testing.T, raw string, bits ...int) (*DB, *sql.DB) { t.Helper() path := createTestDBWithResolvedPath(t, 1, []string{"fixture-relay"}) - w, err := sql.Open("sqlite3", path+"?_journal_mode=WAL&_foreign_keys=on") + w, err := openFixtureSQL(path) if err != nil { t.Fatal(err) } t.Cleanup(func() { w.Close() }) for _, stmt := range []string{ `ALTER TABLE transmissions ADD COLUMN from_pubkey TEXT`, - `CREATE TABLE advert_route_evidence (id INTEGER PRIMARY KEY AUTOINCREMENT, tx_id INTEGER NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, bit INTEGER NOT NULL CHECK (bit IN (1,2)), UNIQUE(tx_id,bit))`, + `CREATE TABLE advert_route_evidence (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, tx_id INTEGER NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, bit INTEGER NOT NULL CHECK (bit IN (1,2)), UNIQUE(tx_id,bit))`, } { if _, err := w.Exec(stmt); err != nil { t.Fatal(err) } } now := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) - if _, err := w.Exec(`UPDATE transmissions SET raw_hex=?, route_type=?, first_seen=?, from_pubkey='fixture-origin'; UPDATE observations SET raw_hex=?, timestamp=?`, raw, int(raw[1]-'0')&3, now, raw, now); err != nil { + if _, err := w.Exec(`UPDATE transmissions SET raw_hex=$1, route_type=$2, first_seen=$3, from_pubkey='fixture-origin'`, raw, int(raw[1]-'0')&3, now); err != nil { + t.Fatal(err) + } + if _, err := w.Exec(`UPDATE observations SET raw_hex=$1, timestamp=$2`, raw, now); err != nil { t.Fatal(err) } for _, bit := range bits { - if _, err := w.Exec(`INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,?)`, bit); err != nil { + if _, err := w.Exec(`INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,$1)`, bit); err != nil { t.Fatal(err) } } - db, err := OpenDB(path) + db, err := openFixtureReader(t, path) if err != nil { t.Fatal(err) } @@ -48,13 +51,13 @@ func advertEvidenceFixture(t *testing.T, raw string, bits ...int) (*DB, *sql.DB) func TestAdvertRouteEvidenceNodeAPIHonorsRequestedLimit(t *testing.T) { db, w := advertEvidenceFixture(t, "1100aa", 1) for id := 2; id <= 24; id++ { - if _, err := w.Exec(`INSERT INTO transmissions(id,hash,raw_hex,first_seen,payload_type,route_type,from_pubkey) SELECT ?,?,'1100aa',first_seen,4,1,'fixture-origin' FROM transmissions WHERE id=1`, id, fmt.Sprintf("node-advert-%d", id)); err != nil { + if _, err := w.Exec(`INSERT INTO transmissions(id,hash,raw_hex,first_seen,payload_type,route_type,from_pubkey) SELECT $1,$2,'1100aa',first_seen,4,1,'fixture-origin' FROM transmissions WHERE id=1`, id, fmt.Sprintf("node-advert-%d", id)); err != nil { t.Fatal(err) } mask := id % 4 for _, bit := range []int{1, 2} { if mask&bit != 0 { - if _, err := w.Exec(`INSERT INTO advert_route_evidence(tx_id,bit) VALUES(?,?)`, id, bit); err != nil { + if _, err := w.Exec(`INSERT INTO advert_route_evidence(tx_id,bit) VALUES($1,$2)`, id, bit); err != nil { t.Fatal(err) } } @@ -79,13 +82,13 @@ func TestAdvertRouteEvidenceNodeAPIHonorsRequestedLimit(t *testing.T) { // Measures the added bulk-load work and an idle feed at a 30K-packet scale. // Both paths touch indexed compact evidence, never observation raw frames. func BenchmarkAdvertEvidence30K(b *testing.B) { - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(b) if err != nil { b.Fatal(err) } conn.SetMaxOpenConns(1) defer conn.Close() - if _, err := conn.Exec(`CREATE TABLE advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit)); + if _, err := conn.Exec(`CREATE TABLE advert_route_evidence(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit)); WITH RECURSIVE ids(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM ids WHERE n<30000) INSERT INTO advert_route_evidence(tx_id,bit) SELECT n,1 FROM ids UNION ALL SELECT n,2 FROM ids`); err != nil { b.Fatal(err) @@ -309,7 +312,21 @@ func TestAdvertRouteEvidencePollSameObservationID(t *testing.T) { maxObs, maxTx := db.GetMaxObservationID(), db.GetMaxTransmissionID() // This matches the ingestor's conflict update: neither observation // ID nor timestamp advances, and the final raw matches the first. - if _, err := w.Exec(`UPDATE observations SET raw_hex=? WHERE id=1; INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,?); UPDATE observations SET raw_hex=? WHERE id=1`, tc.second, tc.secondBit, tc.first); err != nil { + tx, err := w.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + if _, err := tx.Exec(`UPDATE observations SET raw_hex=$1 WHERE id=1`, tc.second); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(`INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,$1)`, tc.secondBit); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(`UPDATE observations SET raw_hex=$1 WHERE id=1`, tc.first); err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { t.Fatal(err) } s.IngestNewFromDB(maxTx, 100) @@ -352,7 +369,7 @@ func TestAdvertRouteEvidenceMissingTableIsUnknown(t *testing.T) { } assertAdvertEvidenceViews(t, s, "other") var count int - if err := w.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE name='advert_route_evidence'`).Scan(&count); err != nil { + if err := w.QueryRow(`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='advert_route_evidence'`).Scan(&count); err != nil { t.Fatal(err) } if count != 0 { @@ -362,10 +379,7 @@ func TestAdvertRouteEvidenceMissingTableIsUnknown(t *testing.T) { func TestAdvertRouteEvidenceTableAppearsAfterStartup(t *testing.T) { db, w := advertEvidenceFixture(t, "1100aa") - var schema string - if err := w.QueryRow(`SELECT sql FROM sqlite_master WHERE name='advert_route_evidence'`).Scan(&schema); err != nil { - t.Fatal(err) - } + const schema = `CREATE TABLE advert_route_evidence (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, tx_id INTEGER NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, bit INTEGER NOT NULL CHECK (bit IN (1,2)), UNIQUE(tx_id,bit))` if _, err := w.Exec(`DROP TABLE advert_route_evidence`); err != nil { t.Fatal(err) } @@ -397,7 +411,7 @@ func TestAdvertRouteEvidencePollRejectsPartialBatch(t *testing.T) { t.Fatal(err) } before := s.advertEvidenceCursor - if _, err := w.Exec(`PRAGMA ignore_check_constraints=ON; INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,2),(1,3)`); err != nil { + if _, err := w.Exec(`ALTER TABLE advert_route_evidence DROP CONSTRAINT advert_route_evidence_bit_check; INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,2),(1,3)`); err != nil { t.Fatal(err) } if err := s.pollAdvertEvidence(500); err == nil { @@ -406,7 +420,7 @@ func TestAdvertRouteEvidencePollRejectsPartialBatch(t *testing.T) { if s.advertEvidenceCursor != before || s.byTxID[1].AdvertRouteEvidence != 1 { t.Fatal("partial failed batch changed evidence or cursor") } - if _, err := w.Exec(`DELETE FROM advert_route_evidence WHERE bit=3; PRAGMA ignore_check_constraints=OFF`); err != nil { + if _, err := w.Exec(`DELETE FROM advert_route_evidence WHERE bit=3; ALTER TABLE advert_route_evidence ADD CONSTRAINT advert_route_evidence_bit_check CHECK (bit IN (1,2))`); err != nil { t.Fatal(err) } if err := s.pollAdvertEvidence(500); err != nil { diff --git a/cmd/server/advert_stats.go b/cmd/server/advert_stats.go index fcde4b54a..835c877b5 100644 --- a/cmd/server/advert_stats.go +++ b/cmd/server/advert_stats.go @@ -63,10 +63,10 @@ func (db *DB) CountFloodAdvertsForNode(pubkey string, windowHours float64, rowCa } floor := time.Now().UTC().Add(-time.Duration(windowHours*float64(time.Hour))).AddDate(0, 0, -1).Format("2006-01-02") rows, err := db.conn.Query( - `SELECT COALESCE(first_seen, ''), ?, COALESCE(hash, '') FROM transmissions t - WHERE from_pubkey=? AND payload_type=? AND first_seen>=? - AND EXISTS(SELECT 1 FROM advert_route_evidence e WHERE e.tx_id=t.id AND e.bit=?) - ORDER BY id DESC LIMIT ?`, + `SELECT COALESCE(first_seen, ''), $1::integer, COALESCE(hash, '') FROM transmissions t + WHERE from_pubkey=$2 AND payload_type=$3 AND first_seen>=$4 + AND EXISTS(SELECT 1 FROM advert_route_evidence e WHERE e.tx_id=t.id AND e.bit=$5) + ORDER BY id DESC LIMIT $6`, packetpath.AdvertFlood, pubkey, payloadTypeAdvert, floor, packetpath.AdvertFlood, rowCap) if err != nil { return 0, err diff --git a/cmd/server/advert_stats_test.go b/cmd/server/advert_stats_test.go index 942a094ae..7332cfd86 100644 --- a/cmd/server/advert_stats_test.go +++ b/cmd/server/advert_stats_test.go @@ -15,16 +15,16 @@ import ( func insertAdvertTx(t *testing.T, db *DB, pubkey, hash string, rt int, ts time.Time) { t.Helper() if _, err := db.conn.Exec( - `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, from_pubkey) VALUES ('00', ?, ?, ?, ?, ?)`, + `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, from_pubkey) VALUES ('00', $1, $2, $3, $4, $5)`, hash, ts.Format("2006-01-02T15:04:05.000Z"), rt, payloadTypeAdvert, pubkey); err != nil { t.Fatalf("insert transmission: %v", err) } // Fixtures represent ingestor output: canonical routing alone is not evidence. - if _, err := db.conn.Exec(`CREATE TABLE IF NOT EXISTS advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit))`); err != nil { + if _, err := db.conn.Exec(`CREATE TABLE IF NOT EXISTS advert_route_evidence(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit))`); err != nil { t.Fatal(err) } if rt == RouteFlood { - if _, err := db.conn.Exec(`INSERT INTO advert_route_evidence(tx_id,bit) SELECT id,1 FROM transmissions WHERE hash=?`, hash); err != nil { + if _, err := db.conn.Exec(`INSERT INTO advert_route_evidence(tx_id,bit) SELECT id,1 FROM transmissions WHERE hash=$1`, hash); err != nil { t.Fatal(err) } } @@ -64,7 +64,7 @@ func TestCountFloodAdverts(t *testing.T) { // counting recent flood adverts only (zero-hop and out-of-window excluded). func TestNodeDetailIncludesFloodAdvertCount(t *testing.T) { srv, router := setupTestServer(t) - if _, err := srv.db.conn.Exec(`CREATE TABLE advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit))`); err != nil { + if _, err := srv.db.conn.Exec(`CREATE TABLE advert_route_evidence(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit))`); err != nil { t.Fatal(err) } now := time.Now().UTC() @@ -131,11 +131,14 @@ func TestNodeDetailFloodCountAgreesWithRouteEvidence(t *testing.T) { for _, firstRoute := range []int{RouteDirect, RouteFlood, RouteTransportFlood} { t.Run(fmt.Sprint(firstRoute), func(t *testing.T) { srv, router := setupTestServer(t) - if _, err := srv.db.conn.Exec(`CREATE TABLE IF NOT EXISTS advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT, tx_id INTEGER, bit INTEGER, UNIQUE(tx_id,bit))`); err != nil { + if _, err := srv.db.conn.Exec(`CREATE TABLE IF NOT EXISTS advert_route_evidence(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, tx_id INTEGER, bit INTEGER, UNIQUE(tx_id,bit))`); err != nil { t.Fatal(err) } const pubkey = "aabbccdd11223344" - if _, err := srv.db.conn.Exec(`DELETE FROM transmissions WHERE from_pubkey=?`, pubkey); err != nil { + if _, err := srv.db.conn.Exec(`DELETE FROM observations WHERE transmission_id IN (SELECT id FROM transmissions WHERE from_pubkey=$1)`, pubkey); err != nil { + t.Fatal(err) + } + if _, err := srv.db.conn.Exec(`DELETE FROM transmissions WHERE from_pubkey=$1`, pubkey); err != nil { t.Fatal(err) } insertAdvertTx(t, srv.db, pubkey, "mixed-advert", firstRoute, time.Now().Add(-time.Hour)) diff --git a/cmd/server/analytics_recompute_after_load_test.go b/cmd/server/analytics_recompute_after_load_test.go index b2a54cb50..7c29d7456 100644 --- a/cmd/server/analytics_recompute_after_load_test.go +++ b/cmd/server/analytics_recompute_after_load_test.go @@ -18,7 +18,6 @@ import ( "encoding/json" "net/http" "net/http/httptest" - "path/filepath" "sync/atomic" "testing" "time" @@ -51,9 +50,9 @@ func waitForTest(t *testing.T, what string, cond func() bool) { // still running (LoadComplete is already true at that point) and close // once RunStartupLoad returns. func TestStartupLoadDone_ClosesOnlyAfterBackgroundFill(t *testing.T) { - dbPath := filepath.Join(t.TempDir(), "test.db") + dbPath := postgresTestDSN(t) createTestDBSpreadOverDays(t, dbPath, 100, 14, time.Now().UTC().Unix()) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -85,9 +84,9 @@ func TestStartupLoadDone_ClosesOnlyAfterBackgroundFill(t *testing.T) { // A failed load is terminal too: nothing more will be loaded, so the // signal must close instead of leaving the recomputers waiting forever. func TestStartupLoadDone_ClosesWhenLoadFails(t *testing.T) { - dbPath := filepath.Join(t.TempDir(), "test.db") + dbPath := postgresTestDSN(t) createTestDBSpreadOverDays(t, dbPath, 10, 1, time.Now().UTC().Unix()) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -367,9 +366,9 @@ func TestDistanceIndexBuild_RefreshesRecomputerBeforeReportingBuilt(t *testing.T // right after the load. func TestAnalyticsRecomputers_FullDataRightAfterStartupLoad(t *testing.T) { const totalRows = 100 - dbPath := filepath.Join(t.TempDir(), "test.db") + dbPath := postgresTestDSN(t) createTestDBSpreadOverDays(t, dbPath, totalRows, 14, time.Now().UTC().Unix()) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/auth_fixture_test.go b/cmd/server/auth_fixture_test.go index 00ea6d0ae..bf0173e79 100644 --- a/cmd/server/auth_fixture_test.go +++ b/cmd/server/auth_fixture_test.go @@ -2,19 +2,18 @@ package main import ( "bytes" - "database/sql" "encoding/json" "io" "net/http" "net/http/httptest" "net/url" - "path/filepath" "regexp" "testing" "time" "github.com/gorilla/mux" "github.com/meshcore-analyzer/mailer" + "github.com/meshcore-analyzer/pgutil/pgtest" "github.com/meshcore-analyzer/users" ) @@ -25,10 +24,11 @@ const ( ) type authFixture struct { - srv *Server - router *mux.Router - fake *mailer.Fake - st *users.Store + ownerURL string + srv *Server + router *mux.Router + fake *mailer.Fake + st *users.Store } // client is a browser: its session cookie and CSRF token. @@ -39,9 +39,32 @@ type client struct { } func newTestAuthService(t *testing.T, adminEmails ...string) (*authService, *mailer.Fake) { + a, fake, _ := newTestAuthServiceWithURL(t, pgtest.NewSchema(t), adminEmails...) + return a, fake +} + +func newTestBackupAuthService(t *testing.T, adminEmails ...string) (*authService, *mailer.Fake) { + a, fake, _ := newTestAuthServiceWithURL(t, pgtest.NewDatabase(t), adminEmails...) + return a, fake +} + +func newTestAuthServiceWithURL(t *testing.T, ownerURL string, adminEmails ...string) (*authService, *mailer.Fake, string) { t.Helper() + owner, err := openFixtureSQL(ownerURL) + if err != nil { + t.Fatal(err) + } + defer owner.Close() + if err := users.Apply(owner); err != nil { + t.Fatal(err) + } + runtimeURL := pgtest.Writer(t, ownerURL) + u, _ := url.Parse(runtimeURL) + if _, err := owner.Exec(`REVOKE INSERT,UPDATE,DELETE ON corescope_schema,schema_version FROM "` + u.User.Username() + `"`); err != nil { + t.Fatal(err) + } set := &userMgmtSettings{ - dbPath: filepath.Join(t.TempDir(), "users.db"), adminEmails: map[string]bool{}, + databaseURL: runtimeURL, adminEmails: map[string]bool{}, sessionTTL: 30 * 24 * time.Hour, provider: "fake", fromEmail: "noreply@example.org", fromName: "CoreScope", webhookSecret: testHook, } @@ -50,7 +73,7 @@ func newTestAuthService(t *testing.T, adminEmails ...string) (*authService, *mai for _, e := range adminEmails { set.adminEmails[e] = true } - st, err := users.Open(set.dbPath) + st, err := users.Open(set.databaseURL) if err != nil { t.Fatal(err) } @@ -60,17 +83,25 @@ func newTestAuthService(t *testing.T, adminEmails ...string) (*authService, *mai a.waitAudits() st.Close() }) - return a, fake + return a, fake, ownerURL } // newAuthFixture builds a Server with auth on and only the auth routes. func newAuthFixture(t *testing.T, adminEmails ...string) *authFixture { + return newAuthFixtureWithURL(t, pgtest.NewSchema(t), adminEmails...) +} + +func newBackupAuthFixture(t *testing.T, adminEmails ...string) *authFixture { + return newAuthFixtureWithURL(t, pgtest.NewDatabase(t), adminEmails...) +} + +func newAuthFixtureWithURL(t *testing.T, databaseURL string, adminEmails ...string) *authFixture { t.Helper() - a, fake := newTestAuthService(t, adminEmails...) + a, fake, ownerURL := newTestAuthServiceWithURL(t, databaseURL, adminEmails...) srv := &Server{cfg: &Config{APIKey: testAPIKey}, perfStats: NewPerfStats(), auth: a} r := mux.NewRouter() srv.registerAuthRoutes(r) - return &authFixture{srv: srv, router: r, fake: fake, st: a.st} + return &authFixture{srv: srv, router: r, fake: fake, st: a.st, ownerURL: ownerURL} } type reqMod func(*http.Request) @@ -187,7 +218,7 @@ func (f *authFixture) login(t *testing.T, email, password string) *client { // store call that touches it fails with a DB error (not ErrNotFound). func (f *authFixture) breakTable(t *testing.T, table string) { t.Helper() - db, err := sql.Open("sqlite", f.srv.auth.set.dbPath) + db, err := openFixtureSQL(f.ownerURL) if err != nil { t.Fatal(err) } @@ -201,13 +232,13 @@ func (f *authFixture) breakTable(t *testing.T, table string) { // from users.db (the raw tokens are not observable when no mail left). func (f *authFixture) unusedTokens(t *testing.T, uid int64, p users.Purpose) int { t.Helper() - db, err := sql.Open("sqlite", f.srv.auth.set.dbPath) + db, err := openFixtureSQL(f.ownerURL) if err != nil { t.Fatal(err) } defer db.Close() var n int - if err := db.QueryRow(`SELECT COUNT(*) FROM tokens WHERE user_id = ? AND purpose = ? AND used_at IS NULL`, uid, string(p)).Scan(&n); err != nil { + if err := db.QueryRow(`SELECT COUNT(*) FROM tokens WHERE user_id = $1 AND purpose = $2 AND used_at IS NULL`, uid, string(p)).Scan(&n); err != nil { t.Fatal(err) } return n @@ -217,7 +248,7 @@ func (f *authFixture) unusedTokens(t *testing.T, uid int64, p users.Purpose) int // simulate a concurrent writer or a failing statement). func (f *authFixture) execDB(t *testing.T, stmt string) { t.Helper() - db, err := sql.Open("sqlite", f.srv.auth.set.dbPath) + db, err := openFixtureSQL(f.ownerURL) if err != nil { t.Fatal(err) } diff --git a/cmd/server/auth_flow_test.go b/cmd/server/auth_flow_test.go index 3776bd79f..5154b313d 100644 --- a/cmd/server/auth_flow_test.go +++ b/cmd/server/auth_flow_test.go @@ -223,8 +223,7 @@ func TestAuthActivateHashChangedMidwayIs409(t *testing.T) { f := newAuthFixture(t) expectStatus(t, f.do("POST", "/api/auth/register", registerRequest{Email: "ned@example.org", DisplayName: "Ned", Password: pw}), 200) tok := f.lastToken(t) - f.execDB(t, `CREATE TRIGGER race AFTER UPDATE OF used_at ON tokens BEGIN - UPDATE users SET password_hash = 'replaced-by-a-re-register' WHERE id = NEW.user_id; END`) + f.execDB(t, `CREATE FUNCTION race() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN UPDATE users SET password_hash='replaced-by-a-re-register' WHERE id=NEW.user_id; RETURN NEW; END $$; CREATE TRIGGER race AFTER UPDATE OF used_at ON tokens FOR EACH ROW EXECUTE FUNCTION race()`) w := f.do("POST", "/api/auth/activate", activateRequest{Token: tok, Password: pw}) expectStatus(t, w, 409) if !strings.Contains(w.Body.String(), "account changed, try again") { diff --git a/cmd/server/auth_login_audit_test.go b/cmd/server/auth_login_audit_test.go index 493c7542e..668c2252a 100644 --- a/cmd/server/auth_login_audit_test.go +++ b/cmd/server/auth_login_audit_test.go @@ -2,7 +2,6 @@ package main import ( "context" - "database/sql" "strings" "testing" "time" @@ -134,7 +133,7 @@ func TestLoginAnswersBeforeTheAuditWrite(t *testing.T) { f := newAuthFixture(t) dave := f.registerAndActivate(t, "dave@example.org", "Dave", pw) - db, err := sql.Open("sqlite", f.srv.auth.set.dbPath) + db, err := openFixtureSQL(f.ownerURL) if err != nil { t.Fatal(err) } @@ -144,7 +143,7 @@ func TestLoginAnswersBeforeTheAuditWrite(t *testing.T) { t.Fatal(err) } defer conn.Close() - if _, err := conn.ExecContext(context.Background(), "BEGIN IMMEDIATE"); err != nil { + if _, err := conn.ExecContext(context.Background(), "BEGIN; LOCK TABLE audit_log IN SHARE MODE"); err != nil { t.Fatal(err) } diff --git a/cmd/server/auth_mail_test.go b/cmd/server/auth_mail_test.go index 1314f77b9..3ef6c2713 100644 --- a/cmd/server/auth_mail_test.go +++ b/cmd/server/auth_mail_test.go @@ -5,7 +5,6 @@ import ( "context" "errors" "log" - "path/filepath" "strings" "testing" @@ -60,22 +59,16 @@ func TestActivationMailMentionsPassword(t *testing.T) { } } -// The startup line shows the absolute users.db path, so a relative path -// resolved against another working directory than the ingestor's is visible. -func TestLogStartupShowsAbsoluteUsersDBPath(t *testing.T) { +func TestLogStartupRedactsDatabaseURL(t *testing.T) { a, _ := newTestAuthService(t) - a.set.dbPath = "users.db" + a.set.databaseURL = "postgresql://private:secret@host/accounts?password=hidden" var buf bytes.Buffer prev := log.Writer() log.SetOutput(&buf) defer log.SetOutput(prev) a.logStartup() - want, err := filepath.Abs("users.db") - if err != nil { - t.Fatal(err) - } - if !strings.Contains(buf.String(), "[users] user management enabled: db="+want+",") { - t.Fatalf("log line = %q, want the absolute path %q", buf.String(), want) + if !strings.Contains(buf.String(), "PostgreSQL") || strings.Contains(buf.String(), "secret") || strings.Contains(buf.String(), "hidden") { + t.Fatalf("unsafe startup log: %q", buf.String()) } } diff --git a/cmd/server/auth_service.go b/cmd/server/auth_service.go index 0a4fc84ac..32c4600da 100644 --- a/cmd/server/auth_service.go +++ b/cmd/server/auth_service.go @@ -58,17 +58,17 @@ func newAuthService(set *userMgmtSettings, st *users.Store, m mailer.Mailer) *au } // initUserManagement builds s.auth when the feature is on. Call it after -// NewServer and before RegisterRoutes. measurementDBPath is passed to -// users.Open as a forbidden path, so users.db can never be the analyzer DB. -func (s *Server) initUserManagement(measurementDBPath string) error { +// NewServer and before RegisterRoutes. measurementDatabaseURL is passed to +// users.Open as a forbidden database, so accounts cannot share telemetry storage. +func (s *Server) initUserManagement(measurementDatabaseURL string) error { if !s.cfg.UserManagementEnabled() { return nil } - set, err := resolveUserManagement(s.cfg.UserManagement, measurementDBPath, os.Getenv) + set, err := resolveUserManagement(s.cfg.UserManagement, runtimeStateDir, os.Getenv) if err != nil { return err } - st, err := users.Open(set.dbPath, measurementDBPath) + st, err := users.Open(set.databaseURL, measurementDatabaseURL) if err != nil { return err } @@ -98,7 +98,7 @@ func (s *Server) initUserManagement(measurementDBPath string) error { return nil } -// closeUserManagement stops the janitor and closes users.db. Safe when off. +// closeUserManagement stops the janitor and closes the account connection pool. Safe when off. func (s *Server) closeUserManagement() { if s.auth == nil { return @@ -114,8 +114,8 @@ func (s *Server) closeUserManagement() { } func (a *authService) logStartup() { - log.Printf("[users] user management enabled: db=%s, %d config admin(s), webhook=%v", - absForLog(a.set.dbPath), len(a.set.adminEmails), a.set.webhookSecret != "") + log.Printf("[users] user management enabled: PostgreSQL, %d config admin(s), webhook=%v", + len(a.set.adminEmails), a.set.webhookSecret != "") admins, err := a.st.List(users.ListFilter{Role: users.RoleAdmin}) if err != nil { log.Printf("[users] list admins: %v", err) diff --git a/cmd/server/backup.go b/cmd/server/backup.go index 91915ed1a..d79c35190 100644 --- a/cmd/server/backup.go +++ b/cmd/server/backup.go @@ -7,29 +7,14 @@ import ( "net/http" "os" "path/filepath" - "strings" "time" + + "github.com/meshcore-analyzer/pgutil" ) -// handleBackup streams a consistent SQLite snapshot of the analyzer DB. -// -// Requires API-key authentication (mounted via requireAPIKey in routes.go). -// -// Strategy: SQLite's `VACUUM INTO 'path'` produces an atomic, defragmented -// copy of the current database into a new file. It runs at READ ISOLATION -// against the source DB (works on our read-only connection) and never -// blocks concurrent writers — the ingestor keeps writing to the WAL while -// the snapshot is taken from a consistent read transaction. -// -// Response: -// -// 200 OK -// Content-Type: application/octet-stream -// Content-Disposition: attachment; filename="corescope-backup-.db" -// -// -// The temp file is removed after the response is fully written, regardless -// of whether the client successfully consumed the stream. +// handleBackup stages a consistent PostgreSQL custom-format pg_dump snapshot. +// pgutil passes credentials through the child environment, bounds its lifetime, +// removes partial output, and returns redacted errors. func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request) { if s.db == nil || s.db.conn == nil { writeError(w, http.StatusServiceUnavailable, "database unavailable") @@ -56,12 +41,9 @@ func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request) { } }() - snapshotPath := filepath.Join(tmpDir, fmt.Sprintf("corescope-backup-%d.db", ts)) + snapshotPath := filepath.Join(tmpDir, fmt.Sprintf("corescope-backup-%d.dump", ts)) - // SQLite parses the path literal — escape any single quotes defensively. - // (mkdtemp output won't contain quotes, but be paranoid for future-proofing.) - escaped := strings.ReplaceAll(snapshotPath, "'", "''") - if _, err := s.db.conn.ExecContext(r.Context(), fmt.Sprintf("VACUUM INTO '%s'", escaped)); err != nil { + if err := pgutil.Dump(r.Context(), s.db.path, snapshotPath); err != nil { writeError(w, http.StatusInternalServerError, "snapshot failed: "+err.Error()) return } @@ -78,8 +60,9 @@ func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Length", fmt.Sprintf("%d", stat.Size())) } w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"corescope-backup-%d.db\"", ts)) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"corescope-backup-%d.dump\"", ts)) w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("Cache-Control", "no-store") w.WriteHeader(http.StatusOK) if _, err := io.Copy(w, f); err != nil { diff --git a/cmd/server/backup_test.go b/cmd/server/backup_test.go index 024ef3cb1..b988a5809 100644 --- a/cmd/server/backup_test.go +++ b/cmd/server/backup_test.go @@ -3,13 +3,16 @@ package main import ( "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "testing" + + "github.com/meshcore-analyzer/pgutil/pgtest" ) -// sqliteMagic is the 16-byte file header identifying a valid SQLite 3 database. -// See https://www.sqlite.org/fileformat.html#magic_header_string -const sqliteMagic = "SQLite format 3\x00" +// postgresMagic is the signature of a PostgreSQL custom-format archive. +const postgresMagic = "PGDMP" func TestBackupRequiresAPIKey(t *testing.T) { _, router := setupTestServerWithAPIKey(t, "test-secret-key-strong-enough") @@ -22,9 +25,18 @@ func TestBackupRequiresAPIKey(t *testing.T) { } } -func TestBackupReturnsValidSQLiteSnapshot(t *testing.T) { +func TestBackupRestoresNativePostgresSnapshot(t *testing.T) { const apiKey = "test-secret-key-strong-enough" - _, router := setupTestServerWithAPIKey(t, apiKey) + srv, router := setupTestServerWithAPIKey(t, apiKey) + srv.db = setupTestDBAtURL(t, pgtest.NewDatabase(t)) + seedTestData(t, srv.db) + writerURL := srv.db.path + reader, err := openFixtureReader(t, writerURL) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + srv.db = reader req := httptest.NewRequest("GET", "/api/backup", nil) req.Header.Set("X-API-Key", apiKey) @@ -41,15 +53,31 @@ func TestBackupReturnsValidSQLiteSnapshot(t *testing.T) { } cd := w.Header().Get("Content-Disposition") - if !strings.HasPrefix(cd, "attachment;") || !strings.Contains(cd, "filename=\"corescope-backup-") || !strings.HasSuffix(cd, ".db\"") { - t.Errorf("expected Content-Disposition attachment with corescope-backup-.db filename, got %q", cd) + if !strings.HasPrefix(cd, "attachment;") || !strings.Contains(cd, "filename=\"corescope-backup-") || !strings.HasSuffix(cd, ".dump\"") { + t.Errorf("expected Content-Disposition attachment with corescope-backup-.dump filename, got %q", cd) } body := w.Body.Bytes() - if len(body) < len(sqliteMagic) { - t.Fatalf("backup body too short (%d bytes) — expected SQLite file", len(body)) + if len(body) < len(postgresMagic) { + t.Fatalf("backup body too short (%d bytes) — expected PostgreSQL file", len(body)) + } + if got := string(body[:len(postgresMagic)]); got != postgresMagic { + t.Fatalf("expected PostgreSQL magic header %q, got %q", postgresMagic, got) + } + path := filepath.Join(t.TempDir(), "telemetry.dump") + if err := os.WriteFile(path, body, 0o600); err != nil { + t.Fatal(err) } - if got := string(body[:len(sqliteMagic)]); got != sqliteMagic { - t.Fatalf("expected SQLite magic header %q, got %q", sqliteMagic, got) + restored := restorePostgresTestBackup(t, path) + var transmissions, observations int + if err := restored.QueryRow(`SELECT COUNT(*) FROM transmissions`).Scan(&transmissions); err != nil { + t.Fatal(err) } + if err := restored.QueryRow(`SELECT COUNT(*) FROM observations`).Scan(&observations); err != nil { + t.Fatal(err) + } + if transmissions != 3 || observations != 4 { + t.Fatalf("restored counts = %d/%d", transmissions, observations) + } + } diff --git a/cmd/server/bounded_load_test.go b/cmd/server/bounded_load_test.go index ef383d549..0d8963879 100644 --- a/cmd/server/bounded_load_test.go +++ b/cmd/server/bounded_load_test.go @@ -1,21 +1,15 @@ package main import ( - "database/sql" "fmt" - "os" - "path/filepath" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // createTestDB creates a temporary SQLite database with N transmissions (1 obs each). func createTestDB(t *testing.T, numTx int) string { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) createTestDBAt(t, dbPath, numTx) return dbPath } @@ -23,7 +17,7 @@ func createTestDB(t *testing.T, numTx int) string { // loadStore creates a PacketStore from a test DB with given maxMemoryMB. func loadStore(t *testing.T, dbPath string, maxMemMB int) *PacketStore { t.Helper() - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -37,7 +31,6 @@ func loadStore(t *testing.T, dbPath string, maxMemMB int) *PacketStore { func TestBoundedLoad_LimitedMemory(t *testing.T) { dbPath := createTestDB(t, 5000) - defer os.RemoveAll(filepath.Dir(dbPath)) // Use 1MB budget — should load far fewer than 5000 packets store := loadStore(t, dbPath, 1) @@ -55,7 +48,6 @@ func TestBoundedLoad_LimitedMemory(t *testing.T) { func TestBoundedLoad_NewestFirst(t *testing.T) { dbPath := createTestDB(t, 5000) - defer os.RemoveAll(filepath.Dir(dbPath)) store := loadStore(t, dbPath, 1) defer store.db.conn.Close() @@ -85,7 +77,6 @@ func TestBoundedLoad_NewestFirst(t *testing.T) { func TestBoundedLoad_OldestLoadedSet(t *testing.T) { dbPath := createTestDB(t, 5000) - defer os.RemoveAll(filepath.Dir(dbPath)) store := loadStore(t, dbPath, 1) defer store.db.conn.Close() @@ -101,7 +92,6 @@ func TestBoundedLoad_OldestLoadedSet(t *testing.T) { func TestBoundedLoad_UnlimitedWithZero(t *testing.T) { dbPath := createTestDB(t, 200) - defer os.RemoveAll(filepath.Dir(dbPath)) store := loadStore(t, dbPath, 0) defer store.db.conn.Close() @@ -113,7 +103,6 @@ func TestBoundedLoad_UnlimitedWithZero(t *testing.T) { func TestBoundedLoad_AscendingOrder(t *testing.T) { dbPath := createTestDB(t, 3000) - defer os.RemoveAll(filepath.Dir(dbPath)) store := loadStore(t, dbPath, 1) defer store.db.conn.Close() @@ -130,7 +119,7 @@ func TestBoundedLoad_AscendingOrder(t *testing.T) { // loadStoreWithRetention creates a PacketStore with retentionHours set. func loadStoreWithRetention(t *testing.T, dbPath string, retentionHours float64) *PacketStore { t.Helper() - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -146,10 +135,9 @@ func loadStoreWithRetention(t *testing.T, dbPath string, retentionHours float64) // the last hour and numOld packets with timestamps 48 hours ago. func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -160,10 +148,10 @@ func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string { t.Fatalf("setup: %v\nSQL: %s", err, s) } } - execOrFail(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`) - execOrFail(`CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`) - execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) - execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) + execOrFail(`CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`) + execOrFail(`CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp BIGINT, raw_hex TEXT)`) + execOrFail(`CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION)`) execOrFail(`CREATE TABLE schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) execOrFail(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`) @@ -174,28 +162,29 @@ func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string { // (auto-commit per Exec fsyncs per row). numOld+numRecent // is small here today, but wrapping keeps the fixture robust if callers scale // it up, and is consistent with the other builders. - if _, err := conn.Exec("BEGIN"); err != nil { + fixtureTx, err := conn.Begin() + if err != nil { t.Fatalf("test DB BEGIN: %v", err) } // Insert old packets (48 hours ago) for i := 0; i < numOld; i++ { oldT := now.Add(-48 * time.Hour).Add(time.Duration(i) * time.Second) ts := oldT.Format(time.RFC3339) - conn.Exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", id, "aa", fmt.Sprintf("old%d", i), ts, `{}`) + fixtureTx.Exec("INSERT INTO transmissions VALUES ($1,$2,$3,$4,0,4,1,$5)", id, "aa", fmt.Sprintf("old%d", i), ts, `{}`) // observations.timestamp is INTEGER (unix seconds) in production schema // — keep the fixture consistent so the RFC3339 subquery matches. - conn.Exec("INSERT INTO observations VALUES (?,?,?,?,?,?,?,?,?,?,?)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, oldT.Unix(), "") + fixtureTx.Exec("INSERT INTO observations VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, oldT.Unix(), "") id++ } // Insert recent packets (within last hour) for i := 0; i < numRecent; i++ { newT := now.Add(-30 * time.Minute).Add(time.Duration(i) * time.Second) ts := newT.Format(time.RFC3339) - conn.Exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", id, "bb", fmt.Sprintf("new%d", i), ts, `{}`) - conn.Exec("INSERT INTO observations VALUES (?,?,?,?,?,?,?,?,?,?,?)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, newT.Unix(), "") + fixtureTx.Exec("INSERT INTO transmissions VALUES ($1,$2,$3,$4,0,4,1,$5)", id, "bb", fmt.Sprintf("new%d", i), ts, `{}`) + fixtureTx.Exec("INSERT INTO observations VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, newT.Unix(), "") id++ } - if _, err := conn.Exec("COMMIT"); err != nil { + if err := fixtureTx.Commit(); err != nil { t.Fatalf("test DB COMMIT: %v", err) } return dbPath @@ -203,7 +192,6 @@ func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string { func TestRetentionLoad_OnlyLoadsRecentPackets(t *testing.T) { dbPath := createTestDBWithAgedPackets(t, 50, 100) - defer os.RemoveAll(filepath.Dir(dbPath)) // retention = 2 hours — should load only the 50 recent packets, not the 100 old ones store := loadStoreWithRetention(t, dbPath, 2) @@ -216,7 +204,6 @@ func TestRetentionLoad_OnlyLoadsRecentPackets(t *testing.T) { func TestRetentionLoad_ZeroRetentionLoadsAll(t *testing.T) { dbPath := createTestDBWithAgedPackets(t, 50, 100) - defer os.RemoveAll(filepath.Dir(dbPath)) // retention = 0 (unlimited) — should load all 150 packets store := loadStoreWithRetention(t, dbPath, 0) @@ -242,13 +229,12 @@ func TestEstimateStoreTxBytesTypical(t *testing.T) { } func BenchmarkLoad_Bounded(b *testing.B) { - dir := b.TempDir() - dbPath := filepath.Join(dir, "bench.db") + dbPath := postgresTestDSN(b) createTestDBAt(b, dbPath, 5000) b.ResetTimer() for i := 0; i < b.N; i++ { - db, _ := OpenDB(dbPath) + db, _ := openFixtureReader(b, dbPath) cfg := &PacketStoreConfig{MaxMemoryMB: 1} store := NewPacketStore(db, cfg) store.Load() @@ -257,13 +243,12 @@ func BenchmarkLoad_Bounded(b *testing.B) { } func BenchmarkLoad_Unlimited(b *testing.B) { - dir := b.TempDir() - dbPath := filepath.Join(dir, "bench.db") + dbPath := postgresTestDSN(b) createTestDBAt(b, dbPath, 5000) b.ResetTimer() for i := 0; i < b.N; i++ { - db, _ := OpenDB(dbPath) + db, _ := openFixtureReader(b, dbPath) cfg := &PacketStoreConfig{MaxMemoryMB: 0} store := NewPacketStore(db, cfg) store.Load() @@ -274,13 +259,12 @@ func BenchmarkLoad_Unlimited(b *testing.B) { // BenchmarkLoad_30K_Bounded benchmarks bounded Load() with 30K transmissions // and realistic observation counts (1–5 per transmission). func BenchmarkLoad_30K_Bounded(b *testing.B) { - dir := b.TempDir() - dbPath := filepath.Join(dir, "bench30k.db") + dbPath := postgresTestDSN(b) createTestDBWithObs(b, dbPath, 30000) b.ResetTimer() for i := 0; i < b.N; i++ { - db, _ := OpenDB(dbPath) + db, _ := openFixtureReader(b, dbPath) cfg := &PacketStoreConfig{MaxMemoryMB: 50} store := NewPacketStore(db, cfg) store.Load() @@ -291,13 +275,12 @@ func BenchmarkLoad_30K_Bounded(b *testing.B) { // BenchmarkLoad_30K_Unlimited benchmarks unlimited Load() with 30K transmissions // and realistic observation counts (1–5 per transmission). func BenchmarkLoad_30K_Unlimited(b *testing.B) { - dir := b.TempDir() - dbPath := filepath.Join(dir, "bench30k.db") + dbPath := postgresTestDSN(b) createTestDBWithObs(b, dbPath, 30000) b.ResetTimer() for i := 0; i < b.N; i++ { - db, _ := OpenDB(dbPath) + db, _ := openFixtureReader(b, dbPath) cfg := &PacketStoreConfig{MaxMemoryMB: 0} store := NewPacketStore(db, cfg) store.Load() @@ -308,7 +291,7 @@ func BenchmarkLoad_30K_Unlimited(b *testing.B) { // createTestDBAt is like createTestDB but writes to a specific path. func createTestDBAt(tb testing.TB, dbPath string, numTx int) { tb.Helper() - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { tb.Fatal(err) } @@ -320,66 +303,41 @@ func createTestDBAt(tb testing.TB, dbPath string, numTx int) { } } execOrFail(`CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT )`) execOrFail(`CREATE TABLE IF NOT EXISTS observations ( - id INTEGER PRIMARY KEY, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`) - execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) execOrFail(`CREATE TABLE IF NOT EXISTS nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, first_seen TEXT, frequency REAL + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, + last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION )`) execOrFail(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) execOrFail(`CREATE INDEX IF NOT EXISTS idx_tx_first_seen ON transmissions(first_seen)`) - txStmt, err := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)") - if err != nil { - tb.Fatalf("test DB prepare transmissions insert: %v", err) - } - obsStmt, err := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") - if err != nil { - tb.Fatalf("test DB prepare observations insert: %v", err) - } - defer txStmt.Close() - defer obsStmt.Close() - base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) - // Wrap the inserts in a single transaction. Without this, SQLite - // (pure-Go driver) auto-commits every Exec → one fsync per row → ~2N fsyncs - // for N transmissions (tx + obs). At numTx=5000 that is ~10k fsyncs and the - // fixture blows past the test timeout (the #1741 hang). A single - // BEGIN/COMMIT makes the whole build one commit, finishing in well under a - // second regardless of numTx. - if _, err := conn.Exec("BEGIN"); err != nil { - tb.Fatalf("test DB BEGIN: %v", err) - } - for i := 1; i <= numTx; i++ { - ts := base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339) - hash := fmt.Sprintf("h%04d", i) - if _, err := txStmt.Exec(i, "aabb", hash, ts, 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%04d"}`, i)); err != nil { - tb.Fatalf("test DB insert transmission %d: %v", i, err) - } - if _, err := obsStmt.Exec(i, i, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `["aa","bb"]`, ts); err != nil { - tb.Fatalf("test DB insert observation %d: %v", i, err) - } - } - if _, err := conn.Exec("COMMIT"); err != nil { - tb.Fatalf("test DB COMMIT: %v", err) - } + copyTestRows(tb, conn, "transmissions", []string{"id", "raw_hex", "hash", "first_seen", "route_type", "payload_type", "payload_version", "decoded_json"}, numTx, func(n int) []any { + i := n + 1 + return []any{i, "aabb", fmt.Sprintf("h%04d", i), base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339), 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%04d"}`, i)} + }) + copyTestRows(tb, conn, "observations", []string{"id", "transmission_id", "observer_id", "observer_name", "direction", "snr", "rssi", "score", "path_json", "timestamp"}, numTx, func(n int) []any { + i := n + 1 + return []any{i, i, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `["aa","bb"]`, base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339)} + }) } // createTestDBWithObs creates a test DB with realistic observation counts (1–5 per tx). func createTestDBWithObs(tb testing.TB, dbPath string, numTx int) { tb.Helper() - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { tb.Fatal(err) } @@ -391,60 +349,41 @@ func createTestDBWithObs(tb testing.TB, dbPath string, numTx int) { } } execOrFail(`CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT )`) execOrFail(`CREATE TABLE IF NOT EXISTS observations ( - id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`) - execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) execOrFail(`CREATE TABLE IF NOT EXISTS nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, first_seen TEXT, frequency REAL + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, + last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION )`) execOrFail(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) execOrFail(`CREATE INDEX IF NOT EXISTS idx_tx_first_seen ON transmissions(first_seen)`) - txStmt, err := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)") - if err != nil { - tb.Fatalf("test DB prepare transmissions: %v", err) - } - obsStmt, err := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") - if err != nil { - tb.Fatalf("test DB prepare observations: %v", err) - } - defer txStmt.Close() - defer obsStmt.Close() - - observers := []string{"obs1", "obs2", "obs3", "obs4", "obs5"} - obsNames := []string{"Alpha", "Bravo", "Charlie", "Delta", "Echo"} - obsID := 1 base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) - // Single transaction for all inserts — see createTestDBAt for the rationale - // (auto-commit per Exec would fsync per row; at numTx=30000 - // the benchmarks would otherwise stall for minutes). One BEGIN/COMMIT. - if _, err := conn.Exec("BEGIN"); err != nil { - tb.Fatalf("test DB BEGIN: %v", err) - } + copyTestRows(tb, conn, "transmissions", []string{"id", "raw_hex", "hash", "first_seen", "route_type", "payload_type", "payload_version", "decoded_json"}, numTx, func(n int) []any { + i := n + 1 + return []any{i, "aabb", fmt.Sprintf("h%06d", i), base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339), 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%06d"}`, i)} + }) + observers := []string{"obs1", "obs2", "obs3", "obs4", "obs5"} + names := []string{"Alpha", "Bravo", "Charlie", "Delta", "Echo"} + totalObs := 0 for i := 1; i <= numTx; i++ { - ts := base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339) - hash := fmt.Sprintf("h%06d", i) - if _, err := txStmt.Exec(i, "aabb", hash, ts, 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%06d"}`, i)); err != nil { - tb.Fatalf("test DB insert transmission %d: %v", i, err) + totalObs += i%5 + 1 + } + txID, j := 1, 0 + copyTestRows(tb, conn, "observations", []string{"id", "transmission_id", "observer_id", "observer_name", "direction", "snr", "rssi", "score", "path_json", "timestamp"}, totalObs, func(n int) []any { + row := []any{n + 1, txID, observers[j], names[j], "RX", -5.0 + float64(j)*2.5, -90.0 + float64(j)*5.0, 5 - j, `["aa","bb"]`, base.Add(time.Duration(txID) * time.Minute).Format(time.RFC3339)} + j++ + if j > txID%5 { + txID++ + j = 0 } - nObs := (i % 5) + 1 // 1–5 observations per transmission - for j := 0; j < nObs; j++ { - snr := -5.0 + float64(j)*2.5 - rssi := -90.0 + float64(j)*5.0 - if _, err := obsStmt.Exec(obsID, i, observers[j], obsNames[j], "RX", snr, rssi, 5-j, `["aa","bb"]`, ts); err != nil { - tb.Fatalf("test DB insert observation %d: %v", obsID, err) - } - obsID++ - } - } - if _, err := conn.Exec("COMMIT"); err != nil { - tb.Fatalf("test DB COMMIT: %v", err) - } + return row + }) } diff --git a/cmd/server/bridge_handle_nodes_test.go b/cmd/server/bridge_handle_nodes_test.go index b88b7b1b6..b08376f45 100644 --- a/cmd/server/bridge_handle_nodes_test.go +++ b/cmd/server/bridge_handle_nodes_test.go @@ -34,7 +34,7 @@ func TestBridgeScore_HandleNodesSurface(t *testing.T) { for _, pk := range pks { if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, 'repeater', 37.5, -122.0, ?, ?, 10)`, + VALUES ($1, $2, 'repeater', 37.5, -122.0, $3, $4, 10)`, pk, "node-"+pk[:4], recent, recent); err != nil { t.Fatal(err) } diff --git a/cmd/server/channel_message_hash_size_test.go b/cmd/server/channel_message_hash_size_test.go index 6a53a345a..c4226915a 100644 --- a/cmd/server/channel_message_hash_size_test.go +++ b/cmd/server/channel_message_hash_size_test.go @@ -44,18 +44,19 @@ func seedChannelHashSize(t *testing.T, db *DB) *DB { } for i, r := range rows { ts := now.Add(time.Duration(i-len(rows)) * time.Minute) - res, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES (?, ?, ?, ?, 5, '{"type":"CHAN","channel":"#hashsize","text":"Alice: msg"}', '#hashsize')`, - r.rawHex, r.hash, ts.Format(time.RFC3339), r.routeType) + var txID int64 + err := db.conn.QueryRow(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) + VALUES ($1, $2, $3, $4, 5, '{"type":"CHAN","channel":"#hashsize","text":"Alice: msg"}', '#hashsize') RETURNING id`, + r.rawHex, r.hash, ts.Format(time.RFC3339), r.routeType).Scan(&txID) if err != nil { t.Fatalf("insert tx %s: %v", r.hash, err) } - txID, _ := res.LastInsertId() + obsSQL := `INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 9.5, -90, '[]', ?)` + VALUES ($1, 1, 9.5, -90, '[]', $2)` if !db.isV3 { obsSQL = `INSERT INTO observations (transmission_id, observer_id, observer_name, snr, rssi, path_json, timestamp) - VALUES (?, 'obs1', 'Observer One', 9.5, -90, '[]', ?)` + VALUES ($1, 'obs1', 'Observer One', 9.5, -90, '[]', $2)` } if _, err := db.conn.Exec(obsSQL, txID, ts.Unix()); err != nil { t.Fatalf("insert obs %s: %v", r.hash, err) diff --git a/cmd/server/channel_message_scope_name_test.go b/cmd/server/channel_message_scope_name_test.go index 3d2af4719..ab834ee5e 100644 --- a/cmd/server/channel_message_scope_name_test.go +++ b/cmd/server/channel_message_scope_name_test.go @@ -54,16 +54,17 @@ func setupChannelScopeDB(t *testing.T) *DB { } for i, r := range rows { ts := now.Add(time.Duration(i-3) * time.Minute) - res, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash, scope_name) - VALUES ('AABB', ?, ?, ?, 5, ?, '#scopetest', ?)`, + var txID int64 + err := db.conn.QueryRow(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash, scope_name) + VALUES ('AABB', $1, $2, $3, 5, $4, '#scopetest', $5) RETURNING id`, r.hash, ts.Format(time.RFC3339), r.routeType, - `{"type":"CHAN","channel":"#scopetest","text":"`+r.text+`"}`, r.scope) + `{"type":"CHAN","channel":"#scopetest","text":"`+r.text+`"}`, r.scope).Scan(&txID) if err != nil { t.Fatalf("insert tx %s: %v", r.hash, err) } - txID, _ := res.LastInsertId() + if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 9.5, -90, '[]', ?)`, txID, ts.Unix()); err != nil { + VALUES ($1, 1, 9.5, -90, '[]', $2)`, txID, ts.Unix()); err != nil { t.Fatalf("insert obs %s: %v", r.hash, err) } } @@ -229,15 +230,16 @@ func TestBroadcastMapsCarryScopeName(t *testing.T) { t.Run("IngestNewFromDB", func(t *testing.T) { txMax := store.MaxTransmissionID() now := time.Now().UTC() - res, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash, scope_name) - VALUES ('AABB', 'cccccccccccccc04', ?, 0, 5, '{"type":"CHAN","channel":"#scopetest","text":"Dave: live"}', '#scopetest', '#belgium')`, - now.Format(time.RFC3339)) + var txID int64 + err := db.conn.QueryRow(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash, scope_name) + VALUES ('AABB', 'cccccccccccccc04', $1, 0, 5, '{"type":"CHAN","channel":"#scopetest","text":"Dave: live"}', '#scopetest', '#belgium') RETURNING id`, + now.Format(time.RFC3339)).Scan(&txID) if err != nil { t.Fatalf("insert tx: %v", err) } - txID, _ := res.LastInsertId() + if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 7.0, -95, '[]', ?)`, txID, now.Unix()); err != nil { + VALUES ($1, 1, 7.0, -95, '[]', $2)`, txID, now.Unix()); err != nil { t.Fatalf("insert obs: %v", err) } maps, _ := store.IngestNewFromDB(txMax, 100) @@ -247,11 +249,11 @@ func TestBroadcastMapsCarryScopeName(t *testing.T) { t.Run("IngestNewObservations", func(t *testing.T) { obsMax := db.GetMaxObservationID() var txID int - if err := db.conn.QueryRow(`SELECT id FROM transmissions WHERE hash = ?`, chScopeHashUnknown).Scan(&txID); err != nil { + if err := db.conn.QueryRow(`SELECT id FROM transmissions WHERE hash = $1`, chScopeHashUnknown).Scan(&txID); err != nil { t.Fatalf("lookup tx: %v", err) } if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 3.0, -110, '["aa"]', ?)`, txID, time.Now().Unix()); err != nil { + VALUES ($1, 1, 3.0, -110, '["aa"]', $2)`, txID, time.Now().Unix()); err != nil { t.Fatalf("insert obs: %v", err) } maps := store.IngestNewObservations(obsMax, 100) diff --git a/cmd/server/channels_message_order_1366_test.go b/cmd/server/channels_message_order_1366_test.go index 0fde9bfbb..5f307443b 100644 --- a/cmd/server/channels_message_order_1366_test.go +++ b/cmd/server/channels_message_order_1366_test.go @@ -34,20 +34,20 @@ func TestChannelMessages_TimestampUsesLatestSeen(t *testing.T) { _ = laterEpoch db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsA', 'ObsA', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, firstSeen) + VALUES ('obsA', 'ObsA', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, firstSeen) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsB', 'ObsB', 'LAX', ?, '2026-01-01T00:00:00Z', 10)`, firstSeen) + VALUES ('obsB', 'ObsB', 'LAX', $1, '2026-01-01T00:00:00Z', 10)`, firstSeen) // One transmission with two observations: T0 (7h ago) and T1 (5m ago). db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('AA01', 'hash_repeated_msg', ?, 1, 5, + VALUES ('AA01', 'hash_repeated_msg', $1, 1, 5, '{"type":"CHAN","channel":"#test","text":"Heartbeat: ping","sender":"Heartbeat","sender_timestamp":`+ strconv.FormatInt(firstSeenEpoch, 10)+`}', '#test')`, firstSeen) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '["aa"]', ?)`, firstSeenEpoch) + VALUES (1, 1, 10.0, -90, '["aa"]', $1)`, firstSeenEpoch) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 2, 11.0, -88, '["bb"]', ?)`, laterEpoch) + VALUES (1, 2, 11.0, -88, '["bb"]', $1)`, laterEpoch) store := NewPacketStore(db, nil) store.Load() @@ -102,14 +102,14 @@ func TestChannelMessages_TimestampNotSenderTimestamp(t *testing.T) { badSenderTs := int64(946684800) // 2000-01-01 UTC db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsX', 'ObsX', 'SJC', ?, '2026-01-01T00:00:00Z', 1)`, firstSeen) + VALUES ('obsX', 'ObsX', 'SJC', $1, '2026-01-01T00:00:00Z', 1)`, firstSeen) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('BB01', 'hash_bad_clock', ?, 1, 5, + VALUES ('BB01', 'hash_bad_clock', $1, 1, 5, '{"type":"CHAN","channel":"#bad","text":"Alice: ping","sender":"Alice","sender_timestamp":`+ strconv.FormatInt(badSenderTs, 10)+`}', '#bad')`, firstSeen) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '["aa"]', ?)`, firstSeenEpoch) + VALUES (1, 1, 10.0, -90, '["aa"]', $1)`, firstSeenEpoch) store := NewPacketStore(db, nil) store.Load() @@ -143,13 +143,13 @@ func TestChannelMessages_TimestampIsUTCZ(t *testing.T) { ep := now.Add(-30 * time.Minute).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsZ', 'ObsZ', 'SJC', ?, '2026-01-01T00:00:00Z', 1)`, fs) + VALUES ('obsZ', 'ObsZ', 'SJC', $1, '2026-01-01T00:00:00Z', 1)`, fs) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('ZZ01', 'hash_zone_check', ?, 1, 5, + VALUES ('ZZ01', 'hash_zone_check', $1, 1, 5, '{"type":"CHAN","channel":"#zone","text":"Carol: ping","sender":"Carol"}', '#zone')`, fs) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 11.0, -89, '["zz"]', ?)`, ep) + VALUES (1, 1, 11.0, -89, '["zz"]', $1)`, ep) store := NewPacketStore(db, nil) store.Load() @@ -197,30 +197,30 @@ func TestChannelMessages_OrderedByLatestSeen_InMemory(t *testing.T) { tNewestStr := tNewest.Format(time.RFC3339) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsO', 'ObsO', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, tOldStr) + VALUES ('obsO', 'ObsO', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, tOldStr) // tx-A: FirstSeen 24h ago, LatestSeen NOW (T-1m). Old insertion order. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('AAAA', 'order_hash_a', ?, 1, 5, + VALUES ('AAAA', 'order_hash_a', $1, 1, 5, '{"type":"CHAN","channel":"#ord","text":"Alpha: hb","sender":"Alpha"}', '#ord')`, tOldStr) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '["aa"]', ?)`, tOld.Unix()) + VALUES (1, 1, 10.0, -90, '["aa"]', $1)`, tOld.Unix()) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 11.0, -88, '["aa"]', ?)`, tFresh.Unix()) + VALUES (1, 1, 11.0, -88, '["aa"]', $1)`, tFresh.Unix()) // tx-B: FirstSeen 1h ago, LatestSeen 1h ago. OLDEST. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('BBBB', 'order_hash_b', ?, 1, 5, + VALUES ('BBBB', 'order_hash_b', $1, 1, 5, '{"type":"CHAN","channel":"#ord","text":"Bravo: msg","sender":"Bravo"}', '#ord')`, tMidStr) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 9.0, -91, '["bb"]', ?)`, tMid.Unix()) + VALUES (2, 1, 9.0, -91, '["bb"]', $1)`, tMid.Unix()) // tx-C: FirstSeen 30m ago, LatestSeen 30m ago. Middle. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('CCCC', 'order_hash_c', ?, 1, 5, + VALUES ('CCCC', 'order_hash_c', $1, 1, 5, '{"type":"CHAN","channel":"#ord","text":"Charlie: msg","sender":"Charlie"}', '#ord')`, tNewestStr) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 9.0, -91, '["cc"]', ?)`, tNewest.Unix()) + VALUES (3, 1, 9.0, -91, '["cc"]', $1)`, tNewest.Unix()) store := NewPacketStore(db, nil) store.Load() @@ -276,34 +276,34 @@ func TestChannelMessages_OrderedByLatestSeen_DB(t *testing.T) { tNewestStr := tNewest.Format(time.RFC3339) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsD', 'ObsD', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, tOldStr) + VALUES ('obsD', 'ObsD', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, tOldStr) // tx-A: FirstSeen 24h ago, observations at T-24h and T-1m (LatestSeen // = T-1m, the FRESHEST). Despite the freshest LatestSeen, a // FirstSeen-DESC selection would push it OFF a small page. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('AADB', 'order_db_hash_a', ?, 1, 5, + VALUES ('AADB', 'order_db_hash_a', $1, 1, 5, '{"type":"CHAN","channel":"#ordb","text":"Alpha: hb","sender":"Alpha"}', '#ordb')`, tOldStr) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '["aa"]', ?)`, tOld.Unix()) + VALUES (1, 1, 10.0, -90, '["aa"]', $1)`, tOld.Unix()) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 11.0, -88, '["aa"]', ?)`, tFresh.Unix()) + VALUES (1, 1, 11.0, -88, '["aa"]', $1)`, tFresh.Unix()) // tx-B: FirstSeen 1h ago, LatestSeen 1h ago. OLDEST LatestSeen. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('BBDB', 'order_db_hash_b', ?, 1, 5, + VALUES ('BBDB', 'order_db_hash_b', $1, 1, 5, '{"type":"CHAN","channel":"#ordb","text":"Bravo: msg","sender":"Bravo"}', '#ordb')`, tMidStr) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 9.0, -91, '["bb"]', ?)`, tMid.Unix()) + VALUES (2, 1, 9.0, -91, '["bb"]', $1)`, tMid.Unix()) // tx-C: FirstSeen 30m ago, LatestSeen 30m ago. Middle LatestSeen. // With FirstSeen-DESC selection + limit=2, page = [tx-C, tx-B] and // tx-A is EXCLUDED — that's the selection bug fix #2 gates. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('CCDB', 'order_db_hash_c', ?, 1, 5, + VALUES ('CCDB', 'order_db_hash_c', $1, 1, 5, '{"type":"CHAN","channel":"#ordb","text":"Charlie: msg","sender":"Charlie"}', '#ordb')`, tNewestStr) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 9.0, -91, '["cc"]', ?)`, tNewest.Unix()) + VALUES (3, 1, 9.0, -91, '["cc"]', $1)`, tNewest.Unix()) msgs, total, err := db.GetChannelMessages("#ordb", 2, 0) if err != nil { diff --git a/cmd/server/channels_no_unknown_bucket_1373_test.go b/cmd/server/channels_no_unknown_bucket_1373_test.go index 3f709f399..d917e4e4b 100644 --- a/cmd/server/channels_no_unknown_bucket_1373_test.go +++ b/cmd/server/channels_no_unknown_bucket_1373_test.go @@ -67,7 +67,7 @@ func TestGetChannels_DB_NoUnknownBucket_1373(t *testing.T) { for i := 0; i < 5; i++ { _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES (?, ?, '2026-05-25T12:00:00Z', 1, 5, + VALUES ($1, $2, '2026-05-25T12:00:00Z', 1, 5, '{"type":"CHAN","channel":"","text":"","sender":""}', NULL)`, "AA", sqlHashFor(i)) if err != nil { @@ -79,7 +79,7 @@ func TestGetChannels_DB_NoUnknownBucket_1373(t *testing.T) { for i := 0; i < 3; i++ { _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES (?, ?, '2026-05-25T12:00:00Z', 1, 5, + VALUES ($1, $2, '2026-05-25T12:00:00Z', 1, 5, '{"type":"CHAN","channel":"#real","text":"Alice: hi","sender":"Alice"}', '#real')`, "BB", sqlHashFor(100+i)) if err != nil { diff --git a/cmd/server/chunked_load.go b/cmd/server/chunked_load.go index 0a82b33f3..f13ab3c4f 100644 --- a/cmd/server/chunked_load.go +++ b/cmd/server/chunked_load.go @@ -22,7 +22,7 @@ package main // logging / tests. // // Concurrency: each chunk acquires s.mu.Lock() ONLY while merging the -// chunk's rows into store-shared maps. SQLite reads run lock-free so +// chunk's rows into store-shared maps. Database reads run without the store lock so // HTTP handlers (which take s.mu.RLock) stay responsive. import ( @@ -102,7 +102,7 @@ func (s *PacketStore) chunkedLoadInit() { // StartupLoadDone returns a channel closed once RunStartupLoad has // returned: LoadChunked AND the background fill loader are finished, -// whether they succeeded or not. Nothing more is loaded from SQLite +// whether they succeeded or not. Nothing more is loaded from the database // after it closes. LoadComplete() is not a substitute: it flips at the // end of the hot window, before the background fill starts. func (s *PacketStore) StartupLoadDone() <-chan struct{} { @@ -265,7 +265,7 @@ func (s *PacketStore) RunStartupLoad(chunkSize int) error { return nil } -// LoadChunked streams transmissions + observations from SQLite into +// LoadChunked streams transmissions + observations from PostgreSQL into // the in-memory store in id-ordered chunks of `chunkSize` rows. Pass // 0 to use the default (10000). // @@ -327,7 +327,7 @@ func (s *PacketStore) LoadChunked(chunkSize int) error { // the prior `t2.first_seen >= cutoff` query loaded only hashes // first-inserted within the window (0.3% of DB on prod). // - // Test/legacy DBs without the column (PRAGMA-detected as + // Test/legacy DBs without the column (schema-detected as // hasLastSeen=false) fall back to the legacy first_seen axis to // keep existing fixtures green. Production goes through // dbschema.AssertReady which fail-fasts when the column is @@ -368,7 +368,7 @@ func (s *PacketStore) LoadChunked(chunkSize int) error { chunkIdx := 0 totalLoaded := 0 - // Start the id cursor BELOW the minimum possible row id so the + // Start the id cursor below the legacy fixture's zero ID so the // first chunk's `t2.id > cursorID` predicate includes id=0. The // e2e fixture seed for issue #1486 inserts the grouped-packet row // with id=0 (so it sorts LAST in the default packets view via @@ -376,12 +376,12 @@ func (s *PacketStore) LoadChunked(chunkSize int) error { // 0 silently excluded that row, leaving the page with no // tr[data-hash] and timing out the playwright wait. Legacy Load() // had no id cursor and loaded id=0 unconditionally — we restore - // that semantic by starting one below SQLite's minimum rowid (-1). + // that semantic by starting at -1. var cursorID int64 = -1 // Relay-hop fallback inputs, fetched ONCE before the chunk-query loop. // getCachedNodesAndPM issues its own DB query, so calling it while a - // chunk cursor is open would deadlock on a single-connection SQLite + // chunk cursor is open would deadlock on a single-connection database // pool. resolved_path is never persisted post-#1287, so scanAndMergeChunk // re-resolves relay hops from path_json using these snapshots. // PR #1643 R1 munger #1: cold load uses unique_prefix-only gate, so @@ -417,7 +417,7 @@ func (s *PacketStore) LoadChunked(chunkSize int) error { chunkSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, t.payload_type, t.payload_version, t.decoded_json, o.id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, - o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')` + obsRawHexCol + rpCol + scopeNameCol + ` + o.snr, o.rssi, o.score, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')` + obsRawHexCol + rpCol + scopeNameCol + ` FROM (SELECT * FROM transmissions t2 ` + whereClause + ` ORDER BY t2.id ASC LIMIT ` + fmt.Sprintf("%d", chunkSize) + `) AS t LEFT JOIN observations o ON o.transmission_id = t.id LEFT JOIN observers obs ON obs.rowid = o.observer_idx @@ -542,7 +542,7 @@ func (s *PacketStore) scanAndMergeChunk(rows *sql.Rows, relayPM *prefixMap, cold var obsID sql.NullInt64 var observerID, observerName, observerIATA, direction, pathJSON, obsTimestamp sql.NullString var snr, rssi sql.NullFloat64 - var score sql.NullInt64 + var score sql.NullFloat64 var obsRawHex sql.NullString var resolvedPathStr sql.NullString var scopeName sql.NullString @@ -620,9 +620,9 @@ func (s *PacketStore) scanAndMergeChunk(rows *sql.Rows, relayPM *prefixMap, cold Direction: nullStrVal(direction), SNR: nullFloatPtr(snr), RSSI: nullFloatPtr(rssi), - Score: nullIntPtr(score), + Score: nullFloatPtr(score), PathJSON: obsPJ, - // Raw frames stay in SQLite; hash equality does not imply route + // Raw frames stay in the database; hash equality does not imply route // equality. Only compact advert evidence is retained per tx. // Packet-detail queries can read the original observation raw. Timestamp: normalizeTimestamp(nullStrVal(obsTimestamp)), diff --git a/cmd/server/chunked_load_id_zero_test.go b/cmd/server/chunked_load_id_zero_test.go index 7b1fb87e7..002c9f11a 100644 --- a/cmd/server/chunked_load_id_zero_test.go +++ b/cmd/server/chunked_load_id_zero_test.go @@ -20,16 +20,14 @@ package main // row into s.byHash. import ( - "database/sql" "fmt" - "path/filepath" "testing" "time" ) func createTestDBWithIDZero(tb testing.TB, dbPath string, extraTx int) { tb.Helper() - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { tb.Fatal(err) } @@ -37,21 +35,21 @@ func createTestDBWithIDZero(tb testing.TB, dbPath string, extraTx int) { stmts := []string{ `CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT )`, `CREATE TABLE IF NOT EXISTS observations ( - id INTEGER PRIMARY KEY, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`, - `CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, + `CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE IF NOT EXISTS nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, first_seen TEXT, frequency REAL + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, + last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION )`, `CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`, `INSERT INTO schema_version (version) VALUES (1)`, @@ -63,8 +61,8 @@ func createTestDBWithIDZero(tb testing.TB, dbPath string, extraTx int) { } } - txStmt, _ := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)") - obsStmt, _ := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") + txStmt, _ := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)") + obsStmt, _ := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)") defer txStmt.Close() defer obsStmt.Close() @@ -89,11 +87,10 @@ func createTestDBWithIDZero(tb testing.TB, dbPath string, extraTx int) { // rows are silently dropped. This breaks the #1486 e2e fixture seed // which uses id=0 to sort the grouped row last in the default view. func TestLoadChunked_IncludesIDZero(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "idzero.db") + dbPath := postgresTestDSN(t) createTestDBWithIDZero(t, dbPath, 10) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/chunked_load_oldest_test.go b/cmd/server/chunked_load_oldest_test.go index 8b3edce7f..fe235223c 100644 --- a/cmd/server/chunked_load_oldest_test.go +++ b/cmd/server/chunked_load_oldest_test.go @@ -19,9 +19,7 @@ package main // transmissions, not the FirstSeen of the first row in s.packets. import ( - "database/sql" "fmt" - "path/filepath" "testing" "time" ) @@ -31,7 +29,7 @@ import ( // This mirrors the freshen-fixture-shifted e2e DB exactly. func createTestDBReverseTime(tb testing.TB, dbPath string, numTx int) { tb.Helper() - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { tb.Fatal(err) } @@ -39,21 +37,21 @@ func createTestDBReverseTime(tb testing.TB, dbPath string, numTx int) { stmts := []string{ `CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT )`, `CREATE TABLE IF NOT EXISTS observations ( - id INTEGER PRIMARY KEY, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`, - `CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, + `CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE IF NOT EXISTS nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, first_seen TEXT, frequency REAL + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, + last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION )`, `CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`, `INSERT INTO schema_version (version) VALUES (1)`, @@ -65,8 +63,8 @@ func createTestDBReverseTime(tb testing.TB, dbPath string, numTx int) { } } - txStmt, _ := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)") - obsStmt, _ := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") + txStmt, _ := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)") + obsStmt, _ := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)") defer txStmt.Close() defer obsStmt.Close() @@ -83,11 +81,10 @@ func createTestDBReverseTime(tb testing.TB, dbPath string, numTx int) { func openReverseTimeStore(t *testing.T, numTx int) *PacketStore { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "rev.db") + dbPath := postgresTestDSN(t) createTestDBReverseTime(t, dbPath, numTx) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/chunked_load_test.go b/cmd/server/chunked_load_test.go index c603acd3f..bd33c2aa4 100644 --- a/cmd/server/chunked_load_test.go +++ b/cmd/server/chunked_load_test.go @@ -19,20 +19,16 @@ package main import ( "net/http" "net/http/httptest" - "os" - "path/filepath" "testing" "time" ) func openChunkedTestStore(t *testing.T, numTx int) *PacketStore { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "chunked.db") + dbPath := postgresTestDSN(t) createTestDBAt(t, dbPath, numTx) - t.Cleanup(func() { os.RemoveAll(dir) }) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/collision_details_test.go b/cmd/server/collision_details_test.go index c59492e27..cbf6e931a 100644 --- a/cmd/server/collision_details_test.go +++ b/cmd/server/collision_details_test.go @@ -21,14 +21,14 @@ func TestCollisionDetailsIncludeNodePairs(t *testing.T) { // Add advert transmissions with hash_size=3 path bytes (0x80 = bits 10 → size 3) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0180aabb11ccdd', 'col_hash_01', ?, 1, 4, '{"pubKey":"aabb11ccdd001122","name":"Node Alpha","type":"ADVERT"}')`, recent) + VALUES ('0180aabb11ccdd', 'col_hash_01', $1, 1, 4, '{"pubKey":"aabb11ccdd001122","name":"Node Alpha","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -91, '["aabb11"]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -91, '["aabb11"]', $1)`, recentEpoch) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0180aabb11eeff', 'col_hash_02', ?, 1, 4, '{"pubKey":"aabb11eeff334455","name":"Node Beta","type":"ADVERT"}')`, recent) + VALUES ('0180aabb11eeff', 'col_hash_02', $1, 1, 4, '{"pubKey":"aabb11eeff334455","name":"Node Beta","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 9.0, -93, '["aabb11"]', ?)`, recentEpoch) + VALUES (2, 1, 9.0, -93, '["aabb11"]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() @@ -102,9 +102,9 @@ func TestCollisionDetailsEmptyWhenNoCollisions(t *testing.T) { db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES ('aabb11ccdd001122', 'Solo Node', 'repeater')`) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0180aabb11ccdd', 'solo_hash_01', ?, 1, 4, '{"pubKey":"aabb11ccdd001122","name":"Solo Node","type":"ADVERT"}')`, recent) + VALUES ('0180aabb11ccdd', 'solo_hash_01', $1, 1, 4, '{"pubKey":"aabb11ccdd001122","name":"Solo Node","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -91, '["aabb11"]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -91, '["aabb11"]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() diff --git a/cmd/server/config.go b/cmd/server/config.go index efa6ed279..d0de5abf2 100644 --- a/cmd/server/config.go +++ b/cmd/server/config.go @@ -37,10 +37,12 @@ type ListLimitsConfig struct { // Config mirrors the Node.js config.json structure (read-only fields). type Config struct { - Port int `json:"port"` - APIKey string `json:"apiKey"` - DBPath string `json:"dbPath"` - ListLimits *ListLimitsConfig `json:"listLimits"` + Port int `json:"port"` + APIKey string `json:"apiKey"` + DBPath string `json:"dbPath,omitempty"` // rejected legacy SQLite setting + DatabaseURL string `json:"databaseURL"` + StateDir string `json:"stateDir"` + ListLimits *ListLimitsConfig `json:"listLimits"` // NodeBlacklist is a list of public keys to exclude from all API responses. // Blacklisted nodes are hidden from node lists, search, detail, map, and stats. @@ -369,7 +371,7 @@ type RetentionConfig struct { MetricsDays int `json:"metricsDays"` } -// DBConfig is the shared SQLite vacuum/maintenance config (#919, #921). +// DBConfig shares startup-load settings and accepts obsolete SQLite config keys. type DBConfig = dbconfig.DBConfig // IncrementalVacuumPages returns the configured pages per vacuum or 1024 default. @@ -735,14 +737,27 @@ func (h HealthThresholds) ToClientMs() map[string]int { } } -func (c *Config) ResolveDBPath(baseDir string) string { - if c.DBPath != "" { - return c.DBPath +// ResolveDatabaseURL refuses legacy paths rather than interpreting them as DSNs. +func (c *Config) ResolveDatabaseURL() (string, error) { + if c.DBPath != "" || os.Getenv("DB_PATH") != "" { + return "", fmt.Errorf("dbPath and DB_PATH are no longer supported; import the SQLite database and configure databaseURL or CORESCOPE_DATABASE_URL") } - if v := os.Getenv("DB_PATH"); v != "" { + v := envOrValue(os.Getenv, "CORESCOPE_READER_DATABASE_URL", envOrValue(os.Getenv, "CORESCOPE_DATABASE_URL", c.DatabaseURL)) + if v == "" { + return "", fmt.Errorf("configure databaseURL or CORESCOPE_DATABASE_URL for the PostgreSQL telemetry reader") + } + return v, nil +} + +func (c *Config) ResolveStateDir(baseDir string) string { + v := envOrValue(os.Getenv, "CORESCOPE_STATE_DIR", c.StateDir) + if v == "" { + v = "data" + } + if filepath.IsAbs(v) { return v } - return filepath.Join(baseDir, "data", "meshcore.db") + return filepath.Join(baseDir, v) } func (c *Config) NormalizeTimestampConfig() { diff --git a/cmd/server/config_mode_test.go b/cmd/server/config_mode_test.go index 601ef7656..175b85a38 100644 --- a/cmd/server/config_mode_test.go +++ b/cmd/server/config_mode_test.go @@ -15,6 +15,11 @@ func TestSaveGeoFilterPreservesFileMode(t *testing.T) { if err := os.WriteFile(path, []byte(`{"apiKey":"secret-key-that-is-long-enough"}`+"\n"), 0600); err != nil { t.Fatal(err) } + before, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + want := before.Mode().Perm() gf := &GeoFilterConfig{Polygon: [][2]float64{{0, 0}, {0, 1}, {1, 1}, {1, 0}}} if err := SaveGeoFilter(dir, gf); err != nil { t.Fatalf("SaveGeoFilter: %v", err) @@ -23,18 +28,20 @@ func TestSaveGeoFilterPreservesFileMode(t *testing.T) { if err != nil { t.Fatal(err) } - if got := fi.Mode().Perm(); got != 0600 { - t.Fatalf("config.json mode after save = %o, want 0600", got) + if got := fi.Mode().Perm(); got != want { + t.Fatalf("config.json mode after save = %o, want %o", got, want) } // Default stays 0644 for a file that was 0644. if err := os.Chmod(path, 0644); err != nil { t.Fatal(err) } + before, _ = os.Stat(path) + want = before.Mode().Perm() if err := SaveGeoFilter(dir, nil); err != nil { t.Fatalf("SaveGeoFilter(nil): %v", err) } fi, _ = os.Stat(path) - if got := fi.Mode().Perm(); got != 0644 { - t.Fatalf("config.json mode after save = %o, want 0644", got) + if got := fi.Mode().Perm(); got != want { + t.Fatalf("config.json mode after save = %o, want %o", got, want) } } diff --git a/cmd/server/config_test.go b/cmd/server/config_test.go index b78968ab6..aa0727e1b 100644 --- a/cmd/server/config_test.go +++ b/cmd/server/config_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "os" "path/filepath" + "strings" "testing" ) @@ -320,33 +321,45 @@ func TestGetHealthMs(t *testing.T) { } } -func TestResolveDBPath(t *testing.T) { - t.Run("DBPath set", func(t *testing.T) { - cfg := &Config{DBPath: "/explicit/path.db"} - got := cfg.ResolveDBPath("/base") - if got != "/explicit/path.db" { - t.Errorf("expected /explicit/path.db, got %s", got) - } - }) - - t.Run("env var", func(t *testing.T) { - cfg := &Config{} - t.Setenv("DB_PATH", "/env/path.db") - got := cfg.ResolveDBPath("/base") - if got != "/env/path.db" { - t.Errorf("expected /env/path.db, got %s", got) - } - }) +func TestResolveDatabaseURL(t *testing.T) { + t.Setenv("CORESCOPE_READER_DATABASE_URL", "") + t.Setenv("DB_PATH", "") + t.Setenv("CORESCOPE_DATABASE_URL", "") + cfg := &Config{DatabaseURL: "postgresql://reader@localhost/telemetry"} + if got, err := cfg.ResolveDatabaseURL(); err != nil || got != cfg.DatabaseURL { + t.Fatalf("config URL: %v", err) + } + t.Setenv("CORESCOPE_DATABASE_URL", "postgresql://reader@localhost/envdb") + if got, err := cfg.ResolveDatabaseURL(); err != nil || got != "postgresql://reader@localhost/envdb" { + t.Fatalf("environment URL: %v", err) + } + t.Setenv("CORESCOPE_READER_DATABASE_URL", "postgresql://reader@localhost/role") + if got, err := cfg.ResolveDatabaseURL(); err != nil || got != "postgresql://reader@localhost/role" { + t.Fatalf("reader URL precedence: %v", err) + } + t.Setenv("CORESCOPE_READER_DATABASE_URL", "") + cfg.DBPath = "private.db" + if _, err := cfg.ResolveDatabaseURL(); err == nil || strings.Contains(err.Error(), cfg.DBPath) { + t.Fatalf("legacy path must be rejected without echoing input: %v", err) + } + cfg.DBPath = "" + t.Setenv("CORESCOPE_DATABASE_URL", "") + cfg.DatabaseURL = "" + if _, err := cfg.ResolveDatabaseURL(); err == nil { + t.Fatal("missing URL accepted") + } +} - t.Run("default", func(t *testing.T) { - cfg := &Config{} - t.Setenv("DB_PATH", "") - got := cfg.ResolveDBPath("/base") - expected := filepath.Join("/base", "data", "meshcore.db") - if got != expected { - t.Errorf("expected %s, got %s", expected, got) - } - }) +func TestResolveStateDir(t *testing.T) { + t.Setenv("CORESCOPE_STATE_DIR", "") + cfg := &Config{} + if got := cfg.ResolveStateDir("base"); got != filepath.Join("base", "data") { + t.Fatal(got) + } + t.Setenv("CORESCOPE_STATE_DIR", "queues") + if got := cfg.ResolveStateDir("base"); got != filepath.Join("base", "queues") { + t.Fatal(got) + } } func TestPropagationBufferMs(t *testing.T) { diff --git a/cmd/server/coverage_test.go b/cmd/server/coverage_test.go index 907ee448e..99be2e2b9 100644 --- a/cmd/server/coverage_test.go +++ b/cmd/server/coverage_test.go @@ -15,14 +15,13 @@ import ( "time" "github.com/gorilla/mux" - _ "github.com/mattn/go-sqlite3" ) // --- helpers --- func setupTestDBv2(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } @@ -31,38 +30,36 @@ func setupTestDBv2(t *testing.T) *DB { schema := ` CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, - lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, - battery_mv INTEGER, temperature_c REAL, foreign_advert INTEGER DEFAULT 0 + lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, + battery_mv INTEGER, temperature_c DOUBLE PRECISION, foreign_advert INTEGER DEFAULT 0 ); - CREATE TABLE observers ( + CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT, packet_count INTEGER DEFAULT 0, model TEXT, firmware TEXT, - client_version TEXT, radio TEXT, battery_mv INTEGER, uptime_secs INTEGER, noise_floor REAL, + client_version TEXT, radio TEXT, battery_mv INTEGER, uptime_secs INTEGER, noise_floor DOUBLE PRECISION, inactive INTEGER DEFAULT 0 ); CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT NOT NULL, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT NOT NULL, hash TEXT NOT NULL UNIQUE, first_seen TEXT NOT NULL, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, - decoded_json TEXT, channel_hash TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL, created_at TEXT DEFAULT (datetime('now')) + decoded_json TEXT, channel_hash TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL, created_at TEXT DEFAULT (to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')) ); CREATE TABLE observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER NOT NULL REFERENCES transmissions(id), observer_id TEXT, observer_name TEXT, direction TEXT, - snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp INTEGER NOT NULL, raw_hex TEXT + snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp INTEGER NOT NULL, raw_hex TEXT ); - CREATE TRIGGER IF NOT EXISTS test_from_pubkey_advert - AFTER INSERT ON transmissions - FOR EACH ROW - WHEN NEW.from_pubkey IS NULL AND NEW.payload_type = 4 AND NEW.decoded_json IS NOT NULL - AND json_extract(NEW.decoded_json, '$.pubKey') IS NOT NULL - AND json_extract(NEW.decoded_json, '$.pubKey') <> '' + CREATE FUNCTION test_from_pubkey_advert() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN - UPDATE transmissions - SET from_pubkey = json_extract(NEW.decoded_json, '$.pubKey') - WHERE id = NEW.id; - END; + IF NEW.from_pubkey IS NULL AND NEW.payload_type = 4 AND NEW.decoded_json IS JSON OBJECT THEN + NEW.from_pubkey := NULLIF(NEW.decoded_json::json ->> 'pubKey', ''); + END IF; + RETURN NEW; + END $$; + CREATE TRIGGER test_from_pubkey_advert BEFORE INSERT ON transmissions + FOR EACH ROW EXECUTE FUNCTION test_from_pubkey_advert(); CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey); ` if _, err := conn.Exec(schema); err != nil { @@ -78,13 +75,13 @@ func seedV2Data(t *testing.T, db *DB) { epoch := now.Add(-1 * time.Hour).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Obs One', 'SJC', ?, '2026-01-01T00:00:00Z', 100)`, recent) + VALUES ('obs1', 'Obs One', 'SJC', $1, '2026-01-01T00:00:00Z', 100)`, recent) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('aabbccdd11223344', 'TestRepeater', 'repeater', 37.5, -122.0, ?, '2026-01-01T00:00:00Z', 50)`, recent) + VALUES ('aabbccdd11223344', 'TestRepeater', 'repeater', 37.5, -122.0, $1, '2026-01-01T00:00:00Z', 50)`, recent) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('AABB', 'abc123def4567890', ?, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT"}')`, recent) + VALUES ('AABB', 'abc123def4567890', $1, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_id, observer_name, snr, rssi, path_json, timestamp) - VALUES (1, 'obs1', 'Obs One', 12.5, -90, '["aa","bb"]', ?)`, epoch) + VALUES (1, 'obs1', 'Obs One', 12.5, -90, '["aa","bb"]', $1)`, epoch) } func setupNoStoreServer(t *testing.T) (*Server, *mux.Router) { @@ -339,13 +336,13 @@ func TestIngestNewFromDB(t *testing.T) { // Insert a new transmission in DB now := time.Now().UTC().Format(time.RFC3339) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('EEFF', 'newhash123456abcd', ?, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now) + VALUES ('EEFF', 'newhash123456abcd', $1, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now) newTxID := 0 db.conn.QueryRow("SELECT MAX(id) FROM transmissions").Scan(&newTxID) // Add observation for the new transmission db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 10.0, -92, '["cc"]', ?)`, newTxID, time.Now().Unix()) + VALUES ($1, 1, 10.0, -92, '["cc"]', $2)`, newTxID, time.Now().Unix()) // Ingest broadcastMaps, newMax := store.IngestNewFromDB(initialMax, 100) @@ -409,11 +406,11 @@ func TestIngestNewFromDBv2(t *testing.T) { now := time.Now().UTC().Format(time.RFC3339) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('EEFF', 'v2newhash12345678', ?, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now) + VALUES ('EEFF', 'v2newhash12345678', $1, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now) newTxID := 0 db.conn.QueryRow("SELECT MAX(id) FROM transmissions").Scan(&newTxID) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_id, observer_name, snr, rssi, path_json, timestamp) - VALUES (?, 'obs1', 'Obs One', 10.0, -92, '["cc"]', ?)`, newTxID, time.Now().Unix()) + VALUES ($1, 'obs1', 'Obs One', 10.0, -92, '["cc"]', $2)`, newTxID, time.Now().Unix()) broadcastMaps, newMax := store.IngestNewFromDB(initialMax, 100) if newMax <= initialMax { @@ -729,15 +726,15 @@ func TestGetChannelMessagesDedupe(t *testing.T) { // Insert a duplicate channel message with the same hash as existing db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('DDEE', 'dupchannelhash1234', ?, 1, 5, '{"type":"CHAN","channel":"#test","text":"Hello: World","sender":"TestUser"}')`, recent) + VALUES ('DDEE', 'dupchannelhash1234', $1, 1, 5, '{"type":"CHAN","channel":"#test","text":"Hello: World","sender":"TestUser"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 11.0, -91, '["aa"]', ?)`, epoch) + VALUES (3, 1, 11.0, -91, '["aa"]', $1)`, epoch) // Insert another dupe same hash as above (should dedup) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('DDFF', 'dupchannelhash5678', ?, 1, 5, '{"type":"CHAN","channel":"#test","text":"Hello: World","sender":"TestUser"}')`, recent) + VALUES ('DDFF', 'dupchannelhash5678', $1, 1, 5, '{"type":"CHAN","channel":"#test","text":"Hello: World","sender":"TestUser"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (4, 2, 9.0, -93, '[]', ?)`, epoch) + VALUES (4, 2, 9.0, -93, '[]', $1)`, epoch) store := NewPacketStore(db, nil) store.Load() @@ -1130,9 +1127,9 @@ func TestPollerStartWithStore(t *testing.T) { // Insert new data while poller running now := time.Now().UTC().Format(time.RFC3339) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES ('FFEE', 'pollerhash12345678', ?, 1, 4)`, now) + VALUES ('FFEE', 'pollerhash12345678', $1, 1, 4)`, now) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES ((SELECT MAX(id) FROM transmissions), 1, 10.0, -92, '[]', ?)`, time.Now().Unix()) + VALUES ((SELECT MAX(id) FROM transmissions), 1, 10.0, -92, '[]', $1)`, time.Now().Unix()) time.Sleep(200 * time.Millisecond) poller.Stop() @@ -1374,7 +1371,7 @@ func TestBuildTransmissionWhereRFC3339(t *testing.T) { // PR #1187 r2: RFC3339 since/until MUST use observations.timestamp // subquery so re-observed packets (older first_seen but recent // observation) are still included. Anything else breaks semantics. - if !strings.Contains(where[0], "observations") || !strings.Contains(where[0], "timestamp >= ?") { + if !strings.Contains(where[0], "observations") || !strings.Contains(where[0], "timestamp >= $1") { t.Errorf("expected observations.timestamp subquery for RFC3339 since, got %q", where[0]) } }) @@ -1388,7 +1385,7 @@ func TestBuildTransmissionWhereRFC3339(t *testing.T) { if len(args) != 1 { t.Errorf("expected 1 arg, got %d", len(args)) } - if !strings.Contains(where[0], "observations") || !strings.Contains(where[0], "timestamp <= ?") { + if !strings.Contains(where[0], "observations") || !strings.Contains(where[0], "timestamp <= $1") { t.Errorf("expected observations.timestamp subquery for RFC3339 until, got %q", where[0]) } }) @@ -2188,21 +2185,21 @@ func setupRichTestDB(t *testing.T) *DB { // route_type 1 = FLOOD, path byte at position 1 (hex index 2..3) // header: 0x01 (route_type=1), path byte: 0x40 (hashSize bits=01 → size 2) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0140aabbccdd', 'hash_with_path_01', ?, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT"}')`, recent) + VALUES ('0140aabbccdd', 'hash_with_path_01', $1, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 10.0, -91, '["aabb","ccdd"]', ?)`, recentEpoch) + VALUES (3, 1, 10.0, -91, '["aabb","ccdd"]', $1)`, recentEpoch) // Another advert with 3-byte hash size: header 0x01, path byte 0x80 (bits=10 → size 3) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0180eeff0011', 'hash_with_path_02', ?, 1, 4, '{"pubKey":"eeff00112233aabb","name":"TestCompanion","type":"ADVERT"}')`, yesterday) + VALUES ('0180eeff0011', 'hash_with_path_02', $1, 1, 4, '{"pubKey":"eeff00112233aabb","name":"TestCompanion","type":"ADVERT"}')`, yesterday) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (4, 2, 8.5, -94, '["eeff","0011","2233"]', ?)`, yesterdayEpoch) + VALUES (4, 2, 8.5, -94, '["eeff","0011","2233"]', $1)`, yesterdayEpoch) // Another channel message with different sender for analytics db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('CC01', 'chan_msg_hash_001', ?, 1, 5, '{"type":"CHAN","channel":"#test","text":"User2: Another msg","sender":"User2","channelHash":"abc123"}')`, recent) + VALUES ('CC01', 'chan_msg_hash_001', $1, 1, 5, '{"type":"CHAN","channel":"#test","text":"User2: Another msg","sender":"User2","channelHash":"abc123"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (5, 1, 14.0, -88, '["aa"]', ?)`, recentEpoch) + VALUES (5, 1, 14.0, -88, '["aa"]', $1)`, recentEpoch) return db } @@ -2556,20 +2553,20 @@ func TestStoreGetAnalyticsChannelsNumericHash(t *testing.T) { // Insert GRP_TXT packets with numeric channelHash (matches decoder.js output) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('DD01', 'grp_num_hash_1', ?, 1, 5, '{"type":"GRP_TXT","channelHash":97,"channelHashHex":"61","decryptionStatus":"no_key"}')`, recent) + VALUES ('DD01', 'grp_num_hash_1', $1, 1, 5, '{"type":"GRP_TXT","channelHash":97,"channelHashHex":"61","decryptionStatus":"no_key"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (4, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (4, 1, 10.0, -90, '[]', $1)`, recentEpoch) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('DD02', 'grp_num_hash_2', ?, 1, 5, '{"type":"GRP_TXT","channelHash":42,"channelHashHex":"2A","decryptionStatus":"no_key"}')`, recent) + VALUES ('DD02', 'grp_num_hash_2', $1, 1, 5, '{"type":"GRP_TXT","channelHash":42,"channelHashHex":"2A","decryptionStatus":"no_key"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (5, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (5, 1, 10.0, -90, '[]', $1)`, recentEpoch) // Also a decrypted CHAN with numeric channelHash — use hash 198 which is the real hash for #general db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('DD03', 'chan_num_hash_3', ?, 1, 5, '{"type":"CHAN","channel":"general","channelHash":198,"channelHashHex":"C6","text":"hello","sender":"Alice"}')`, recent) + VALUES ('DD03', 'chan_num_hash_3', $1, 1, 5, '{"type":"CHAN","channel":"general","channelHash":198,"channelHashHex":"C6","text":"hello","sender":"Alice"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (6, 1, 12.0, -88, '[]', ?)`, recentEpoch) + VALUES (6, 1, 12.0, -88, '[]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() @@ -3220,64 +3217,64 @@ func TestGetNodeHashSizeInfoEdgeCases(t *testing.T) { // Observers db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Obs', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs1', 'Obs', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, recent) // Adverts with various edge cases // 1. Valid advert with pubKey db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0140aabbccdd', 'hs_valid_1', ?, 1, 4, '{"pubKey":"aabbccdd11223344","name":"NodeA","type":"ADVERT"}')`, recent) + VALUES ('0140aabbccdd', 'hs_valid_1', $1, 1, 4, '{"pubKey":"aabbccdd11223344","name":"NodeA","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -90, '[]', $1)`, recentEpoch) // 2. Short raw_hex (< 4 chars) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('01', 'hs_short_hex', ?, 1, 4, '{"pubKey":"eeff00112233aabb","name":"NodeB","type":"ADVERT"}')`, recent) + VALUES ('01', 'hs_short_hex', $1, 1, 4, '{"pubKey":"eeff00112233aabb","name":"NodeB","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (2, 1, 10.0, -90, '[]', $1)`, recentEpoch) // 3. Invalid hex in path byte position db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('01GGHHII', 'hs_bad_hex', ?, 1, 4, '{"pubKey":"1122334455667788","name":"NodeC","type":"ADVERT"}')`, recent) + VALUES ('01GGHHII', 'hs_bad_hex', $1, 1, 4, '{"pubKey":"1122334455667788","name":"NodeC","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (3, 1, 10.0, -90, '[]', $1)`, recentEpoch) // 4. Invalid JSON db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0140aabb', 'hs_bad_json', ?, 1, 4, 'not-json')`, recent) + VALUES ('0140aabb', 'hs_bad_json', $1, 1, 4, 'not-json')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (4, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (4, 1, 10.0, -90, '[]', $1)`, recentEpoch) // 5. JSON with public_key field instead of pubKey db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0180eeff', 'hs_alt_key', ?, 1, 4, '{"public_key":"aabbccdd11223344","name":"NodeA","type":"ADVERT"}')`, recent) + VALUES ('0180eeff', 'hs_alt_key', $1, 1, 4, '{"public_key":"aabbccdd11223344","name":"NodeA","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (5, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (5, 1, 10.0, -90, '[]', $1)`, recentEpoch) // 6. JSON with no pubKey at all db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('01C0ffee', 'hs_no_pk', ?, 1, 4, '{"name":"NodeZ","type":"ADVERT"}')`, recent) + VALUES ('01C0ffee', 'hs_no_pk', $1, 1, 4, '{"name":"NodeZ","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (6, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (6, 1, 10.0, -90, '[]', $1)`, recentEpoch) // 7. Empty decoded_json db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0140bbcc', 'hs_empty_json', ?, 1, 4, '')`, recent) + VALUES ('0140bbcc', 'hs_empty_json', $1, 1, 4, '')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (7, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (7, 1, 10.0, -90, '[]', $1)`, recentEpoch) // 8-10. Multiple adverts for same node with different hash sizes (flip-flop test) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0140dd01', 'hs_flip_1', ?, 1, 4, '{"pubKey":"ffff000011112222","name":"Flipper","type":"ADVERT"}')`, recent) + VALUES ('0140dd01', 'hs_flip_1', $1, 1, 4, '{"pubKey":"ffff000011112222","name":"Flipper","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (8, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (8, 1, 10.0, -90, '[]', $1)`, recentEpoch) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0180dd02', 'hs_flip_2', ?, 1, 4, '{"pubKey":"ffff000011112222","name":"Flipper","type":"ADVERT"}')`, recent) + VALUES ('0180dd02', 'hs_flip_2', $1, 1, 4, '{"pubKey":"ffff000011112222","name":"Flipper","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (9, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (9, 1, 10.0, -90, '[]', $1)`, recentEpoch) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0140dd03', 'hs_flip_3', ?, 1, 4, '{"pubKey":"ffff000011112222","name":"Flipper","type":"ADVERT"}')`, recent) + VALUES ('0140dd03', 'hs_flip_3', $1, 1, 4, '{"pubKey":"ffff000011112222","name":"Flipper","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (10, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (10, 1, 10.0, -90, '[]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() @@ -3321,7 +3318,7 @@ func TestHashSizeTransportRoutePathByteOffset(t *testing.T) { recentEpoch := now.Add(-1 * time.Hour).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Obs', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs1', 'Obs', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, recent) // Route type 0 (TRANSPORT_FLOOD): header=0x04 (payload_type=1, route_type=0) // 4 transport bytes + path byte at offset 5. @@ -3329,25 +3326,25 @@ func TestHashSizeTransportRoutePathByteOffset(t *testing.T) { // If bug is present, code reads byte 1 (0xAA) → hash_size bits = 10 → size 3 (coincidence) // Use path byte 0x40 (hash_size=2) and transport byte 0x01 at offset 1 (hash_size=1 if misread) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('100102030440aabb', 'tf_offset', ?, 0, 4, '{"pubKey":"aaaa000000000001","name":"TF-Node","type":"ADVERT"}')`, recent) + VALUES ('100102030440aabb', 'tf_offset', $1, 0, 4, '{"pubKey":"aaaa000000000001","name":"TF-Node","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -90, '[]', $1)`, recentEpoch) // Route type 3 (TRANSPORT_DIRECT): header=0x13 (payload_type=4, route_type=3) // 4 transport bytes + path byte at offset 5. // Path byte 0xC1 → hash_size bits = 11 → size 4, hop_count = 1 (not zero-hop) // Byte 1 = 0x05 → hash_size bits = 00 → size 1 if misread db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1305060708C1bbcc', 'td_offset', ?, 3, 4, '{"pubKey":"aaaa000000000002","name":"TD-Node","type":"ADVERT"}')`, recent) + VALUES ('1305060708C1bbcc', 'td_offset', $1, 3, 4, '{"pubKey":"aaaa000000000002","name":"TD-Node","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (2, 1, 10.0, -90, '[]', $1)`, recentEpoch) // Route type 1 (FLOOD): header=0x11 (payload_type=4, route_type=1) // Path byte at offset 1. Path byte 0x80 → hash_size bits = 10 → size 3 db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1180aabbccdd', 'flood_offset', ?, 1, 4, '{"pubKey":"aaaa000000000003","name":"Flood-Node","type":"ADVERT"}')`, recent) + VALUES ('1180aabbccdd', 'flood_offset', $1, 1, 4, '{"pubKey":"aaaa000000000003","name":"Flood-Node","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (3, 1, 10.0, -90, '[]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() @@ -3386,7 +3383,7 @@ func TestHashSizeTransportDirectZeroHopSkipped(t *testing.T) { recentEpoch := now.Add(-1 * time.Hour).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Obs', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs1', 'Obs', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, recent) // RouteDirect (2) zero-hop with an all-zero path byte: the sender wiped the // size bits, so there is nothing to read. Should be skipped. @@ -3394,23 +3391,23 @@ func TestHashSizeTransportDirectZeroHopSkipped(t *testing.T) { // a deliberate declaration — see TestZeroHopDirectAdvertDeclaredSizeIsEvidence // — so the skip is keyed on the byte's content, not on the route type.) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1200aabbccdd', 'direct_zh', ?, 2, 4, '{"pubKey":"bbbb000000000001","name":"Direct-ZH","type":"ADVERT"}')`, recent) + VALUES ('1200aabbccdd', 'direct_zh', $1, 2, 4, '{"pubKey":"bbbb000000000001","name":"Direct-ZH","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -90, '[]', $1)`, recentEpoch) // RouteTransportDirect (3) zero-hop: 4 transport bytes + all-zero path byte // Should ALSO be skipped (this was the missing case in #747) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('130102030400aabb', 'tdirect_zh', ?, 3, 4, '{"pubKey":"bbbb000000000002","name":"TDirect-ZH","type":"ADVERT"}')`, recent) + VALUES ('130102030400aabb', 'tdirect_zh', $1, 3, 4, '{"pubKey":"bbbb000000000002","name":"TDirect-ZH","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (2, 1, 10.0, -90, '[]', $1)`, recentEpoch) // RouteDirect (2) non-zero-hop: path byte 0x41 → hop_count=1 // Should NOT be skipped db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1241aabbccdd', 'direct_1h', ?, 2, 4, '{"pubKey":"bbbb000000000003","name":"Direct-1H","type":"ADVERT"}')`, recent) + VALUES ('1241aabbccdd', 'direct_1h', $1, 2, 4, '{"pubKey":"bbbb000000000003","name":"Direct-1H","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (3, 1, 10.0, -90, '[]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() @@ -3446,10 +3443,10 @@ func TestAnalyticsHashSizesZeroHopSkip(t *testing.T) { recentEpoch := now.Add(-1 * time.Hour).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Obs', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs1', 'Obs', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, recent) pk := "cccc000000000001" - db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES (?, 'ZH-Analytics', 'repeater')`, pk) + db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES ($1, 'ZH-Analytics', 'repeater')`, pk) decoded := `{"pubKey":"` + pk + `","name":"ZH-Analytics","type":"ADVERT"}` @@ -3457,16 +3454,16 @@ func TestAnalyticsHashSizesZeroHopSkip(t *testing.T) { // header 0x11 = route_type 1 (flood), payload_type 4 // pathByte 0x41 = hashSize bits 01 → size 2, hop_count 1 db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1141aabbccdd', 'az_flood', ?, 1, 4, ?)`, recent, decoded) + VALUES ('1141aabbccdd', 'az_flood', $1, 1, 4, $2)`, recent, decoded) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '["aabb"]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -90, '["aabb"]', $1)`, recentEpoch) // Second: a direct zero-hop advert with pathByte=0x00 → would give hashSize=1 // header 0x12 = route_type 2 (direct), payload_type 4 db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1200aabbccdd', 'az_direct', ?, 2, 4, ?)`, recent, decoded) + VALUES ('1200aabbccdd', 'az_direct', $1, 2, 4, $2)`, recent, decoded) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (2, 1, 10.0, -90, '[]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() @@ -3600,32 +3597,32 @@ func TestGetChannelMessagesDedupeRepeats(t *testing.T) { recentEpoch := now.Add(-1 * time.Hour).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Obs1', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs1', 'Obs1', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, recent) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs2', 'Obs2', 'LAX', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs2', 'Obs2', 'LAX', $1, '2026-01-01T00:00:00Z', 10)`, recent) // Insert two copies of same CHAN message (same hash, different observers) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('CC01', 'dedup_chan_1', ?, 1, 5, '{"type":"CHAN","channel":"#general","text":"Alice: hello","sender":"Alice"}')`, recent) + VALUES ('CC01', 'dedup_chan_1', $1, 1, 5, '{"type":"CHAN","channel":"#general","text":"Alice: hello","sender":"Alice"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 12.0, -88, '["aa"]', ?)`, recentEpoch) + VALUES (1, 1, 12.0, -88, '["aa"]', $1)`, recentEpoch) // Same sender + hash → different observation (simulates dedup) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('CC02', 'dedup_chan_1', ?, 1, 5, '{"type":"CHAN","channel":"#general","text":"Alice: hello","sender":"Alice"}')`, recent) + VALUES ('CC02', 'dedup_chan_1', $1, 1, 5, '{"type":"CHAN","channel":"#general","text":"Alice: hello","sender":"Alice"}')`, recent) // Note: won't load due to UNIQUE constraint on hash → tests the code path with single tx having multiple obs // Second different message db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('CC03', 'dedup_chan_2', ?, 1, 5, '{"type":"CHAN","channel":"#general","text":"Bob: world","sender":"Bob"}')`, recent) + VALUES ('CC03', 'dedup_chan_2', $1, 1, 5, '{"type":"CHAN","channel":"#general","text":"Bob: world","sender":"Bob"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 2, 10.0, -90, '["bb"]', ?)`, recentEpoch) + VALUES (2, 2, 10.0, -90, '["bb"]', $1)`, recentEpoch) // GRP_TXT (not CHAN) — should be skipped by GetChannelMessages db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('DD01', 'grp_msg_hash_1', ?, 1, 5, '{"type":"GRP_TXT","channelHash":"42","text":"encrypted"}')`, recent) + VALUES ('DD01', 'grp_msg_hash_1', $1, 1, 5, '{"type":"GRP_TXT","channelHash":"42","text":"encrypted"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (3, 1, 10.0, -90, '[]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() @@ -3947,16 +3944,16 @@ func TestIngestNewFromDBDuplicateObs(t *testing.T) { // Insert new transmission with same hash as existing (should merge into existing tx) now := time.Now().UTC().Format(time.RFC3339) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('AABB', 'dedup_test_hash_01', ?, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now) + VALUES ('AABB', 'dedup_test_hash_01', $1, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now) newTxID := 0 db.conn.QueryRow("SELECT MAX(id) FROM transmissions").Scan(&newTxID) // Add observation db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 11.0, -89, '["dd"]', ?)`, newTxID, time.Now().Unix()) + VALUES ($1, 1, 11.0, -89, '["dd"]', $2)`, newTxID, time.Now().Unix()) // Add duplicate observation (same observer_id + path_json) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 11.0, -89, '["dd"]', ?)`, newTxID, time.Now().Unix()) + VALUES ($1, 1, 11.0, -89, '["dd"]', $2)`, newTxID, time.Now().Unix()) _, newMax := store.IngestNewFromDB(initialMax, 100) if newMax <= initialMax { @@ -3989,7 +3986,7 @@ func TestIngestNewObservations(t *testing.T) { // Simulate a new observation arriving for the existing transmission AFTER // the poller has already advanced past its transmission ID db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 2, 5.0, -100, '["aa","bb","cc"]', ?)`, time.Now().Unix()) + VALUES (1, 2, 5.0, -100, '["aa","bb","cc"]', $1)`, time.Now().Unix()) // Verify IngestNewFromDB does NOT pick up the new observation (tx id hasn't changed) txMax := store.MaxTransmissionID() @@ -4027,7 +4024,7 @@ func TestIngestNewObservations(t *testing.T) { t.Run("dedup by observer+path", func(t *testing.T) { // Insert duplicate observation (same observer + path as existing) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 12.5, -90, '["aa","bb"]', ?)`, time.Now().Unix()) + VALUES (1, 1, 12.5, -90, '["aa","bb"]', $1)`, time.Now().Unix()) prevCount := initialTx.ObservationCount maps := store.IngestNewObservations(db.GetMaxObservationID()-1, 500) if initialTx.ObservationCount != prevCount { @@ -4061,7 +4058,7 @@ func TestIngestNewObservationsV2(t *testing.T) { // Add new observation for existing transmission db.conn.Exec(`INSERT INTO observations (transmission_id, observer_id, observer_name, snr, rssi, path_json, timestamp) - VALUES (1, 'obs2', 'Obs Two', 6.0, -98, '["dd","ee"]', ?)`, time.Now().Unix()) + VALUES (1, 'obs2', 'Obs Two', 6.0, -98, '["dd","ee"]', $1)`, time.Now().Unix()) newMaps := store.IngestNewObservations(maxObsID, 500) if len(newMaps) != 1 { @@ -4113,7 +4110,7 @@ func TestHandleNodeAnalyticsNameless(t *testing.T) { // Insert a node without a name db.conn.Exec(`INSERT INTO nodes (public_key, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('nameless_node_pk_1', 'repeater', 37.5, -122.0, ?, '2026-01-01', 1)`, + VALUES ('nameless_node_pk_1', 'repeater', 37.5, -122.0, $1, '2026-01-01', 1)`, time.Now().UTC().Format(time.RFC3339)) cfg := &Config{Port: 3000} @@ -4237,8 +4234,8 @@ func TestGetDBSizeStatsMemory(t *testing.T) { seedTestData(t, db) stats := db.GetDBSizeStats() - if stats["dbSizeMB"] != float64(0) { - t.Errorf("expected 0 for in-memory, got %v", stats["dbSizeMB"]) + if stats["dbSizeMB"].(float64) <= 0 { + t.Errorf("expected positive PostgreSQL database size, got %v", stats["dbSizeMB"]) } } @@ -4261,11 +4258,11 @@ func TestGetChannelMessagesAfterIngest(t *testing.T) { now := time.Now().UTC() nowStr := now.Format(time.RFC3339) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('FF01', 'newchannelmsg19800', ?, 1, 5, '{"type":"CHAN","channel":"#test","text":"NewUser: brand new message","sender":"NewUser"}')`, nowStr) + VALUES ('FF01', 'newchannelmsg19800', $1, 1, 5, '{"type":"CHAN","channel":"#test","text":"NewUser: brand new message","sender":"NewUser"}')`, nowStr) newTxID := 0 db.conn.QueryRow("SELECT MAX(id) FROM transmissions").Scan(&newTxID) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 12.0, -88, '[]', ?)`, newTxID, now.Unix()) + VALUES ($1, 1, 12.0, -88, '[]', $2)`, newTxID, now.Unix()) // Ingest the new data _, newMax := store.IngestNewFromDB(initialMax, 100) @@ -4559,8 +4556,8 @@ func TestBuildTransmissionWhereMultiObserver(t *testing.T) { t.Fatalf("expected 1 WHERE clause, got %d", len(where)) } clause := where[0] - if !strings.Contains(clause, "IN (?,?)") { - t.Errorf("expected IN (?,?) in clause, got: %s", clause) + if !strings.Contains(clause, "IN ($1,$2)") { + t.Errorf("expected IN ($1,$2) in clause, got: %s", clause) } if len(args) != 2 { t.Fatalf("expected 2 args, got %d", len(args)) @@ -4587,8 +4584,8 @@ func TestBuildTransmissionWhereMultiObserver(t *testing.T) { if len(where) != 1 { t.Fatalf("expected 1 WHERE clause, got %d", len(where)) } - if !strings.Contains(where[0], "IN (?)") { - t.Errorf("expected IN (?) for single observer, got: %s", where[0]) + if !strings.Contains(where[0], "IN ($1)") { + t.Errorf("expected IN ($1) for single observer, got: %s", where[0]) } if len(args) != 1 || args[0] != "obs1" { t.Errorf("expected [obs1], got %v", args) @@ -4612,7 +4609,7 @@ func TestDistanceIncrementalUpdate(t *testing.T) { // Insert a new observation with a different path to trigger an incremental update. maxObsID := db.GetMaxObservationID() db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 2, 5.0, -100, '["xx","yy","zz"]', ?)`, time.Now().Unix()) + VALUES (1, 2, 5.0, -100, '["xx","yy","zz"]', $1)`, time.Now().Unix()) store.IngestNewObservations(maxObsID, 500) @@ -4624,7 +4621,7 @@ func TestDistanceIncrementalUpdate(t *testing.T) { // Insert another observation with yet another path. maxObsID = db.GetMaxObservationID() db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 2, 7.0, -95, '["aa","bb","cc","dd"]', ?)`, time.Now().Unix()) + VALUES (1, 2, 7.0, -95, '["aa","bb","cc","dd"]', $1)`, time.Now().Unix()) store.IngestNewObservations(maxObsID, 500) @@ -4724,11 +4721,11 @@ func TestIngestTraceBroadcastIncludesPath(t *testing.T) { traceHex := "2502AABB010000000200000000DEADBEEF" now := time.Now().UTC().Format(time.RFC3339) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES (?, 'tracehash683test', ?, 1, 9, '')`, traceHex, now) + VALUES ($1, 'tracehash683test', $2, 1, 9, '')`, traceHex, now) newTxID := 0 db.conn.QueryRow("SELECT MAX(id) FROM transmissions").Scan(&newTxID) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 5.0, -100, '["aa"]', ?)`, newTxID, time.Now().Unix()) + VALUES ($1, 1, 5.0, -100, '["aa"]', $2)`, newTxID, time.Now().Unix()) broadcastMaps, _ := store.IngestNewFromDB(initialMax, 100) if len(broadcastMaps) < 1 { diff --git a/cmd/server/db.go b/cmd/server/db.go index 5851dce4f..4dcaa54af 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -7,17 +7,16 @@ import ( "fmt" "log" "math" - "os" "sort" "strings" "sync" "sync/atomic" "time" - _ "github.com/mattn/go-sqlite3" "github.com/meshcore-analyzer/dbschema" "github.com/meshcore-analyzer/geofilter" "github.com/meshcore-analyzer/packetpath" + "github.com/meshcore-analyzer/pgutil" "golang.org/x/sync/singleflight" ) @@ -36,12 +35,13 @@ const routeTypeTransportSQL = "route_type IN (0, 3)" // per MeshCore protocol (#1838). const routeTypeNonTransportSQL = "route_type IN (1, 2)" -// DB wraps a read-only connection to the MeshCore SQLite database. +// DB wraps a read-only connection to the CoreScope PostgreSQL database. type DB struct { advertEvidenceTable atomic.Bool advertEvidenceReadHook func() // test-only: immediately before a bulk mask query conn *sql.DB - path string // filesystem path to the database file + path string // PostgreSQL URL; never log or use as a filesystem path + stateDir string // local queues and small sidecars isV3 bool // v3 schema: observer_idx in observations (vs observer_id in v2) hasResolvedPath bool // observations table has resolved_path column hasObsRawHex bool // observations table has raw_hex column (#881) @@ -58,6 +58,13 @@ type DB struct { // startup probe can lose the race. See declaredRegionsTablePresent. declaredRegionsTableLate atomic.Bool + pgStatsMu sync.Mutex + pgStatsCache postgresDatabaseSample + pgStatsExpires time.Time + pgStatsSF singleflight.Group + pgStatsMissHook func() // test-only: after an initial cache miss + pgStatsQueryHook func() // test-only: immediately before the shared sample query + // Channel list caches, keyed by region param — avoids repeated GROUP BY // scans (#762). Keyed per-region (not a single slot) so mixed-region // traffic doesn't evict and re-run the query on every request. @@ -128,42 +135,22 @@ type channelMessagesCacheEntry struct { exp time.Time } -// OpenDB opens a read-only SQLite connection. -// -// The DSN used to pass _journal_mode=WAL and _busy_timeout=5000, which -// modernc.org/sqlite silently ignored: it understood only the -// _pragma=name(value) form. Under github.com/mattn/go-sqlite3 those parameters -// are honoured, and setting journal_mode on a read-only handle is a write — it -// would succeed only because the database is already WAL. Both are gone: -// mattn's own busy_timeout default is already 5000ms, so the read handle keeps -// the timeout (which it had silently lacked) without the pointless write. -// -// What remains is deliberate. mode=ro is the read-only invariant from -// #1283/#1289 and works because mattn's C wrapper ORs SQLITE_OPEN_URI into the -// open flags — see TestOpenDBRefusesMissingDatabase, which fails if that ever -// stops holding. _cache_size pins SQLite's C-allocated page cache at 2 MiB per -// connection; it sits outside GOMEMLIMIT, so it is bounded here rather than -// left to the driver's default (see memlimit.go). -func OpenDB(path string) (*DB, error) { - dsn := fmt.Sprintf("file:%s?mode=ro&_cache_size=-2000", path) - conn, err := sql.Open("sqlite3", dsn) +// OpenDB opens a restricted PostgreSQL reader. Schema creation and writes are +// exclusively bootstrap/ingestor responsibilities. The session flag is defense +// in depth; effective grants are checked so changing it cannot enable writes. +func OpenDB(databaseURL string) (*DB, error) { + conn, err := pgutil.Open(databaseURL, true) if err != nil { return nil, err } conn.SetMaxOpenConns(4) - conn.SetMaxIdleConns(2) - if err := conn.Ping(); err != nil { + if err := pgutil.AssertReadOnly(conn); err != nil { conn.Close() - return nil, fmt.Errorf("ping failed: %w", err) - } - d := &DB{conn: conn, path: path} - // Detect the on-disk schema on a single pinned connection and fail loudly if - // it cannot be probed. A swallowed detection failure would silently cache the - // wrong schema mode (isV3=false against a v3 DB) for the entire process - // lifetime, breaking every read path until a manual restart (#1901). Aborting - // here lets the supervisor restart us, which clears any transient cause (e.g. - // a WAL recovery racing the read-only open). - ctx := context.Background() + return nil, err + } + d := &DB{conn: conn, path: databaseURL} + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() sc, err := conn.Conn(ctx) if err != nil { conn.Close() @@ -175,8 +162,6 @@ func OpenDB(path string) (*DB, error) { conn.Close() return nil, fmt.Errorf("schema detection failed: %w", derr) } - // Statements are prepared after schema detection so they can never be - // compiled against a schema mode that turned out to be wrong (#1901). if err := d.prepareStatements(); err != nil { conn.Close() return nil, fmt.Errorf("prepare statements: %w", err) @@ -195,7 +180,7 @@ func (db *DB) stmtQueryRow(stmt *sql.Stmt, fallbackSQL string, args ...interface } // prepareStatements creates prepared statements for frequently-called queries. -// SQLite compiles and caches the query plan once, avoiding re-parsing per request. +// PostgreSQL prepares these queries once per connection. func (db *DB) prepareStatements() error { var err error prepare := func(query string) *sql.Stmt { @@ -209,15 +194,15 @@ func (db *DB) prepareStatements() error { db.stmtCountTransmissions = prepare("SELECT COUNT(*) FROM transmissions") db.stmtCountObservations = prepare("SELECT COUNT(*) FROM observations") - db.stmtCountNodesActive = prepare("SELECT COUNT(*) FROM nodes WHERE last_seen > ?") + db.stmtCountNodesActive = prepare("SELECT COUNT(*) FROM nodes WHERE last_seen > $1") db.stmtCountNodesAll = prepare("SELECT COUNT(*) FROM nodes") db.stmtCountObservers = prepare("SELECT COUNT(*) FROM observers WHERE inactive IS NULL OR inactive = 0") - db.stmtCountObsLastHour = prepare("SELECT COUNT(*) FROM observations WHERE timestamp > ?") - db.stmtCountObsLastDay = prepare("SELECT COUNT(*) FROM observations WHERE timestamp > ?") - db.stmtNodeLookup = prepare("SELECT public_key FROM nodes WHERE public_key = ? OR name = ? LIMIT 1") - db.stmtTxByHash = prepare("SELECT id FROM transmissions WHERE hash = ?") - db.stmtCountNodesByRole = prepare("SELECT COUNT(*) FROM nodes WHERE role = ? AND last_seen > ?") - db.stmtCountNodesByRoleAll = prepare("SELECT COUNT(*) FROM nodes WHERE role = ?") + db.stmtCountObsLastHour = prepare("SELECT COUNT(*) FROM observations WHERE timestamp > $1") + db.stmtCountObsLastDay = prepare("SELECT COUNT(*) FROM observations WHERE timestamp > $1") + db.stmtNodeLookup = prepare("SELECT public_key FROM nodes WHERE public_key = $1 OR name = $2 LIMIT 1") + db.stmtTxByHash = prepare("SELECT id FROM transmissions WHERE hash = $1") + db.stmtCountNodesByRole = prepare("SELECT COUNT(*) FROM nodes WHERE role = $1 AND last_seen > $2") + db.stmtCountNodesByRoleAll = prepare("SELECT COUNT(*) FROM nodes WHERE role = $1") db.stmtMaxTxID = prepare("SELECT COALESCE(MAX(id), 0) FROM transmissions") db.stmtMaxObsID = prepare("SELECT COALESCE(MAX(id), 0) FROM observations") return err @@ -238,11 +223,6 @@ func (db *DB) Close() error { s.Close() } } - // No WAL checkpoint here. The connection is opened read-only (mode=ro in - // OpenDB), so PRAGMA wal_checkpoint(TRUNCATE) always fails with "disk I/O - // error (778)" on it. Attempting it emitted a misleading storage-fault line - // on every shutdown that wasted incident investigation time (#1901); the - // ingestor (the writer) owns checkpointing. return db.conn.Close() } @@ -251,7 +231,7 @@ func (db *DB) Close() error { // It returns an error if any probe query fails. Previously these failures were // swallowed with a bare return, leaving isV3 (and the feature flags) at their // zero value for the whole process lifetime: a single transient failure of the -// first PRAGMA silently ran v2 SQL against a v3 database until the process was +// first schema probe silently ran v2 SQL against a v3 database until the process was // restarted (#1901). Detection is now all-or-nothing — on any probe error the // caller aborts startup so the supervisor can retry. func (db *DB) detectSchema(ctx context.Context, q rowQuerier) error { @@ -300,24 +280,18 @@ type rowQuerier interface { QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) } -// schemaColumns returns the set of column names present on the given table via -// PRAGMA table_info. Unlike the previous inline scans, a query or scan error is -// returned rather than swallowed (#1901). The table name is a trusted literal -// supplied by the caller, not user input. +// schemaColumns reads only the selected PostgreSQL schema's visible columns. func schemaColumns(ctx context.Context, q rowQuerier, table string) (map[string]bool, error) { - rows, err := q.QueryContext(ctx, fmt.Sprintf("PRAGMA table_info(%s)", table)) + rows, err := q.QueryContext(ctx, `SELECT column_name FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = $1`, table) if err != nil { return nil, err } defer rows.Close() cols := make(map[string]bool) for rows.Next() { - var cid int var name string - var ctype sql.NullString - var notnull, pk int - var dflt sql.NullString - if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil { + if err := rows.Scan(&name); err != nil { return nil, err } cols[name] = true @@ -348,7 +322,7 @@ func (db *DB) transmissionBaseSQL() (selectCols, observerJoin string) { o.snr, o.rssi, o.path_json, o.direction` observerJoin = `LEFT JOIN observations o ON o.id = ( SELECT id FROM observations WHERE transmission_id = t.id - ORDER BY length(COALESCE(path_json,'')) DESC LIMIT 1 + ORDER BY length(COALESCE(path_json,'')) DESC NULLS LAST LIMIT 1 ) LEFT JOIN observers obs ON obs.rowid = o.observer_idx` } else { @@ -358,7 +332,7 @@ func (db *DB) transmissionBaseSQL() (selectCols, observerJoin string) { o.snr, o.rssi, o.path_json, o.direction` observerJoin = `LEFT JOIN observations o ON o.id = ( SELECT id FROM observations WHERE transmission_id = t.id - ORDER BY length(COALESCE(path_json,'')) DESC LIMIT 1 + ORDER BY length(COALESCE(path_json,'')) DESC NULLS LAST LIMIT 1 ) LEFT JOIN observers obs2 ON obs2.id = o.observer_id` } @@ -480,7 +454,7 @@ type Observation struct { Direction *string `json:"direction"` SNR *float64 `json:"snr"` RSSI *float64 `json:"rssi"` - Score *int `json:"score"` + Score *float64 `json:"score"` Hash *string `json:"hash"` RouteType *int `json:"route_type"` PayloadType *int `json:"payload_type"` @@ -514,127 +488,51 @@ func (db *DB) GetStats() (*Stats, error) { db.stmtQueryRow(db.stmtCountObservations, "SELECT COUNT(*) FROM observations").Scan(&s.TotalObservations) // Node.js uses 7-day active nodes for totalNodes sevenDaysAgo := time.Now().Add(-7 * 24 * time.Hour).Format(time.RFC3339) - db.stmtQueryRow(db.stmtCountNodesActive, "SELECT COUNT(*) FROM nodes WHERE last_seen > ?", sevenDaysAgo).Scan(&s.TotalNodes) + db.stmtQueryRow(db.stmtCountNodesActive, "SELECT COUNT(*) FROM nodes WHERE last_seen > $1", sevenDaysAgo).Scan(&s.TotalNodes) db.stmtQueryRow(db.stmtCountNodesAll, "SELECT COUNT(*) FROM nodes").Scan(&s.TotalNodesAllTime) db.stmtQueryRow(db.stmtCountObservers, "SELECT COUNT(*) FROM observers WHERE inactive IS NULL OR inactive = 0").Scan(&s.TotalObservers) oneHourAgo := time.Now().Add(-1 * time.Hour).Unix() - db.stmtQueryRow(db.stmtCountObsLastHour, "SELECT COUNT(*) FROM observations WHERE timestamp > ?", oneHourAgo).Scan(&s.PacketsLastHour) + db.stmtQueryRow(db.stmtCountObsLastHour, "SELECT COUNT(*) FROM observations WHERE timestamp > $1", oneHourAgo).Scan(&s.PacketsLastHour) oneDayAgo := time.Now().Add(-24 * time.Hour).Unix() - db.stmtQueryRow(db.stmtCountObsLastDay, "SELECT COUNT(*) FROM observations WHERE timestamp > ?", oneDayAgo).Scan(&s.PacketsLast24h) + db.stmtQueryRow(db.stmtCountObsLastDay, "SELECT COUNT(*) FROM observations WHERE timestamp > $1", oneDayAgo).Scan(&s.PacketsLast24h) return s, nil } -// GetDBSizeStats returns SQLite file sizes and row counts (matching Node.js /api/perf sqlite shape). +// GetDBSizeStats retains the legacy map caller while using PostgreSQL metrics. func (db *DB) GetDBSizeStats() map[string]interface{} { - result := map[string]interface{}{} - - // DB file size - var dbSizeMB float64 - if db.path != "" && db.path != ":memory:" { - if info, err := os.Stat(db.path); err == nil { - dbSizeMB = math.Round(float64(info.Size())/1048576*10) / 10 - } - } - result["dbSizeMB"] = dbSizeMB - - // WAL file size - var walSizeMB float64 - if db.path != "" && db.path != ":memory:" { - if info, err := os.Stat(db.path + "-wal"); err == nil { - walSizeMB = math.Round(float64(info.Size())/1048576*10) / 10 - } + stats := db.GetDBSizeStatsTyped() + var size any + var rows map[string]int + if stats.DbSizeMB != nil { + size = *stats.DbSizeMB } - result["walSizeMB"] = walSizeMB - - // Freelist size via PRAGMA (matches Node.js: page_size * freelist_count) - var pageSize, freelistCount int64 - db.conn.QueryRow("PRAGMA page_size").Scan(&pageSize) - db.conn.QueryRow("PRAGMA freelist_count").Scan(&freelistCount) - freelistMB := math.Round(float64(pageSize*freelistCount)/1048576*10) / 10 - result["freelistMB"] = freelistMB - - // WAL checkpoint info (matches Node.js: PRAGMA wal_checkpoint(PASSIVE)) - var walBusy, walLog, walCheckpointed int - err := db.conn.QueryRow("PRAGMA wal_checkpoint(PASSIVE)").Scan(&walBusy, &walLog, &walCheckpointed) - if err == nil { - result["walPages"] = map[string]interface{}{ - "total": walLog, - "checkpointed": walCheckpointed, - "busy": walBusy, - } - } else { - result["walPages"] = map[string]interface{}{ - "total": 0, - "checkpointed": 0, - "busy": 0, - } + if stats.Rows != nil { + rows = map[string]int{"transmissions": stats.Rows.Transmissions, + "observations": stats.Rows.Observations, "nodes": stats.Rows.Nodes, + "observers": stats.Rows.Observers} } - - // Row counts per table - rows := map[string]int{} - for _, table := range []string{"transmissions", "observations", "nodes", "observers"} { - var count int - db.conn.QueryRow("SELECT COUNT(*) FROM " + table).Scan(&count) - rows[table] = count + return map[string]interface{}{ + "engine": stats.Engine, "dbSizeMB": size, "rows": rows, "error": stats.Error, } - result["rows"] = rows - - return result } -// GetDBSizeStatsTyped returns SQLite file sizes and row counts as a typed struct. -func (db *DB) GetDBSizeStatsTyped() SqliteStats { - result := SqliteStats{} - - if db.path != "" && db.path != ":memory:" { - if info, err := os.Stat(db.path); err == nil { - result.DbSizeMB = math.Round(float64(info.Size())/1048576*10) / 10 - } +// GetDBSizeStatsTyped reports native PostgreSQL database size and table counts. +// WAL is cluster-wide and cannot be attributed to this database by a reader. +func (db *DB) GetDBSizeStatsTyped() PostgresStats { + sample, err := db.postgresStats(context.Background()) + result := PostgresStats{ + Engine: "postgresql", SampledAt: postgresSampleStamp(sample), + SampleIntervalSeconds: int(postgresStatsTTL / time.Second), Stale: err != nil, } - - if db.path != "" && db.path != ":memory:" { - if info, err := os.Stat(db.path + "-wal"); err == nil { - result.WalSizeMB = math.Round(float64(info.Size())/1048576*10) / 10 - } - } - - var pageSize, freelistCount int64 - db.conn.QueryRow("PRAGMA page_size").Scan(&pageSize) - db.conn.QueryRow("PRAGMA freelist_count").Scan(&freelistCount) - result.FreelistMB = math.Round(float64(pageSize*freelistCount)/1048576*10) / 10 - - var walBusy, walLog, walCheckpointed int - err := db.conn.QueryRow("PRAGMA wal_checkpoint(PASSIVE)").Scan(&walBusy, &walLog, &walCheckpointed) - if err == nil { - result.WalPages = &WalPages{ - Total: walLog, - Checkpointed: walCheckpointed, - Busy: walBusy, - } - } else { - result.WalPages = &WalPages{} - } - - rows := &SqliteRowCounts{} - for _, table := range []string{"transmissions", "observations", "nodes", "observers"} { - var count int - db.conn.QueryRow("SELECT COUNT(*) FROM " + table).Scan(&count) - switch table { - case "transmissions": - rows.Transmissions = count - case "observations": - rows.Observations = count - case "nodes": - rows.Nodes = count - case "observers": - rows.Observers = count - } + if sample.SampledAt.IsZero() { + result.Error = "database diagnostics unavailable" + return result } - result.Rows = rows - + mb := math.Round(float64(sample.DatabaseBytes)/1048576*10) / 10 + result.DbSizeMB, result.Rows = &mb, &sample.Rows return result } @@ -644,7 +542,7 @@ func (db *DB) GetRoleCounts() map[string]int { counts := map[string]int{} for _, role := range []string{"repeater", "room", "companion", "sensor"} { var c int - db.stmtQueryRow(db.stmtCountNodesByRole, "SELECT COUNT(*) FROM nodes WHERE role = ? AND last_seen > ?", role, sevenDaysAgo).Scan(&c) + db.stmtQueryRow(db.stmtCountNodesByRole, "SELECT COUNT(*) FROM nodes WHERE role = $1 AND last_seen > $2", role, sevenDaysAgo).Scan(&c) counts[role+"s"] = c } return counts @@ -655,7 +553,7 @@ func (db *DB) GetAllRoleCounts() map[string]int { counts := map[string]int{} for _, role := range []string{"repeater", "room", "companion", "sensor"} { var c int - db.stmtQueryRow(db.stmtCountNodesByRoleAll, "SELECT COUNT(*) FROM nodes WHERE role = ?", role).Scan(&c) + db.stmtQueryRow(db.stmtCountNodesByRoleAll, "SELECT COUNT(*) FROM nodes WHERE role = $1", role).Scan(&c) counts[role+"s"] = c } return counts @@ -689,6 +587,7 @@ type PacketResult struct { // QueryPackets returns paginated, filtered packets as transmissions (matching Node.js shape). func (db *DB) QueryPackets(q PacketQuery) (*PacketResult, error) { + q.Offset = max(0, q.Offset) if q.Limit <= 0 { q.Limit = 50 } @@ -718,8 +617,8 @@ func (db *DB) QueryPackets(q PacketQuery) (*PacketResult, error) { // is what the packets page is showing. The `since=` filter still uses // first_seen / observation timestamp, preserving "received-by-radio since X." selectCols, observerJoin := db.transmissionBaseSQL() - querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s %s ORDER BY t.id %s LIMIT ? OFFSET ?", - selectCols, observerJoin, w, q.Order) + querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s %s ORDER BY t.id %s LIMIT $%d OFFSET $%d", + selectCols, observerJoin, w, q.Order, len(args)+1, len(args)+2) qArgs := make([]interface{}, len(args)) copy(qArgs, args) @@ -744,6 +643,7 @@ func (db *DB) QueryPackets(q PacketQuery) (*PacketResult, error) { // QueryGroupedPackets groups by hash (transmissions) — queries transmissions table directly for performance. func (db *DB) QueryGroupedPackets(q PacketQuery) (*PacketResult, error) { + q.Offset = max(0, q.Offset) if q.Limit <= 0 { q.Limit = 50 } @@ -779,32 +679,32 @@ func (db *DB) QueryGroupedPackets(q PacketQuery) (*PacketResult, error) { querySQL = fmt.Sprintf(`SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type, COALESCE((SELECT COUNT(*) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS count, COALESCE((SELECT COUNT(DISTINCT oi.observer_idx) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS observer_count, - COALESCE((SELECT MAX(strftime('%%Y-%%m-%%dT%%H:%%M:%%fZ', oi.timestamp, 'unixepoch')) FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen) AS latest, + COALESCE((SELECT MAX(to_char(to_timestamp(oi.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')) FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen) AS latest, obs.id AS observer_id, obs.name AS observer_name, COALESCE(obs.iata, '') AS observer_iata, o.snr, o.rssi, o.path_json, - COALESCE((SELECT GROUP_CONCAT(DISTINCT obi.iata) FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas`+scopeNameCol+` + COALESCE((SELECT string_agg(DISTINCT obi.iata, ',') FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas`+scopeNameCol+` FROM transmissions t LEFT JOIN observations o ON o.id = ( SELECT id FROM observations WHERE transmission_id = t.id - ORDER BY length(COALESCE(path_json,'')) DESC LIMIT 1 + ORDER BY length(COALESCE(path_json,'')) DESC NULLS LAST LIMIT 1 ) LEFT JOIN observers obs ON obs.rowid = o.observer_idx - %s ORDER BY latest DESC LIMIT ? OFFSET ?`, w) + %s ORDER BY latest DESC NULLS LAST LIMIT $%d OFFSET $%d`, w, len(args)+1, len(args)+2) } else { querySQL = fmt.Sprintf(`SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type, COALESCE((SELECT COUNT(*) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS count, COALESCE((SELECT COUNT(DISTINCT oi.observer_id) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS observer_count, - COALESCE((SELECT MAX(oi.timestamp) FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen) AS latest, + COALESCE((SELECT MAX(oi.timestamp)::text FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen) AS latest, o.observer_id, o.observer_name, COALESCE(obs2.iata, '') AS observer_iata, o.snr, o.rssi, o.path_json, - COALESCE((SELECT GROUP_CONCAT(DISTINCT obi.iata) FROM observations oi JOIN observers obi ON obi.id = oi.observer_id WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas`+scopeNameCol+` + COALESCE((SELECT string_agg(DISTINCT obi.iata, ',') FROM observations oi JOIN observers obi ON obi.id = oi.observer_id WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas`+scopeNameCol+` FROM transmissions t LEFT JOIN observations o ON o.id = ( SELECT id FROM observations WHERE transmission_id = t.id - ORDER BY length(COALESCE(path_json,'')) DESC LIMIT 1 + ORDER BY length(COALESCE(path_json,'')) DESC NULLS LAST LIMIT 1 ) LEFT JOIN observers obs2 ON obs2.id = o.observer_id - %s ORDER BY latest DESC LIMIT ? OFFSET ?`, w) + %s ORDER BY latest DESC NULLS LAST LIMIT $%d OFFSET $%d`, w, len(args)+1, len(args)+2) } qArgs := make([]interface{}, len(args)) @@ -860,7 +760,7 @@ func (db *DB) QueryGroupedPackets(q PacketQuery) (*PacketResult, error) { return &PacketResult{Packets: packets, Total: total}, nil } -// parseDistinctIatasCSV turns SQLite GROUP_CONCAT output ("SJC,SFO,OAK") into +// parseDistinctIatasCSV turns PostgreSQL string_agg output ("SJC,SFO,OAK") into // a sorted, deduped []string. Returns an empty (non-nil) slice when the input // is empty/nil so JSON serialization stays consistent (`[]` not `null`). func parseDistinctIatasCSV(v interface{}) []string { @@ -888,38 +788,38 @@ func (db *DB) buildPacketWhere(q PacketQuery) ([]string, []interface{}) { var args []interface{} if q.Type != nil { - where = append(where, "payload_type = ?") + where = append(where, fmt.Sprintf("payload_type = $%d", len(args)+1)) args = append(args, *q.Type) } if q.Route != nil { - where = append(where, "route_type = ?") + where = append(where, fmt.Sprintf("route_type = $%d", len(args)+1)) args = append(args, *q.Route) } if q.Observer != "" { - where = append(where, "observer_id = ?") + where = append(where, fmt.Sprintf("observer_id = $%d", len(args)+1)) args = append(args, q.Observer) } if q.Hash != "" { - where = append(where, "hash = ?") + where = append(where, fmt.Sprintf("hash = $%d", len(args)+1)) args = append(args, strings.ToLower(q.Hash)) } if q.Since != "" { - where = append(where, "timestamp > ?") + where = append(where, fmt.Sprintf("timestamp > $%d", len(args)+1)) args = append(args, q.Since) } if q.Until != "" { - where = append(where, "timestamp < ?") + where = append(where, fmt.Sprintf("timestamp < $%d", len(args)+1)) args = append(args, q.Until) } if q.Region != "" { - where = append(where, "observer_id IN (SELECT id FROM observers WHERE iata = ?)") + where = append(where, fmt.Sprintf("observer_id IN (SELECT id FROM observers WHERE iata = $%d)", len(args)+1)) args = append(args, q.Region) } if q.Node != "" { pk := db.resolveNodePubkey(q.Node) // #1143: exact-match on the dedicated from_pubkey column instead of // LIKE-on-JSON substring (adversarial spoof + same-name false positives). - where = append(where, "from_pubkey = ?") + where = append(where, fmt.Sprintf("from_pubkey = $%d", len(args)+1)) args = append(args, pk) } return where, args @@ -932,15 +832,15 @@ func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) { var args []interface{} if q.Type != nil { - where = append(where, "t.payload_type = ?") + where = append(where, fmt.Sprintf("t.payload_type = $%d", len(args)+1)) args = append(args, *q.Type) } if q.Route != nil { - where = append(where, "t.route_type = ?") + where = append(where, fmt.Sprintf("t.route_type = $%d", len(args)+1)) args = append(args, *q.Route) } if q.Hash != "" { - where = append(where, "t.hash = ?") + where = append(where, fmt.Sprintf("t.hash = $%d", len(args)+1)) args = append(args, strings.ToLower(q.Hash)) } if q.Since != "" { @@ -949,37 +849,36 @@ func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) { // but which have observations inside the window) are still included. // Non-RFC3339 falls back to t.first_seen string compare. if ts, err := time.Parse(time.RFC3339Nano, q.Since); err == nil { - where = append(where, "t.id IN (SELECT DISTINCT transmission_id FROM observations WHERE timestamp >= ?)") + where = append(where, fmt.Sprintf("t.id IN (SELECT DISTINCT transmission_id FROM observations WHERE timestamp >= $%d)", len(args)+1)) args = append(args, ts.Unix()) } else { - where = append(where, "t.first_seen > ?") + where = append(where, fmt.Sprintf("t.first_seen > $%d", len(args)+1)) args = append(args, q.Since) } } if q.Until != "" { if ts, err := time.Parse(time.RFC3339Nano, q.Until); err == nil { - where = append(where, "t.id IN (SELECT DISTINCT transmission_id FROM observations WHERE timestamp <= ?)") + where = append(where, fmt.Sprintf("t.id IN (SELECT DISTINCT transmission_id FROM observations WHERE timestamp <= $%d)", len(args)+1)) args = append(args, ts.Unix()) } else { - where = append(where, "t.first_seen < ?") + where = append(where, fmt.Sprintf("t.first_seen < $%d", len(args)+1)) args = append(args, q.Until) } } if q.Node != "" { pk := db.resolveNodePubkey(q.Node) // #1143: exact-match on dedicated from_pubkey column. - where = append(where, "t.from_pubkey = ?") + where = append(where, fmt.Sprintf("t.from_pubkey = $%d", len(args)+1)) args = append(args, pk) } if q.Channel != "" { // channel_hash column is indexed for payload_type = 5; filter is exact match. - where = append(where, "t.channel_hash = ?") + where = append(where, fmt.Sprintf("t.channel_hash = $%d", len(args)+1)) args = append(args, q.Channel) } if q.Observer != "" { ids := strings.Split(q.Observer, ",") - placeholders := strings.Repeat("?,", len(ids)) - placeholders = placeholders[:len(placeholders)-1] + placeholders := sqlPlaceholders(len(ids), len(args)+1) if db.isV3 { where = append(where, "EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.id IN ("+placeholders+"))") } else { @@ -991,9 +890,9 @@ func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) { } if q.Region != "" { if db.isV3 { - where = append(where, "EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.iata = ?)") + where = append(where, fmt.Sprintf("EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.iata = $%d)", len(args)+1)) } else { - where = append(where, "EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.id = oi.observer_id WHERE oi.transmission_id = t.id AND obi.iata = ?)") + where = append(where, fmt.Sprintf("EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.id = oi.observer_id WHERE oi.transmission_id = t.id AND obi.iata = $%d)", len(args)+1)) } args = append(args, q.Region) } @@ -1002,7 +901,7 @@ func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) { func (db *DB) resolveNodePubkey(nodeIDOrName string) string { var pk string - err := db.stmtQueryRow(db.stmtNodeLookup, "SELECT public_key FROM nodes WHERE public_key = ? OR name = ? LIMIT 1", nodeIDOrName, nodeIDOrName).Scan(&pk) + err := db.stmtQueryRow(db.stmtNodeLookup, "SELECT public_key FROM nodes WHERE public_key = $1 OR name = $2 LIMIT 1", nodeIDOrName, nodeIDOrName).Scan(&pk) if err != nil { return nodeIDOrName } @@ -1012,7 +911,7 @@ func (db *DB) resolveNodePubkey(nodeIDOrName string) string { // GetTransmissionByID fetches from transmissions table with observer data. func (db *DB) GetTransmissionByID(id int) (map[string]interface{}, error) { selectCols, observerJoin := db.transmissionBaseSQL() - querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.id = ?", selectCols, observerJoin) + querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.id = $1", selectCols, observerJoin) rows, err := db.conn.Query(querySQL, id) if err != nil { @@ -1028,7 +927,7 @@ func (db *DB) GetTransmissionByID(id int) (map[string]interface{}, error) { // GetPacketByHash fetches a transmission by content hash with observer data. func (db *DB) GetPacketByHash(hash string) (map[string]interface{}, error) { selectCols, observerJoin := db.transmissionBaseSQL() - querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.hash = ?", selectCols, observerJoin) + querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.hash = $1", selectCols, observerJoin) rows, err := db.conn.Query(querySQL, strings.ToLower(hash)) if err != nil { @@ -1046,7 +945,7 @@ func (db *DB) GetPacketByHash(hash string) (map[string]interface{}, error) { // when the in-memory PacketStore has pruned the entry but the DB still has it. func (db *DB) GetObservationsForHash(hash string) []map[string]interface{} { var txID int - err := db.stmtQueryRow(db.stmtTxByHash, "SELECT id FROM transmissions WHERE hash = ?", strings.ToLower(hash)).Scan(&txID) + err := db.stmtQueryRow(db.stmtTxByHash, "SELECT id FROM transmissions WHERE hash = $1", strings.ToLower(hash)).Scan(&txID) if err != nil { return nil } @@ -1076,13 +975,13 @@ func (db *DB) ObservationRawHexForHash(hash string) map[int]string { return nil } var txID int - if err := db.stmtQueryRow(db.stmtTxByHash, "SELECT id FROM transmissions WHERE hash = ?", + if err := db.stmtQueryRow(db.stmtTxByHash, "SELECT id FROM transmissions WHERE hash = $1", strings.ToLower(hash)).Scan(&txID); err != nil { return nil } rows, err := db.conn.Query( `SELECT id, raw_hex FROM observations - WHERE transmission_id = ? AND raw_hex IS NOT NULL AND raw_hex <> ''`, txID) + WHERE transmission_id = $1 AND raw_hex IS NOT NULL AND raw_hex <> ''`, txID) if err != nil { return nil } @@ -1122,7 +1021,7 @@ type NodeQuery struct { // GetNodes returns filtered, paginated node list. func (db *DB) GetNodes(nq NodeQuery) ([]map[string]interface{}, int, map[string]int, error) { - limit, offset := nq.Limit, nq.Offset + limit, offset := nq.Limit, max(0, nq.Offset) role, search, before, lastHeard, sortBy, region := nq.Role, nq.Search, nq.Before, nq.LastHeard, nq.SortBy, nq.Region if nq.RegionPubkeys != nil { region = "" @@ -1131,15 +1030,15 @@ func (db *DB) GetNodes(nq NodeQuery) ([]map[string]interface{}, int, map[string] var args []interface{} if role != "" { - where = append(where, "role = ?") + where = append(where, fmt.Sprintf("role = $%d", len(args)+1)) args = append(args, role) } if search != "" { - where = append(where, "name LIKE ?") + where = append(where, fmt.Sprintf("name ILIKE $%d ESCAPE ''", len(args)+1)) args = append(args, "%"+search+"%") } if before != "" { - where = append(where, "first_seen <= ?") + where = append(where, fmt.Sprintf("first_seen <= $%d", len(args)+1)) args = append(args, before) } if lastHeard != "" { @@ -1149,7 +1048,7 @@ func (db *DB) GetNodes(nq NodeQuery) ([]map[string]interface{}, int, map[string] } if ms, ok := durations[lastHeard]; ok { since := time.Now().Add(-time.Duration(ms) * time.Millisecond).Format(time.RFC3339) - where = append(where, "last_seen > ?") + where = append(where, fmt.Sprintf("last_seen > $%d", len(args)+1)) args = append(args, since) } } @@ -1160,7 +1059,7 @@ func (db *DB) GetNodes(nq NodeQuery) ([]map[string]interface{}, int, map[string] placeholders := make([]string, len(codes)) regionArgs := make([]interface{}, len(codes)) for i, c := range codes { - placeholders[i] = "?" + placeholders[i] = fmt.Sprintf("$%d", len(args)+i+1) regionArgs[i] = c } joinCond := "obs.rowid = o.observer_idx" @@ -1184,7 +1083,7 @@ func (db *DB) GetNodes(nq NodeQuery) ([]map[string]interface{}, int, map[string] } if nq.RegionPubkeys != nil { if len(nq.RegionPubkeys) == 0 { - where = append(where, "0") + where = append(where, "FALSE") } else { // One JSON parameter rather than one placeholder per key: a region // can hold thousands of nodes, and the IN list stays a primary-key @@ -1193,7 +1092,7 @@ func (db *DB) GetNodes(nq NodeQuery) ([]map[string]interface{}, int, map[string] if err != nil { return nil, 0, nil, err } - where = append(where, "public_key IN (SELECT value FROM json_each(?))") + where = append(where, fmt.Sprintf("public_key IN (SELECT value FROM json_array_elements_text($%d::json))", len(args)+1)) args = append(args, string(keysJSON)) } } @@ -1204,9 +1103,9 @@ func (db *DB) GetNodes(nq NodeQuery) ([]map[string]interface{}, int, map[string] } sortMap := map[string]string{ - "name": "name ASC", "lastSeen": "last_seen DESC", "packetCount": "advert_count DESC", + "name": "name ASC NULLS FIRST", "lastSeen": "last_seen DESC NULLS LAST", "packetCount": "advert_count DESC NULLS LAST", } - order := "last_seen DESC" + order := "last_seen DESC NULLS LAST" if s, ok := sortMap[sortBy]; ok { order = s } @@ -1218,7 +1117,7 @@ func (db *DB) GetNodes(nq NodeQuery) ([]map[string]interface{}, int, map[string] var total int db.conn.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM nodes %s", w), args...).Scan(&total) - querySQL := fmt.Sprintf("SELECT %s FROM nodes %s ORDER BY %s LIMIT ? OFFSET ?", db.nodeSelectCols(), w, order) + querySQL := fmt.Sprintf("SELECT %s FROM nodes %s ORDER BY %s LIMIT $%d OFFSET $%d", db.nodeSelectCols(), w, order, len(args)+1, len(args)+2) qArgs := append(args, limit, offset) rows, err := db.conn.Query(querySQL, qArgs...) @@ -1244,7 +1143,7 @@ func (db *DB) SearchNodes(query string, limit int) ([]map[string]interface{}, er if limit <= 0 { limit = 10 } - rows, err := db.conn.Query(fmt.Sprintf("SELECT %s FROM nodes WHERE name LIKE ? OR public_key LIKE ? ORDER BY last_seen DESC LIMIT ?", db.nodeSelectCols()), + rows, err := db.conn.Query(fmt.Sprintf("SELECT %s FROM nodes WHERE name ILIKE $1 ESCAPE '' OR public_key ILIKE $2 ESCAPE '' ORDER BY last_seen DESC NULLS LAST LIMIT $3", db.nodeSelectCols()), "%"+query+"%", query+"%", limit) if err != nil { return nil, err @@ -1282,7 +1181,7 @@ func (db *DB) GetNodeByPrefix(prefix string) (map[string]interface{}, bool, erro } } rows, err := db.conn.Query( - fmt.Sprintf("SELECT %s FROM nodes WHERE public_key LIKE ? LIMIT 2", db.nodeSelectCols()), + fmt.Sprintf("SELECT %s FROM nodes WHERE public_key ILIKE $1 ESCAPE '' LIMIT 2", db.nodeSelectCols()), prefix+"%", ) if err != nil { @@ -1311,7 +1210,7 @@ func (db *DB) GetNodeByPrefix(prefix string) (map[string]interface{}, bool, erro // GetNodeByPubkey returns a single node. func (db *DB) GetNodeByPubkey(pubkey string) (map[string]interface{}, error) { - rows, err := db.conn.Query(fmt.Sprintf("SELECT %s FROM nodes WHERE public_key = ?", db.nodeSelectCols()), pubkey) + rows, err := db.conn.Query(fmt.Sprintf("SELECT %s FROM nodes WHERE public_key = $1", db.nodeSelectCols()), pubkey) if err != nil { return nil, err } @@ -1338,7 +1237,7 @@ func (db *DB) GetRecentTransmissionsForNode(pubkey string, limit int) ([]map[str // #1345: order by ingest id, not first_seen (=rxTime). Buffered observer // uploads with old rxTime would otherwise displace fresh activity from // the "recent transmissions for node" list. - querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.from_pubkey = ? ORDER BY t.id DESC LIMIT ?", + querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.from_pubkey = $1 ORDER BY t.id DESC LIMIT $2", selectCols, observerJoin) args := []interface{}{pubkey, limit} @@ -1403,14 +1302,14 @@ func (db *DB) getObservationsForTransmissions(txIDs []int) map[int][]map[string] placeholders := make([]string, len(txIDs)) args := make([]interface{}, len(txIDs)) for i, id := range txIDs { - placeholders[i] = "?" + placeholders[i] = fmt.Sprintf("$%d", i+1) args[i] = id } var querySQL string if db.isV3 { querySQL = fmt.Sprintf(`SELECT o.transmission_id, o.id, obs.id AS observer_id, obs.name AS observer_name, COALESCE(obs.iata, '') AS observer_iata, - o.direction, o.snr, o.rssi, o.path_json, strftime('%%Y-%%m-%%dT%%H:%%M:%%fZ', o.timestamp, 'unixepoch') AS obs_timestamp + o.direction, o.snr, o.rssi, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') AS obs_timestamp FROM observations o LEFT JOIN observers obs ON obs.rowid = o.observer_idx WHERE o.transmission_id IN (%s) @@ -1482,7 +1381,7 @@ func (db *DB) GetObservers() ([]Observer, error) { model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, last_packet_at, clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at, ` + canRelayClause + `, ` + canRelaySeenClause + ` - FROM observers WHERE inactive IS NULL OR inactive = 0 ORDER BY last_seen DESC`) + FROM observers WHERE inactive IS NULL OR inactive = 0 ORDER BY last_seen DESC NULLS LAST`) if err != nil { return nil, err } @@ -1600,7 +1499,7 @@ func (db *DB) GetObserverByID(id string) (*Observer, error) { model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, last_packet_at, clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at, `+canRelayClause+`, `+canRelaySeenClause+` - FROM observers WHERE id = ?`, id). + FROM observers WHERE id = $1`, id). Scan(&o.ID, &o.Name, &o.IATA, &o.LastSeen, &o.FirstSeen, &o.PacketCount, &o.Model, &o.Firmware, &o.ClientVersion, &o.Radio, &batteryMv, &uptimeSecs, &noiseFloor, &o.LastPacketAt, &clockSkewSec, &clockSkewCount, &o.ClockLastNaiveAt, &canRelay, &canRelaySeen) @@ -1640,7 +1539,7 @@ func (db *DB) GetObserverIdsForRegion(regionParam string) ([]string, error) { placeholders := make([]string, len(codes)) args := make([]interface{}, len(codes)) for i, c := range codes { - placeholders[i] = "?" + placeholders[i] = fmt.Sprintf("$%d", i+1) args[i] = c } rows, err := db.conn.Query(fmt.Sprintf("SELECT id FROM observers WHERE UPPER(TRIM(iata)) IN (%s)", strings.Join(placeholders, ",")), args...) @@ -1760,20 +1659,20 @@ func (db *DB) GetTraces(hash string) ([]map[string]interface{}, error) { var querySQL string if db.isV3 { querySQL = `SELECT obs.id AS observer_id, obs.name AS observer_name, - strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch') AS timestamp, + to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') AS timestamp, o.snr, o.rssi, o.path_json FROM observations o JOIN transmissions t ON t.id = o.transmission_id LEFT JOIN observers obs ON obs.rowid = o.observer_idx - WHERE t.hash = ? + WHERE t.hash = $1 ORDER BY o.timestamp ASC` } else { querySQL = `SELECT o.observer_id, o.observer_name, - strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch') AS timestamp, + to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') AS timestamp, o.snr, o.rssi, o.path_json FROM observations o JOIN transmissions t ON t.id = o.transmission_id - WHERE t.hash = ? + WHERE t.hash = $1 ORDER BY o.timestamp ASC` } rows, err := db.conn.Query(querySQL, strings.ToLower(hash)) @@ -1876,14 +1775,11 @@ func (db *DB) GetChannels(region ...string) ([]map[string]interface{}, error) { if len(regionCodes) > 0 { placeholders := make([]string, len(regionCodes)) for i, code := range regionCodes { - placeholders[i] = "?" + placeholders[i] = fmt.Sprintf("$%d", i+1) args = append(args, code) } regionPlaceholder := strings.Join(placeholders, ",") - // #1899: the sample_json subquery is region-scoped too, so its placeholders - // appear FIRST in the statement (it sits in the SELECT list, ahead of the - // WHERE). Bind the codes twice, subquery set first. - args = append(append(make([]interface{}, 0, len(regionCodes)*2), args...), args...) + // Reuse the same native parameters for sample and channel region filters. if db.isV3 { querySQL = fmt.Sprintf(`SELECT t.channel_hash, COUNT(*) AS msg_count, @@ -1902,7 +1798,7 @@ func (db *DB) GetChannels(region ...string) ([]map[string]interface{}, error) { AND t.channel_hash NOT LIKE 'enc_%%' AND obs.rowid IS NOT NULL AND UPPER(TRIM(obs.iata)) IN (%s) GROUP BY t.channel_hash - ORDER BY last_activity DESC`, regionPlaceholder, regionPlaceholder) + ORDER BY last_activity DESC NULLS LAST`, regionPlaceholder, regionPlaceholder) } else { querySQL = fmt.Sprintf(`SELECT t.channel_hash, COUNT(*) AS msg_count, @@ -1927,7 +1823,7 @@ func (db *DB) GetChannels(region ...string) ([]map[string]interface{}, error) { AND UPPER(TRIM(obs.iata)) IN (%s) ) GROUP BY t.channel_hash - ORDER BY last_activity DESC`, regionPlaceholder, regionPlaceholder) + ORDER BY last_activity DESC NULLS LAST`, regionPlaceholder, regionPlaceholder) } } else { querySQL = `SELECT channel_hash, @@ -1941,7 +1837,7 @@ func (db *DB) GetChannels(region ...string) ([]map[string]interface{}, error) { AND channel_hash IS NOT NULL AND channel_hash NOT LIKE 'enc_%%' GROUP BY channel_hash - ORDER BY last_activity DESC` + ORDER BY last_activity DESC NULLS LAST` } rows, err := db.conn.Query(querySQL, args...) @@ -2028,7 +1924,7 @@ func (db *DB) GetEncryptedChannels(region ...string) ([]map[string]interface{}, if len(regionCodes) > 0 { placeholders := make([]string, len(regionCodes)) for i, code := range regionCodes { - placeholders[i] = "?" + placeholders[i] = fmt.Sprintf("$%d", i+1) args = append(args, code) } regionPlaceholder := strings.Join(placeholders, ",") @@ -2043,7 +1939,7 @@ func (db *DB) GetEncryptedChannels(region ...string) ([]map[string]interface{}, AND t.channel_hash LIKE 'enc_%%' AND obs.rowid IS NOT NULL AND UPPER(TRIM(obs.iata)) IN (%s) GROUP BY t.channel_hash - ORDER BY last_activity DESC`, regionPlaceholder) + ORDER BY last_activity DESC NULLS LAST`, regionPlaceholder) } else { querySQL = fmt.Sprintf(`SELECT t.channel_hash, COUNT(*) AS msg_count, @@ -2058,7 +1954,7 @@ func (db *DB) GetEncryptedChannels(region ...string) ([]map[string]interface{}, AND UPPER(TRIM(obs.iata)) IN (%s) ) GROUP BY t.channel_hash - ORDER BY last_activity DESC`, regionPlaceholder) + ORDER BY last_activity DESC NULLS LAST`, regionPlaceholder) } } else { querySQL = `SELECT channel_hash, @@ -2068,7 +1964,7 @@ func (db *DB) GetEncryptedChannels(region ...string) ([]map[string]interface{}, WHERE payload_type = 5 AND channel_hash LIKE 'enc_%%' GROUP BY channel_hash - ORDER BY last_activity DESC` + ORDER BY last_activity DESC NULLS LAST` } rows, err := db.conn.Query(querySQL, args...) @@ -2165,7 +2061,7 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . if len(regionCodes) > 0 { placeholders := make([]string, len(regionCodes)) for i, code := range regionCodes { - placeholders[i] = "?" + placeholders[i] = fmt.Sprintf("$%d", i+2) regionArgs = append(regionArgs, code) } regionPlaceholders = strings.Join(placeholders, ",") @@ -2192,7 +2088,7 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . // 1) Total count (after region filter, before pagination). countSQL := `SELECT COUNT(*) FROM transmissions t - WHERE t.channel_hash = ? AND t.payload_type = 5` + regionFilter + WHERE t.channel_hash = $1 AND t.payload_type = 5` + regionFilter countArgs := []interface{}{channelHash} countArgs = append(countArgs, regionArgs...) var total int @@ -2222,16 +2118,16 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . pageSQL := `SELECT t.id, COALESCE((SELECT MAX(timestamp) FROM observations WHERE transmission_id = t.id), 0) AS latest_obs_epoch FROM transmissions t - WHERE t.channel_hash = ? AND t.payload_type = 5 - ORDER BY latest_obs_epoch DESC, t.id DESC - LIMIT ? OFFSET ?` + WHERE t.channel_hash = $1 AND t.payload_type = 5 + ORDER BY latest_obs_epoch DESC NULLS LAST, t.id DESC + LIMIT ` + fmt.Sprintf("$%d OFFSET $%d", len(regionArgs)+2, len(regionArgs)+3) if len(regionCodes) > 0 { pageSQL = `SELECT t.id, COALESCE((SELECT MAX(timestamp) FROM observations WHERE transmission_id = t.id), 0) AS latest_obs_epoch FROM transmissions t - WHERE t.channel_hash = ? AND t.payload_type = 5` + regionFilter + ` - ORDER BY latest_obs_epoch DESC, t.id DESC - LIMIT ? OFFSET ?` + WHERE t.channel_hash = $1 AND t.payload_type = 5` + regionFilter + ` + ORDER BY latest_obs_epoch DESC NULLS LAST, t.id DESC + LIMIT ` + fmt.Sprintf("$%d OFFSET $%d", len(regionArgs)+2, len(regionArgs)+3) } pageArgs := []interface{}{channelHash} pageArgs = append(pageArgs, regionArgs...) @@ -2263,7 +2159,7 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . idPlaceholders := make([]string, len(pageIDs)) obsArgs := make([]interface{}, len(pageIDs)) for i, id := range pageIDs { - idPlaceholders[i] = "?" + idPlaceholders[i] = fmt.Sprintf("$%d", i+1) obsArgs[i] = id } // #1851: scope_name lives on the transmission row, so appending it as the @@ -2424,13 +2320,17 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . return messages, total, nil } +// IDs are NOT NULL. Keep native index ordering so LIMIT stops the cursor +// after its page, without sorting every row beyond the cursor. +const newTransmissionsSinceSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, t.payload_type, t.payload_version, t.decoded_json + FROM transmissions t WHERE t.id > $1 ORDER BY t.id ASC LIMIT $2` + // GetNewTransmissionsSince returns new transmissions after a given ID for WebSocket polling. func (db *DB) GetNewTransmissionsSince(lastID int, limit int) ([]map[string]interface{}, error) { if limit <= 0 { limit = 100 } - rows, err := db.conn.Query(`SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, t.payload_type, t.payload_version, t.decoded_json - FROM transmissions t WHERE t.id > ? ORDER BY t.id ASC LIMIT ?`, lastID, limit) + rows, err := db.conn.Query(newTransmissionsSinceSQL, lastID, limit) if err != nil { return nil, err } @@ -2479,12 +2379,12 @@ func (db *DB) GetObserverPacketCounts(sinceEpoch int64) map[string]int { rows, err = db.conn.Query(`SELECT obs.id, COUNT(*) as cnt FROM observations o JOIN observers obs ON obs.rowid = o.observer_idx - WHERE o.timestamp > ? + WHERE o.timestamp > $1 GROUP BY obs.id`, sinceEpoch) } else { rows, err = db.conn.Query(`SELECT o.observer_id, COUNT(*) as cnt FROM observations o - WHERE o.observer_id IS NOT NULL AND o.timestamp > ? + WHERE o.observer_id IS NOT NULL AND o.timestamp > $1 GROUP BY o.observer_id`, sinceEpoch) } if err != nil { @@ -2532,7 +2432,7 @@ func (db *DB) GetNodeLocationsByKeys(keys []string) map[string]map[string]interf placeholders := make([]string, len(keys)) args := make([]interface{}, len(keys)) for i, k := range keys { - placeholders[i] = "?" + placeholders[i] = fmt.Sprintf("$%d", i+1) args[i] = strings.ToLower(k) } // #1481 P0-3: drop LOWER(public_key) — that wrap is non-sargable and @@ -2560,6 +2460,7 @@ func (db *DB) GetNodeLocationsByKeys(keys []string) map[string]map[string]interf // QueryMultiNodePackets returns transmissions referencing any of the given pubkeys. func (db *DB) QueryMultiNodePackets(pubkeys []string, limit, offset int, order, since, until string) (*PacketResult, error) { + offset = max(0, offset) if len(pubkeys) == 0 { return &PacketResult{Packets: []map[string]interface{}{}, Total: 0}, nil } @@ -2577,17 +2478,17 @@ func (db *DB) QueryMultiNodePackets(pubkeys []string, limit, offset int, order, for _, pk := range pubkeys { resolved := db.resolveNodePubkey(pk) args = append(args, resolved) - placeholders = append(placeholders, "?") + placeholders = append(placeholders, fmt.Sprintf("$%d", len(args))) } pkWhere := "t.from_pubkey IN (" + strings.Join(placeholders, ",") + ")" var timeFilters []string if since != "" { - timeFilters = append(timeFilters, "t.first_seen >= ?") + timeFilters = append(timeFilters, fmt.Sprintf("t.first_seen >= $%d", len(args)+1)) args = append(args, since) } if until != "" { - timeFilters = append(timeFilters, "t.first_seen <= ?") + timeFilters = append(timeFilters, fmt.Sprintf("t.first_seen <= $%d", len(args)+1)) args = append(args, until) } @@ -2601,8 +2502,8 @@ func (db *DB) QueryMultiNodePackets(pubkeys []string, limit, offset int, order, selectCols, observerJoin := db.transmissionBaseSQL() // #1345: order by ingest id (see QueryPackets comment above). - querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s %s ORDER BY t.id %s LIMIT ? OFFSET ?", - selectCols, observerJoin, w, order) + querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s %s ORDER BY t.id %s LIMIT $%d OFFSET $%d", + selectCols, observerJoin, w, order, len(args)+1, len(args)+2) qArgs := make([]interface{}, len(args)) copy(qArgs, args) @@ -2629,8 +2530,8 @@ func (db *DB) QueryMultiNodePackets(pubkeys []string, limit, offset int, order, func scanPacketRow(rows *sql.Rows) map[string]interface{} { var id int var rawHex, ts, obsID, obsName, direction, hash, pathJSON, decodedJSON, createdAt sql.NullString - var snr, rssi sql.NullFloat64 - var score, routeType, payloadType, payloadVersion sql.NullInt64 + var snr, rssi, score sql.NullFloat64 + var routeType, payloadType, payloadVersion sql.NullInt64 if err := rows.Scan(&id, &rawHex, &ts, &obsID, &obsName, &direction, &snr, &rssi, &score, &hash, &routeType, &payloadType, &payloadVersion, &pathJSON, &decodedJSON, &createdAt); err != nil { return nil @@ -2644,7 +2545,7 @@ func scanPacketRow(rows *sql.Rows) map[string]interface{} { "direction": nullStr(direction), "snr": nullFloat(snr), "rssi": nullFloat(rssi), - "score": nullInt(score), + "score": nullFloat(score), "hash": nullStr(hash), "route_type": nullInt(routeType), "payload_type": nullInt(payloadType), @@ -2761,7 +2662,7 @@ func nullInt(ni sql.NullInt64) interface{} { // PruneOldPackets, PruneOldMetrics, and RemoveStaleObservers were // removed in #1283 — they are write operations and now live on the // ingestor's *Store (cmd/ingestor/maintenance.go and cmd/ingestor/db.go). -// The server is the read path; it must not hold the SQLite write lock. +// The server is the read path; it must not write telemetry. // MetricsSample represents a single row from observer_metrics with computed deltas. type MetricsSample struct { @@ -2815,35 +2716,35 @@ func (db *DB) GetObserverMetrics(observerID, since, until, resolution string, sa // post-reboot baseline, not the pre-reboot high-water mark. query = `SELECT ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv FROM ( SELECT - strftime('%Y-%m-%dT%H:00:00Z', timestamp) as ts, + to_char(timestamp::timestamptz AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:00:00"Z"') as ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv, - ROW_NUMBER() OVER (PARTITION BY observer_id, strftime('%Y-%m-%dT%H:00:00Z', timestamp) ORDER BY timestamp DESC) as rn - FROM observer_metrics WHERE observer_id = ?` + ROW_NUMBER() OVER (PARTITION BY observer_id, to_char(timestamp::timestamptz AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:00:00"Z"') ORDER BY timestamp DESC) as rn + FROM observer_metrics WHERE observer_id = $1` case "1d": bucketSizeSec = 86400 query = `SELECT ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv FROM ( SELECT - strftime('%Y-%m-%dT00:00:00Z', timestamp) as ts, + to_char(timestamp::timestamptz AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"00:00:00"Z"') as ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv, - ROW_NUMBER() OVER (PARTITION BY observer_id, strftime('%Y-%m-%dT00:00:00Z', timestamp) ORDER BY timestamp DESC) as rn - FROM observer_metrics WHERE observer_id = ?` + ROW_NUMBER() OVER (PARTITION BY observer_id, to_char(timestamp::timestamptz AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"00:00:00"Z"') ORDER BY timestamp DESC) as rn + FROM observer_metrics WHERE observer_id = $1` default: // "5m" or raw query = `SELECT timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv - FROM observer_metrics WHERE observer_id = ?` + FROM observer_metrics WHERE observer_id = $1` } if since != "" { - query += " AND timestamp >= ?" + query += fmt.Sprintf(" AND timestamp >= $%d", len(args)+1) args = append(args, since) } if until != "" { - query += " AND timestamp <= ?" + query += fmt.Sprintf(" AND timestamp <= $%d", len(args)+1) args = append(args, until) } switch resolution { case "1h", "1d": - query += ") WHERE rn = 1 ORDER BY ts ASC" + query += ") WHERE rn = 1 ORDER BY ts ASC NULLS FIRST" default: query += " ORDER BY timestamp ASC" } @@ -3013,7 +2914,7 @@ func (db *DB) GetMetricsSummary(since string) ([]MetricsSummaryRow, error) { SELECT observer_id, noise_floor, battery_mv, ROW_NUMBER() OVER (PARTITION BY observer_id ORDER BY timestamp DESC) as rn FROM observer_metrics - WHERE timestamp >= ? + WHERE timestamp >= $1 ) SELECT m.observer_id, o.name, COALESCE(o.iata, '') as iata, r.noise_floor as current_nf, @@ -3024,9 +2925,9 @@ func (db *DB) GetMetricsSummary(since string) ([]MetricsSummaryRow, error) { FROM observer_metrics m LEFT JOIN observers o ON o.id = m.observer_id LEFT JOIN ranked r ON r.observer_id = m.observer_id AND r.rn = 1 - WHERE m.timestamp >= ? - GROUP BY m.observer_id - ORDER BY max_nf DESC + WHERE m.timestamp >= $2 + GROUP BY m.observer_id, o.name, o.iata, r.noise_floor, r.battery_mv + ORDER BY max_nf DESC NULLS LAST ` rows, err := db.conn.Query(query, since, since) if err != nil { @@ -3049,7 +2950,7 @@ func (db *DB) GetMetricsSummary(since string) ([]MetricsSummaryRow, error) { // Fetch sparkline data (noise_floor series) for all observers in one query if len(result) > 0 { sparkQuery := `SELECT observer_id, noise_floor FROM observer_metrics - WHERE timestamp >= ? ORDER BY observer_id, timestamp ASC` + WHERE timestamp >= $1 ORDER BY observer_id, timestamp ASC` sparkRows, err := db.conn.Query(sparkQuery, since) if err != nil { return nil, err @@ -3091,17 +2992,17 @@ func (db *DB) GetDroppedPackets(limit int, observerID, nodePubkey string) ([]map var conditions []string var args []interface{} if observerID != "" { - conditions = append(conditions, "observer_id = ?") + conditions = append(conditions, fmt.Sprintf("observer_id = $%d", len(args)+1)) args = append(args, observerID) } if nodePubkey != "" { - conditions = append(conditions, "node_pubkey = ?") + conditions = append(conditions, fmt.Sprintf("node_pubkey = $%d", len(args)+1)) args = append(args, nodePubkey) } if len(conditions) > 0 { query += " WHERE " + strings.Join(conditions, " AND ") } - query += " ORDER BY dropped_at DESC LIMIT ?" + query += fmt.Sprintf(" ORDER BY dropped_at DESC NULLS LAST LIMIT $%d", len(args)+1) args = append(args, limit) rows, err := db.conn.Query(query, args...) @@ -3199,16 +3100,16 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { case "1h": since = time.Now().Add(-1 * time.Hour).UTC().Format(time.RFC3339) // 5-minute buckets - bucketExpr = `strftime('%Y-%m-%dT%H:', first_seen) || printf('%02d', (CAST(strftime('%M', first_seen) AS INTEGER) / 5) * 5) || ':00Z'` + bucketExpr = `to_char(date_bin('5 minutes', (CASE WHEN pg_input_is_valid(first_seen, 'timestamp with time zone') THEN first_seen::timestamptz END), '2000-01-01T00:00:00Z'::timestamptz) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')` case "7d": since = time.Now().Add(-7 * 24 * time.Hour).UTC().Format(time.RFC3339) // 6-hour buckets - bucketExpr = `strftime('%Y-%m-%dT', first_seen) || printf('%02d', (CAST(strftime('%H', first_seen) AS INTEGER) / 6) * 6) || ':00:00Z'` + bucketExpr = `to_char(date_bin('6 hours', (CASE WHEN pg_input_is_valid(first_seen, 'timestamp with time zone') THEN first_seen::timestamptz END), '2000-01-01T00:00:00Z'::timestamptz) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')` default: // "24h" window = "24h" since = time.Now().Add(-24 * time.Hour).UTC().Format(time.RFC3339) // 1-hour buckets - bucketExpr = `strftime('%Y-%m-%dT%H:00:00Z', first_seen)` + bucketExpr = `to_char((CASE WHEN pg_input_is_valid(first_seen, 'timestamp with time zone') THEN first_seen::timestamptz END) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:00:00"Z"')` } resp := &ScopeStatsResponse{Window: window} @@ -3221,7 +3122,7 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { COALESCE(SUM(CASE WHEN scope_name IS NULL THEN 1 ELSE 0 END), 0) AS unscoped, COALESCE(SUM(CASE WHEN scope_name = '' THEN 1 ELSE 0 END), 0) AS unknown_scope FROM transmissions - WHERE `+routeTypeTransportSQL+` AND first_seen >= ? + WHERE `+routeTypeTransportSQL+` AND first_seen >= $1 `, since) if err := row.Scan( &resp.Summary.TransportTotal, @@ -3239,7 +3140,7 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { var nonTransportUnscoped int if err := db.conn.QueryRow(` SELECT COUNT(*) FROM transmissions - WHERE `+routeTypeNonTransportSQL+` AND first_seen >= ? + WHERE `+routeTypeNonTransportSQL+` AND first_seen >= $1 `, since).Scan(&nonTransportUnscoped); err != nil { return nil, fmt.Errorf("scope non-transport count query: %w", err) } @@ -3249,9 +3150,9 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { rows, err := db.conn.Query(` SELECT scope_name, COUNT(*) AS cnt FROM transmissions - WHERE `+routeTypeTransportSQL+` AND scope_name IS NOT NULL AND scope_name != '' AND first_seen >= ? + WHERE `+routeTypeTransportSQL+` AND scope_name IS NOT NULL AND scope_name != '' AND first_seen >= $1 GROUP BY scope_name - ORDER BY cnt DESC + ORDER BY cnt DESC NULLS LAST `, since) if err != nil { return nil, fmt.Errorf("scope byRegion query: %w", err) @@ -3276,7 +3177,7 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { COUNT(scope_name) AS scoped, SUM(CASE WHEN scope_name IS NULL THEN 1 ELSE 0 END) AS unscoped FROM transmissions - WHERE `+routeTypeTransportSQL+` AND first_seen >= ? + WHERE `+routeTypeTransportSQL+` AND first_seen >= $1 GROUP BY bucket ORDER BY bucket `, bucketExpr) @@ -3311,9 +3212,9 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { SUM(CASE WHEN t.scope_name != '' THEN 1 ELSE 0 END) AS named FROM transmissions t LEFT JOIN nodes n ON n.public_key = t.from_pubkey - WHERE +t.payload_type = ? AND t.route_type IN (0, 1) AND t.first_seen >= ? + WHERE +t.payload_type = $1 AND t.route_type IN (0, 1) AND t.first_seen >= $2 GROUP BY 1 - ORDER BY COUNT(*) DESC, 1 + ORDER BY COUNT(*) DESC NULLS LAST, 1 `, payloadTypeAdvert, since) if err != nil { return nil, fmt.Errorf("scope advertsByRole query: %w", err) diff --git a/cmd/server/db_channel_messages_perf_test.go b/cmd/server/db_channel_messages_perf_test.go index b96fc8d42..9a7a0d1dd 100644 --- a/cmd/server/db_channel_messages_perf_test.go +++ b/cmd/server/db_channel_messages_perf_test.go @@ -37,39 +37,15 @@ func TestGetChannelMessagesPerfLargeChannel(t *testing.T) { const numTx = 1500 const obsPerTx = 50 - tx, err := db.conn.Begin() - if err != nil { - t.Fatal(err) - } - txStmt, err := tx.Prepare(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES (?, ?, ?, 1, 5, ?, '#perf')`) - if err != nil { - t.Fatal(err) - } - obsStmt, err := tx.Prepare(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 10.0, -90, '[]', ?)`) - if err != nil { - t.Fatal(err) - } base := time.Now().UTC().Add(-24 * time.Hour) - for i := 0; i < numTx; i++ { - ts := base.Add(time.Duration(i) * time.Second).Format(time.RFC3339) - hash := fmt.Sprintf("perfhash%08d", i) + copyTestRows(t, db.conn, "transmissions", []string{"id", "raw_hex", "hash", "first_seen", "route_type", "payload_type", "decoded_json", "channel_hash"}, numTx, func(i int) []any { body := fmt.Sprintf(`{"type":"CHAN","channel":"#perf","text":"Sender%d: msg %d","sender":"Sender%d"}`, i%10, i, i%10) - res, err := txStmt.Exec(fmt.Sprintf("%04X", i), hash, ts, body) - if err != nil { - t.Fatal(err) - } - txID, _ := res.LastInsertId() - for o := 0; o < obsPerTx; o++ { - if _, err := obsStmt.Exec(txID, base.Unix()+int64(i*100+o)); err != nil { - t.Fatal(err) - } - } - } - if err := tx.Commit(); err != nil { - t.Fatal(err) - } + return []any{i + 1, fmt.Sprintf("%04X", i), fmt.Sprintf("perfhash%08d", i), base.Add(time.Duration(i) * time.Second).Format(time.RFC3339), 1, 5, body, "#perf"} + }) + copyTestRows(t, db.conn, "observations", []string{"id", "transmission_id", "observer_idx", "snr", "rssi", "path_json", "timestamp"}, numTx*obsPerTx, func(n int) []any { + i, o := n/obsPerTx, n%obsPerTx + return []any{n + 1, i + 1, 1, 10.0, -90.0, "[]", base.Unix() + int64(i*100+o)} + }) // Warm-up call to amortize first-run prepare cost. if _, _, err := db.GetChannelMessages("#perf", 50, 0); err != nil { diff --git a/cmd/server/db_test.go b/cmd/server/db_test.go index 5f0d656f3..dd1703cec 100644 --- a/cmd/server/db_test.go +++ b/cmd/server/db_test.go @@ -10,18 +10,21 @@ import ( "strings" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // setupTestDB creates an in-memory SQLite database with the v3 schema. func setupTestDB(t testing.TB) *DB { + return setupTestDBAtURL(t, postgresTestDSN(t)) +} + +func setupTestDBAtURL(t testing.TB, dsn string) *DB { t.Helper() - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openFixtureSQL(dsn) if err != nil { t.Fatal(err) } - // Force single connection so all goroutines share the same in-memory DB + t.Cleanup(func() { conn.Close() }) + // One fixture connection keeps write and assertion order deterministic. conn.SetMaxOpenConns(1) // Create schema matching MeshCore Analyzer v3 @@ -30,17 +33,17 @@ func setupTestDB(t testing.TB) *DB { public_key TEXT PRIMARY KEY, name TEXT, role TEXT, - lat REAL, - lon REAL, + lat DOUBLE PRECISION, + lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, battery_mv INTEGER, - temperature_c REAL, + temperature_c DOUBLE PRECISION, foreign_advert INTEGER DEFAULT 0 ); - CREATE TABLE observers ( + CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT PRIMARY KEY, name TEXT, iata TEXT, @@ -53,7 +56,7 @@ func setupTestDB(t testing.TB) *DB { radio TEXT, battery_mv INTEGER, uptime_secs INTEGER, - noise_floor REAL, + noise_floor DOUBLE PRECISION, inactive INTEGER DEFAULT 0, last_packet_at TEXT DEFAULT NULL, clock_skew_seconds INTEGER DEFAULT NULL, @@ -62,7 +65,7 @@ func setupTestDB(t testing.TB) *DB { ); CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT NOT NULL, hash TEXT NOT NULL UNIQUE, first_seen TEXT NOT NULL, @@ -72,16 +75,16 @@ func setupTestDB(t testing.TB) *DB { decoded_json TEXT, channel_hash TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL, - created_at TEXT DEFAULT (datetime('now')) + created_at TEXT DEFAULT (to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')) ); CREATE TABLE observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER NOT NULL REFERENCES transmissions(id), observer_idx INTEGER, direction TEXT, - snr REAL, - rssi REAL, + snr DOUBLE PRECISION, + rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp INTEGER NOT NULL, @@ -92,7 +95,7 @@ func setupTestDB(t testing.TB) *DB { CREATE TABLE IF NOT EXISTS observer_metrics ( observer_id TEXT NOT NULL, timestamp TEXT NOT NULL, - noise_floor REAL, + noise_floor DOUBLE PRECISION, tx_air_secs INTEGER, rx_air_secs INTEGER, recv_errors INTEGER, @@ -115,17 +118,15 @@ func setupTestDB(t testing.TB) *DB { -- non-empty string. json_extract returns NULL for missing keys, so -- the explicit IS NOT NULL AND <> empty-string guard catches the empty-string -- case too. UPDATE only when we have something to write. - CREATE TRIGGER IF NOT EXISTS test_from_pubkey_advert - AFTER INSERT ON transmissions - FOR EACH ROW - WHEN NEW.from_pubkey IS NULL AND NEW.payload_type = 4 AND NEW.decoded_json IS NOT NULL - AND json_extract(NEW.decoded_json, '$.pubKey') IS NOT NULL - AND json_extract(NEW.decoded_json, '$.pubKey') <> '' + CREATE FUNCTION test_from_pubkey_advert() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN - UPDATE transmissions - SET from_pubkey = json_extract(NEW.decoded_json, '$.pubKey') - WHERE id = NEW.id; - END; + IF NEW.from_pubkey IS NULL AND NEW.payload_type = 4 AND NEW.decoded_json IS JSON OBJECT THEN + NEW.from_pubkey := NULLIF(NEW.decoded_json::json ->> 'pubKey', ''); + END IF; + RETURN NEW; + END $$; + CREATE TRIGGER test_from_pubkey_advert BEFORE INSERT ON transmissions + FOR EACH ROW EXECUTE FUNCTION test_from_pubkey_advert(); CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey); -- Mirror prod indexes from internal/dbschema/dbschema.go so query plans @@ -142,7 +143,7 @@ func setupTestDB(t testing.TB) *DB { t.Fatal(err) } - return &DB{conn: conn, isV3: true, hasResolvedPath: true} + return &DB{conn: conn, path: dsn, stateDir: t.TempDir(), isV3: true, hasResolvedPath: true} } func seedTestData(t *testing.T, db *DB) { @@ -157,37 +158,37 @@ func seedTestData(t *testing.T, db *DB) { // Seed observers db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Observer One', 'SJC', ?, '2026-01-01T00:00:00Z', 100)`, recent) + VALUES ('obs1', 'Observer One', 'SJC', $1, '2026-01-01T00:00:00Z', 100)`, recent) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs2', 'Observer Two', 'SFO', ?, '2026-01-01T00:00:00Z', 50)`, yesterday) + VALUES ('obs2', 'Observer Two', 'SFO', $1, '2026-01-01T00:00:00Z', 50)`, yesterday) // Seed nodes db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('aabbccdd11223344', 'TestRepeater', 'repeater', 37.5, -122.0, ?, '2026-01-01T00:00:00Z', 50)`, recent) + VALUES ('aabbccdd11223344', 'TestRepeater', 'repeater', 37.5, -122.0, $1, '2026-01-01T00:00:00Z', 50)`, recent) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('eeff00112233aabb', 'TestCompanion', 'companion', 37.6, -122.1, ?, '2026-01-01T00:00:00Z', 10)`, yesterday) + VALUES ('eeff00112233aabb', 'TestCompanion', 'companion', 37.6, -122.1, $1, '2026-01-01T00:00:00Z', 10)`, yesterday) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('1122334455667788', 'TestRoom', 'room', 37.4, -121.9, ?, '2026-01-01T00:00:00Z', 5)`, twoDaysAgo) + VALUES ('1122334455667788', 'TestRoom', 'room', 37.4, -121.9, $1, '2026-01-01T00:00:00Z', 5)`, twoDaysAgo) // Seed transmissions db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash, from_pubkey) - VALUES ('AABB', 'abc123def4567890', ?, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT","timestamp":1700000000,"timestampISO":"2023-11-14T22:13:20.000Z","signature":"abcdef","flags":{"isRepeater":true},"lat":37.5,"lon":-122.0}', '#test', 'aabbccdd11223344')`, recent) + VALUES ('AABB', 'abc123def4567890', $1, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT","timestamp":1700000000,"timestampISO":"2023-11-14T22:13:20.000Z","signature":"abcdef","flags":{"isRepeater":true},"lat":37.5,"lon":-122.0}', '#test', 'aabbccdd11223344')`, recent) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('CCDD', '1234567890abcdef', ?, 1, 5, '{"type":"CHAN","channel":"#test","text":"Hello: World","sender":"TestUser"}', '#test')`, yesterday) + VALUES ('CCDD', '1234567890abcdef', $1, 1, 5, '{"type":"CHAN","channel":"#test","text":"Hello: World","sender":"TestUser"}', '#test')`, yesterday) // Second ADVERT for same node with different hash_size (raw_hex byte 0x1F → hs=1 vs 0xBB → hs=3) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, from_pubkey) - VALUES ('AA1F', 'def456abc1230099', ?, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT","timestamp":1700000100,"timestampISO":"2023-11-14T22:14:40.000Z","signature":"fedcba","flags":{"isRepeater":true},"lat":37.5,"lon":-122.0}', 'aabbccdd11223344')`, yesterday) + VALUES ('AA1F', 'def456abc1230099', $1, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT","timestamp":1700000100,"timestampISO":"2023-11-14T22:14:40.000Z","signature":"fedcba","flags":{"isRepeater":true},"lat":37.5,"lon":-122.0}', 'aabbccdd11223344')`, yesterday) // Seed observations (use unix timestamps) // resolved_path contains full pubkeys parallel to path_json hops db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) - VALUES (1, 1, 12.5, -90, '["aa","bb"]', ?, '["aabbccdd11223344","eeff00112233aabb"]')`, recentEpoch) + VALUES (1, 1, 12.5, -90, '["aa","bb"]', $1, '["aabbccdd11223344","eeff00112233aabb"]')`, recentEpoch) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) - VALUES (1, 2, 8.0, -95, '["aa"]', ?, '["aabbccdd11223344"]')`, recentEpoch-100) + VALUES (1, 2, 8.0, -95, '["aa"]', $1, '["aabbccdd11223344"]')`, recentEpoch-100) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 15.0, -85, '[]', ?)`, yesterdayEpoch) + VALUES (2, 1, 15.0, -85, '[]', $1)`, yesterdayEpoch) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) - VALUES (3, 1, 10.0, -92, '["cc"]', ?, '["1122334455667788"]')`, yesterdayEpoch) + VALUES (3, 1, 10.0, -92, '["cc"]', $1, '["1122334455667788"]')`, yesterdayEpoch) } func TestGetStats(t *testing.T) { @@ -238,8 +239,8 @@ func TestGetDBSizeStats(t *testing.T) { stats := db.GetDBSizeStats() // In-memory DB has dbSizeMB=0 and walSizeMB=0 - if stats["dbSizeMB"] != float64(0) { - t.Errorf("expected dbSizeMB=0 for in-memory DB, got %v", stats["dbSizeMB"]) + if stats["dbSizeMB"].(float64) <= 0 { + t.Errorf("expected positive PostgreSQL database size, got %v", stats["dbSizeMB"]) } rows, ok := stats["rows"].(map[string]int) @@ -259,17 +260,12 @@ func TestGetDBSizeStats(t *testing.T) { t.Errorf("expected 2 observers rows, got %d", rows["observers"]) } - // Verify new PRAGMA-based fields - if _, ok := stats["freelistMB"]; !ok { - t.Error("expected freelistMB in DB size stats") - } - walPages, ok := stats["walPages"].(map[string]interface{}) - if !ok { - t.Fatal("expected walPages object in DB size stats") + if stats["engine"] != "postgresql" { + t.Fatal("database engine not identified") } - for _, key := range []string{"total", "checkpointed", "busy"} { - if _, ok := walPages[key]; !ok { - t.Errorf("expected %s in walPages", key) + for _, obsolete := range []string{"freelistMB", "walPages", "walSizeMB"} { + if _, ok := stats[obsolete]; ok { + t.Fatalf("invented SQLite metric %s", obsolete) } } } @@ -413,7 +409,7 @@ func TestGetObservers_ExcludesInactive(t *testing.T) { defer db.Close() seedTestData(t, db) // Mark obs2 inactive — soft delete simulating a stale-observer prune. - if _, err := db.conn.Exec(`UPDATE observers SET inactive = 1 WHERE id = ?`, "obs2"); err != nil { + if _, err := db.conn.Exec(`UPDATE observers SET inactive = 1 WHERE id = $1`, "obs2"); err != nil { t.Fatalf("update inactive: %v", err) } observers, err := db.GetObservers() @@ -455,7 +451,7 @@ func TestGetObserverLastPacketAt(t *testing.T) { // Set last_packet_at for obs1 ts := "2026-04-24T12:00:00Z" - db.conn.Exec(`UPDATE observers SET last_packet_at = ? WHERE id = ?`, ts, "obs1") + db.conn.Exec(`UPDATE observers SET last_packet_at = $1 WHERE id = $2`, ts, "obs1") // Verify via GetObservers observers, err := db.GetObservers() @@ -767,7 +763,7 @@ func TestGetObserverIdsForRegion(t *testing.T) { t.Run("case and trim normalization", func(t *testing.T) { db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs3', 'Observer Three', ' sjc ', ?, '2026-01-01T00:00:00Z', 1)`, time.Now().UTC().Format(time.RFC3339)) + VALUES ('obs3', 'Observer Three', ' sjc ', $1, '2026-01-01T00:00:00Z', 1)`, time.Now().UTC().Format(time.RFC3339)) ids, err := db.GetObserverIdsForRegion(" sjc ") if err != nil { t.Fatal(err) @@ -853,15 +849,15 @@ func TestGetChannelMessagesRegionFiltering(t *testing.T) { db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs1', 'Observer One', 'SJC')`) db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs2', 'Observer Two', ' sfo ')`) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('AA', 'chanregion0001', ?, 1, 5, + VALUES ('AA', 'chanregion0001', $1, 1, 5, '{"type":"CHAN","channel":"#region","text":"SjcUser: One","sender":"SjcUser"}', '#region')`, ts1) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('BB', 'chanregion0002', ?, 1, 5, + VALUES ('BB', 'chanregion0002', $1, 1, 5, '{"type":"CHAN","channel":"#region","text":"SfoUser: Two","sender":"SfoUser"}', '#region')`, ts2) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '[]', ?)`, epoch1) + VALUES (1, 1, 10.0, -90, '[]', $1)`, epoch1) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 2, 9.0, -91, '[]', ?)`, epoch2) + VALUES (2, 2, 9.0, -91, '[]', $1)`, epoch2) msgsSJC, totalSJC, err := db.GetChannelMessages("#region", 100, 0, " sjc ") if err != nil { @@ -1198,7 +1194,7 @@ func TestGetNodesFiltering(t *testing.T) { // where observations use observer_id TEXT instead of observer_idx INTEGER. func setupTestDBV2(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } @@ -1209,17 +1205,17 @@ func setupTestDBV2(t *testing.T) *DB { public_key TEXT PRIMARY KEY, name TEXT, role TEXT, - lat REAL, - lon REAL, + lat DOUBLE PRECISION, + lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, battery_mv INTEGER, - temperature_c REAL, + temperature_c DOUBLE PRECISION, foreign_advert INTEGER DEFAULT 0 ); - CREATE TABLE observers ( + CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT PRIMARY KEY, name TEXT, iata TEXT, @@ -1231,7 +1227,7 @@ func setupTestDBV2(t *testing.T) *DB { ); CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT NOT NULL, hash TEXT NOT NULL UNIQUE, first_seen TEXT NOT NULL, @@ -1241,34 +1237,32 @@ func setupTestDBV2(t *testing.T) *DB { decoded_json TEXT, channel_hash TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL, - created_at TEXT DEFAULT (datetime('now')) + created_at TEXT DEFAULT (to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')) ); CREATE TABLE observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER NOT NULL REFERENCES transmissions(id), observer_id TEXT, observer_name TEXT, direction TEXT, - snr REAL, - rssi REAL, + snr DOUBLE PRECISION, + rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp INTEGER NOT NULL, raw_hex TEXT ); - CREATE TRIGGER IF NOT EXISTS test_from_pubkey_advert - AFTER INSERT ON transmissions - FOR EACH ROW - WHEN NEW.from_pubkey IS NULL AND NEW.payload_type = 4 AND NEW.decoded_json IS NOT NULL - AND json_extract(NEW.decoded_json, '$.pubKey') IS NOT NULL - AND json_extract(NEW.decoded_json, '$.pubKey') <> '' + CREATE FUNCTION test_from_pubkey_advert() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN - UPDATE transmissions - SET from_pubkey = json_extract(NEW.decoded_json, '$.pubKey') - WHERE id = NEW.id; - END; + IF NEW.from_pubkey IS NULL AND NEW.payload_type = 4 AND NEW.decoded_json IS JSON OBJECT THEN + NEW.from_pubkey := NULLIF(NEW.decoded_json::json ->> 'pubKey', ''); + END IF; + RETURN NEW; + END $$; + CREATE TRIGGER test_from_pubkey_advert BEFORE INSERT ON transmissions + FOR EACH ROW EXECUTE FUNCTION test_from_pubkey_advert(); CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey); ` if _, err := conn.Exec(schema); err != nil { @@ -1288,19 +1282,19 @@ func TestGetNodesRegionFilterV2(t *testing.T) { // Seed observer with IATA code db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs-v2-1', 'V2 Observer', 'LAX', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs-v2-1', 'V2 Observer', 'LAX', $1, '2026-01-01T00:00:00Z', 10)`, recent) // Seed a node db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('v2pubkey11223344', 'V2Node', 'repeater', 34.0, -118.0, ?, '2026-01-01T00:00:00Z', 5)`, recent) + VALUES ('v2pubkey11223344', 'V2Node', 'repeater', 34.0, -118.0, $1, '2026-01-01T00:00:00Z', 5)`, recent) // Seed an ADVERT transmission for the node db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('AABB', 'v2hash0001', ?, 1, 4, '{"pubKey":"v2pubkey11223344","name":"V2Node","type":"ADVERT"}')`, recent) + VALUES ('AABB', 'v2hash0001', $1, 1, 4, '{"pubKey":"v2pubkey11223344","name":"V2Node","type":"ADVERT"}')`, recent) // Seed v2-style observation: observer_id references observers.id directly db.conn.Exec(`INSERT INTO observations (transmission_id, observer_id, observer_name, snr, rssi, path_json, timestamp) - VALUES (1, 'obs-v2-1', 'V2 Observer', 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (1, 'obs-v2-1', 'V2 Observer', 10.0, -90, '[]', $1)`, recentEpoch) t.Run("v2 region filter match", func(t *testing.T) { nodes, total, _, err := db.GetNodes(NodeQuery{Limit: 50, Region: "LAX"}) @@ -1411,9 +1405,9 @@ func TestGetNetworkStatusDateFormats(t *testing.T) { // Insert nodes with different date formats db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen) - VALUES ('node1111', 'NodeRFC', 'repeater', ?)`, time.Now().Format(time.RFC3339)) + VALUES ('node1111', 'NodeRFC', 'repeater', $1)`, time.Now().Format(time.RFC3339)) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen) - VALUES ('node2222', 'NodeSQL', 'companion', ?)`, time.Now().Format("2006-01-02 15:04:05")) + VALUES ('node2222', 'NodeSQL', 'companion', $1)`, time.Now().Format("2006-01-02 15:04:05")) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('node3333', 'NodeNull', 'room', NULL)`) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen) @@ -1451,15 +1445,14 @@ func TestGetNetworkStatusDateFormats(t *testing.T) { func TestOpenDBValid(t *testing.T) { // Create a real SQLite database file - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) // Create DB with a table using a writable connection first - conn, err := sql.Open("sqlite3", dbPath) + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } - _, err = conn.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT)`) + _, err = conn.Exec(`CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`) if err != nil { conn.Close() t.Fatal(err) @@ -1468,7 +1461,7 @@ func TestOpenDBValid(t *testing.T) { conn.Close() // Now test OpenDB (read-only) - database, err := OpenDB(dbPath) + database, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB failed: %v", err) } @@ -1493,16 +1486,15 @@ func TestOpenDBInvalidPath(t *testing.T) { // The existing ScopeStats tests set these flags manually — this test ensures // the flag-setting code itself is covered. func TestDetectSchemaScopeName(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "detect.db") + dbPath := postgresTestDSN(t) // Create file-based DB with the scope_name and default_scope columns. - conn, err := sql.Open("sqlite3", dbPath) + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } conn.SetMaxOpenConns(1) - if _, err := conn.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT, scope_name TEXT)`); err != nil { + if _, err := conn.Exec(`CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT, scope_name TEXT)`); err != nil { conn.Close() t.Fatalf("create transmissions: %v", err) } @@ -1510,14 +1502,14 @@ func TestDetectSchemaScopeName(t *testing.T) { conn.Close() t.Fatalf("create nodes: %v", err) } - if _, err := conn.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY)`); err != nil { + if _, err := conn.Exec(`CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY)`); err != nil { conn.Close() t.Fatalf("create observations: %v", err) } ensurePreparable(t, conn) conn.Close() - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -1531,19 +1523,19 @@ func TestDetectSchemaScopeName(t *testing.T) { } // Verify the flags stay false when the columns are absent. - dbPath2 := filepath.Join(dir, "detect2.db") - conn2, err := sql.Open("sqlite3", dbPath2) + dbPath2 := postgresTestDSN(t) + conn2, err := openFixtureSQL(dbPath2) if err != nil { t.Fatal(err) } conn2.SetMaxOpenConns(1) - conn2.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT)`) + conn2.Exec(`CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`) conn2.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) - conn2.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY)`) + conn2.Exec(`CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY)`) ensurePreparable(t, conn2) conn2.Close() - db2, err := OpenDB(dbPath2) + db2, err := openFixtureReader(t, dbPath2) if err != nil { t.Fatalf("OpenDB2: %v", err) } @@ -1813,6 +1805,9 @@ func TestNodeTelemetryFields(t *testing.T) { } func TestMain(m *testing.M) { + // Server fixtures need stable build metadata, not a Git subprocess per + // instance. TestResolveCommit still exercises the actual fallback. + Commit = "unit-test" os.Exit(m.Run()) } @@ -1825,13 +1820,13 @@ func TestGetObserverMetrics(t *testing.T) { t2 := now.Add(-1 * time.Hour).Format(time.RFC3339) t3 := now.Format(time.RFC3339) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv) VALUES (?, ?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv) VALUES ($1, $2, $3, $4, $5, $6, $7)", "obs1", t1, -112.5, 100, 500, 3, 3720) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors) VALUES (?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors) VALUES ($1, $2, $3, $4, $5, $6)", "obs1", t2, -110.0, 200, 800, 5) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors) VALUES (?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors) VALUES ($1, $2, $3, $4, $5, $6)", "obs1", t3, -108.0, 300, 1100, 8) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES (?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES ($1, $2, $3)", "obs2", t1, -115.0) // Query all for obs1 @@ -1882,11 +1877,11 @@ func TestGetMetricsSummary(t *testing.T) { t1 := now.Add(-2 * time.Hour).Format(time.RFC3339) t2 := now.Add(-1 * time.Hour).Format(time.RFC3339) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, battery_mv) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, battery_mv) VALUES ($1, $2, $3, $4)", "obs1", t1, -112.0, 3720) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES (?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES ($1, $2, $3)", "obs1", t2, -108.0) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES (?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES ($1, $2, $3)", "obs2", t1, -115.0) since := now.Add(-24 * time.Hour).Format(time.RFC3339) @@ -1932,7 +1927,7 @@ func TestObserverMetricsAPIEndpoints(t *testing.T) { now := time.Now().UTC() t1 := now.Add(-1 * time.Hour).Format(time.RFC3339) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES (?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES ($1, $2, $3)", "obs1", t1, -112.0) // Query directly to verify @@ -2056,13 +2051,13 @@ func TestGetObserverMetricsResolution(t *testing.T) { db := setupTestDB(t) seedTestData(t, db) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES ($1, $2, $3, $4)", "obs1", "2026-04-05T00:00:00Z", -112.0, 100) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES ($1, $2, $3, $4)", "obs1", "2026-04-05T00:05:00Z", -110.0, 200) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES ($1, $2, $3, $4)", "obs1", "2026-04-05T01:00:00Z", -108.0, 500) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES ($1, $2, $3, $4)", "obs1", "2026-04-05T01:05:00Z", -106.0, 600) // 5m resolution: all 4 rows @@ -2099,9 +2094,9 @@ func TestHourlyResolutionDeltasNotNull(t *testing.T) { // Two hourly buckets, each with one sample. With old MAX+hardcoded gap threshold, // the 3600s gap would exceed sampleInterval*2 (600s) and deltas would be null. - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", "obs_hr", "2026-04-05T10:00:00Z", -110.0, 100, 200, 5, 50, 100) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", "obs_hr", "2026-04-05T11:00:00Z", -108.0, 200, 400, 10, 80, 200) m, _, err := db.GetObserverMetrics("obs_hr", "2026-04-04T00:00:00Z", "", "1h", 300) @@ -2124,15 +2119,15 @@ func TestLastValuePreservesReboot(t *testing.T) { // Hour bucket with two samples: pre-reboot (high) and post-reboot (low). // With MAX(), the pre-reboot value wins and the reboot is hidden. // With LAST (latest timestamp), the post-reboot value wins. - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", "obs_rb", "2026-04-05T10:00:00Z", -110.0, 1000, 2000, 500, 400, 800) // pre-reboot baseline - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", "obs_rb", "2026-04-05T10:20:00Z", -110.0, 5000, 6000, 900, 700, 1200) // pre-reboot peak - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", "obs_rb", "2026-04-05T10:40:00Z", -110.0, 10, 20, 1, 5, 10) // post-reboot (counter reset) // Next hour bucket - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", "obs_rb", "2026-04-05T11:00:00Z", -108.0, 100, 120, 5, 20, 50) m, reboots, err := db.GetObserverMetrics("obs_rb", "2026-04-04T00:00:00Z", "", "1h", 300) @@ -2196,14 +2191,14 @@ func TestPerObservationRawHexEnrich(t *testing.T) { // Insert transmission with raw_hex txHex := "deadbeef" - db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen) VALUES (?, 'hash1', '2026-04-21T10:00:00Z')`, txHex) + db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen) VALUES ($1, 'hash1', '2026-04-21T10:00:00Z')`, txHex) // Insert two observations: A has its own raw_hex, B has NULL (historical) obsAHex := "c0ffee01" db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, raw_hex) - VALUES (1, ?, -5.0, -90.0, '[]', 1745236800, ?)`, rowA, obsAHex) + VALUES (1, $1, -5.0, -90.0, '[]', 1745236800, $2)`, rowA, obsAHex) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, ?, -3.0, -85.0, '["aabb"]', 1745236801)`, rowB) + VALUES (1, $1, -3.0, -85.0, '["aabb"]', 1745236801)`, rowB) store := NewPacketStore(db, nil) if err := store.Load(); err != nil { @@ -2237,7 +2232,7 @@ func TestPerObservationRawHexEnrich(t *testing.T) { } func TestGetScopeStats(t *testing.T) { - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatalf("sql.Open: %v", err) } @@ -2247,7 +2242,7 @@ func TestGetScopeStats(t *testing.T) { // Create minimal schema db.conn.Exec(`CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT, scope_name TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL @@ -2258,15 +2253,15 @@ func TestGetScopeStats(t *testing.T) { now := time.Now().UTC().Format(time.RFC3339) // Transport scoped, known region - db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('a', ?, 0, '#belgium')`, now) + db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('a', $1, 0, '#belgium')`, now) // Transport scoped, unknown - db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('b', ?, 0, '')`, now) + db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('b', $1, 0, '')`, now) // Transport unscoped (NULL) - db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('c', ?, 0, NULL)`, now) + db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('c', $1, 0, NULL)`, now) // Non-transport FLOOD (route_type=1) — inherently unscoped per MeshCore protocol (#1838) - db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('d', ?, 1, NULL)`, now) + db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('d', $1, 1, NULL)`, now) // Non-transport DIRECT (route_type=2) — inherently unscoped per MeshCore protocol (#1838) - db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('e', ?, 2, NULL)`, now) + db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('e', $1, 2, NULL)`, now) stats, err := db.GetScopeStats("24h") if err != nil { @@ -2297,7 +2292,7 @@ func TestGetScopeStats(t *testing.T) { // scope_name states (NULL is unscoped, an empty string is unknown scope, a // name is a named scope). func TestGetScopeStatsAdvertsByRole(t *testing.T) { - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatalf("sql.Open: %v", err) } @@ -2307,7 +2302,7 @@ func TestGetScopeStatsAdvertsByRole(t *testing.T) { for _, stmt := range []string{ `CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, scope_name TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL )`, @@ -2357,7 +2352,7 @@ func TestGetScopeStatsAdvertsByRole(t *testing.T) { } for _, r := range rows { if _, err := db.conn.Exec( - `INSERT INTO transmissions (hash, first_seen, route_type, payload_type, scope_name, from_pubkey) VALUES (?, ?, ?, ?, ?, ?)`, + `INSERT INTO transmissions (hash, first_seen, route_type, payload_type, scope_name, from_pubkey) VALUES ($1, $2, $3, $4, $5, $6)`, r.hash, r.seen, r.route, r.ptype, r.scope, r.fromKey, ); err != nil { t.Fatalf("seed %s: %v", r.hash, err) @@ -2381,14 +2376,14 @@ func TestGetScopeStatsAdvertsByRole(t *testing.T) { } func TestGetScopeStatsAdvertsByRoleEmpty(t *testing.T) { - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatalf("sql.Open: %v", err) } conn.SetMaxOpenConns(1) db := &DB{conn: conn, hasScopeName: true} defer db.conn.Close() - db.conn.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, first_seen TEXT, route_type INTEGER, payload_type INTEGER, scope_name TEXT, from_pubkey TEXT)`) + db.conn.Exec(`CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, first_seen TEXT, route_type INTEGER, payload_type INTEGER, scope_name TEXT, from_pubkey TEXT)`) db.conn.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, role TEXT)`) stats, err := db.GetScopeStats("1h") @@ -2419,12 +2414,12 @@ func TestLoadIndexesRelayHopsFromResolvedPath(t *testing.T) { const senderPubkey = "sender00112233445566" db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('FF01', 'relaytest0001hash', ?, 1, 4, ?)`, + VALUES ('FF01', 'relaytest0001hash', $1, 1, 4, $2)`, now, `{"pubKey":"`+senderPubkey+`","name":"Sender","type":"ADVERT"}`) // Observer hears the packet via the relay node. db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) - VALUES (1, 1, 10.0, -90, '["rr"]', ?, ?)`, + VALUES (1, 1, 10.0, -90, '["rr"]', $1, $2)`, epoch, `["`+relayPubkey+`"]`) store := NewPacketStore(db, nil) @@ -2455,7 +2450,7 @@ type failingQuerier struct { } func (f *failingQuerier) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) { - if f.failOn != "" && strings.Contains(query, f.failOn) { + if f.failOn != "" && (strings.Contains(query, f.failOn) || (len(args) > 0 && args[0] == f.failOn)) { return nil, f.err } return f.real.QueryContext(ctx, query, args...) @@ -2466,28 +2461,28 @@ func (f *failingQuerier) QueryContext(ctx context.Context, query string, args .. // instead of being silently swallowed and leaving the server in v2 mode against // a v3 database for the whole process lifetime. func TestDetectSchemaFailsLoudOnProbeError(t *testing.T) { - dbPath := filepath.Join(t.TempDir(), "v3.db") - conn, err := sql.Open("sqlite3", dbPath) + dbPath := postgresTestDSN(t) + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } conn.SetMaxOpenConns(1) // v3-shaped: observations carries observer_idx. - if _, err := conn.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY, observer_idx INTEGER)`); err != nil { + if _, err := conn.Exec(`CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, observer_idx INTEGER)`); err != nil { conn.Close() t.Fatal(err) } - conn.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY)`) + conn.Exec(`CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY)`) conn.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) conn.Close() - real, err := sql.Open("sqlite3", "file:"+dbPath+"?mode=ro") + real, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } defer real.Close() - q := &failingQuerier{real: real, failOn: "table_info(observations)", err: errors.New("injected probe failure")} + q := &failingQuerier{real: real, failOn: "observations", err: errors.New("injected probe failure")} db := &DB{} if err := db.detectSchema(context.Background(), q); err == nil { t.Fatal("detectSchema must return an error when a schema probe fails (#1901)") @@ -2500,20 +2495,19 @@ func TestDetectSchemaFailsLoudOnProbeError(t *testing.T) { // TestDetectSchemaV3AndV2 verifies detection sets isV3 correctly for both schema // shapes via the real OpenDB path, and that both open cleanly. func TestDetectSchemaV3AndV2(t *testing.T) { - dir := t.TempDir() - v3 := filepath.Join(dir, "v3.db") - c, err := sql.Open("sqlite3", v3) + v3 := postgresTestDSN(t) + c, err := openFixtureSQL(v3) if err != nil { t.Fatal(err) } c.SetMaxOpenConns(1) - c.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY, observer_idx INTEGER)`) - c.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT)`) + c.Exec(`CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, observer_idx INTEGER)`) + c.Exec(`CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`) c.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) ensurePreparable(t, c) c.Close() - db, err := OpenDB(v3) + db, err := openFixtureReader(t, v3) if err != nil { t.Fatalf("OpenDB v3: %v", err) } @@ -2522,18 +2516,18 @@ func TestDetectSchemaV3AndV2(t *testing.T) { } db.Close() - v2 := filepath.Join(dir, "v2.db") - c2, err := sql.Open("sqlite3", v2) + v2 := postgresTestDSN(t) + c2, err := openFixtureSQL(v2) if err != nil { t.Fatal(err) } c2.SetMaxOpenConns(1) - c2.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY, observer_id INTEGER)`) - c2.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT)`) + c2.Exec(`CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, observer_id INTEGER)`) + c2.Exec(`CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`) c2.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) ensurePreparable(t, c2) c2.Close() - db2, err := OpenDB(v2) + db2, err := openFixtureReader(t, v2) if err != nil { t.Fatalf("OpenDB v2: %v", err) } diff --git a/cmd/server/decoder.go b/cmd/server/decoder.go index 8f7981ef5..875e6052a 100644 --- a/cmd/server/decoder.go +++ b/cmd/server/decoder.go @@ -1,7 +1,6 @@ package main import ( - "crypto/sha256" "encoding/binary" "encoding/hex" "encoding/json" @@ -669,55 +668,7 @@ func DecodePacket(hexString string, validateSignatures bool) (*DecodedPacket, er // route-independent identifier for the same logical packet. For TRACE packets, // path_len is included in the hash to match firmware behavior. func ComputeContentHash(rawHex string) string { - buf, err := hex.DecodeString(rawHex) - if err != nil || len(buf) < 2 { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex - } - - headerByte := buf[0] - offset := 1 - if isTransportRoute(int(headerByte & 0x03)) { - offset += 4 - } - if offset >= len(buf) { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex - } - pathByte := buf[offset] - offset++ - hashSize := int((pathByte>>6)&0x3) + 1 - hashCount := int(pathByte & 0x3F) - pathBytes := hashSize * hashCount - - payloadStart := offset + pathBytes - if payloadStart > len(buf) { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex - } - - payload := buf[payloadStart:] - - // Hash payload-type byte only (bits 2-5 of header), not the full header. - // Firmware: SHA256(payload_type + [path_len for TRACE] + payload) - // Using the full header caused different hashes for the same logical packet - // when route type or version bits differed. See issue #786. - payloadType := (headerByte >> 2) & 0x0F - toHash := []byte{payloadType} - if int(payloadType) == PayloadTRACE { - // Firmware uses uint16_t path_len (2 bytes, little-endian) - toHash = append(toHash, pathByte, 0x00) - } - toHash = append(toHash, payload...) - - h := sha256.Sum256(toHash) - return hex.EncodeToString(h[:])[:16] + return packetpath.ContentHash(rawHex) } // PayloadJSON serializes the payload to JSON for DB storage. diff --git a/cmd/server/encrypted_channels_test.go b/cmd/server/encrypted_channels_test.go index d8632b886..2c19b296b 100644 --- a/cmd/server/encrypted_channels_test.go +++ b/cmd/server/encrypted_channels_test.go @@ -16,15 +16,15 @@ func seedEncryptedChannelData(t *testing.T, db *DB) { // Two encrypted GRP_TXT packets on channel hash "A1B2" db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('EE01', 'enc_hash_001', ?, 1, 5, '{"type":"GRP_TXT","channelHashHex":"A1B2","decryptionStatus":"no_key"}', 'enc_A1B2')`, recent) + VALUES ('EE01', 'enc_hash_001', $1, 1, 5, '{"type":"GRP_TXT","channelHashHex":"A1B2","decryptionStatus":"no_key"}', 'enc_A1B2')`, recent) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('EE02', 'enc_hash_002', ?, 1, 5, '{"type":"GRP_TXT","channelHashHex":"A1B2","decryptionStatus":"no_key"}', 'enc_A1B2')`, recent) + VALUES ('EE02', 'enc_hash_002', $1, 1, 5, '{"type":"GRP_TXT","channelHashHex":"A1B2","decryptionStatus":"no_key"}', 'enc_A1B2')`, recent) // Observations for both db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES ((SELECT id FROM transmissions WHERE hash='enc_hash_001'), 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES ((SELECT id FROM transmissions WHERE hash='enc_hash_001'), 1, 10.0, -90, '[]', $1)`, recentEpoch) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES ((SELECT id FROM transmissions WHERE hash='enc_hash_002'), 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES ((SELECT id FROM transmissions WHERE hash='enc_hash_002'), 1, 10.0, -90, '[]', $1)`, recentEpoch) } func TestGetEncryptedChannels(t *testing.T) { diff --git a/cmd/server/eviction_test.go b/cmd/server/eviction_test.go index cc80bd8c2..e16014616 100644 --- a/cmd/server/eviction_test.go +++ b/cmd/server/eviction_test.go @@ -295,9 +295,9 @@ func TestEvictStale_CleansResolvedPathNodeIndexes(t *testing.T) { tx.Observations = append(tx.Observations, obs) // Insert into DB so on-demand SQL fetch works during eviction - db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (?, '', ?, ?)", + db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES ($1, '', $2, $3)", txID, tx.Hash, tx.FirstSeen) - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (?, ?, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES ($1, $2, 1, $3, $4, $5)", obsID, txID, `["aa"]`, now.Add(-48*time.Hour).Unix(), `["`+relayPK+`"]`) store.packets = append(store.packets, tx) diff --git a/cmd/server/first_seen_1166_test.go b/cmd/server/first_seen_1166_test.go index f056c1274..1c7f386ba 100644 --- a/cmd/server/first_seen_1166_test.go +++ b/cmd/server/first_seen_1166_test.go @@ -24,7 +24,7 @@ func TestFirstSeen_1166_HandleNodesSurface(t *testing.T) { last := time.Now().UTC().Format("2006-01-02T15:04:05.000Z") if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'rpt', 'repeater', 37.5, -122.0, ?, ?, 5)`, + VALUES ($1, 'rpt', 'repeater', 37.5, -122.0, $2, $3, 5)`, pk, last, first); err != nil { t.Fatal(err) } diff --git a/cmd/server/go.mod b/cmd/server/go.mod index badbe96db..b0e5a9ddf 100644 --- a/cmd/server/go.mod +++ b/cmd/server/go.mod @@ -1,6 +1,6 @@ module github.com/corescope/server -go 1.22 +go 1.25.0 require ( github.com/gorilla/mux v1.8.1 @@ -38,9 +38,8 @@ require github.com/meshcore-analyzer/prunequeue v0.0.0 replace github.com/meshcore-analyzer/prunequeue => ../../internal/prunequeue require ( - github.com/mattn/go-sqlite3 v1.14.52 github.com/meshcore-analyzer/mbcapqueue v0.0.0 - golang.org/x/sync v0.10.0 + golang.org/x/sync v0.13.0 ) replace github.com/meshcore-analyzer/mbcapqueue => ../../internal/mbcapqueue @@ -52,22 +51,26 @@ require ( replace github.com/meshcore-analyzer/users => ../../internal/users -require github.com/meshcore-analyzer/channel v0.0.0 +require ( + github.com/jackc/pgx/v5 v5.7.6 + github.com/meshcore-analyzer/channel v0.0.0 +) + +require ( + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + golang.org/x/text v0.24.0 // indirect +) replace github.com/meshcore-analyzer/channel => ../../internal/channel require ( - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/ncruces/go-strftime v0.1.9 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/crypto v0.31.0 // indirect - golang.org/x/sys v0.28.0 // indirect - modernc.org/libc v1.55.3 // indirect - modernc.org/mathutil v1.6.0 // indirect - modernc.org/memory v1.8.0 // indirect - modernc.org/sqlite v1.34.5 // indirect + github.com/meshcore-analyzer/pgutil v0.0.0 + golang.org/x/crypto v0.37.0 // indirect + golang.org/x/sys v0.32.0 // indirect ) replace github.com/meshcore-analyzer/mailer => ../../internal/mailer + +replace github.com/meshcore-analyzer/pgutil => ../../internal/pgutil diff --git a/cmd/server/go.sum b/cmd/server/go.sum index 74e587a22..4bd844326 100644 --- a/cmd/server/go.sum +++ b/cmd/server/go.sum @@ -1,53 +1,34 @@ -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= -github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= -modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= -modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= -modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= -modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= -modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= -modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= -modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= -modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= -modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= -modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= -modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= -modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= -modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= -modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= -modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= -modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= -modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= -modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cmd/server/hash_migrate.go b/cmd/server/hash_migrate.go index a85d58bc1..1dd52090c 100644 --- a/cmd/server/hash_migrate.go +++ b/cmd/server/hash_migrate.go @@ -5,132 +5,34 @@ import ( "time" ) -// migrateContentHashesAsync recomputes content hashes in batches after the -// server is already serving HTTP. Packets whose hash changes are updated in -// both the DB and the in-memory byHash index. The migration is idempotent: -// once all hashes match the current formula it completes instantly. -func migrateContentHashesAsync(store *PacketStore, batchSize int, yieldDuration time.Duration) { - // #1856: every DB failure below continues to the next batch, so the loop - // always reaches the deferred completion. Setting the flag unconditionally - // therefore reported success after migrating nothing, which is exactly what - // happens on the read-only handle the server holds since #1283: begin, - // prepare and commit all fail, every batch is skipped, and /api/stats then - // answers hashMigrationComplete: true. - failedBatches := 0 - defer func() { - if r := recover(); r != nil { - log.Printf("[hash-migrate] panic recovered: %v", r) - failedBatches++ - } - if failedBatches > 0 { - log.Printf("[hash-migrate] INCOMPLETE: %d batch(es) could not be written; "+ - "hashMigrationComplete stays false. On a read-only DB handle this is expected "+ - "and the migration belongs in the ingestor (#1856).", failedBatches) - return - } - store.hashMigrationComplete.Store(true) - }() - - // Snapshot the packet slice length under lock (packets only grow). +// verifyContentHashesAsync reports whether the loaded hashes are current. The +// server never rewrites telemetry or its hash index independently of storage. +// Legacy hash repair is a separate offline operation; import preserves hashes. +func verifyContentHashesAsync(store *PacketStore, batchSize int, yieldDuration time.Duration) { + if batchSize <= 0 { + batchSize = 5000 + } store.mu.RLock() total := len(store.packets) store.mu.RUnlock() - - migrated := 0 for offset := 0; offset < total; offset += batchSize { - end := offset + batchSize - if end > total { - end = total - } - - // Collect stale hashes in this batch under RLock. - type hashUpdate struct { - tx *StoreTx - oldHash string - newHash string - } - var updates []hashUpdate - store.mu.RLock() - for _, tx := range store.packets[offset:end] { - if tx.RawHex == "" { - continue - } - newHash := ComputeContentHash(tx.RawHex) - if newHash != tx.Hash { - updates = append(updates, hashUpdate{tx: tx, oldHash: tx.Hash, newHash: newHash}) + end := min(offset+batchSize, min(total, len(store.packets))) + stale := false + for _, tx := range store.packets[min(offset, end):end] { + if tx.RawHex != "" && ComputeContentHash(tx.RawHex) != tx.Hash { + stale = true + break } } store.mu.RUnlock() - - if len(updates) == 0 { - continue - } - - // Write batch to DB in a single transaction. - dbTx, err := store.db.conn.Begin() - if err != nil { - log.Printf("[hash-migrate] begin tx: %v", err) - failedBatches++ - continue - } - stmt, err := dbTx.Prepare("UPDATE transmissions SET hash = ? WHERE id = ?") - if err != nil { - log.Printf("[hash-migrate] prepare: %v", err) - dbTx.Rollback() - failedBatches++ - continue - } - - for _, u := range updates { - if _, err := stmt.Exec(u.newHash, u.tx.ID); err != nil { - // UNIQUE constraint = two old hashes map to the same new hash (duplicate). - // Merge observations to the surviving tx, delete the duplicate. - log.Printf("[hash-migrate] tx %d collides — merging duplicate", u.tx.ID) - var survID int - if err2 := dbTx.QueryRow("SELECT id FROM transmissions WHERE hash = ?", u.newHash).Scan(&survID); err2 == nil { - dbTx.Exec("UPDATE observations SET transmission_id = ? WHERE transmission_id = ?", survID, u.tx.ID) - dbTx.Exec("DELETE FROM transmissions WHERE id = ?", u.tx.ID) - u.newHash = "" // mark for in-memory removal only - } - } + if stale { + log.Print("[hash-verify] stale content hashes found; perform explicit offline hash repair on a working copy of the legacy snapshot before import. The importer preserves existing hashes") + return } - stmt.Close() - - if err := dbTx.Commit(); err != nil { - log.Printf("[hash-migrate] commit: %v", err) - failedBatches++ - continue + if yieldDuration > 0 { + time.Sleep(yieldDuration) } - - // Update in-memory index under write lock. - store.mu.Lock() - for _, u := range updates { - delete(store.byHash, u.oldHash) - if u.newHash == "" { - // Merged duplicate — remove from packets slice and indexes. - delete(store.byTxID, u.tx.ID) - // Move observations to survivor if present. - if surv := store.byHash[ComputeContentHash(u.tx.RawHex)]; surv != nil { - for _, obs := range u.tx.Observations { - surv.Observations = append(surv.Observations, obs) - surv.ObservationCount++ - } - } - } else { - u.tx.Hash = u.newHash - store.byHash[u.newHash] = u.tx - } - } - store.mu.Unlock() - - migrated += len(updates) - - // Yield to let HTTP handlers run. - time.Sleep(yieldDuration) - } - - if migrated > 0 { - log.Printf("[hash-migrate] Migrated %d content hashes to new formula", migrated) } + store.hashMigrationComplete.Store(true) } diff --git a/cmd/server/hash_migrate_test.go b/cmd/server/hash_migrate_test.go index 0cc7f21e6..60a61a656 100644 --- a/cmd/server/hash_migrate_test.go +++ b/cmd/server/hash_migrate_test.go @@ -5,7 +5,7 @@ import ( "time" ) -func TestMigrateContentHashesAsync(t *testing.T) { +func TestVerifyContentHashesDoesNotWrite(t *testing.T) { db := setupTestDBv2(t) store := NewPacketStore(db, nil) @@ -15,7 +15,7 @@ func TestMigrateContentHashesAsync(t *testing.T) { wrongHash := "deadbeef12345678" _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES (?, ?, datetime('now'), 0, 2)`, rawHex, wrongHash) + VALUES ($1, $2, to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), 0, 2)`, rawHex, wrongHash) if err != nil { t.Fatal(err) } @@ -28,25 +28,25 @@ func TestMigrateContentHashesAsync(t *testing.T) { t.Fatal("expected packet under wrong hash before migration") } - migrateContentHashesAsync(store, 100, time.Millisecond) + verifyContentHashesAsync(store, 100, time.Millisecond) - if !store.hashMigrationComplete.Load() { - t.Error("expected hashMigrationComplete to be true") + if store.hashMigrationComplete.Load() { + t.Error("stale hashes must not report migration complete") } - if store.byHash[wrongHash] != nil { - t.Error("old hash should be removed from index") + if store.byHash[wrongHash] == nil { + t.Error("read-only verification must preserve the old hash index") } - if store.byHash[correctHash] == nil { - t.Error("new hash should be in index") + if store.byHash[correctHash] != nil { + t.Error("verification must not invent a new hash") } var dbHash string - err = db.conn.QueryRow("SELECT hash FROM transmissions WHERE raw_hex = ?", rawHex).Scan(&dbHash) + err = db.conn.QueryRow("SELECT hash FROM transmissions WHERE raw_hex = $1", rawHex).Scan(&dbHash) if err != nil { t.Fatal(err) } - if dbHash != correctHash { - t.Errorf("DB hash = %s, want %s", dbHash, correctHash) + if dbHash != wrongHash { + t.Errorf("DB hash = %s, want unchanged %s", dbHash, wrongHash) } } @@ -58,7 +58,7 @@ func TestMigrateContentHashesAsync_NoOp(t *testing.T) { correctHash := ComputeContentHash(rawHex) _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES (?, ?, datetime('now'), 0, 2)`, rawHex, correctHash) + VALUES ($1, $2, to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), 0, 2)`, rawHex, correctHash) if err != nil { t.Fatal(err) } @@ -67,7 +67,7 @@ func TestMigrateContentHashesAsync_NoOp(t *testing.T) { t.Fatal(err) } - migrateContentHashesAsync(store, 100, time.Millisecond) + verifyContentHashesAsync(store, 100, time.Millisecond) if !store.hashMigrationComplete.Load() { t.Error("expected hashMigrationComplete to be true") @@ -94,7 +94,7 @@ func TestMigrateContentHashesAsyncDoesNotClaimCompletionWhenWritesFail(t *testin rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976" wrongHash := "deadbeef12345678" if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES (?, ?, datetime('now'), 0, 2)`, rawHex, wrongHash); err != nil { + VALUES ($1, $2, to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), 0, 2)`, rawHex, wrongHash); err != nil { t.Fatal(err) } if err := store.Load(); err != nil { @@ -109,7 +109,7 @@ func TestMigrateContentHashesAsyncDoesNotClaimCompletionWhenWritesFail(t *testin t.Fatal(err) } - migrateContentHashesAsync(store, 100, time.Millisecond) + verifyContentHashesAsync(store, 100, time.Millisecond) if store.hashMigrationComplete.Load() { t.Error("hashMigrationComplete must stay false when no batch could be written; " + diff --git a/cmd/server/hidden_name_prefix_1181_endpoints_test.go b/cmd/server/hidden_name_prefix_1181_endpoints_test.go index 8b30bb020..dab2224b6 100644 --- a/cmd/server/hidden_name_prefix_1181_endpoints_test.go +++ b/cmd/server/hidden_name_prefix_1181_endpoints_test.go @@ -23,7 +23,7 @@ func TestHiddenNamePrefix_1181_NodeHealth(t *testing.T) { pk := "deadbeef00001184" if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, $3, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, pk, "🚫 health me", "companion"); err != nil { t.Fatalf("insert: %v", err) } @@ -57,7 +57,7 @@ func TestHiddenNamePrefix_1181_BulkHealth(t *testing.T) { pk := "deadbeef00001185" if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, $3, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, pk, "🚫 bulk me", "companion"); err != nil { t.Fatalf("insert: %v", err) } @@ -91,7 +91,7 @@ func TestHiddenNamePrefix_1181_Paths(t *testing.T) { pk := "deadbeef00001186" if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, $3, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, pk, "🚫 paths me", "companion"); err != nil { t.Fatalf("insert: %v", err) } @@ -113,7 +113,7 @@ func TestHiddenNamePrefix_1181_Analytics(t *testing.T) { pk := "deadbeef00001187" if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, $3, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, pk, "🚫 analytics me", "companion"); err != nil { t.Fatalf("insert: %v", err) } diff --git a/cmd/server/hidden_name_prefix_1181_test.go b/cmd/server/hidden_name_prefix_1181_test.go index d449fcae6..9dee2da4a 100644 --- a/cmd/server/hidden_name_prefix_1181_test.go +++ b/cmd/server/hidden_name_prefix_1181_test.go @@ -19,7 +19,7 @@ func TestHiddenNamePrefix_1181_NodesList(t *testing.T) { // Insert a node whose name starts with the configured 🚫 prefix. _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, $3, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, "deadbeef00001181", "🚫 ban me", "companion") if err != nil { t.Fatalf("insert hidden node: %v", err) @@ -70,7 +70,7 @@ func TestHiddenNamePrefix_1181_Search(t *testing.T) { if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, $3, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, "deadbeef00001182", "🚫 search me", "companion"); err != nil { t.Fatalf("insert: %v", err) } @@ -105,7 +105,7 @@ func TestHiddenNamePrefix_1181_Detail(t *testing.T) { pk := "deadbeef00001183" if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, $3, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, pk, "🚫 detail me", "companion"); err != nil { t.Fatalf("insert: %v", err) } diff --git a/cmd/server/hop_disambig_e2e_test.go b/cmd/server/hop_disambig_e2e_test.go index 41ad1f127..4e993f9c3 100644 --- a/cmd/server/hop_disambig_e2e_test.go +++ b/cmd/server/hop_disambig_e2e_test.go @@ -76,7 +76,7 @@ func t1201InsertNode(t *testing.T, db *DB, n t1201Node) { // "Berlin would win tier-3" premise of this fixture weakens silently — // update both this insert and the candidate scoring assertions. _, err := db.conn.Exec( - `INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) VALUES (?, ?, 'repeater', ?, ?, ?, '2026-01-01T00:00:00Z', ?)`, + `INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) VALUES ($1, $2, 'repeater', $3, $4, $5, '2026-01-01T00:00:00Z', $6)`, n.pk, "node-"+n.pk[:4], n.lat, n.lon, "2026-05-01T00:00:00Z", n.obsCount, ) if err != nil { @@ -109,7 +109,7 @@ func TestTopHopsRespectsContextAcrossAllCallSites(t *testing.T) { // Insert observer row (referenced by observations via observer_idx). if _, err := db.conn.Exec( - `INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES (?, ?, ?, '2026-01-01T00:00:00Z', 100)`, + `INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ($1, $2, $3, '2026-01-01T00:00:00Z', 100)`, t1201Observer, "obs-ca", "2026-05-01T00:00:00Z", ); err != nil { t.Fatal(err) @@ -128,17 +128,18 @@ func TestTopHopsRespectsContextAcrossAllCallSites(t *testing.T) { for i := 0; i < 50; i++ { ts := baseTime.Add(time.Duration(i) * time.Minute).Format(time.RFC3339) hash := fmt.Sprintf("hash1201_%03d", i) - res, err := tx.Exec( - `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES (?, ?, ?, 1, 1, ?)`, + var txID int64 + err = tx.QueryRow( + `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES ($1, $2, $3, 1, 1, $4) RETURNING id`, "AA", hash, ts, string(decoded), - ) + ).Scan(&txID) if err != nil { _ = tx.Rollback() t.Fatal(err) } - txID, _ := res.LastInsertId() + if _, err := tx.Exec( - `INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (?, 1, 12.0, -90, ?, ?)`, + `INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES ($1, 1, 12.0, -90, $2, $3)`, txID, pathJSON, baseTime.Add(time.Duration(i)*time.Minute).Unix(), ); err != nil { _ = tx.Rollback() diff --git a/cmd/server/hot_startup_consistency_test.go b/cmd/server/hot_startup_consistency_test.go index 709333993..50929d6c5 100644 --- a/cmd/server/hot_startup_consistency_test.go +++ b/cmd/server/hot_startup_consistency_test.go @@ -26,7 +26,7 @@ func TestHotStartup_loadChunk_IndexSliceConsistency(t *testing.T) { // spans 3 batches, widening the inconsistency window for the reader. dbPath := createTestDBWithAgedPackets(t, 10, 1200) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } diff --git a/cmd/server/hot_startup_test.go b/cmd/server/hot_startup_test.go index 238369c3c..f30ed72f0 100644 --- a/cmd/server/hot_startup_test.go +++ b/cmd/server/hot_startup_test.go @@ -1,19 +1,16 @@ package main import ( - "database/sql" "encoding/json" "fmt" "net/http" "net/http/httptest" - "path/filepath" "sync" "sync/atomic" "testing" "time" "github.com/gorilla/mux" - _ "github.com/mattn/go-sqlite3" ) // createTestDBMultiDay creates a test DB with packets spread across numDays days. @@ -21,10 +18,9 @@ import ( // Packets within each day are spaced 1 minute apart. func createTestDBMultiDay(t *testing.T, numDays, txPerDay int) string { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -35,33 +31,25 @@ func createTestDBMultiDay(t *testing.T, numDays, txPerDay int) string { t.Fatalf("createTestDBMultiDay setup: %v", err) } } - execOrFail(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`) - execOrFail(`CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`) - execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) - execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) + execOrFail(`CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`) + execOrFail(`CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`) + execOrFail(`CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION)`) execOrFail(`CREATE TABLE schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) execOrFail(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`) - id := 1 now := time.Now().UTC() - for day := numDays; day >= 1; day-- { - // Offset by +30 minutes so day boundaries don't coincide exactly with - // hotStartupHours/retentionHours cutoffs, preventing timing-boundary flakiness. - // E.g. for numDays=3: day3 starts at now-71.5h, day2 at now-47.5h, day1 at now-23.5h. - base := now.Add(-time.Duration(day)*24*time.Hour + 30*time.Minute) - for i := 0; i < txPerDay; i++ { - ts := base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339) - hash := fmt.Sprintf("hash%06d", id) - if _, err := conn.Exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", id, "aa", hash, ts, `{}`); err != nil { - t.Fatalf("createTestDBMultiDay insert tx: %v", err) - } - if _, err := conn.Exec("INSERT INTO observations VALUES (?,?,?,?,?,?,?,?,?,?,?)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, ts, ""); err != nil { - t.Fatalf("createTestDBMultiDay insert obs: %v", err) - } - id++ - } + stamp := func(n int) string { + day, offset := numDays-n/txPerDay, n%txPerDay + return now.Add(-time.Duration(day)*24*time.Hour + 30*time.Minute + time.Duration(offset)*time.Minute).Format(time.RFC3339) } + copyTestRows(t, conn, "transmissions", []string{"id", "raw_hex", "hash", "first_seen", "route_type", "payload_type", "payload_version", "decoded_json"}, numDays*txPerDay, func(n int) []any { + return []any{n + 1, "aa", fmt.Sprintf("hash%06d", n+1), stamp(n), 0, 4, 1, "{}"} + }) + copyTestRows(t, conn, "observations", []string{"id", "transmission_id", "observer_id", "observer_name", "direction", "snr", "rssi", "score", "path_json", "timestamp", "raw_hex"}, numDays*txPerDay, func(n int) []any { + return []any{n + 1, n + 1, "obs1", "Obs1", "RX", -10.0, -80.0, 5, "[]", stamp(n), ""} + }) return dbPath } @@ -81,7 +69,7 @@ func waitForBackgroundLoad(t *testing.T, store *PacketStore, timeout time.Durati func TestHotStartupConfig_Clamp(t *testing.T) { dbPath := createTestDB(t, 10) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -100,7 +88,7 @@ func TestHotStartupConfig_Clamp(t *testing.T) { func TestHotStartupConfig_ZeroIsDisabled(t *testing.T) { dbPath := createTestDB(t, 10) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -119,7 +107,7 @@ func TestHotStartup_LoadsOnlyHotWindow(t *testing.T) { // 50 old packets (48h ago), 10 recent (30min ago) dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -156,7 +144,7 @@ func TestHotStartup_DisabledWhenZero(t *testing.T) { // 50 old (48h ago), 10 recent (30min ago) — all within 72h retention dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -180,7 +168,7 @@ func TestHotStartup_loadChunk_AddsOlderData(t *testing.T) { // 50 old packets (48h ago), 10 recent (30min ago) dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -229,7 +217,7 @@ func TestHotStartup_BackgroundFillsToRetention(t *testing.T) { // 3 days × 50 tx/day = 150 total dbPath := createTestDBMultiDay(t, 3, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -269,7 +257,7 @@ func TestHotStartup_BackgroundFillsToRetention(t *testing.T) { func TestHotStartup_ChunkErrorRecovery(t *testing.T) { dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -314,7 +302,7 @@ func TestHotStartup_SQLFallback_TriggeredForOldDate(t *testing.T) { // 50 old packets (48h ago), 10 recent (30min ago) dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -345,7 +333,7 @@ func TestHotStartup_SQLFallback_TriggeredForOldDate(t *testing.T) { func TestHotStartup_PerfStats(t *testing.T) { dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -376,7 +364,7 @@ func TestHotStartup_SQLFallback_NotTriggeredForRecentDate(t *testing.T) { // 50 old packets (48h ago), 10 recent (30min ago) dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -405,7 +393,7 @@ func TestHotStartup_SQLFallback_Until(t *testing.T) { // 50 old packets (48h ago), 10 recent (30min ago) dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -436,7 +424,7 @@ func TestHotStartup_SQLFallback_Until(t *testing.T) { func TestHotStartup_PerfStoreHTTP(t *testing.T) { dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -485,7 +473,7 @@ func TestHotStartup_ConcurrentQueryDuringBackgroundLoad(t *testing.T) { // large enough to give pollers >=1 query during the background fill. dbPath := createTestDBMultiDay(t, 5, 200) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -583,7 +571,7 @@ func (a *atomicSamples) get() int64 { func TestHotStartup_BackgroundLoadFailureSurfacesInPerf(t *testing.T) { dbPath := createTestDBMultiDay(t, 3, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } diff --git a/cmd/server/issue1189_distinct_iatas_test.go b/cmd/server/issue1189_distinct_iatas_test.go index 3a096fe95..2bf397336 100644 --- a/cmd/server/issue1189_distinct_iatas_test.go +++ b/cmd/server/issue1189_distinct_iatas_test.go @@ -27,22 +27,22 @@ func TestQueryGroupedPacketsReturnsDistinctIATAs(t *testing.T) { // Observers: SJC + SFO + a third with no IATA (should be excluded). db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsA', 'A', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, now.Format(time.RFC3339)) + VALUES ('obsA', 'A', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, now.Format(time.RFC3339)) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsB', 'B', 'SFO', ?, '2026-01-01T00:00:00Z', 10)`, now.Format(time.RFC3339)) + VALUES ('obsB', 'B', 'SFO', $1, '2026-01-01T00:00:00Z', 10)`, now.Format(time.RFC3339)) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsC', 'C', '', ?, '2026-01-01T00:00:00Z', 10)`, now.Format(time.RFC3339)) + VALUES ('obsC', 'C', '', $1, '2026-01-01T00:00:00Z', 10)`, now.Format(time.RFC3339)) // One transmission with 3 observations (SJC, SFO, no-IATA). db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('AABB', 'deadbeefcafef00d', ?, 1, 4, '{}')`, now.Format(time.RFC3339)) + VALUES ('AABB', 'deadbeefcafef00d', $1, 1, 4, '{}')`, now.Format(time.RFC3339)) // v3 schema: observer_idx = observers.rowid (auto-assigned 1,2,3 in insert order). db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 12.0, -80, '["aa"]', ?)`, recentEpoch) + VALUES (1, 1, 12.0, -80, '["aa"]', $1)`, recentEpoch) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 2, 8.0, -90, '["aa"]', ?)`, recentEpoch-30) + VALUES (1, 2, 8.0, -90, '["aa"]', $1)`, recentEpoch-30) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 3, 5.0, -95, '["aa"]', ?)`, recentEpoch-60) + VALUES (1, 3, 5.0, -95, '["aa"]', $1)`, recentEpoch-60) result, err := db.QueryGroupedPackets(PacketQuery{Limit: 50}) if err != nil { @@ -79,11 +79,11 @@ func TestQueryGroupedPacketsDistinctIATAsEmptyWhenNoIATA(t *testing.T) { recentEpoch := now.Add(-1 * time.Hour).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsX', 'X', '', ?, '2026-01-01T00:00:00Z', 1)`, now.Format(time.RFC3339)) + VALUES ('obsX', 'X', '', $1, '2026-01-01T00:00:00Z', 1)`, now.Format(time.RFC3339)) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('AA', '1111222233334444', ?, 1, 4, '{}')`, now.Format(time.RFC3339)) + VALUES ('AA', '1111222233334444', $1, 1, 4, '{}')`, now.Format(time.RFC3339)) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -85, '[]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -85, '[]', $1)`, recentEpoch) result, err := db.QueryGroupedPackets(PacketQuery{Limit: 50}) if err != nil { diff --git a/cmd/server/issue1690_cold_load_test.go b/cmd/server/issue1690_cold_load_test.go index 1668e04c9..23f6cba39 100644 --- a/cmd/server/issue1690_cold_load_test.go +++ b/cmd/server/issue1690_cold_load_test.go @@ -14,12 +14,9 @@ package main import ( "encoding/json" - "path/filepath" "strings" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // createTestDBWithLastSeen seeds a DB with the post-fix schema (last_seen @@ -49,15 +46,14 @@ func createTestDBWithLastSeen(t *testing.T, dbPath string, numTx, obsPerTx int, // loads zero rows; the post-fix code (filtering on last_seen) must load // all 1000. func Test1690_ColdLoad_TimeAxis(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) nowSec := time.Now().UTC().Unix() createTestDBWithLastSeen(t, dbPath, 1000, 1, nowSec, 30*24*time.Hour, // first_seen = 30d ago 30*time.Minute) // last_seen = 30min ago - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -86,8 +82,7 @@ func Test1690_ColdLoad_TimeAxis(t *testing.T) { // loadBackgroundChunks runs, backgroundLoadDone must be FALSE and // backgroundLoadFailed must be TRUE because actual coverage is < 90%. func Test1690_BackgroundLoadHonesty(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) nowSec := time.Now().UTC().Unix() // 5000 rows; chunkSize=500 + maxMemoryMB=1 (→ maxPackets ≈ 1000) so @@ -96,7 +91,7 @@ func Test1690_BackgroundLoadHonesty(t *testing.T) { createTestDBWithLastSeen(t, dbPath, 5000, 1, nowSec, 30*time.Minute, 30*time.Minute) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -132,14 +127,13 @@ func Test1690_BackgroundLoadHonesty(t *testing.T) { // Test1690_PerfStats_NewFields asserts the typed perf payload exposes the // retention/coverage fields needed for prod observability. func Test1690_PerfStats_NewFields(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) nowSec := time.Now().UTC().Unix() createTestDBWithLastSeen(t, dbPath, 10, 1, nowSec, 30*time.Minute, 30*time.Minute) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/issue1809_bg_load_race_test.go b/cmd/server/issue1809_bg_load_race_test.go index eca19727c..6a0eea56a 100644 --- a/cmd/server/issue1809_bg_load_race_test.go +++ b/cmd/server/issue1809_bg_load_race_test.go @@ -21,9 +21,7 @@ package main // kept intact; we add the coverage assertions on top. import ( - "database/sql" "fmt" - "path/filepath" "testing" "time" ) @@ -34,8 +32,7 @@ import ( // produces backgroundLoadFailed=true deterministically because the bg // loader reads oldestLoaded="" and bails. func Test1809_StartupLoad_BgLoaderSeesOldestLoaded(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) nowSec := time.Now().UTC().Unix() // Seed 100 rows spread over 14 days with last_seen also spread, so @@ -47,7 +44,7 @@ func Test1809_StartupLoad_BgLoaderSeesOldestLoaded(t *testing.T) { const spanDays = 14 createTestDBSpreadOverDays(t, dbPath, totalRows, spanDays, nowSec) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -138,7 +135,7 @@ func createTestDBSpreadOverDays(t *testing.T, dbPath string, numTx, spanDays int // (adv #11: prior code carried a duplicated annotation on every line)." func seedTestDBRows(t *testing.T, dbPath string, numTx, obsPerTx int, rowTimes func(i int) (string, int64)) { t.Helper() - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -151,7 +148,7 @@ func seedTestDBRows(t *testing.T, dbPath string, numTx, obsPerTx int, rowTimes f } // PREFLIGHT: async=true reason="unit-test fixture seeder; in-memory ephemeral SQLite; all CREATE/INSERT below covered by this block annotation (pr-preflight 25-line lookback window)" execOrFail(`CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT, @@ -159,14 +156,14 @@ func seedTestDBRows(t *testing.T, dbPath string, numTx, obsPerTx int, rowTimes f )`) // PREFLIGHT: async=true reason="unit-test fixture seeder" execOrFail(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`) // PREFLIGHT: async=true reason="unit-test fixture seeder" - execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + execOrFail(`CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) // PREFLIGHT: async=true reason="unit-test fixture seeder" - execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) + execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION)`) // PREFLIGHT: async=true reason="unit-test fixture seeder" execOrFail(`CREATE TABLE schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) @@ -175,30 +172,15 @@ func seedTestDBRows(t *testing.T, dbPath string, numTx, obsPerTx int, rowTimes f // PREFLIGHT: async=true reason="unit-test fixture seeder" execOrFail(`CREATE INDEX idx_tx_last_seen ON transmissions(last_seen)`) - txStmt, err := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, last_seen) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)") - if err != nil { - t.Fatalf("prepare tx: %v", err) - } - defer txStmt.Close() - obsStmt, err := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") - if err != nil { - t.Fatalf("prepare obs: %v", err) - } - defer obsStmt.Close() - - obsID := 1 - for i := 1; i <= numTx; i++ { - firstSeenStr, lastSeenUnix := rowTimes(i) - hash := fmt.Sprintf("h%06d", i) - if _, err := txStmt.Exec(i, "aabb", hash, firstSeenStr, 0, 4, 1, "{}", lastSeenUnix); err != nil { - t.Fatalf("insert tx %d: %v", i, err) - } - for j := 0; j < obsPerTx; j++ { - obsTs := time.Unix(lastSeenUnix, 0).UTC().Add(-time.Duration(j) * time.Minute).Format(time.RFC3339) - if _, err := obsStmt.Exec(obsID, i, "obs1", "Obs1", "RX", -10.0, -80.0, 5, "[]", obsTs); err != nil { - t.Fatalf("insert obs: %v", err) - } - obsID++ - } - } + copyTestRows(t, conn, "transmissions", []string{"id", "raw_hex", "hash", "first_seen", "route_type", "payload_type", "payload_version", "decoded_json", "last_seen"}, numTx, func(n int) []any { + i := n + 1 + firstSeen, lastSeen := rowTimes(i) + return []any{i, "aabb", fmt.Sprintf("h%06d", i), firstSeen, 0, 4, 1, "{}", lastSeen} + }) + copyTestRows(t, conn, "observations", []string{"id", "transmission_id", "observer_id", "observer_name", "direction", "snr", "rssi", "score", "path_json", "timestamp"}, numTx*obsPerTx, func(n int) []any { + i, j := n/obsPerTx+1, n%obsPerTx + _, lastSeen := rowTimes(i) + stamp := time.Unix(lastSeen, 0).UTC().Add(-time.Duration(j) * time.Minute).Format(time.RFC3339) + return []any{n + 1, i, "obs1", "Obs1", "RX", -10.0, -80.0, 5, "[]", stamp} + }) } diff --git a/cmd/server/issue673_test.go b/cmd/server/issue673_test.go index 85337c9d7..0295270b8 100644 --- a/cmd/server/issue673_test.go +++ b/cmd/server/issue673_test.go @@ -4,8 +4,6 @@ import ( "encoding/json" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) const issue673NodePK = "7502f19f44cad6d7b626e1d811c00a914af452636182ccded3fd019803395ec9" @@ -18,7 +16,7 @@ func setupIssue673Store(t *testing.T) (*PacketStore, *DB) { db := setupTestDB(t) _, err := db.conn.Exec( - "INSERT INTO nodes (public_key, name, role) VALUES (?, ?, ?)", + "INSERT INTO nodes (public_key, name, role) VALUES ($1, $2, $3)", issue673NodePK, "Quail Hollow Park", "repeater", ) if err != nil { diff --git a/cmd/server/issue804_repeater_region_test.go b/cmd/server/issue804_repeater_region_test.go index 05e7a7a5c..a6bd86600 100644 --- a/cmd/server/issue804_repeater_region_test.go +++ b/cmd/server/issue804_repeater_region_test.go @@ -34,19 +34,19 @@ func TestIssue804_AnalyticsAttributesByRepeaterRegion(t *testing.T) { // Observers: one in PDX, one in SJC db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs-pdx', 'Obs PDX', 'PDX', ?, '2026-01-01T00:00:00Z', 100)`, recent) + VALUES ('obs-pdx', 'Obs PDX', 'PDX', $1, '2026-01-01T00:00:00Z', 100)`, recent) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs-sjc', 'Obs SJC', 'SJC', ?, '2026-01-01T00:00:00Z', 100)`, recent) + VALUES ('obs-sjc', 'Obs SJC', 'SJC', $1, '2026-01-01T00:00:00Z', 100)`, recent) // PDX-Repeater node (lives in Portland) pdxPK := "pdx0000000000001" db.conn.Exec(`INSERT INTO nodes (public_key, name, role) - VALUES (?, 'PDX-Repeater', 'repeater')`, pdxPK) + VALUES ($1, 'PDX-Repeater', 'repeater')`, pdxPK) // SJC-Repeater node (lives in San Jose) — sanity baseline sjcPK := "sjc0000000000001" db.conn.Exec(`INSERT INTO nodes (public_key, name, role) - VALUES (?, 'SJC-Repeater', 'repeater')`, sjcPK) + VALUES ($1, 'SJC-Repeater', 'repeater')`, sjcPK) pdxDecoded := `{"pubKey":"` + pdxPK + `","name":"PDX-Repeater","type":"ADVERT","flags":{"isRepeater":true}}` sjcDecoded := `{"pubKey":"` + sjcPK + `","name":"SJC-Repeater","type":"ADVERT","flags":{"isRepeater":true}}` @@ -56,31 +56,31 @@ func TestIssue804_AnalyticsAttributesByRepeaterRegion(t *testing.T) { // raw_hex header 0x12 = route_type 2 (direct), payload_type 4 // pathByte 0x40 (hashSize bits=01 → 2, hop_count=0) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1240aabbccdd', 'pdx_zh_direct', ?, 2, 4, ?)`, recent, pdxDecoded) + VALUES ('1240aabbccdd', 'pdx_zh_direct', $1, 2, 4, $2)`, recent, pdxDecoded) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 12.0, -85, '[]', ?)`, recentEpoch) + VALUES (1, 1, 12.0, -85, '[]', $1)`, recentEpoch) // 2) PDX-Repeater FLOOD advert with hashSize=2 (reliable). // Heard ONLY by obs-SJC via a relay path (this is the polluting case). // raw_hex header 0x11 = route_type 1 (flood), payload_type 4 // pathByte 0x41 (hashSize bits=01 → 2, hop_count=1) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1141aabbccdd', 'pdx_flood', ?, 1, 4, ?)`, recent, pdxDecoded) + VALUES ('1141aabbccdd', 'pdx_flood', $1, 1, 4, $2)`, recent, pdxDecoded) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 2, 8.0, -95, '["aa11"]', ?)`, recentEpoch) + VALUES (2, 2, 8.0, -95, '["aa11"]', $1)`, recentEpoch) // 3) SJC-Repeater zero-hop DIRECT advert heard only by obs-SJC. // Establishes SJC as the repeater's home region. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1240ccddeeff', 'sjc_zh_direct', ?, 2, 4, ?)`, recent, sjcDecoded) + VALUES ('1240ccddeeff', 'sjc_zh_direct', $1, 2, 4, $2)`, recent, sjcDecoded) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 2, 14.0, -82, '[]', ?)`, recentEpoch) + VALUES (3, 2, 14.0, -82, '[]', $1)`, recentEpoch) // 4) SJC-Repeater FLOOD advert with hashSize=2, heard by obs-SJC. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1141ccddeeff', 'sjc_flood', ?, 1, 4, ?)`, recent, sjcDecoded) + VALUES ('1141ccddeeff', 'sjc_flood', $1, 1, 4, $2)`, recent, sjcDecoded) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (4, 2, 11.0, -88, '["cc22"]', ?)`, recentEpoch) + VALUES (4, 2, 11.0, -88, '["cc22"]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() diff --git a/cmd/server/issue810_repro_test.go b/cmd/server/issue810_repro_test.go index c5e6e9a85..2bb34f357 100644 --- a/cmd/server/issue810_repro_test.go +++ b/cmd/server/issue810_repro_test.go @@ -18,14 +18,14 @@ func TestRepro810(t *testing.T) { now := time.Now().UTC() recent := now.Add(-1 * time.Hour).Format(time.RFC3339) recentEpoch := now.Add(-1 * time.Hour).Unix() - db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs1','O1',?, '2026-01-01T00:00:00Z', 100)`, recent) - db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs2','O2',?, '2026-01-01T00:00:00Z', 100)`, recent) - db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ('aabbccdd11223344','R','repeater',?, '2026-01-01T00:00:00Z', 1)`, recent) - db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES ('AABB','testhash00000001',?,1,4,'{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, recent) + db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs1','O1',$1, '2026-01-01T00:00:00Z', 100)`, recent) + db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs2','O2',$1, '2026-01-01T00:00:00Z', 100)`, recent) + db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ('aabbccdd11223344','R','repeater',$1, '2026-01-01T00:00:00Z', 1)`, recent) + db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES ('AABB','testhash00000001',$1,1,4,'{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, recent) // Longest-path obs WITHOUT resolved_path - db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (1,1,12.5,-90,'["aa","bb","cc"]',?)`, recentEpoch) + db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (1,1,12.5,-90,'["aa","bb","cc"]',$1)`, recentEpoch) // Shorter-path obs WITH resolved_path - db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) VALUES (1,2,8.0,-95,'["aa","bb"]',?,'["aabbccdd11223344","eeff00112233aabb"]')`, recentEpoch-100) + db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) VALUES (1,2,8.0,-95,'["aa","bb"]',$1,'["aabbccdd11223344","eeff00112233aabb"]')`, recentEpoch-100) cfg := &Config{Port: 3000} hub := NewHub() diff --git a/cmd/server/issue871_test.go b/cmd/server/issue871_test.go index 43f2768aa..2539df728 100644 --- a/cmd/server/issue871_test.go +++ b/cmd/server/issue871_test.go @@ -19,7 +19,7 @@ func TestIssue871_NoNullHashOrTimestamp(t *testing.T) { // Insert bad legacy data: packet with empty hash now := time.Now().UTC().Add(-30 * time.Minute).Format(time.RFC3339) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('DEAD', '', ?, 1, 4, '{}')`, now) + VALUES ('DEAD', '', $1, 1, 4, '{}')`, now) // Insert bad legacy data: packet with NULL first_seen (timestamp) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES ('BEEF', 'aa11bb22cc33dd44', NULL, 1, 4, '{}')`) diff --git a/cmd/server/load_graph_invariant_test.go b/cmd/server/load_graph_invariant_test.go index 07ae649f0..511db94bd 100644 --- a/cmd/server/load_graph_invariant_test.go +++ b/cmd/server/load_graph_invariant_test.go @@ -1,12 +1,8 @@ package main import ( - "database/sql" - "path/filepath" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // TestLoad_PanicsWhenGraphNotLoadedAndEdgesExist pins the startup-ordering @@ -19,10 +15,9 @@ import ( // Load() must panic when neighbor_edges has rows but s.graph.Load() returns // nil. Fast-fail at startup beats silently-wrong attribution. func TestLoad_PanicsWhenGraphNotLoadedAndEdgesExist(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) - rw, err := sql.Open("sqlite3", "file:"+dbPath+"?_journal_mode=WAL") + rw, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -36,23 +31,23 @@ func TestLoad_PanicsWhenGraphNotLoadedAndEdgesExist(t *testing.T) { // Minimal CoreScope schema. PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" exec(`CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT )`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" exec(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, transmission_id INTEGER, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT, resolved_path TEXT )`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + exec(`CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" exec(`CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0 )`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" @@ -67,10 +62,10 @@ func TestLoad_PanicsWhenGraphNotLoadedAndEdgesExist(t *testing.T) { PRIMARY KEY (node_a, node_b) )`) now := time.Now().UTC().Format(time.RFC3339) - exec(`INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES (?, ?, ?, ?)`, + exec(`INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES ($1, $2, $3, $4)`, "aaa", "bbb", 5, now) - d, err := OpenDB(dbPath) + d, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/loadchunk_ambiguous_prefix_test.go b/cmd/server/loadchunk_ambiguous_prefix_test.go index 3c0e1596b..250ff5705 100644 --- a/cmd/server/loadchunk_ambiguous_prefix_test.go +++ b/cmd/server/loadchunk_ambiguous_prefix_test.go @@ -1,13 +1,9 @@ package main import ( - "database/sql" "fmt" - "path/filepath" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // createTestDBAmbiguousPrefix builds a fixture where TWO repeaters share the @@ -21,10 +17,9 @@ import ( // picks today's tier-4 winner against ~7-day-old observations. func createTestDBAmbiguousPrefix(t *testing.T, relayA, relayB, hop, firstSeen string) string { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -38,26 +33,26 @@ func createTestDBAmbiguousPrefix(t *testing.T, relayA, relayB, hop, firstSeen st // PREFLIGHT: async=true reason="test fixture: in-memory t.TempDir SQLite, never touches a real DB." exec(`CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT )`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" exec(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT, resolved_path TEXT )`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + exec(`CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" exec(`CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0 )`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" @@ -69,15 +64,15 @@ func createTestDBAmbiguousPrefix(t *testing.T, relayA, relayB, hop, firstSeen st // Two repeaters sharing the same 2-char prefix `hop`. // Different advert_counts so tier-4 tiebreak deterministically picks one // (proving the bug: it over-attributes to the higher-count node). - exec(`INSERT INTO nodes (public_key, name, role, advert_count) VALUES (?,?,?,?)`, + exec(`INSERT INTO nodes (public_key, name, role, advert_count) VALUES ($1,$2,$3,$4)`, relayA, "Relay A", "repeater", 50) - exec(`INSERT INTO nodes (public_key, name, role, advert_count) VALUES (?,?,?,?)`, + exec(`INSERT INTO nodes (public_key, name, role, advert_count) VALUES ($1,$2,$3,$4)`, relayB, "Relay B", "repeater", 10) // Aged 48h so it lands in the background window (loadChunk path). - exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", + exec("INSERT INTO transmissions VALUES ($1,$2,$3,$4,0,4,1,$5)", 1, "aa", "hashamb_1", firstSeen, `{}`) - exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) VALUES (?,?,?,?,?,?,?,?,?,?,?,NULL)", + exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,NULL)", 1, 1, "obs1", "Obs1", "RX", -10.0, -80.0, 5, fmt.Sprintf(`[%q]`, hop), firstSeen, "") return dbPath @@ -97,7 +92,7 @@ func TestLoadChunk_AmbiguousPrefix_SkipsAttribution(t *testing.T) { aged := time.Now().UTC().Add(-48 * time.Hour).Format(time.RFC3339) dbPath := createTestDBAmbiguousPrefix(t, relayA, relayB, hop, aged) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -148,7 +143,7 @@ func TestLoad_AmbiguousPrefix_SkipsAttribution(t *testing.T) { ts := time.Now().UTC().Format(time.RFC3339) dbPath := createTestDBAmbiguousPrefix(t, relayA, relayB, hop, ts) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } diff --git a/cmd/server/loadchunk_pathjson_fallback_test.go b/cmd/server/loadchunk_pathjson_fallback_test.go index 9d28b6203..3a63213ff 100644 --- a/cmd/server/loadchunk_pathjson_fallback_test.go +++ b/cmd/server/loadchunk_pathjson_fallback_test.go @@ -1,13 +1,9 @@ package main import ( - "database/sql" "fmt" - "path/filepath" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // createTestDBPathJSONNoResolvedPath builds a fixture that mirrors the LIVE @@ -22,10 +18,9 @@ import ( // s.byNode is via path_json → resolvePathForObs relay-hop resolution. func createTestDBPathJSONNoResolvedPath(t *testing.T, relayPubkey, hopPrefix, firstSeen string) string { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -39,7 +34,7 @@ func createTestDBPathJSONNoResolvedPath(t *testing.T, relayPubkey, hopPrefix, fi // PREFLIGHT: async=true reason="test fixture: in-memory t.TempDir SQLite, never touches a real DB. Tables are CREATE-from-empty in a one-shot OpenDB call, not a schema migration over existing data." exec(`CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT @@ -47,21 +42,21 @@ func createTestDBPathJSONNoResolvedPath(t *testing.T, relayPubkey, hopPrefix, fi // resolved_path column present (matches live schema) but left NULL. // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" exec(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT, resolved_path TEXT )`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + exec(`CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) // Production nodes schema uses public_key (not pubkey) — getAllNodes / // buildPrefixMap reads public_key, role, advert_count, first_seen. // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" exec(`CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0 )`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" @@ -71,13 +66,13 @@ func createTestDBPathJSONNoResolvedPath(t *testing.T, relayPubkey, hopPrefix, fi exec(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`) // Repeater node so canAppearInPath() admits it to the prefix map. - exec(`INSERT INTO nodes (public_key, name, role, advert_count) VALUES (?,?,?,?)`, + exec(`INSERT INTO nodes (public_key, name, role, advert_count) VALUES ($1,$2,$3,$4)`, relayPubkey, "Relay One", "repeater", 10) - exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", + exec("INSERT INTO transmissions VALUES ($1,$2,$3,$4,0,4,1,$5)", 1, "aa", "hashpjf_1", firstSeen, `{}`) // resolved_path explicitly NULL; path_json carries the relay hop prefix. - exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) VALUES (?,?,?,?,?,?,?,?,?,?,?,NULL)", + exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,NULL)", 1, 1, "obs1", "Obs1", "RX", -10.0, -80.0, 5, fmt.Sprintf(`[%q]`, hopPrefix), firstSeen, "") return dbPath @@ -103,7 +98,7 @@ func TestLoadChunked_ResolvesRelayHopsFromPathJSON_WhenResolvedPathEmpty(t *test ts := time.Now().UTC().Format(time.RFC3339) dbPath := createTestDBPathJSONNoResolvedPath(t, relayPK, hop, ts) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -146,7 +141,7 @@ func TestLoadChunk_ResolvesRelayHopsFromPathJSON_WhenResolvedPathEmpty(t *testin aged := time.Now().UTC().Add(-48 * time.Hour).Format(time.RFC3339) dbPath := createTestDBPathJSONNoResolvedPath(t, relayPK, hop, aged) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } diff --git a/cmd/server/loadchunk_resolved_path_1558_test.go b/cmd/server/loadchunk_resolved_path_1558_test.go index 9cd83ecd5..99d5fbeb1 100644 --- a/cmd/server/loadchunk_resolved_path_1558_test.go +++ b/cmd/server/loadchunk_resolved_path_1558_test.go @@ -1,13 +1,9 @@ package main import ( - "database/sql" "fmt" - "path/filepath" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // createTestDBWithResolvedPath creates a fixture DB containing numTx old @@ -17,10 +13,9 @@ import ( // column so loadChunk's hasResolvedPath branch is exercised. func createTestDBWithResolvedPath(t *testing.T, numTx int, relayPubkeys []string) string { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -33,22 +28,22 @@ func createTestDBWithResolvedPath(t *testing.T, numTx int, relayPubkeys []string } exec(`CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT )`) exec(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT, resolved_path TEXT )`) - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) - exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) + exec(`CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION)`) exec(`CREATE TABLE schema_version (version INTEGER)`) exec(`INSERT INTO schema_version (version) VALUES (1)`) exec(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`) @@ -67,9 +62,9 @@ func createTestDBWithResolvedPath(t *testing.T, numTx int, relayPubkeys []string for i := 0; i < numTx; i++ { ts := now.Add(-48 * time.Hour).Add(time.Duration(i) * time.Second).Format(time.RFC3339) hash := fmt.Sprintf("hash1558_%d", i) - exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", + exec("INSERT INTO transmissions VALUES ($1,$2,$3,$4,0,4,1,$5)", i+1, "aa", hash, ts, `{}`) - exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)", i+1, i+1, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, ts, "", rpJSON) } return dbPath @@ -107,7 +102,7 @@ func TestLoadChunk_IndexesResolvedPathPubkeys_Issue1558(t *testing.T) { dbPath := createTestDBWithResolvedPath(t, 3, []string{relayPK1, relayPK2}) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } diff --git a/cmd/server/main.go b/cmd/server/main.go index 58a679825..3356fb4f4 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -2,7 +2,6 @@ package main import ( "context" - "database/sql" "flag" "fmt" "log" @@ -86,18 +85,22 @@ func main() { } var ( - configDir string - port int - dbPath string - publicDir string - pollMs int + configDir string + port int + legacyDB string + databaseURL string + stateDir string + publicDir string + pollMs int ) flag.StringVar(&configDir, "config-dir", ".", "Directory containing config.json") flag.IntVar(&port, "port", 0, "HTTP port (overrides config)") - flag.StringVar(&dbPath, "db", "", "SQLite database path (overrides config/env)") + flag.StringVar(&legacyDB, "db", "", "Removed: import SQLite and use -database-url") + flag.StringVar(&databaseURL, "database-url", "", "PostgreSQL telemetry reader URL (overrides config/env)") + flag.StringVar(&stateDir, "state-dir", "", "Local directory for queues and sidecars") flag.StringVar(&publicDir, "public", "public", "Directory to serve static files from") - flag.IntVar(&pollMs, "poll-ms", 1000, "SQLite poll interval for WebSocket broadcast (ms)") + flag.IntVar(&pollMs, "poll-ms", 1000, "Database poll interval for WebSocket broadcast (ms)") flag.Parse() // Load config @@ -113,8 +116,8 @@ func main() { if cfg.Port == 0 { cfg.Port = 3000 } - if dbPath != "" { - cfg.DBPath = dbPath + if legacyDB != "" { + log.Fatal("[config] -db is no longer supported; import SQLite and use -database-url") } if cfg.APIKey == "" { log.Printf("[security] WARNING: no apiKey configured — write endpoints are BLOCKED (set apiKey in config.json to enable them)") @@ -157,9 +160,19 @@ func main() { warnIfMemlimitUnderprovisioned(limit) } - // Resolve DB path - resolvedDB := cfg.ResolveDBPath(configDir) - log.Printf("[config] port=%d db=%s public=%s", cfg.Port, resolvedDB, publicDir) + // Resolve connection separately from local state. Never log a database URL. + resolvedDB, err := cfg.ResolveDatabaseURL() + if databaseURL != "" && cfg.DBPath == "" && os.Getenv("DB_PATH") == "" { + resolvedDB, err = databaseURL, nil + } + if err != nil { + log.Fatalf("[config] %v", err) + } + runtimeStateDir = cfg.ResolveStateDir(configDir) + if stateDir != "" { + runtimeStateDir = stateDir + } + log.Printf("[config] port=%d database=postgresql public=%s", cfg.Port, publicDir) if len(cfg.NodeBlacklist) > 0 { log.Printf("[config] nodeBlacklist: %d node(s) will be hidden from API", len(cfg.NodeBlacklist)) for _, pk := range cfg.NodeBlacklist { @@ -172,8 +185,9 @@ func main() { // Open database database, err := OpenDB(resolvedDB) if err != nil { - log.Fatalf("[db] failed to open %s: %v", resolvedDB, err) + log.Fatalf("[db] failed to open PostgreSQL: %v", err) } + database.stateDir = runtimeStateDir var dbCloseOnce sync.Once dbClose := func() error { var err error @@ -182,13 +196,6 @@ func main() { } defer dbClose() - // Verify DB has expected tables - var tableName string - err = database.conn.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name='transmissions'").Scan(&tableName) - if err == sql.ErrNoRows { - log.Fatalf("[db] table 'transmissions' not found — is this a CoreScope database?") - } - stats, err := database.GetStats() if err != nil { log.Printf("[db] warning: could not read stats: %v", err) @@ -205,7 +212,7 @@ func main() { // column/index/table is missing, the operator must restart the // ingestor (which owns dbschema.Apply) before this server can start. if err := dbschema.AssertReady(database.conn); err != nil { - log.Fatalf("[db] schema not ready (ingestor must run migrations first): %v", err) + log.Fatalf("[db] schema not ready (run the PostgreSQL migration/bootstrap command first): %v", err) } // In-memory packet store @@ -285,7 +292,6 @@ func main() { // loaded above, before the packet load. Per #1287 schema migrations // all live in the ingestor; the server only reads the snapshot and // then refreshes it via the recompNeighborGraph slot every 60s. - dbPath = database.path database.hasResolvedPath = true // dbschema.AssertReady above already verified observations.resolved_path exists // WaitGroup for background init steps that gate /api/healthz readiness. @@ -392,7 +398,7 @@ func main() { }) } - // Start SQLite poller for WebSocket broadcast + // Start PostgreSQL poller for WebSocket broadcast poller := NewPoller(database, hub, time.Duration(pollMs)*time.Millisecond) poller.store = store go poller.Start() @@ -534,7 +540,7 @@ func main() { // 1c. Stop steady-state analytics recomputers (issue #1240). // Must happen before dbClose so any in-flight compute that - // reaches into SQLite has finished. + // queries PostgreSQL has finished. if stopAnalyticsRecomp != nil { stopAnalyticsRecomp() } @@ -552,7 +558,7 @@ func main() { // 3b. Close users.db (user management, opt-in; no-op when off). srv.closeUserManagement() - // 4. Close database (release SQLite WAL lock) + // 4. Close the database connection pool if err := dbClose(); err != nil { log.Printf("[server] DB close error: %v", err) } @@ -566,8 +572,8 @@ func main() { // process. The server reads the results via the periodic // recompNeighborGraph / fetchResolvedPathForObs paths. - // Migrate old content hashes in background (one-time, idempotent). - go migrateContentHashesAsync(store, 5000, 100*time.Millisecond) + // Verify loaded content hashes without issuing telemetry writes. + go verifyContentHashesAsync(store, 5000, 100*time.Millisecond) if err := httpServer.ListenAndServe(); err != http.ErrServerClosed { log.Fatalf("[server] %v", err) diff --git a/cmd/server/memlimit.go b/cmd/server/memlimit.go index f8e3df5c8..4cbfa1d3b 100644 --- a/cmd/server/memlimit.go +++ b/cmd/server/memlimit.go @@ -37,20 +37,9 @@ func applyMemoryLimit(maxMemoryMB int, envSet bool) (int64, string) { // transient peaks (cold-load row-scan / decode pipeline, Go's NextGC // trigger at ~2x live heap). See issue #836 heap profile. // - // GOMEMLIMIT covers memory the Go runtime manages — heap, stacks and - // runtime structures — and nothing else. Since the SQLite driver became - // cgo (github.com/mattn/go-sqlite3), what SQLite allocates in C is - // outside it, so this limit is not an RSS ceiling: actual RSS is this - // plus the native allocations plus whatever the runtime has reserved - // but not returned. - // - // The page cache, at least, is deliberately bounded: both DSNs pin - // _cache_size=-2000, i.e. ~2 MiB per connection, ~8 MiB across - // SetMaxOpenConns(4). That fits inside the headroom above rather than - // eating into the store budget. It does not bound SQLite's other native - // allocations, so if the connection count or _cache_size grows - // materially, or a workload starts holding many prepared statements, - // this derivation needs revisiting against measured RSS. + // GOMEMLIMIT covers Go-managed memory, not an RSS ceiling. PostgreSQL + // runs in another process and needs its own memory budget; its buffers + // and work memory are not represented by this packet-store limit. limit := int64(maxMemoryMB) * 1024 * 1024 * 3 / 2 debug.SetMemoryLimit(limit) return limit, "derived" diff --git a/cmd/server/memory.go b/cmd/server/memory.go index 8ad52b516..2b43a67c0 100644 --- a/cmd/server/memory.go +++ b/cmd/server/memory.go @@ -32,22 +32,10 @@ import ( // eviction; both can shrink when packets age out. goHeapInuseMB and goSysMB // fluctuate with GC. // -// There is no cgo-allocation field, but for a different reason than before: -// the driver is github.com/mattn/go-sqlite3 now, so SQLite's page cache IS a C -// allocation — the Go runtime simply exposes no byte counter for it. -// -// processRSSMB does include it, since VmRSS counts every resident page whoever -// allocated it. Resist the temptation to read `processRSSMB - goSysMB` as "the -// C share": goSysMB is address space the runtime has reserved, not what is -// resident, so the subtraction mixes two different things and can even go -// negative. It is a smell test, not a measurement. To size native usage, -// measure RSS against a build with the store disabled, or profile. -// -// What is bounded is SQLite's page cache specifically: both DSNs pin -// _cache_size=-2000, i.e. ~2 MiB per connection, so roughly 8 MiB across -// SetMaxOpenConns(4) here and 2 MiB in the ingestor. That is a cap on the page -// cache, not on everything SQLite allocates — statement and schema memory sit -// outside it. See memlimit.go for why any of it matters to GOMEMLIMIT. +// These are server-process measurements. PostgreSQL buffers and query work +// memory live in the database process and must be measured separately. +// processRSSMB - goSysMB is not an allocation counter: goSysMB includes address +// space reserved but not resident, so that subtraction can be negative. type MemorySnapshot struct { ProcessRSSMB float64 `json:"processRSSMB"` GoHeapInuseMB float64 `json:"goHeapInuseMB"` diff --git a/cmd/server/multibyte_capability_test.go b/cmd/server/multibyte_capability_test.go index 53267533f..d447e5701 100644 --- a/cmd/server/multibyte_capability_test.go +++ b/cmd/server/multibyte_capability_test.go @@ -1,14 +1,11 @@ package main import ( - "database/sql" "encoding/json" "fmt" "strings" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // recentTS returns a timestamp string N hours ago, ensuring test data @@ -20,17 +17,17 @@ func recentTS(hoursAgo int) string { // setupCapabilityTestDB creates a minimal in-memory DB with nodes table. func setupCapabilityTestDB(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } conn.SetMaxOpenConns(1) conn.Exec(`CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, - lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, - advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL + lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, + advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c DOUBLE PRECISION )`) - conn.Exec(`CREATE TABLE observers ( + conn.Exec(`CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT, packet_count INTEGER DEFAULT 0, model TEXT, firmware TEXT, client_version TEXT, radio TEXT, battery_mv INTEGER, @@ -86,7 +83,7 @@ func TestMultiByteCapability_Confirmed(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepA", "repeater", recentTS(24)) store := NewPacketStore(db, nil) @@ -113,7 +110,7 @@ func TestMultiByteCapability_Suspected(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepB", "repeater", recentTS(48)) store := NewPacketStore(db, nil) @@ -151,7 +148,7 @@ func TestMultiByteCapability_Unknown(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepC", "repeater", recentTS(72)) store := NewPacketStore(db, nil) @@ -179,9 +176,9 @@ func TestMultiByteCapability_PrefixCollision(t *testing.T) { defer db.conn.Close() // Two repeaters sharing 1-byte prefix "aa" - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabb000000000001", "RepConfirmed", "repeater", recentTS(24)) - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aacc000000000002", "RepOther", "repeater", recentTS(24)) store := NewPacketStore(db, nil) @@ -227,7 +224,7 @@ func TestMultiByteCapability_TraceExcluded(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepTrace", "repeater", recentTS(48)) store := NewPacketStore(db, nil) @@ -259,7 +256,7 @@ func TestMultiByteCapability_NonTraceStillSuspected(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepNonTrace", "repeater", recentTS(48)) store := NewPacketStore(db, nil) @@ -291,7 +288,7 @@ func TestMultiByteCapability_ConfirmedUnaffectedByTraceExclusion(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepConfirmedTrace", "repeater", recentTS(24)) store := NewPacketStore(db, nil) @@ -326,7 +323,7 @@ func TestMultiByteCapability_CompanionConfirmed(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "CompA", "companion", recentTS(24)) store := NewPacketStore(db, nil) @@ -353,11 +350,11 @@ func TestMultiByteCapability_RoleColumnPopulated(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabb000000000001", "Rep1", "repeater", recentTS(24)) - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "ccdd000000000002", "Comp1", "companion", recentTS(24)) - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "eeff000000000003", "Room1", "room_server", recentTS(24)) store := NewPacketStore(db, nil) @@ -392,7 +389,7 @@ func TestMultiByteCapability_AdopterEvidenceTakesPrecedence(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepAdopter", "repeater", recentTS(24)) store := NewPacketStore(db, nil) @@ -446,22 +443,22 @@ func TestMultiByteCapability_AdopterEvidenceTakesPrecedence(t *testing.T) { // setupPersistTestDB creates an in-memory DB with multibyte_sup/multibyte_evidence columns. func setupPersistTestDB(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } conn.SetMaxOpenConns(1) conn.Exec(`CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, - lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, - advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL, + lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, + advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c DOUBLE PRECISION, foreign_advert INTEGER DEFAULT 0, default_scope TEXT, multibyte_sup INTEGER NOT NULL DEFAULT 0, multibyte_evidence TEXT )`) conn.Exec(`CREATE TABLE inactive_nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, - lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, - advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL, + lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, + advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c DOUBLE PRECISION, foreign_advert INTEGER DEFAULT 0, default_scope TEXT, multibyte_sup INTEGER NOT NULL DEFAULT 0, multibyte_evidence TEXT )`) diff --git a/cmd/server/multibyte_region_filter_test.go b/cmd/server/multibyte_region_filter_test.go index a66fedc13..6fe8d6719 100644 --- a/cmd/server/multibyte_region_filter_test.go +++ b/cmd/server/multibyte_region_filter_test.go @@ -30,9 +30,9 @@ func TestMultiByteCapability_RegionFiltered_PreservesConfirmedStatus(t *testing. // Two observers in different regions. db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs-sjc', 'Obs SJC', 'SJC', ?, '2026-01-01T00:00:00Z', 100)`, recent) + VALUES ('obs-sjc', 'Obs SJC', 'SJC', $1, '2026-01-01T00:00:00Z', 100)`, recent) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs-jkg', 'Obs JKG', 'JKG', ?, '2026-01-01T00:00:00Z', 100)`, recent) + VALUES ('obs-jkg', 'Obs JKG', 'JKG', $1, '2026-01-01T00:00:00Z', 100)`, recent) // Node A: a JKG-region repeater that advertises multi-byte (hash_size=2). // Its zero-hop direct advert is only heard by obs-SJC (e.g. an out-of-region @@ -42,7 +42,7 @@ func TestMultiByteCapability_RegionFiltered_PreservesConfirmedStatus(t *testing. // to "confirmed" via the global advert evidence. pkA := "aaa0000000000001" db.conn.Exec(`INSERT INTO nodes (public_key, name, role) - VALUES (?, 'Node-A', 'repeater')`, pkA) + VALUES ($1, 'Node-A', 'repeater')`, pkA) decodedA := `{"pubKey":"` + pkA + `","name":"Node-A","type":"ADVERT","flags":{"isRepeater":true}}` @@ -50,17 +50,17 @@ func TestMultiByteCapability_RegionFiltered_PreservesConfirmedStatus(t *testing. // pathByte 0x40 → hash_size bits 01 → 2 bytes. // Heard by obs-SJC ONLY. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1240aabbccdd', 'a_zh_direct', ?, 2, 4, ?)`, recent, decodedA) + VALUES ('1240aabbccdd', 'a_zh_direct', $1, 2, 4, $2)`, recent, decodedA) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 12.0, -85, '[]', ?)`, recentEpoch) + VALUES (1, 1, 12.0, -85, '[]', $1)`, recentEpoch) // Node A also appears as a path hop in a JKG-observed packet, so it // shows up in the JKG region's node list. // route_type=1 (flood), payload_type=4, pathByte 0x41 (hs=2, hops=1) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1141aabbccdd', 'a_jkg_relay', ?, 1, 4, ?)`, recent, decodedA) + VALUES ('1141aabbccdd', 'a_jkg_relay', $1, 1, 4, $2)`, recent, decodedA) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 2, 8.0, -95, '["aa"]', ?)`, recentEpoch) + VALUES (2, 2, 8.0, -95, '["aa"]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() diff --git a/cmd/server/neighbor_api_test.go b/cmd/server/neighbor_api_test.go index e3ab0b96c..b5ccb9725 100644 --- a/cmd/server/neighbor_api_test.go +++ b/cmd/server/neighbor_api_test.go @@ -1,7 +1,6 @@ package main import ( - "database/sql" "encoding/json" "net/http" "net/http/httptest" @@ -9,7 +8,6 @@ import ( "time" "github.com/gorilla/mux" - _ "github.com/mattn/go-sqlite3" ) // ─── Helpers ─────────────────────────────────────────────────────────────────── @@ -464,10 +462,9 @@ func TestNeighborGraphAPI_ResponseShape(t *testing.T) { func TestBuildNodeInfoMap_ObserverEnrichment(t *testing.T) { // Create a temp SQLite DB with nodes and observers tables. - tmpDir := t.TempDir() - dbPath := tmpDir + "/test.db" + dbPath := postgresTestDSN(t) - conn, err := sql.Open("sqlite3", dbPath) + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -475,8 +472,8 @@ func TestBuildNodeInfoMap_ObserverEnrichment(t *testing.T) { // Create tables for _, stmt := range []string{ - "CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, lat REAL, lon REAL)", - "CREATE TABLE observers (id TEXT, name TEXT, iata TEXT, inactive INTEGER)", + "CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION)", + "CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, name TEXT, iata TEXT, inactive INTEGER)", "INSERT INTO nodes VALUES ('AAAA1111', 'Repeater-1', 'repeater', 0, 0)", "INSERT INTO observers (id, name, iata) VALUES ('BBBB2222', 'Observer-Alpha', '')", "INSERT INTO observers (id, name, iata) VALUES ('AAAA1111', 'Obs-also-repeater', '')", @@ -489,7 +486,7 @@ func TestBuildNodeInfoMap_ObserverEnrichment(t *testing.T) { conn.Close() // Open via our DB wrapper - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -539,18 +536,17 @@ func TestBuildNodeInfoMap_ObserverEnrichment(t *testing.T) { // existing 30s node cache, both calls return the original value — same as // every other nodeInfo field that comes from getAllNodes(). func TestBuildNodeInfoMap_FirstSeenIsCached(t *testing.T) { - tmpDir := t.TempDir() - dbPath := tmpDir + "/test.db" + dbPath := postgresTestDSN(t) // Seed via rw connection. - rw, err := sql.Open("sqlite3", dbPath) + rw, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } defer rw.Close() for _, stmt := range []string{ - "CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER)", - "CREATE TABLE observers (id TEXT, name TEXT, iata TEXT, inactive INTEGER)", + "CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER)", + "CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, name TEXT, iata TEXT, inactive INTEGER)", "INSERT INTO nodes VALUES ('AAAA1111', 'Repeater-1', 'repeater', 0, 0, '', '2024-01-01T00:00:00Z', 0)", } { if _, err := rw.Exec(stmt); err != nil { @@ -559,7 +555,7 @@ func TestBuildNodeInfoMap_FirstSeenIsCached(t *testing.T) { } ensurePreparable(t, rw) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -606,18 +602,17 @@ func TestBuildNodeInfoMap_FirstSeenIsCached(t *testing.T) { // values must have empty FirstSeen with no panic. Regression coverage for the // existing fallback branch (#1632 review loop 1). func TestGetAllNodes_FirstSeenSchemaFallback(t *testing.T) { - tmpDir := t.TempDir() - dbPath := tmpDir + "/test.db" + dbPath := postgresTestDSN(t) // Seed a nodes table WITHOUT first_seen (advert_count + last_seen present). - rw, err := sql.Open("sqlite3", dbPath) + rw, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } defer rw.Close() for _, stmt := range []string{ - "CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, advert_count INTEGER)", - "CREATE TABLE observers (id TEXT, name TEXT, iata TEXT, inactive INTEGER)", + "CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, advert_count INTEGER)", + "CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, name TEXT, iata TEXT, inactive INTEGER)", "INSERT INTO nodes VALUES ('BBBB2222', 'Repeater-2', 'repeater', 0, 0, '2024-02-02T00:00:00Z', 3)", } { if _, err := rw.Exec(stmt); err != nil { @@ -626,7 +621,7 @@ func TestGetAllNodes_FirstSeenSchemaFallback(t *testing.T) { } ensurePreparable(t, rw) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } diff --git a/cmd/server/neighbor_persist_legacy_test.go b/cmd/server/neighbor_persist_legacy_test.go index 798b96e4c..051c7ba64 100644 --- a/cmd/server/neighbor_persist_legacy_test.go +++ b/cmd/server/neighbor_persist_legacy_test.go @@ -1,12 +1,8 @@ package main import ( - "database/sql" - "path/filepath" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // TestNeighborPersist_LegacyEdgeInvariant (#1638 adv-#1): edges loaded from @@ -16,9 +12,8 @@ import ( // pre-existing observations land in bucket 0 (legacy/unknown, conservative // weight in the JS confidence indicator). func TestNeighborPersist_LegacyEdgeInvariant(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "neighbor_legacy.db") - rw, err := sql.Open("sqlite3", "file:"+dbPath+"?_journal_mode=WAL") + dbPath := postgresTestDSN(t) + rw, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -34,7 +29,7 @@ func TestNeighborPersist_LegacyEdgeInvariant(t *testing.T) { } now := time.Now().UTC().Format(time.RFC3339) if _, err := rw.Exec( - `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES (?, ?, ?, ?)`, + `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES ($1, $2, $3, $4)`, "aaaa", "bbbb", 7, now, ); err != nil { t.Fatal(err) @@ -77,9 +72,8 @@ func TestNeighborPersist_LegacyEdgeInvariant(t *testing.T) { // in the snapshot" path (loader's else-branch): subsequent counts must // accumulate into bucket 0 too, preserving the invariant. func TestNeighborPersist_LegacyEdgeMergeOnReload(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "neighbor_legacy_merge.db") - rw, err := sql.Open("sqlite3", "file:"+dbPath+"?_journal_mode=WAL") + dbPath := postgresTestDSN(t) + rw, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -97,7 +91,7 @@ func TestNeighborPersist_LegacyEdgeMergeOnReload(t *testing.T) { now := time.Now().UTC().Format(time.RFC3339) for _, cnt := range []int{3, 4} { if _, err := rw.Exec( - `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES (?, ?, ?, ?)`, + `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES ($1, $2, $3, $4)`, "aaaa", "bbbb", cnt, now, ); err != nil { t.Fatal(err) diff --git a/cmd/server/neighbor_recomputer_test.go b/cmd/server/neighbor_recomputer_test.go index cdcf7b8f6..1673de975 100644 --- a/cmd/server/neighbor_recomputer_test.go +++ b/cmd/server/neighbor_recomputer_test.go @@ -1,12 +1,9 @@ package main import ( - "database/sql" - "path/filepath" "testing" "time" - _ "github.com/mattn/go-sqlite3" "github.com/meshcore-analyzer/dbschema" ) @@ -15,11 +12,10 @@ import ( // snapshot the ingestor writes. After a write to neighbor_edges (here // done synthetically), the recomputer's atomic-swap must reflect it. func TestNeighborGraphRecomputerLoadsSnapshot(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "neighbor_recomp.db") + dbPath := postgresTestDSN(t) // Bootstrap a WAL DB with the neighbor_edges table. - rw, err := sql.Open("sqlite3", "file:"+dbPath+"?_journal_mode=WAL") + rw, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -38,14 +34,14 @@ func TestNeighborGraphRecomputerLoadsSnapshot(t *testing.T) { // Stage one edge. now := time.Now().UTC().Format(time.RFC3339) if _, err := rw.Exec( - `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES (?, ?, ?, ?)`, + `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES ($1, $2, $3, $4)`, "aaa", "bbb", 5, now, ); err != nil { t.Fatal(err) } // Server opens read-only and refreshes via the recomputer. - d, err := OpenDB(dbPath) + d, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -64,7 +60,7 @@ func TestNeighborGraphRecomputerLoadsSnapshot(t *testing.T) { // Add another row, refresh, assert the new total. if _, err := rw.Exec( - `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES (?, ?, ?, ?)`, + `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES ($1, $2, $3, $4)`, "ccc", "ddd", 2, now, ); err != nil { t.Fatal(err) @@ -81,8 +77,7 @@ func TestNeighborGraphRecomputerLoadsSnapshot(t *testing.T) { // AssertReady on a DB missing the required columns returns an error // listing every missing surface; main.go then calls log.Fatalf. func TestServerStartupRequiresMigratedSchema(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "unmigrated.db") + dbPath := postgresTestDSN(t) // Bootstrap with ONLY transmissions/observations (the things // server tries to read) but WITHOUT the columns dbschema asserts @@ -94,15 +89,15 @@ func TestServerStartupRequiresMigratedSchema(t *testing.T) { // them selects on inactive, so a fixture without it cannot reach // AssertReady at all. The other missing surfaces above still make // AssertReady fail, which is what this test asserts. - rw, err := sql.Open("sqlite3", "file:"+dbPath+"?_journal_mode=WAL") + rw, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } defer rw.Close() for _, s := range []string{ - `CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT, payload_type INTEGER)`, - `CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER)`, - `CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`, + `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT, payload_type INTEGER)`, + `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER)`, + `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`, `CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`, `CREATE TABLE inactive_nodes (public_key TEXT PRIMARY KEY)`, } { @@ -115,7 +110,7 @@ func TestServerStartupRequiresMigratedSchema(t *testing.T) { // Open the read-only server handle and call AssertReady directly // (production path: main.go does this before any business logic). - d, err := OpenDB(dbPath) + d, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/node_battery.go b/cmd/server/node_battery.go index 7e66ac579..c8e8f4ff7 100644 --- a/cmd/server/node_battery.go +++ b/cmd/server/node_battery.go @@ -53,9 +53,9 @@ func (db *DB) GetNodeBatteryHistory(pubkey, since string) ([]NodeBatterySample, rows, err := db.conn.Query(` SELECT timestamp, battery_mv FROM observer_metrics - WHERE LOWER(observer_id) = ? + WHERE LOWER(observer_id) = $1 AND battery_mv IS NOT NULL - AND timestamp >= ? + AND timestamp >= $2 ORDER BY timestamp ASC`, pk, since) if err != nil { return nil, err diff --git a/cmd/server/node_battery_test.go b/cmd/server/node_battery_test.go index 247bd963a..55edf96d5 100644 --- a/cmd/server/node_battery_test.go +++ b/cmd/server/node_battery_test.go @@ -20,19 +20,19 @@ func TestGetNodeBatteryHistory_FromObserverMetrics(t *testing.T) { // node + observer with matching pubkey (cases differ on purpose) pkLower := "deadbeefcafef00d11223344" idUpper := strings.ToUpper(pkLower) - db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen) VALUES (?, 'BatNode', 'repeater', ?, ?)`, + db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen) VALUES ($1, 'BatNode', 'repeater', $2, $3)`, pkLower, now.Format(time.RFC3339), now.Add(-72*time.Hour).Format(time.RFC3339)) - db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen) VALUES (?, 'BatNode', ?, ?)`, + db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen) VALUES ($1, 'BatNode', $2, $3)`, idUpper, now.Format(time.RFC3339), now.Add(-72*time.Hour).Format(time.RFC3339)) // 3 metrics samples: 3700, 3500, 3200 mV for i, mv := range []int{3700, 3500, 3200} { ts := now.Add(time.Duration(-2+i) * time.Hour).Format(time.RFC3339) - db.conn.Exec(`INSERT INTO observer_metrics (observer_id, timestamp, battery_mv) VALUES (?, ?, ?)`, + db.conn.Exec(`INSERT INTO observer_metrics (observer_id, timestamp, battery_mv) VALUES ($1, $2, $3)`, idUpper, ts, mv) } // One sample with NULL battery should be skipped - db.conn.Exec(`INSERT INTO observer_metrics (observer_id, timestamp) VALUES (?, ?)`, + db.conn.Exec(`INSERT INTO observer_metrics (observer_id, timestamp) VALUES ($1, $2)`, idUpper, now.Add(-3*time.Hour).Format(time.RFC3339)) since := now.Add(-24 * time.Hour).Format(time.RFC3339) @@ -57,11 +57,11 @@ func TestNodeBatteryEndpoint(t *testing.T) { now := time.Now().UTC() pkLower := "aabbccdd11223344" idUpper := strings.ToUpper(pkLower) - db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen) VALUES (?, 'TestRepeater', ?, ?)`, + db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen) VALUES ($1, 'TestRepeater', $2, $3)`, idUpper, now.Format(time.RFC3339), now.Add(-72*time.Hour).Format(time.RFC3339)) for i, mv := range []int{3800, 3600, 3200} { ts := now.Add(time.Duration(-2+i) * time.Hour).Format(time.RFC3339) - db.conn.Exec(`INSERT INTO observer_metrics (observer_id, timestamp, battery_mv) VALUES (?, ?, ?)`, + db.conn.Exec(`INSERT INTO observer_metrics (observer_id, timestamp, battery_mv) VALUES ($1, $2, $3)`, idUpper, ts, mv) } diff --git a/cmd/server/node_blacklist_test.go b/cmd/server/node_blacklist_test.go index a6d877693..bc1d6ba4c 100644 --- a/cmd/server/node_blacklist_test.go +++ b/cmd/server/node_blacklist_test.go @@ -71,8 +71,8 @@ func TestConfigBlacklistEmptyEntries(t *testing.T) { func TestBlacklistFiltersHandleNodes(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', datetime('now'))") - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', datetime('now'))") + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING") + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING") cfg := &Config{ NodeBlacklist: []string{"badnode"}, @@ -105,7 +105,7 @@ func TestBlacklistFiltersHandleNodes(t *testing.T) { func TestBlacklistFiltersNodeDetail(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', datetime('now'))") + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING") cfg := &Config{ NodeBlacklist: []string{"badnode"}, @@ -124,8 +124,8 @@ func TestBlacklistFiltersNodeDetail(t *testing.T) { func TestBlacklistFiltersNodeSearch(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'TrollNode', 'companion', datetime('now'))") - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', datetime('now'))") + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'TrollNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING") + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING") cfg := &Config{ NodeBlacklist: []string{"badnode"}, @@ -155,7 +155,7 @@ func TestBlacklistFiltersNodeSearch(t *testing.T) { func TestNoBlacklistPassesAll(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('somenode', 'SomeNode', 'companion', datetime('now'))") + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('somenode', 'SomeNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING") cfg := &Config{} srv := NewServer(db, cfg, NewHub()) @@ -234,7 +234,7 @@ func TestBlacklistFiltersNeighborGraph(t *testing.T) { func TestBlacklistFiltersResolveHops(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', datetime('now'))") + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING") cfg := &Config{ NodeBlacklist: []string{"badnode"}, diff --git a/cmd/server/node_declared_regions_api_test.go b/cmd/server/node_declared_regions_api_test.go index 480ee6b46..3c79ed471 100644 --- a/cmd/server/node_declared_regions_api_test.go +++ b/cmd/server/node_declared_regions_api_test.go @@ -163,7 +163,7 @@ func TestScopeAuditAndNodesAgreeOnDeclaredRegions(t *testing.T) { for pk, csv := range cases { if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, configured_scope, configured_scope_at) - VALUES (?, ?, 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, ?, '2026-01-01T00:00:00Z')`, + VALUES ($1, $2, 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, $3, '2026-01-01T00:00:00Z')`, pk, "rp-"+pk, csv, ); err != nil { t.Fatal(err) diff --git a/cmd/server/node_health_can_relay_case_1290_test.go b/cmd/server/node_health_can_relay_case_1290_test.go index 59d0aa6fc..0a3159dd6 100644 --- a/cmd/server/node_health_can_relay_case_1290_test.go +++ b/cmd/server/node_health_can_relay_case_1290_test.go @@ -36,7 +36,7 @@ func TestNodeHealth_CanRelayCaseInsensitive_Issue1290(t *testing.T) { } if _, err := srv.store.db.conn.Exec( `INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, can_relay, can_relay_seen) - VALUES (?, 'ListenerOnly', 'SJC', ?, '2026-01-01T00:00:00Z', 1, 0, 1)`, + VALUES ($1, 'ListenerOnly', 'SJC', $2, '2026-01-01T00:00:00Z', 1, 0, 1)`, obsIDLower, now); err != nil { t.Fatalf("seed observer: %v", err) } diff --git a/cmd/server/node_hop_analytics_test.go b/cmd/server/node_hop_analytics_test.go index 5c3dd2c1c..a0a41606f 100644 --- a/cmd/server/node_hop_analytics_test.go +++ b/cmd/server/node_hop_analytics_test.go @@ -287,16 +287,16 @@ func TestNodeHopPackets_UniquenessIsPerPrefixLength(t *testing.T) { func hopInsertNode(t *testing.T, db *DB, pk, name string, adverts int) { t.Helper() mustExec(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) - VALUES (?, ?, 'repeater', ?, '2026-01-01', ?)`, pk, name, time.Now().UTC().Format(time.RFC3339), adverts) + VALUES ($1, $2, 'repeater', $3, '2026-01-01', $4)`, pk, name, time.Now().UTC().Format(time.RFC3339), adverts) } func hopInsertTx(t *testing.T, db *DB, id int, hash string, routeType, payloadType int, paths ...string) { t.Helper() ts := time.Now().Add(-1 * time.Hour) - mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (?, 'AA', ?, ?, ?, ?)`, + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES ($1, 'AA', $2, $3, $4, $5)`, id, hash, ts.UTC().Format(time.RFC3339), routeType, payloadType) for _, p := range paths { - mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (?, NULL, ?, ?, NULL)`, + mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES ($1, NULL, $2, $3, NULL)`, id, p, ts.Unix()) } } @@ -352,7 +352,7 @@ func TestNodeHopAnalytics_SameResultAfterRestart(t *testing.T) { db := setupTestDB(t) mustExec(t, db, `CREATE TABLE neighbor_edges (node_a TEXT NOT NULL, node_b TEXT NOT NULL, count INTEGER DEFAULT 1, last_seen TEXT, PRIMARY KEY (node_a, node_b))`) - mustExec(t, db, `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES (?, ?, 50, ?)`, + mustExec(t, db, `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES ($1, $2, 50, $3)`, hopOther, hopTarget, time.Now().UTC().Format(time.RFC3339)) hopInsertNode(t, db, hopTarget, "Target", 1) hopInsertNode(t, db, hopCollider, "Collider", 500) @@ -403,14 +403,14 @@ func TestHandleNodeHopAnalytics(t *testing.T) { hopInsertNode(t, db, hopOther, "Other", 1) hopInsertNode(t, db, hopCollider, "Collider", 1) - mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (1, 'AA', 'hop_recent_2', ?, 1, 5)`, recent) - mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (1, NULL, '["77AA","77AA","AB12"]', ?)`, recentEpoch) - mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (2, 'BB', 'hop_recent_1byte', ?, 0, 4)`, recent) - mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (2, NULL, '["AB"]', ?)`, recentEpoch) - mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (3, 'CC', 'hop_scoped_advert', ?, 0, 4)`, recent) - mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (3, NULL, '["AB12"]', ?)`, recentEpoch) - mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (4, 'DD', 'hop_old', ?, 1, 5)`, old) - mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (4, NULL, '["AB12"]', ?)`, oldEpoch) + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (1, 'AA', 'hop_recent_2', $1, 1, 5)`, recent) + mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (1, NULL, '["77AA","77AA","AB12"]', $1)`, recentEpoch) + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (2, 'BB', 'hop_recent_1byte', $1, 0, 4)`, recent) + mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (2, NULL, '["AB"]', $1)`, recentEpoch) + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (3, 'CC', 'hop_scoped_advert', $1, 0, 4)`, recent) + mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (3, NULL, '["AB12"]', $1)`, recentEpoch) + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (4, 'DD', 'hop_old', $1, 1, 5)`, old) + mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (4, NULL, '["AB12"]', $1)`, oldEpoch) srv := NewServer(db, &Config{Port: 3000}, NewHub()) router := mux.NewRouter() diff --git a/cmd/server/node_reach.go b/cmd/server/node_reach.go index 0cfa97896..e4e4d1daa 100644 --- a/cmd/server/node_reach.go +++ b/cmd/server/node_reach.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/json" "errors" + "fmt" "log" "net/http" "sort" @@ -217,7 +218,7 @@ func (s *Server) reachNodeScopes(ctx context.Context, pubkey string) (string, st args[i] = &vals[i] } row := s.db.conn.QueryRowContext(ctx, - "SELECT "+strings.Join(cols, ", ")+" FROM nodes WHERE public_key = ?", pubkey) + "SELECT "+strings.Join(cols, ", ")+" FROM nodes WHERE public_key = $1", pubkey) if err := row.Scan(args...); err != nil { return "", "", "" } @@ -335,7 +336,7 @@ type reachState struct { // buildSem bounds how many cold-cache reach scans run at the same time. // singleflight only collapses identical keys; distinct (pubkey, days) // keys each start a scan, and the endpoint is unauthenticated, so - // without a cap a handful of requests can hold every SQLite reader. + // without a cap a handful of requests can hold every database reader. buildSemOnce sync.Once buildSem chan struct{} // sf dedups concurrent cold-cache requests for the same key so N @@ -760,15 +761,15 @@ func (s *Server) scanReachRows(ctx context.Context, tokens map[string]bool, sinc } sort.Strings(toks) for _, tok := range toks { - likes = append(likes, "o.path_json LIKE ?") + likes = append(likes, fmt.Sprintf("o.path_json ILIKE $%d ESCAPE ''", len(args)+1)) args = append(args, "%\""+tok+"\"%") } q := `SELECT LOWER(COALESCE(obs.id,'')), LOWER(COALESCE(t.from_pubkey,'')), COALESCE(t.payload_type,0), o.path_json, o.snr FROM observations o JOIN transmissions t ON t.id = o.transmission_id LEFT JOIN observers obs ON obs.rowid = o.observer_idx - WHERE o.timestamp >= ? AND (` + strings.Join(likes, " OR ") + `) - LIMIT ?` + WHERE o.timestamp >= $1 AND (` + strings.Join(likes, " OR ") + `) + LIMIT ` + fmt.Sprintf("$%d", len(args)+1) args = append(args, reachScanRowLimit) rows, err := s.db.conn.QueryContext(ctx, q, args...) if err != nil { @@ -811,7 +812,7 @@ func (s *Server) scanReachRows(ctx context.Context, tokens map[string]bool, sinc } // reachMaxConcurrentBuilds is the number of cold-cache reach scans allowed at -// once. Two keeps half of the default 4-connection SQLite pool free for +// once. Two keeps half of the default 4-connection PostgreSQL pool free for // every other handler while a scan runs. const reachMaxConcurrentBuilds = 2 diff --git a/cmd/server/node_reach_bench_test.go b/cmd/server/node_reach_bench_test.go index bbf24a347..5397553de 100644 --- a/cmd/server/node_reach_bench_test.go +++ b/cmd/server/node_reach_bench_test.go @@ -2,11 +2,8 @@ package main import ( "context" - "database/sql" "fmt" "testing" - - _ "github.com/mattn/go-sqlite3" ) // benchReachDB builds an in-memory DB with nObs observations. matchEvery @@ -24,14 +21,14 @@ func benchReachDB(b *testing.B, nObs, matchEvery int, lowerHops bool) *DB { // path_json is uppercase; this measures the worst case Carmack flagged). matchPath, fillerPath = `["aa","01fa","bb"]`, `["aa","cc","bb"]` } - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(b) if err != nil { b.Fatal(err) } schema := []string{ - `CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT, first_seen TEXT, payload_type INTEGER, from_pubkey TEXT)`, - `CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`, - `CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr REAL, path_json TEXT, timestamp INTEGER)`, + `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT, first_seen TEXT, payload_type INTEGER, from_pubkey TEXT)`, + `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`, + `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`, `CREATE INDEX idx_obs_ts ON observations(timestamp)`, } for _, s := range schema { @@ -47,7 +44,7 @@ func benchReachDB(b *testing.B, nObs, matchEvery int, lowerHops bool) *DB { b.Fatal(err) } for i := 0; i < nObs; i++ { - if _, err := tx.Exec(`INSERT INTO transmissions (id, hash, first_seen, payload_type, from_pubkey) VALUES (?,?,?,5,'')`, + if _, err := tx.Exec(`INSERT INTO transmissions (id, hash, first_seen, payload_type, from_pubkey) VALUES ($1,$2,$3,5,'')`, i, fmt.Sprintf("h%d", i), "2026-06-07T00:00:00Z"); err != nil { b.Fatal(err) } @@ -55,7 +52,7 @@ func benchReachDB(b *testing.B, nObs, matchEvery int, lowerHops bool) *DB { if i%matchEvery == 0 { path = matchPath } - if _, err := tx.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, snr, path_json, timestamp) VALUES (?,?,1,-7.0,?,?)`, + if _, err := tx.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, snr, path_json, timestamp) VALUES ($1,$2,1,-7.0,$3,$4)`, i, i, path, 1000); err != nil { b.Fatal(err) } @@ -155,7 +152,7 @@ func BenchmarkNodeReachAttribute(b *testing.B) { // must surface an error, not a swallowed nil. Lives in this file because // the bench callers in the same file rely on the same signature. func TestScanReachRows_ErrorReturn(t *testing.T) { - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatalf("open: %v", err) } diff --git a/cmd/server/node_reach_endpoint_test.go b/cmd/server/node_reach_endpoint_test.go index a7fc6d4aa..d189c263a 100644 --- a/cmd/server/node_reach_endpoint_test.go +++ b/cmd/server/node_reach_endpoint_test.go @@ -1,7 +1,6 @@ package main import ( - "database/sql" "encoding/json" "net/http" "net/http/httptest" @@ -11,7 +10,6 @@ import ( "time" "github.com/gorilla/mux" - _ "github.com/mattn/go-sqlite3" ) func serveReach(srv *Server, path string) *httptest.ResponseRecorder { @@ -57,7 +55,7 @@ func resetReachState(t *testing.T, servers ...*Server) { // build the zero-reach case (identifiable node, no matching observations). func newReachIntegrationDB(t *testing.T, obsPath string) (*DB, string) { t.Helper() - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } @@ -66,10 +64,10 @@ func newReachIntegrationDB(t *testing.T, obsPath string) (*DB, string) { b := pk64("ccdd") // successor → B hears us now := time.Now().Unix() stmts := []string{ - `CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER)`, - `CREATE TABLE transmissions (id INTEGER PRIMARY KEY, from_pubkey TEXT, payload_type INTEGER)`, - `CREATE TABLE observers (id TEXT, inactive INTEGER)`, - `CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr REAL, path_json TEXT, timestamp INTEGER)`, + `CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER)`, + `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, from_pubkey TEXT, payload_type INTEGER)`, + `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, inactive INTEGER)`, + `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`, `CREATE TABLE neighbor_edges (node_a TEXT, node_b TEXT, count INTEGER)`, } for _, s := range stmts { @@ -81,12 +79,12 @@ func newReachIntegrationDB(t *testing.T, obsPath string) (*DB, string) { q string args []interface{} }{ - {`INSERT INTO nodes VALUES (?, 'N', 'repeater', 50.9, 5.4, ?, '2026-06-01T00:00:00Z', 3)`, []interface{}{n, "2026-06-07T00:00:00Z"}}, - {`INSERT INTO nodes VALUES (?, 'A', 'repeater', 51.0, 5.5, ?, '2026-06-01T00:00:00Z', 1)`, []interface{}{a, "2026-06-07T00:00:00Z"}}, - {`INSERT INTO nodes VALUES (?, 'B', 'repeater', 51.1, 5.6, ?, '2026-06-01T00:00:00Z', 1)`, []interface{}{b, "2026-06-07T00:00:00Z"}}, + {`INSERT INTO nodes VALUES ($1, 'N', 'repeater', 50.9, 5.4, $2, '2026-06-01T00:00:00Z', 3)`, []interface{}{n, "2026-06-07T00:00:00Z"}}, + {`INSERT INTO nodes VALUES ($1, 'A', 'repeater', 51.0, 5.5, $2, '2026-06-01T00:00:00Z', 1)`, []interface{}{a, "2026-06-07T00:00:00Z"}}, + {`INSERT INTO nodes VALUES ($1, 'B', 'repeater', 51.1, 5.6, $2, '2026-06-01T00:00:00Z', 1)`, []interface{}{b, "2026-06-07T00:00:00Z"}}, {`INSERT INTO observers (id) VALUES ('OBS1')`, nil}, {`INSERT INTO transmissions (id, from_pubkey, payload_type) VALUES (1, '', 5)`, nil}, - {`INSERT INTO observations (id, transmission_id, observer_idx, snr, path_json, timestamp) VALUES (1,1,1,-7.0,?,?)`, []interface{}{obsPath, now}}, + {`INSERT INTO observations (id, transmission_id, observer_idx, snr, path_json, timestamp) VALUES (1,1,1,-7.0,$1,$2)`, []interface{}{obsPath, now}}, } for _, in := range ins { if _, err := conn.Exec(in.q, in.args...); err != nil { @@ -233,10 +231,10 @@ func TestNodeReach_ShapeAndClamp(t *testing.T) { mustExecDB(t, db, `DROP TABLE observations`) // PREFLIGHT: async=true reason="test-only in-memory schema rebuild; not a production migration" mustExecDB(t, db, `CREATE TABLE observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, - snr REAL, + snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER )`) diff --git a/cmd/server/node_reach_test.go b/cmd/server/node_reach_test.go index 55938919f..4179c7506 100644 --- a/cmd/server/node_reach_test.go +++ b/cmd/server/node_reach_test.go @@ -2,25 +2,22 @@ package main import ( "context" - "database/sql" "strconv" "testing" - - _ "github.com/mattn/go-sqlite3" ) // newReachScanTestDB builds a minimal observer_idx-schema DB with two rows whose // path contains "01FA" and one that does not, for scanReachRows coverage. func newReachScanTestDB(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } stmts := []string{ - `CREATE TABLE transmissions (id INTEGER PRIMARY KEY, from_pubkey TEXT, payload_type INTEGER)`, - `CREATE TABLE observers (id TEXT, inactive INTEGER)`, - `CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr REAL, path_json TEXT, timestamp INTEGER)`, + `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, from_pubkey TEXT, payload_type INTEGER)`, + `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, inactive INTEGER)`, + `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`, `INSERT INTO observers (id) VALUES ('OBS1')`, // rowid 1 `INSERT INTO transmissions (id, from_pubkey, payload_type) VALUES (1,'FF00',4),(2,'',5),(3,'',5)`, `INSERT INTO observations (id, transmission_id, observer_idx, snr, path_json, timestamp) VALUES diff --git a/cmd/server/node_resolve.go b/cmd/server/node_resolve.go index bf5bd5ff6..6567dac49 100644 --- a/cmd/server/node_resolve.go +++ b/cmd/server/node_resolve.go @@ -41,7 +41,7 @@ func (s *Server) handleResolvePrefix(w http.ResponseWriter, r *http.Request) { } // LIMIT 2: we only need to know unique vs ambiguous. nodes.public_key is the // PK and stored lowercase; pfx is validated hex so the LIKE pattern is safe. - rows, err := s.db.conn.Query(`SELECT public_key, COALESCE(name,'') FROM nodes WHERE public_key LIKE ? LIMIT 2`, pfx+"%") + rows, err := s.db.conn.Query(`SELECT public_key, COALESCE(name,'') FROM nodes WHERE public_key LIKE $1 LIMIT 2`, pfx+"%") if err != nil { http.Error(w, "query failed", http.StatusInternalServerError) return diff --git a/cmd/server/nodes_pagination_has_more_test.go b/cmd/server/nodes_pagination_has_more_test.go index de1eada42..9bbb13ce6 100644 --- a/cmd/server/nodes_pagination_has_more_test.go +++ b/cmd/server/nodes_pagination_has_more_test.go @@ -40,7 +40,7 @@ func TestNodesHasMoreSurvivesPostLimitFiltering(t *testing.T) { lastSeen := fmt.Sprintf("2026-06-0%dT00:00:00Z", 7-i) if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, 'repeater', 0, 0, ?, '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, 'repeater', 0, 0, $3, '2026-06-01T00:00:00Z', 1)`, fmt.Sprintf("deadbeef0000200%d", i), name, lastSeen); err != nil { t.Fatalf("insert %d: %v", i, err) } diff --git a/cmd/server/nodes_perf_limit2000_test.go b/cmd/server/nodes_perf_limit2000_test.go index e03a7ad38..7ac64d86b 100644 --- a/cmd/server/nodes_perf_limit2000_test.go +++ b/cmd/server/nodes_perf_limit2000_test.go @@ -41,7 +41,7 @@ func TestHandleNodesLimit2000ColdMiss(t *testing.T) { } stmt, err := tx.Prepare(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, foreign_advert) - VALUES (?, ?, ?, 0, 0, ?, '2026-01-01T00:00:00Z', 1, 0)`) + VALUES ($1, $2, $3, 0, 0, $4, '2026-01-01T00:00:00Z', 1, 0)`) if err != nil { t.Fatal(err) } diff --git a/cmd/server/nodes_perf_test.go b/cmd/server/nodes_perf_test.go index c7e1c5722..b1ac1acb4 100644 --- a/cmd/server/nodes_perf_test.go +++ b/cmd/server/nodes_perf_test.go @@ -43,7 +43,7 @@ func TestHandleNodesPerfLargeFleet(t *testing.T) { } stmt, err := tx.Prepare(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, foreign_advert) - VALUES (?, ?, ?, 0, 0, ?, '2026-01-01T00:00:00Z', 1, 0)`) + VALUES ($1, $2, $3, 0, 0, $4, '2026-01-01T00:00:00Z', 1, 0)`) if err != nil { t.Fatal(err) } diff --git a/cmd/server/notify_service_test.go b/cmd/server/notify_service_test.go index c9b8b32fc..1804516cf 100644 --- a/cmd/server/notify_service_test.go +++ b/cmd/server/notify_service_test.go @@ -3,7 +3,6 @@ package main import ( "context" "errors" - "path/filepath" "regexp" "strings" "testing" @@ -11,6 +10,7 @@ import ( "github.com/gorilla/mux" "github.com/meshcore-analyzer/mailer" + "github.com/meshcore-analyzer/pgutil/pgtest" "github.com/meshcore-analyzer/users" ) @@ -91,7 +91,7 @@ type notifyFixture struct { // admin@example.org is a config admin. func newNotifyFixture(t *testing.T, ns notifySettings) *notifyFixture { t.Helper() - a, fake := newTestAuthService(t, "admin@example.org") + a, fake, ownerURL := newTestAuthServiceWithURL(t, pgtest.NewSchema(t), "admin@example.org") a.set.notify = ns clk := ¬ifyClock{t: time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)} a.st.SetClock(clk.Now) @@ -103,7 +103,7 @@ func newNotifyFixture(t *testing.T, ns notifySettings) *notifyFixture { srv := &Server{cfg: &Config{APIKey: testAPIKey}, perfStats: NewPerfStats(), auth: a} r := mux.NewRouter() srv.registerAuthRoutes(r) - return ¬ifyFixture{authFixture: &authFixture{srv: srv, router: r, fake: fake, st: a.st}, n: a.notify, src: src, clk: clk} + return ¬ifyFixture{authFixture: &authFixture{srv: srv, router: r, fake: fake, st: a.st, ownerURL: ownerURL}, n: a.notify, src: src, clk: clk} } // watcher registers and activates a user with default prefs watching pubkeys. @@ -312,13 +312,13 @@ func TestNotifierStateWriteFailureSendsNothing(t *testing.T) { f.watcher(t, "pat@example.org", "Pat", evPkA) f.setNode(evPkA, "Alpha", "companion", time.Hour, nil) f.tick() - f.execDB(t, `CREATE TRIGGER notify_state_fail BEFORE UPDATE ON notification_state BEGIN SELECT RAISE(ABORT, 'boom'); END`) + f.execDB(t, `CREATE FUNCTION notify_state_fail() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'boom'; END $$; CREATE TRIGGER notify_state_fail BEFORE UPDATE ON notification_state FOR EACH ROW EXECUTE FUNCTION notify_state_fail()`) f.clk.Advance(25 * time.Hour) f.tick() if m := f.notifyMails(); len(m) != 0 { t.Fatalf("mailed although the states were not written: %+v", m) } - f.execDB(t, `DROP TRIGGER notify_state_fail`) + f.execDB(t, `DROP TRIGGER notify_state_fail ON notification_state`) f.tick() if m := f.notifyMails(); len(m) != 1 { t.Fatalf("mails after the write works again = %d; want 1", len(m)) @@ -499,13 +499,16 @@ func TestNotifySkipReason(t *testing.T) { func TestInitUserManagementStartsTheNotifierOnlyWhenEnabled(t *testing.T) { for _, on := range []bool{false, true} { - um := &UserManagementConfig{Enabled: true, PublicBaseURL: testBase, - Mail: UserMailConfig{BrevoAPIKey: "k", FromEmail: "noreply@example.org"}} + a, _ := newTestAuthService(t) + um := validUM() + um.DatabaseURL = a.set.databaseURL + backupOff := false + um.Backup = &UsersBackupConfig{Enabled: &backupOff} if on { um.Notifications = &NotificationsConfig{Enabled: true} } srv := &Server{cfg: &Config{UserManagement: um}} - if err := srv.initUserManagement(filepath.Join(t.TempDir(), "meshcore.db")); err != nil { + if err := srv.initUserManagement(pgtest.NewDatabase(t)); err != nil { t.Fatal(err) } if (srv.auth.notify != nil) != on { diff --git a/cmd/server/notify_source_test.go b/cmd/server/notify_source_test.go index 1d5f025a5..46fef0941 100644 --- a/cmd/server/notify_source_test.go +++ b/cmd/server/notify_source_test.go @@ -45,7 +45,7 @@ func TestDBNotifyNodesChunksLongLists(t *testing.T) { keys = append(keys, fmt.Sprintf("%064x", i)) } for _, i := range []int{3, 600, 1199} { - if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, role) VALUES (?, 'companion')`, keys[i]); err != nil { + if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, role) VALUES ($1, 'companion')`, keys[i]); err != nil { t.Fatal(err) } } diff --git a/cmd/server/obs_raw_hex_test.go b/cmd/server/obs_raw_hex_test.go index 45d9a45e0..557e50ffc 100644 --- a/cmd/server/obs_raw_hex_test.go +++ b/cmd/server/obs_raw_hex_test.go @@ -33,11 +33,11 @@ func seedDistinctFrames(t *testing.T, db *DB, hash string) (string, []int) { now := time.Now().UTC().Format(time.RFC3339) if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('CANON0000', ?, ?, 1, 5, '{"type":"CHAN"}')`, hash, now); err != nil { + VALUES ('CANON0000', $1, $2, 1, 5, '{"type":"CHAN"}')`, hash, now); err != nil { t.Fatalf("insert transmission: %v", err) } var txID int - if err := db.conn.QueryRow("SELECT id FROM transmissions WHERE hash = ?", hash).Scan(&txID); err != nil { + if err := db.conn.QueryRow("SELECT id FROM transmissions WHERE hash = $1", hash).Scan(&txID); err != nil { t.Fatalf("lookup tx id: %v", err) } @@ -52,14 +52,15 @@ func seedDistinctFrames(t *testing.T, db *DB, hash string) (string, []int) { } var ids []int for i, f := range frames { - res, err := db.conn.Exec(`INSERT INTO observations + var id int64 + err := db.conn.QueryRow(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, raw_hex) - VALUES (?, 1, 5.0, -90, ?, ?, ?)`, - txID, f.pathJSON, time.Now().Unix()-int64(i), f.rawHex) + VALUES ($1, 1, 5.0, -90, $2, $3, $4) RETURNING id`, + txID, f.pathJSON, time.Now().Unix()-int64(i), f.rawHex).Scan(&id) if err != nil { t.Fatalf("insert observation %d: %v", i, err) } - id, err := res.LastInsertId() + if err != nil { t.Fatalf("last insert id: %v", err) } diff --git a/cmd/server/observer_blacklist_test.go b/cmd/server/observer_blacklist_test.go index ec6b2b473..0afea186e 100644 --- a/cmd/server/observer_blacklist_test.go +++ b/cmd/server/observer_blacklist_test.go @@ -48,8 +48,8 @@ func TestConfigIsObserverBlacklistedNil(t *testing.T) { func TestObserverBlacklistFiltersHandleObservers(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO observers (id, name, iata, last_seen) VALUES ('goodobs', 'GoodObs', 'SFO', datetime('now'))") - db.conn.Exec("INSERT OR IGNORE INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', datetime('now'))") + db.conn.Exec("INSERT INTO observers (id, name, iata, last_seen) VALUES ('goodobs', 'GoodObs', 'SFO', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING") + db.conn.Exec("INSERT INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING") cfg := &Config{ ObserverBlacklist: []string{"badobs"}, @@ -89,7 +89,7 @@ func TestObserverBlacklistFiltersHandleObservers(t *testing.T) { func TestObserverBlacklistFiltersObserverDetail(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', datetime('now'))") + db.conn.Exec("INSERT INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING") cfg := &Config{ ObserverBlacklist: []string{"badobs"}, @@ -108,7 +108,7 @@ func TestObserverBlacklistFiltersObserverDetail(t *testing.T) { func TestNoObserverBlacklistPassesAll(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO observers (id, name, iata, last_seen) VALUES ('someobs', 'SomeObs', 'SFO', datetime('now'))") + db.conn.Exec("INSERT INTO observers (id, name, iata, last_seen) VALUES ('someobs', 'SomeObs', 'SFO', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING") cfg := &Config{} srv := NewServer(db, cfg, NewHub()) diff --git a/cmd/server/observer_count_1888_test.go b/cmd/server/observer_count_1888_test.go index 89d4bd225..65aa4d29c 100644 --- a/cmd/server/observer_count_1888_test.go +++ b/cmd/server/observer_count_1888_test.go @@ -27,18 +27,18 @@ func seedObserversForCount(t *testing.T, db *DB) { live := []string{"live1", "live2", "live3"} for _, id := range live { if _, err := db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count, inactive) - VALUES (?, ?, ?, ?, 10, 0)`, id, "Observer "+id, now, now); err != nil { + VALUES ($1, $2, $3, $4, 10, 0)`, id, "Observer "+id, now, now); err != nil { t.Fatalf("insert %s: %v", id, err) } } // One explicitly soft-deleted, one with a NULL flag — GetObservers treats // NULL as live, so only the inactive=1 row must be excluded. if _, err := db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count, inactive) - VALUES ('gone1', 'Gone One', ?, ?, 5, 1)`, now, now); err != nil { + VALUES ('gone1', 'Gone One', $1, $2, 5, 1)`, now, now); err != nil { t.Fatalf("insert gone1: %v", err) } if _, err := db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count, inactive) - VALUES ('nullflag', 'Null Flag', ?, ?, 5, NULL)`, now, now); err != nil { + VALUES ('nullflag', 'Null Flag', $1, $2, 5, NULL)`, now, now); err != nil { t.Fatalf("insert nullflag: %v", err) } } diff --git a/cmd/server/observer_naive_clock_1478_test.go b/cmd/server/observer_naive_clock_1478_test.go index e0d369c24..5e749367f 100644 --- a/cmd/server/observer_naive_clock_1478_test.go +++ b/cmd/server/observer_naive_clock_1478_test.go @@ -30,7 +30,7 @@ func TestHandleObservers_Issue1478_SurfacesRecentNaiveSkew(t *testing.T) { _, err := srv.db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, "naive-obs-1", "California Pi", "SFO", now.Format(time.RFC3339), now.Add(-7*24*time.Hour).Format(time.RFC3339), 42, -28800, 17, recent) @@ -91,7 +91,7 @@ func TestHandleObservers_Issue1478_DecaysAfter24h(t *testing.T) { _, err := srv.db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, "naive-obs-old", "Fixed Pi", "LAX", now.Format(time.RFC3339), now.Add(-30*24*time.Hour).Format(time.RFC3339), 99, -28800, 5, stale) @@ -136,7 +136,7 @@ func TestHandleObserverDetail_Issue1478_IncludesClockNaiveFields(t *testing.T) { _, err := srv.db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, "naive-obs-detail", "Detail Pi", "SJC", now.Format(time.RFC3339), now.Add(-2*24*time.Hour).Format(time.RFC3339), 7, 25200, 3, recent) diff --git a/cmd/server/observers_canrelay_tristate_1290_test.go b/cmd/server/observers_canrelay_tristate_1290_test.go index 146f7ae9e..5073edcda 100644 --- a/cmd/server/observers_canrelay_tristate_1290_test.go +++ b/cmd/server/observers_canrelay_tristate_1290_test.go @@ -39,19 +39,19 @@ func TestObservers_CanRelayTriState_Issue1290(t *testing.T) { // Legacy: never received repeat field. can_relay=DEFAULT 1, seen=0. if _, err := srv.store.db.conn.Exec( `INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('legacy-obs', 'Legacy', 'SJC', ?, '2026-01-01T00:00:00Z', 1)`, now); err != nil { + VALUES ('legacy-obs', 'Legacy', 'SJC', $1, '2026-01-01T00:00:00Z', 1)`, now); err != nil { t.Fatalf("seed legacy: %v", err) } // Repeater: ingestor wrote can_relay=1, seen=1. if _, err := srv.store.db.conn.Exec( `INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, can_relay, can_relay_seen) - VALUES ('rep-obs', 'Repeater', 'SFO', ?, '2026-01-01T00:00:00Z', 1, 1, 1)`, now); err != nil { + VALUES ('rep-obs', 'Repeater', 'SFO', $1, '2026-01-01T00:00:00Z', 1, 1, 1)`, now); err != nil { t.Fatalf("seed repeater: %v", err) } // Listener: ingestor wrote can_relay=0, seen=1. if _, err := srv.store.db.conn.Exec( `INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, can_relay, can_relay_seen) - VALUES ('lst-obs', 'Listener', 'OAK', ?, '2026-01-01T00:00:00Z', 1, 0, 1)`, now); err != nil { + VALUES ('lst-obs', 'Listener', 'OAK', $1, '2026-01-01T00:00:00Z', 1, 0, 1)`, now); err != nil { t.Fatalf("seed listener: %v", err) } diff --git a/cmd/server/openapi.go b/cmd/server/openapi.go index 3f2f29dcd..1dc807db4 100644 --- a/cmd/server/openapi.go +++ b/cmd/server/openapi.go @@ -49,14 +49,15 @@ func routeDescriptions() map[string]routeMeta { "GET /api/config/geo-filter": {Summary: "Get geo-filter configuration", Tag: "config"}, // Admin / system - "GET /api/health": {Summary: "Health check", Description: "Returns server health, uptime, and memory stats.", Tag: "admin"}, - "GET /api/stats": {Summary: "Network statistics", Description: "Returns aggregate stats (node counts, packet counts, observer counts). Cached for 10s.", Tag: "admin"}, - "GET /api/perf": {Summary: "Performance statistics", Description: "Returns per-endpoint request timing and slow query log.", Tag: "admin"}, - "GET /api/mqtt/status": {Summary: "MQTT source status", Description: "Returns per-MQTT-source connection state and counters (lastConnectUnix, lastPacketUnix, packetsTotal, etc.). Broker URL passwords are masked. Sourced from the ingestor stats file; empty list when unavailable. (#1043)", Tag: "admin"}, - "POST /api/perf/reset": {Summary: "Reset performance stats", Tag: "admin", Auth: true, Session: true}, + "GET /api/health": {Summary: "Health check", Description: "Returns server health, uptime, and memory stats.", Tag: "admin"}, + "GET /api/stats": {Summary: "Network statistics", Description: "Returns aggregate stats (node counts, packet counts, observer counts). Cached for 10s.", Tag: "admin"}, + "GET /api/perf": {Summary: "Performance statistics", Description: "Returns per-endpoint request timing and slow query log.", Tag: "admin"}, + "GET /api/perf/postgres": {Summary: "PostgreSQL database statistics", Description: "Returns databaseBytes and PostgreSQL block-cache hit ratio (cacheHitRate, 0..1 or null without available reads), sampled with table counts once per 30 seconds and shared with /api/perf. sampledAt and sampleIntervalSeconds disclose freshness; stale marks a failed refresh. Before a first successful sample, unavailable size/counts are null and error is set. After a refresh failure, the last storage/count sample remains with stale=true and the unavailable block ratio is null. Pool usage and waits remain live. Block-cache counters are cumulative since the PostgreSQL statistics reset and exclude the operating-system cache.", Tag: "admin"}, + "GET /api/mqtt/status": {Summary: "MQTT source status", Description: "Returns per-MQTT-source connection state and counters (lastConnectUnix, lastPacketUnix, packetsTotal, etc.). Broker URL passwords are masked. Sourced from the ingestor stats file; empty list when unavailable. (#1043)", Tag: "admin"}, + "POST /api/perf/reset": {Summary: "Reset performance stats", Tag: "admin", Auth: true, Session: true}, // "POST /api/admin/prune" removed in #1283 (ingestor owns prune). "GET /api/debug/affinity": {Summary: "Debug neighbor affinity scores", Tag: "admin", Auth: true, Session: true}, - "GET /api/backup": {Summary: "Download SQLite backup", Description: "Streams a consistent SQLite snapshot of the analyzer DB (VACUUM INTO). Response is application/octet-stream with attachment filename corescope-backup-.db.", Tag: "admin", Auth: true, Session: true}, + "GET /api/backup": {Summary: "Download PostgreSQL backup", Description: "Streams a consistent PostgreSQL custom-format pg_dump archive of telemetry. Restore with pg_restore into a separate database. Response is application/octet-stream with attachment filename corescope-backup-.dump.", Tag: "admin", Auth: true, Session: true}, // User management (optional; routes exist only when userManagement.enabled) "POST /api/auth/register": {Summary: "Register an account", Description: "Creates a pending account and mails an activation link. The response is identical whether or not the address is already registered.", Tag: "users"}, @@ -88,7 +89,7 @@ func routeDescriptions() map[string]routeMeta { "POST /api/admin/users/{id}/mail/{mailId}/refresh": {Summary: "Pull delivery events for one mail from the provider (admin)", Tag: "users", Session: true}, "GET /api/admin/audit": {Summary: "Global audit log (admin)", Description: "Newest first, keyset-paginated: {entries, next}. Pass next as before for the following page; next is null on the last page. Entry: {id, at, action, actor, target, detail}; actor and target are {id, displayName, email}, {id, deleted: true} for a removed account, or null for the system. Login rows (user.login, user.login.failed) are kept 90 days. 400 on an invalid parameter.", Tag: "users", Session: true, QueryParams: []paramMeta{{Name: "action", Description: "One action, or a group ending in .* (user.login.* matches user.login and user.login.failed)", Type: "string"}, {Name: "user", Description: "User id, as actor or target", Type: "integer"}, {Name: "from", Description: "RFC 3339 time, inclusive", Type: "string"}, {Name: "to", Description: "RFC 3339 time, inclusive", Type: "string"}, {Name: "before", Description: "Only entries with a smaller id (next of the previous page)", Type: "integer"}, {Name: "limit", Description: "Page size, default 100, capped at 500", Type: "integer"}}}, "GET /api/admin/stats": {Summary: "User figures for the admin overview (admin)", Description: "Accounts by status, active admins, pending accounts older than 24 hours (stuckPending), bouncing addresses, registrations in 7 and 30 days and per UTC day for 30 days (newPerDay, oldest first), users active in 7 and 30 days, logins and failed logins in 24 hours, mail sent in 7 days by latest delivery status (mail7d), and accounts with 5 or more failed logins in 24 hours (guessing: [{userId, displayName, failed}]).", Tag: "users", Session: true}, - "GET /api/admin/users-backup": {Summary: "Download a users.db snapshot (admin)", Description: "A fresh, consistent copy of users.db (VACUUM INTO) as application/octet-stream, attachment corescope-users-.db (UTC). It holds every password hash and address: store it encrypted. Records a user.backup audit row. The analyzer database has its own route, GET /api/backup.", Tag: "users", Session: true}, + "GET /api/admin/users-backup": {Summary: "Download accounts backup (admin)", Description: "A consistent PostgreSQL custom-format pg_dump archive of accounts as application/octet-stream, attachment corescope-users-.dump (UTC). It holds every password hash and address: store it encrypted. Records a user.backup audit row. The analyzer database has its own route, GET /api/backup.", Tag: "users", Session: true}, "POST /api/proposals": {Summary: "Propose a hashtag channel", Description: "Body {kind: \"hashtag_channel\", subject}. subject is trimmed and gets a leading # when missing; at most 31 UTF-8 bytes including the #, no control or invisible characters (ZWJ allowed), not Public. 201 with the proposal {id, kind, subject, status, note, createdAt, decidedAt}. 400 invalid kind or name, 409 already pending, already approved or rejected, or already decrypted on this instance (a name in config.json hashChannels or channelKeys, case-sensitive), 429 when the caller created userManagement.channelProposals.perUserPerDay proposals in 24 hours or maxPending proposals wait for review. Registered only when userManagement.channelProposals.enabled.", Tag: "users", Session: true}, "GET /api/account/proposals": {Summary: "List own proposals", Description: "The caller's proposals, newest first: [{id, kind, subject, status, note, createdAt, decidedAt}]. note is the reviewer's reason.", Tag: "users", Session: true}, "GET /api/admin/proposals": {Summary: "List proposals (admin)", Description: "Newest first, at most 500: the proposal fields plus proposer and reviewer ({id, displayName, email}, or null once the account is deleted). 400 on an invalid filter.", Tag: "users", Session: true, QueryParams: []paramMeta{{Name: "status", Description: "pending | approved | rejected | revoked", Type: "string"}, {Name: "kind", Description: "hashtag_channel", Type: "string"}}}, diff --git a/cmd/server/packet_scope_name_test.go b/cmd/server/packet_scope_name_test.go index cbf920b7d..00caf0be7 100644 --- a/cmd/server/packet_scope_name_test.go +++ b/cmd/server/packet_scope_name_test.go @@ -71,15 +71,15 @@ func TestPacketDetailExposesScopeName(t *testing.T) { now := time.Now().UTC().Format(time.RFC3339) // route_type 1 = FLOOD (never transport-scoped → NULL); 0 = TRANSPORT_FLOOD. if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES ('AABB', 'aaaaaaaaaaaaaaa1', ?, 1, 4)`, now); err != nil { + VALUES ('AABB', 'aaaaaaaaaaaaaaa1', $1, 1, 4)`, now); err != nil { t.Fatalf("insert unscoped: %v", err) } if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, scope_name) - VALUES ('AABB', 'aaaaaaaaaaaaaaa2', ?, 0, 4, '')`, now); err != nil { + VALUES ('AABB', 'aaaaaaaaaaaaaaa2', $1, 0, 4, '')`, now); err != nil { t.Fatalf("insert unknown-scope: %v", err) } if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, scope_name) - VALUES ('AABB', 'aaaaaaaaaaaaaaa3', ?, 0, 4, '#belgium')`, now); err != nil { + VALUES ('AABB', 'aaaaaaaaaaaaaaa3', $1, 0, 4, '#belgium')`, now); err != nil { t.Fatalf("insert matched-scope: %v", err) } @@ -165,15 +165,15 @@ func TestGroupedPacketsEndpointExposesScopeName(t *testing.T) { now := time.Now().UTC().Format(time.RFC3339) if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES ('AABB', 'bbbbbbbbbbbbbbb1', ?, 1, 4)`, now); err != nil { + VALUES ('AABB', 'bbbbbbbbbbbbbbb1', $1, 1, 4)`, now); err != nil { t.Fatalf("insert unscoped: %v", err) } if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, scope_name) - VALUES ('AABB', 'bbbbbbbbbbbbbbb2', ?, 0, 4, '')`, now); err != nil { + VALUES ('AABB', 'bbbbbbbbbbbbbbb2', $1, 0, 4, '')`, now); err != nil { t.Fatalf("insert unknown-scope: %v", err) } if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, scope_name) - VALUES ('AABB', 'bbbbbbbbbbbbbbb3', ?, 0, 4, '#belgium')`, now); err != nil { + VALUES ('AABB', 'bbbbbbbbbbbbbbb3', $1, 0, 4, '#belgium')`, now); err != nil { t.Fatalf("insert matched-scope: %v", err) } diff --git a/cmd/server/packets_order_test.go b/cmd/server/packets_order_test.go index 8d62efb99..853eaf803 100644 --- a/cmd/server/packets_order_test.go +++ b/cmd/server/packets_order_test.go @@ -32,11 +32,11 @@ func TestQueryPacketsOrdersByIngestID(t *testing.T) { bufferedFirstSeen := now.Add(-6 * time.Hour).Format(time.RFC3339) if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) - VALUES ('AA', 'hashfresh00000001', ?, 4)`, freshFirstSeen); err != nil { + VALUES ('AA', 'hashfresh00000001', $1, 4)`, freshFirstSeen); err != nil { t.Fatal(err) } if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) - VALUES ('BB', 'hashbuffered00002', ?, 4)`, bufferedFirstSeen); err != nil { + VALUES ('BB', 'hashbuffered00002', $1, 4)`, bufferedFirstSeen); err != nil { t.Fatal(err) } @@ -78,24 +78,24 @@ func TestQueryPacketsSinceFilterUsesFirstSeen(t *testing.T) { oldEpoch := now.Add(-6 * time.Hour).Unix() if _, err := db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Obs1', ?, ?, 1)`, recent, recent); err != nil { + VALUES ('obs1', 'Obs1', $1, $2, 1)`, recent, recent); err != nil { t.Fatal(err) } if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) - VALUES ('AA', 'recentrx00000001', ?, 4)`, recent); err != nil { + VALUES ('AA', 'recentrx00000001', $1, 4)`, recent); err != nil { t.Fatal(err) } // Buffered upload — ingested SECOND, but rxTime is 6h ago. if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) - VALUES ('BB', 'oldrxbuffered001', ?, 4)`, old); err != nil { + VALUES ('BB', 'oldrxbuffered001', $1, 4)`, old); err != nil { t.Fatal(err) } if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10, -90, '[]', ?)`, recentEpoch); err != nil { + VALUES (1, 1, 10, -90, '[]', $1)`, recentEpoch); err != nil { t.Fatal(err) } if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 10, -90, '[]', ?)`, oldEpoch); err != nil { + VALUES (2, 1, 10, -90, '[]', $1)`, oldEpoch); err != nil { t.Fatal(err) } diff --git a/cmd/server/parity_test.go b/cmd/server/parity_test.go index 3abe15e70..23d707f08 100644 --- a/cmd/server/parity_test.go +++ b/cmd/server/parity_test.go @@ -1,8 +1,9 @@ package main // parity_test.go — Golden fixture shape tests. -// Validates that Go API responses match the shape of Node.js API responses. -// Shapes were captured from the production Node.js server and stored in +// Validates the public API shape contract captured from the original server. +// PostgreSQL diagnostics deliberately replace SQLite-only metrics; telemetry +// and account response contracts remain unchanged. Shapes are stored in // testdata/golden/shapes.json. import ( @@ -200,7 +201,7 @@ func TestParityShapes(t *testing.T) { mismatches := validateShape(body, spec, ep.path) if len(mismatches) > 0 { - t.Errorf("Go %s has %d shape mismatches vs Node.js golden:\n %s", + t.Errorf("Go %s has %d shape mismatches vs golden contract:\n %s", ep.path, len(mismatches), strings.Join(mismatches, "\n ")) } }) @@ -231,7 +232,7 @@ func TestParityNodeDetail(t *testing.T) { mismatches := validateShape(body, spec, "/api/nodes/{pubkey}") if len(mismatches) > 0 { - t.Errorf("Go node detail has %d shape mismatches vs Node.js golden:\n %s", + t.Errorf("Go node detail has %d shape mismatches vs golden contract:\n %s", len(mismatches), strings.Join(mismatches, "\n ")) } } @@ -441,7 +442,7 @@ func TestParityWSMultiObserverGolden(t *testing.T) { now := time.Now().UTC().Format(time.RFC3339) if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('BEEF', 'goldenstarburst237', ?, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now); err != nil { + VALUES ('BEEF', 'goldenstarburst237', $1, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now); err != nil { t.Fatalf("insert tx failed: %v", err) } var txID int @@ -450,9 +451,9 @@ func TestParityWSMultiObserverGolden(t *testing.T) { } ts := time.Now().Unix() if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 11.0, -88, '["p1"]', ?), - (?, 2, 9.0, -92, '["p1","p2"]', ?), - (?, 1, 7.0, -96, '["p1","p2","p3"]', ?)`, + VALUES ($1, 1, 11.0, -88, '["p1"]', $2), + ($3, 2, 9.0, -92, '["p1","p2"]', $4), + ($5, 1, 7.0, -96, '["p1","p2","p3"]', $6)`, txID, ts, txID, ts+1, txID, ts+2); err != nil { t.Fatalf("insert obs failed: %v", err) } diff --git a/cmd/server/pathhop_dedupe_2108_test.go b/cmd/server/pathhop_dedupe_2108_test.go index b77b2fe5c..c9c6d326a 100644 --- a/cmd/server/pathhop_dedupe_2108_test.go +++ b/cmd/server/pathhop_dedupe_2108_test.go @@ -62,11 +62,11 @@ func phdSeed(t testing.TB, db *DB, observers int) { t.Helper() ts := time.Now().UTC().Add(-time.Hour).Format(time.RFC3339) for i, pk := range phdRelays { - phdExec(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES (?, ?, 'repeater', ?, '2026-01-01', 1)`, + phdExec(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ($1, $2, 'repeater', $3, '2026-01-01', 1)`, pk, fmt.Sprintf("phdRelay-%d", i), ts) } for i := 1; i <= observers; i++ { - phdExec(t, db, `INSERT INTO observers (rowid, id, name, iata) VALUES (?, ?, ?, 'TST')`, + phdExec(t, db, `INSERT INTO observers (rowid, id, name, iata) VALUES ($1, $2, $3, 'TST')`, i, fmt.Sprintf("phdObs-%02d", i), fmt.Sprintf("Observer %d", i)) } } @@ -82,7 +82,7 @@ func phdExec(t testing.TB, db *DB, q string, args ...any) { func phdInsertTx(t testing.TB, db *DB, i int, firstSeen time.Time) { t.Helper() phdExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES (?, 'CAFE', ?, ?, 1, 2, '{"type":"TXT_MSG"}')`, i+1, fmt.Sprintf("phdTx-%03d", i), firstSeen.Format(time.RFC3339)) + VALUES ($1, 'CAFE', $2, $3, 1, 2, '{"type":"TXT_MSG"}')`, i+1, fmt.Sprintf("phdTx-%03d", i), firstSeen.Format(time.RFC3339)) } // phdInsertObs inserts one observation of transmission i through its path, @@ -102,7 +102,7 @@ func phdInsertObsVia(t testing.TB, db *DB, i int, path []int, obsIdx int, ts tim rp = phdResolvedJSON(path) } phdExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) - VALUES (?, ?, 5, -90, ?, ?, ?)`, i+1, obsIdx, phdRawHopsJSON(path), ts.Unix(), rp) + VALUES ($1, $2, 5, -90, $3, $4, $5)`, i+1, obsIdx, phdRawHopsJSON(path), ts.Unix(), rp) } func phdMaxObsID(t testing.TB, db *DB) int { @@ -580,7 +580,7 @@ func TestPathHopConsumersIgnoreDuplicateEntries_2108(t *testing.T) { func TestMultiByteCapabilityIgnoresDuplicateEntries_2108(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepB", "repeater", recentTS(48)) store := NewPacketStore(db, nil) pt := 1 diff --git a/cmd/server/paths_anchor_bias_test.go b/cmd/server/paths_anchor_bias_test.go index 1a966c5df..a50f88744 100644 --- a/cmd/server/paths_anchor_bias_test.go +++ b/cmd/server/paths_anchor_bias_test.go @@ -43,23 +43,23 @@ func TestHandleNodePaths_AnchorBiasInconsistency_Issue1278(t *testing.T) { nodeGPSPK := "c0ffeec700000000000000000000000000000000000000000000000000000001" if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'NodeNoGPS', 'repeater', 0, 0, ?, '2026-01-01', 1)`, nodeNoGPSPK, recent); err != nil { + VALUES ($1, 'NodeNoGPS', 'repeater', 0, 0, $2, '2026-01-01', 1)`, nodeNoGPSPK, recent); err != nil { t.Fatalf("insert nodeNoGPS: %v", err) } if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'NodeGPS', 'repeater', 37.5, -122.0, ?, '2026-01-01', 1)`, nodeGPSPK, recent); err != nil { + VALUES ($1, 'NodeGPS', 'repeater', 37.5, -122.0, $2, '2026-01-01', 1)`, nodeGPSPK, recent); err != nil { t.Fatalf("insert nodeGPS: %v", err) } if _, err := db.conn.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) - VALUES (100, 'AA', 'hash_collision', ?)`, recent); err != nil { + VALUES (100, 'AA', 'hash_collision', $1)`, recent); err != nil { t.Fatalf("insert tx: %v", err) } // obs1: SHORTER path_json (single hop), resolved → nodeNoGPS. // (Without this row, the membership index wouldn't list nodeNoGPS at all // and the tx would be cleanly excluded — the bug needs the index hit.) if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (100, NULL, '["c0"]', ?, ?)`, recentEpoch, `["`+nodeNoGPSPK+`"]`); err != nil { + VALUES (100, NULL, '["c0"]', $1, $2)`, recentEpoch, `["`+nodeNoGPSPK+`"]`); err != nil { t.Fatalf("insert obs1: %v", err) } // obs2: LONGER path_json (two hops, first hop is what packets page shows @@ -67,7 +67,7 @@ func TestHandleNodePaths_AnchorBiasInconsistency_Issue1278(t *testing.T) { // canonical because it has the longer path_json. Its resolved_path // picks nodeGPS for "c0", NOT nodeNoGPS. if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (100, NULL, '["c0","ee"]', ?, ?)`, recentEpoch, `["`+nodeGPSPK+`","ee00000000000000000000000000000000000000000000000000000000000000"]`); err != nil { + VALUES (100, NULL, '["c0","ee"]', $1, $2)`, recentEpoch, `["`+nodeGPSPK+`","ee00000000000000000000000000000000000000000000000000000000000000"]`); err != nil { t.Fatalf("insert obs2: %v", err) } diff --git a/cmd/server/paths_hop_name_test.go b/cmd/server/paths_hop_name_test.go index 56d6408e1..a1f13780f 100644 --- a/cmd/server/paths_hop_name_test.go +++ b/cmd/server/paths_hop_name_test.go @@ -29,14 +29,14 @@ func TestHandleNodePaths_HopName_CanonicalPathShowsTarget_1144(t *testing.T) { siblingPK := "37bb000011112222" // has GPS — biased resolver picks this without fix mustExec(t, db, `INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'CJS SF Mission', 'repeater', 0, 0, ?, '2026-01-01', 1)`, targetPK, recent) + VALUES ($1, 'CJS SF Mission', 'repeater', 0, 0, $2, '2026-01-01', 1)`, targetPK, recent) mustExec(t, db, `INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'Templeton Hills', 'repeater', 35.5, -120.7, ?, '2026-01-01', 1)`, siblingPK, recent) + VALUES ($1, 'Templeton Hills', 'repeater', 35.5, -120.7, $2, '2026-01-01', 1)`, siblingPK, recent) // TX: resolved_path = [targetPK] → canonical path (Option A) → lookupNode(targetPK) - mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, 'AA', 'hash1144', ?)`, recent) + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, 'AA', 'hash1144', $1)`, recent) mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (1, NULL, '["37"]', ?, ?)`, recentEpoch, `["`+targetPK+`"]`) + VALUES (1, NULL, '["37"]', $1, $2)`, recentEpoch, `["`+targetPK+`"]`) cfg := &Config{Port: 3000} hub := NewHub() diff --git a/cmd/server/paths_sort_test.go b/cmd/server/paths_sort_test.go index aad84c1f5..8dbf5c53a 100644 --- a/cmd/server/paths_sort_test.go +++ b/cmd/server/paths_sort_test.go @@ -39,7 +39,7 @@ func TestHandleNodePaths_SortByRecency_1145(t *testing.T) { // Only the target node needs to be in the nodes table. // Relay pubkeys appear only in resolved_path; they don't need a nodes row. mustExec(t, db, `INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'Target', 'repeater', 0, 0, '2026-05-01T00:00:00Z', '2026-01-01T00:00:00Z', 1)`, targetPK) + VALUES ($1, 'Target', 'repeater', 0, 0, '2026-05-01T00:00:00Z', '2026-01-01T00:00:00Z', 1)`, targetPK) // -- Path A (via relay1): 3 txs, last seen 2026-01-03 → group sig "relay1PK→targetPK" -- for txID, ts := range map[int]string{ @@ -47,17 +47,17 @@ func TestHandleNodePaths_SortByRecency_1145(t *testing.T) { 2: "2026-01-02T00:00:00Z", 3: "2026-01-03T00:00:00Z", } { - mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (?, 'AA', ?, ?)`, + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES ($1, 'AA', $2, $3)`, txID, "hashA"+string(rune('0'+txID)), ts) mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (?, NULL, '["11", "aa"]', ?, ?)`, + VALUES ($1, NULL, '["11", "aa"]', $2, $3)`, txID, epoch(ts), `["`+relay1PK+`", "`+targetPK+`"]`) } // -- Path B (via relay2): 1 tx, last seen 2026-05-01 → group sig "relay2PK→targetPK" -- mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (4, 'BB', 'hashB1', '2026-05-01T00:00:00Z')`) mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (4, NULL, '["22", "aa"]', ?, ?)`, + VALUES (4, NULL, '["22", "aa"]', $1, $2)`, epoch("2026-05-01T00:00:00Z"), `["`+relay2PK+`", "`+targetPK+`"]`) // -- Path C (direct — target is sole hop): 2 txs, last seen 2026-03-02 -- @@ -65,10 +65,10 @@ func TestHandleNodePaths_SortByRecency_1145(t *testing.T) { 5: "2026-03-01T00:00:00Z", 6: "2026-03-02T00:00:00Z", } { - mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (?, 'CC', ?, ?)`, + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES ($1, 'CC', $2, $3)`, txID, "hashC"+string(rune('0'+txID)), ts) mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (?, NULL, '["aa"]', ?, ?)`, + VALUES ($1, NULL, '["aa"]', $2, $3)`, txID, epoch(ts), `["`+targetPK+`"]`) } @@ -130,7 +130,7 @@ func TestHandleNodePaths_SortCountTiebreaker_1145(t *testing.T) { } mustExec(t, db, `INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'Tgt', 'repeater', 0, 0, ?, '2026-01-01T00:00:00Z', 1)`, targetPK, sameTS) + VALUES ($1, 'Tgt', 'repeater', 0, 0, $2, '2026-01-01T00:00:00Z', 1)`, targetPK, sameTS) // Path X: 3 txs, all at sameTS → higher count for txID, ts := range map[int]string{ @@ -138,17 +138,17 @@ func TestHandleNodePaths_SortCountTiebreaker_1145(t *testing.T) { 11: "2026-04-15T11:30:00Z", 12: sameTS, } { - mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (?, 'XX', ?, ?)`, + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES ($1, 'XX', $2, $3)`, txID, "hashX"+string(rune('0'+txID)), ts) mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (?, NULL, '["aa", "cc"]', ?, ?)`, + VALUES ($1, NULL, '["aa", "cc"]', $2, $3)`, txID, epoch(ts), `["`+relay1PK+`", "`+targetPK+`"]`) } // Path Y: 1 tx, at sameTS → lower count - mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (20, 'YY', 'hashY1', ?)`, sameTS) + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (20, 'YY', 'hashY1', $1)`, sameTS) mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (20, NULL, '["bb", "cc"]', ?, ?)`, + VALUES (20, NULL, '["bb", "cc"]', $1, $2)`, epoch(sameTS), `["`+relay2PK+`", "`+targetPK+`"]`) cfg := &Config{Port: 3000} diff --git a/cmd/server/paths_through_collision_1352_test.go b/cmd/server/paths_through_collision_1352_test.go index 00545bde8..52bfaa7cb 100644 --- a/cmd/server/paths_through_collision_1352_test.go +++ b/cmd/server/paths_through_collision_1352_test.go @@ -67,11 +67,11 @@ func setupCollisionScenario(t *testing.T, withGPS bool) *collisionScenario { } mustExec(t, db, `INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'NodeA', 'repeater', ?, ?, ?, '2026-01-01', 1)`, sc.nodeAPK, aLat, aLon, recent) + VALUES ($1, 'NodeA', 'repeater', $2, $3, $4, '2026-01-01', 1)`, sc.nodeAPK, aLat, aLon, recent) mustExec(t, db, `INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'NodeB', 'repeater', ?, ?, ?, '2026-01-01', 1)`, sc.nodeBPK, bLat, bLon, recent) + VALUES ($1, 'NodeB', 'repeater', $2, $3, $4, '2026-01-01', 1)`, sc.nodeBPK, bLat, bLon, recent) mustExec(t, db, `INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'NodeC', 'repeater', ?, ?, ?, '2026-01-01', 1)`, sc.nodeCPK, cLat, cLon, recent) + VALUES ($1, 'NodeC', 'repeater', $2, $3, $4, '2026-01-01', 1)`, sc.nodeCPK, cLat, cLon, recent) cfg := &Config{Port: 3000} hub := NewHub() @@ -133,9 +133,9 @@ func (sc *collisionScenario) query(t *testing.T, pk string) NodePathsResponse { // must include the tx. paths-through-A and paths-through-C must exclude it. func TestHandleNodePaths_PrefixCollision_1352(t *testing.T) { sc := setupCollisionScenario(t, false /* only B has GPS */) - mustExec(t, sc.db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (42, 'DEAD', 'hash_1352', ?)`, sc.recent) + mustExec(t, sc.db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (42, 'DEAD', 'hash_1352', $1)`, sc.recent) mustExec(t, sc.db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (42, NULL, '["c0"]', ?, ?)`, sc.recentEpoch, `["`+sc.nodeBPK+`"]`) + VALUES (42, NULL, '["c0"]', $1, $2)`, sc.recentEpoch, `["`+sc.nodeBPK+`"]`) sc.reloadStore(t) respA := sc.query(t, sc.nodeAPK) @@ -173,9 +173,9 @@ func TestHandleNodePaths_PrefixCollision_1352(t *testing.T) { // all three = 3. Fixed: ≤1, and we tighten further to ≤1 explicitly. func TestHandleNodePaths_PrefixCollision_1352_FallbackBranch(t *testing.T) { sc := setupCollisionScenario(t, true /* all three have GPS */) - mustExec(t, sc.db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (43, 'BEEF', 'hash_1352_fb', ?)`, sc.recent) + mustExec(t, sc.db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (43, 'BEEF', 'hash_1352_fb', $1)`, sc.recent) mustExec(t, sc.db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (43, NULL, '["c0"]', ?, NULL)`, sc.recentEpoch) + VALUES (43, NULL, '["c0"]', $1, NULL)`, sc.recentEpoch) sc.reloadStore(t) a := sc.query(t, sc.nodeAPK).TotalTransmissions @@ -213,10 +213,10 @@ func TestHandleNodePaths_FallbackUniquePrefix_1352(t *testing.T) { pk := "abcdef0123456789" mustExec(t, db, `INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'UniqueNode', 'repeater', 37.78, -122.4, ?, '2026-01-01', 1)`, pk, recent) - mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (44, 'CAFE', 'hash_1352_unique', ?)`, recent) + VALUES ($1, 'UniqueNode', 'repeater', 37.78, -122.4, $2, '2026-01-01', 1)`, pk, recent) + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (44, 'CAFE', 'hash_1352_unique', $1)`, recent) mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (44, NULL, '["ab"]', ?, NULL)`, recentEpoch) + VALUES (44, NULL, '["ab"]', $1, NULL)`, recentEpoch) cfg := &Config{Port: 3000} hub := NewHub() @@ -274,14 +274,14 @@ func TestHandleNodePaths_FallbackPreconfirmed_1352(t *testing.T) { // tx 50: best obs has NULL resolved_path (fallback branch). A SECOND // obs persists resolved_path = [B] which populates the byPathHop index // for B's full pubkey AND lets confirmedBySQL hit via INSTR. - mustExec(t, sc.db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (50, 'F00D', 'hash_1352_pre', ?)`, sc.recent) + mustExec(t, sc.db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (50, 'F00D', 'hash_1352_pre', $1)`, sc.recent) mustExec(t, sc.db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (50, NULL, '["c0"]', ?, NULL)`, sc.recentEpoch) + VALUES (50, NULL, '["c0"]', $1, NULL)`, sc.recentEpoch) // Second observation (different observer) — same tx, persisted resolved_path = [B]. // This populates byPathHop[B] during Load(), so confirmedByFullKey is true // when paths-through-B is queried. mustExec(t, sc.db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (50, 1, '["c0"]', ?, ?)`, sc.recentEpoch+1, `["`+sc.nodeBPK+`"]`) + VALUES (50, 1, '["c0"]', $1, $2)`, sc.recentEpoch+1, `["`+sc.nodeBPK+`"]`) sc.reloadStore(t) respA := sc.query(t, sc.nodeAPK) @@ -324,9 +324,9 @@ func TestHandleNodePaths_FallbackPreconfirmed_1352(t *testing.T) { func TestHandleNodePaths_FallbackUnresolvableHop_1352(t *testing.T) { sc := setupCollisionScenario(t, false /* only B has GPS */) - mustExec(t, sc.db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (60, 'FEED', 'hash_1352_unres', ?)`, sc.recent) + mustExec(t, sc.db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (60, 'FEED', 'hash_1352_unres', $1)`, sc.recent) mustExec(t, sc.db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (60, NULL, '["c0"]', ?, NULL)`, sc.recentEpoch) + VALUES (60, NULL, '["c0"]', $1, NULL)`, sc.recentEpoch) sc.reloadStore(t) // Query A (no GPS): biased resolver in the fallback branch picks B via diff --git a/cmd/server/paths_through_test.go b/cmd/server/paths_through_test.go index ee049b850..43c4aadbf 100644 --- a/cmd/server/paths_through_test.go +++ b/cmd/server/paths_through_test.go @@ -28,19 +28,19 @@ func TestHandleNodePaths_PrefixCollisionExclusion(t *testing.T) { nodeBPK := "7aff2222ccccdddd" // same "7a" prefix, has GPS so resolveHop("7a") picks B db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'NodeA', 'repeater', 0, 0, ?, '2026-01-01', 1)`, nodeAPK, recent) + VALUES ($1, 'NodeA', 'repeater', 0, 0, $2, '2026-01-01', 1)`, nodeAPK, recent) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'NodeB', 'repeater', 37.5, -122.0, ?, '2026-01-01', 1)`, nodeBPK, recent) + VALUES ($1, 'NodeB', 'repeater', 37.5, -122.0, $2, '2026-01-01', 1)`, nodeBPK, recent) // tx1: no resolved_path — should be excluded by hop-level check - db.conn.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (10, 'AA', 'hash_fp', ?)`, recent) + db.conn.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (10, 'AA', 'hash_fp', $1)`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (10, NULL, '["7a"]', ?, NULL)`, recentEpoch) + VALUES (10, NULL, '["7a"]', $1, NULL)`, recentEpoch) // tx2: resolved_path confirms nodeA — must be included - db.conn.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (11, 'BB', 'hash_tp', ?)`, recent) + db.conn.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (11, 'BB', 'hash_tp', $1)`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (11, NULL, '["7a"]', ?, ?)`, recentEpoch, `["`+nodeAPK+`"]`) + VALUES (11, NULL, '["7a"]', $1, $2)`, recentEpoch, `["`+nodeAPK+`"]`) cfg := &Config{Port: 3000} hub := NewHub() diff --git a/cmd/server/perf_io.go b/cmd/server/perf_io.go index dcf1372eb..f11894ba8 100644 --- a/cmd/server/perf_io.go +++ b/cmd/server/perf_io.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "os" + "path/filepath" "sync" "sync/atomic" "time" @@ -36,14 +37,20 @@ type PerfIOResponse struct { // drift between the publisher (ingestor) and the consumer (server) (#1167). type PerfIOSample = perfio.Sample -// PerfSqliteResponse holds SQLite-specific perf metrics. -type PerfSqliteResponse struct { - WalSizeMB float64 `json:"walSizeMB"` - WalSize int64 `json:"walSize"` - PageCount int64 `json:"pageCount"` - PageSize int64 `json:"pageSize"` - CacheSize int64 `json:"cacheSize"` - CacheHitRate float64 `json:"cacheHitRate"` +// PerfPostgresResponse combines sampled database metrics with live pool usage. +type PerfPostgresResponse struct { + Engine string `json:"engine"` + DatabaseBytes *int64 `json:"databaseBytes"` + OpenConnections int `json:"openConnections"` + InUseConnections int `json:"inUseConnections"` + IdleConnections int `json:"idleConnections"` + ConnectionWaitCount int64 `json:"connectionWaitCount"` + ConnectionWaitMs float64 `json:"connectionWaitMs"` + CacheHitRate *float64 `json:"cacheHitRate"` + SampledAt string `json:"sampledAt,omitempty"` + SampleIntervalSeconds int `json:"sampleIntervalSeconds"` + Stale bool `json:"stale"` + Error string `json:"error,omitempty"` } // procIOSample is a snapshot of /proc/self/io counters. @@ -246,36 +253,20 @@ func readIngestorIOSample() *PerfIOSample { return st.ProcIO } -// handlePerfSqlite returns SQLite WAL size + cache hit-rate stats. -func (s *Server) handlePerfSqlite(w http.ResponseWriter, r *http.Request) { - resp := PerfSqliteResponse{} +// handlePerfPostgres reports read-only, database-scoped PostgreSQL metrics. +func (s *Server) handlePerfPostgres(w http.ResponseWriter, r *http.Request) { + resp := PerfPostgresResponse{Engine: "postgresql", SampleIntervalSeconds: int(postgresStatsTTL / time.Second), Stale: true, Error: "database diagnostics unavailable"} if s.db != nil && s.db.conn != nil { - var pageCount, pageSize int64 - _ = s.db.conn.QueryRow("PRAGMA page_count").Scan(&pageCount) - _ = s.db.conn.QueryRow("PRAGMA page_size").Scan(&pageSize) - var cacheSize int64 - _ = s.db.conn.QueryRow("PRAGMA cache_size").Scan(&cacheSize) - resp.PageCount = pageCount - resp.PageSize = pageSize - resp.CacheSize = cacheSize - - // Cache hit rate: derived from PacketStore cache (rw_cache). We don't - // have a direct SQLite cache counter through the driver, so we - // surface the closest available proxy — the in-process row cache. - if s.store != nil { - cs := s.store.GetCacheStatsTyped() - total := cs.Hits + cs.Misses - if total > 0 { - resp.CacheHitRate = float64(cs.Hits) / float64(total) - } - } - - if s.db.path != "" && s.db.path != ":memory:" { - if info, err := os.Stat(s.db.path + "-wal"); err == nil { - resp.WalSize = info.Size() - resp.WalSizeMB = float64(info.Size()) / 1048576 - } + sample, err := s.db.postgresStats(r.Context()) + resp.CacheHitRate = sample.CacheHitRate + resp.SampledAt, resp.Stale = postgresSampleStamp(sample), err != nil + if !sample.SampledAt.IsZero() { + resp.DatabaseBytes, resp.Error = &sample.DatabaseBytes, "" } + pool := s.db.conn.Stats() + resp.OpenConnections, resp.InUseConnections, resp.IdleConnections = pool.OpenConnections, pool.InUse, pool.Idle + resp.ConnectionWaitCount = pool.WaitCount + resp.ConnectionWaitMs = float64(pool.WaitDuration) / float64(time.Millisecond) } writeJSON(w, resp) } @@ -297,7 +288,7 @@ type IngestorStats struct { // ProcIO is the ingestor's own /proc/self/io rates (since its previous // sample). Optional — older ingestor builds don't publish this. See #1120. ProcIO *PerfIOSample `json:"procIO,omitempty"` - // WriterPerf is the per-component SQLite writer-lock latency + // WriterPerf is the per-component serialized-writer latency // snapshot (#1340). Optional — older ingestor builds don't // publish this. Surfaced under .writer_perf by // handlePerfWriteSources. @@ -340,7 +331,7 @@ func IngestorStatsPath() string { if p := os.Getenv("CORESCOPE_INGESTOR_STATS"); p != "" { return p } - return "/tmp/corescope-ingestor-stats.json" + return filepath.Join(runtimeStateDir, "ingestor-stats.json") } // readIngestorSourceLiveness returns the per-source receipt/write-path @@ -482,7 +473,7 @@ func (s *Server) handlePerfWriteSources(w http.ResponseWriter, r *http.Request) } out["sources"] = sources out["sampleAt"] = st.SampledAt - // Surface per-component SQLite writer-lock latency histograms + // Surface per-component serialized-writer latency histograms // (#1340) under .writer_perf so operators can see when a // component (e.g. neighbor_builder) is starving the writer. // Empty map when the ingestor is too old to publish this field. diff --git a/cmd/server/perf_io_test.go b/cmd/server/perf_io_test.go index ecb4f884f..ed944f1c0 100644 --- a/cmd/server/perf_io_test.go +++ b/cmd/server/perf_io_test.go @@ -9,6 +9,48 @@ import ( "testing" ) +func TestPerfPostgresCacheIsNotPacketStoreCache(t *testing.T) { + srv, _ := setupTestServer(t) + get := func() map[string]any { + w := httptest.NewRecorder() + srv.handlePerfPostgres(w, httptest.NewRequest("GET", "/api/perf/postgres", nil)) + var body map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + return body + } + srv.store.cacheMu.Lock() + srv.store.cacheHits, srv.store.cacheMisses = 0, 1 + srv.store.cacheMu.Unlock() + first := get() + srv.store.cacheMu.Lock() + srv.store.cacheHits, srv.store.cacheMisses = 1, 0 + srv.store.cacheMu.Unlock() + second := get() + if first["cacheHitRate"] != second["cacheHitRate"] { + t.Fatalf("database cache metric changed with the PacketStore cache: %v -> %v", first["cacheHitRate"], second["cacheHitRate"]) + } + if first["sampledAt"] == nil || first["sampleIntervalSeconds"] != float64(30) { + t.Fatalf("diagnostic sample freshness is not disclosed: %v", first) + } +} + +func TestPerfPostgresUnavailableCacheIsNull(t *testing.T) { + w := httptest.NewRecorder() + (&Server{}).handlePerfPostgres(w, httptest.NewRequest("GET", "/api/perf/postgres", nil)) + var body map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if v, present := body["cacheHitRate"]; !present || v != nil { + t.Fatalf("unavailable database cache metric must be null, got %v", v) + } + if v, present := body["databaseBytes"]; !present || v != nil { + t.Fatalf("unavailable database size must be null, got %v", v) + } +} + func TestPerfIOEndpoint_ReturnsValidJSON(t *testing.T) { _, router := setupTestServer(t) @@ -55,10 +97,10 @@ func TestPerfIOEndpoint_ReturnsValidJSON(t *testing.T) { } } -func TestPerfSqliteEndpoint_ReturnsValidJSON(t *testing.T) { +func TestPerfPostgresEndpoint_ReturnsValidJSON(t *testing.T) { _, router := setupTestServer(t) - req := httptest.NewRequest("GET", "/api/perf/sqlite", nil) + req := httptest.NewRequest("GET", "/api/perf/postgres", nil) w := httptest.NewRecorder() router.ServeHTTP(w, req) @@ -69,14 +111,14 @@ func TestPerfSqliteEndpoint_ReturnsValidJSON(t *testing.T) { if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { t.Fatalf("invalid JSON: %v", err) } - for _, field := range []string{"walSize", "pageCount", "pageSize", "cacheHitRate"} { + for _, field := range []string{"engine", "databaseBytes", "openConnections", "cacheHitRate"} { if _, ok := body[field]; !ok { t.Errorf("missing field %q", field) } } - // pageSize must be > 0 for any open SQLite DB - if v, ok := body["pageSize"].(float64); !ok || v <= 0 { - t.Errorf("expected pageSize > 0, got %v", body["pageSize"]) + // A PostgreSQL database has a positive physical size. + if v, ok := body["databaseBytes"].(float64); !ok || v <= 0 { + t.Errorf("expected databaseBytes > 0, got %v", body["databaseBytes"]) } } diff --git a/cmd/server/postgres_fixture_test.go b/cmd/server/postgres_fixture_test.go new file mode 100644 index 000000000..68367bfbd --- /dev/null +++ b/cmd/server/postgres_fixture_test.go @@ -0,0 +1,85 @@ +package main + +import ( + "context" + "database/sql" + "os/exec" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/stdlib" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +// copyTestRows streams a bounded synthetic fixture through native PostgreSQL +// COPY. Each COPY remains an atomic durable statement; no test durability +// settings are changed and there is no INSERT round trip per fixture row. +func copyTestRows(t testing.TB, db *sql.DB, table string, columns []string, n int, row func(int) []any) { + t.Helper() + ctx := context.Background() + conn, err := db.Conn(ctx) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + err = conn.Raw(func(driver any) error { + count, err := driver.(*stdlib.Conn).Conn().CopyFrom(ctx, pgx.Identifier{table}, columns, + pgx.CopyFromSlice(n, func(i int) ([]any, error) { return row(i), nil })) + if err == nil && count != int64(n) { + t.Errorf("copied %d/%d rows into %s", count, n, table) + } + return err + }) + if err != nil { + t.Fatalf("seed %s: %v", table, err) + } +} + +func postgresTestDSN(t testing.TB) string { + t.Helper() + return pgtest.NewSchema(t) +} + +func openFixtureSQL(dsn string) (*sql.DB, error) { return pgutil.Open(dsn, false) } + +func openPostgresTestDB(t testing.TB) (*sql.DB, error) { + t.Helper() + db, err := openFixtureSQL(postgresTestDSN(t)) + if err == nil { + t.Cleanup(func() { db.Close() }) + } + return db, err +} + +func openFixtureReader(t testing.TB, dsn string) (*DB, error) { + t.Helper() + return OpenDB(pgtest.ReadOnly(t, dsn)) +} + +func restorePostgresTestBackup(t testing.TB, path string) *sql.DB { + t.Helper() + dsn := pgtest.NewDatabase(t) + config, err := pgutil.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + env, err := pgutil.CommandEnv(dsn) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + cmd := exec.CommandContext(ctx, "pg_restore", "--exit-on-error", "--no-owner", "--no-privileges", "--dbname="+config.Database, path) + cmd.Env = env + if err := cmd.Run(); err != nil { + t.Fatal("restore native PostgreSQL backup failed") + } + db, err := openFixtureSQL(dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + return db +} diff --git a/cmd/server/postgres_stats.go b/cmd/server/postgres_stats.go new file mode 100644 index 000000000..2cc68ad29 --- /dev/null +++ b/cmd/server/postgres_stats.go @@ -0,0 +1,80 @@ +package main + +import ( + "context" + "time" +) + +const postgresStatsTTL = 30 * time.Second +const postgresBlockCacheRateSQL = `blks_hit::double precision / NULLIF(blks_hit::double precision + blks_read, 0)` + +// Storage traversal and full row counts are diagnostic samples, independent of +// ingest and PacketStore invalidation. Keep one bounded immutable sample per DB. +type postgresDatabaseSample struct { + DatabaseBytes int64 + CacheHitRate *float64 + Rows DatabaseRowCounts + SampledAt time.Time +} + +func (db *DB) cachedPostgresStats() (postgresDatabaseSample, bool) { + db.pgStatsMu.Lock() + sample, valid := db.pgStatsCache, time.Now().Before(db.pgStatsExpires) + db.pgStatsMu.Unlock() + return sample, valid +} + +func (db *DB) postgresStats(ctx context.Context) (postgresDatabaseSample, error) { + cached, valid := db.cachedPostgresStats() + if valid { + return cached, nil + } + if db.pgStatsMissHook != nil { + db.pgStatsMissHook() + } + flight := db.pgStatsSF.DoChan("database", func() (any, error) { + if sample, valid := db.cachedPostgresStats(); valid { + return sample, nil + } + if db.pgStatsQueryHook != nil { + db.pgStatsQueryHook() + } + // One canceled dashboard request must not cancel the shared sample. + // The work itself is bounded, and no cache mutex is held during SQL. + queryCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + var sample postgresDatabaseSample + err := db.conn.QueryRowContext(queryCtx, `SELECT pg_database_size(current_database()), `+postgresBlockCacheRateSQL+`, + (SELECT COUNT(*) FROM transmissions), (SELECT COUNT(*) FROM observations), + (SELECT COUNT(*) FROM nodes), (SELECT COUNT(*) FROM observers) + FROM pg_catalog.pg_stat_database WHERE datname=current_database()`).Scan( + &sample.DatabaseBytes, &sample.CacheHitRate, &sample.Rows.Transmissions, + &sample.Rows.Observations, &sample.Rows.Nodes, &sample.Rows.Observers) + if err != nil { + return nil, err + } + sample.SampledAt = time.Now() + db.pgStatsMu.Lock() + db.pgStatsCache, db.pgStatsExpires = sample, sample.SampledAt.Add(postgresStatsTTL) + db.pgStatsMu.Unlock() + return sample, nil + }) + select { + case result := <-flight: + if result.Err == nil { + return result.Val.(postgresDatabaseSample), nil + } + cached.CacheHitRate = nil + return cached, result.Err + case <-ctx.Done(): + cached.CacheHitRate = nil + return cached, ctx.Err() + } +} + +func postgresSampleStamp(sample postgresDatabaseSample) string { + if sample.SampledAt.IsZero() { + return "" + } + return sample.SampledAt.UTC().Format(time.RFC3339Nano) +} diff --git a/cmd/server/postgres_stats_test.go b/cmd/server/postgres_stats_test.go new file mode 100644 index 000000000..701b946d9 --- /dev/null +++ b/cmd/server/postgres_stats_test.go @@ -0,0 +1,186 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "math" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestPostgresInitialDiagnosticFailureIsExplicit(t *testing.T) { + db := setupTestDB(t) + db.conn.Close() + bytes, err := json.Marshal(db.GetDBSizeStatsTyped()) + if err != nil { + t.Fatal(err) + } + var body map[string]any + if err := json.Unmarshal(bytes, &body); err != nil { + t.Fatal(err) + } + if body["error"] == nil || body["error"] == "" || body["rows"] != nil || body["dbSizeMB"] != nil { + t.Fatalf("unknown database size/counts must not become zeros: %s", bytes) + } + w := httptest.NewRecorder() + (&Server{db: db}).handlePerfPostgres(w, httptest.NewRequest("GET", "/api/perf/postgres", nil)) + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if body["error"] == nil || body["databaseBytes"] != nil || body["cacheHitRate"] != nil { + t.Fatalf("unknown database measurements must be null: %s", w.Body.String()) + } +} + +func TestPostgresBlockCacheRatioNullAndZero(t *testing.T) { + db := setupTestDB(t) + nineTenths, zero, one := 0.9, 0.0, 1.0 + for _, tc := range []struct { + name string + hits, reads any + want *float64 + }{ + {"hits", 9, 1, &nineTenths}, {"misses", 0, 9, &zero}, + {"all cached", 9, 0, &one}, {"no reads", 0, 0, nil}, + {"unavailable", nil, 9, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + var got *float64 + err := db.conn.QueryRow(`SELECT `+postgresBlockCacheRateSQL+` FROM (VALUES ($1::bigint,$2::bigint)) AS counters(blks_hit,blks_read)`, tc.hits, tc.reads).Scan(&got) + if err != nil { + t.Fatal(err) + } + if (got == nil) != (tc.want == nil) { + t.Fatalf("ratio = %v, want %v", got, tc.want) + } + if got != nil && math.Abs(*got-*tc.want) > 1e-12 { + t.Fatalf("ratio = %g, want %g", *got, *tc.want) + } + }) + } +} + +func TestPostgresDiagnosticRequestsCoalesce(t *testing.T) { + db := setupTestDB(t) + seedTestData(t, db) + const requests = 12 + misses := make(chan struct{}, requests) + release := make(chan struct{}) + var queries atomic.Int64 + db.pgStatsMissHook = func() { misses <- struct{}{} } + db.pgStatsQueryHook = func() { queries.Add(1); <-release } + var wg sync.WaitGroup + for i := 0; i < requests; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + if i%2 == 0 { + if got := db.GetDBSizeStatsTyped(); got.Rows.Transmissions != 3 { + t.Errorf("sample rows = %d", got.Rows.Transmissions) + } + } else { + w := httptest.NewRecorder() + (&Server{db: db}).handlePerfPostgres(w, httptest.NewRequest("GET", "/api/perf/postgres", nil)) + if w.Code != 200 { + t.Errorf("status = %d", w.Code) + } + } + }(i) + } + // Every request must have observed the empty cache before the first query + // completes. This proves coalescing without sleeps or timing assumptions. + for i := 0; i < requests; i++ { + <-misses + } + close(release) + wg.Wait() + if got := queries.Load(); got != 1 { + t.Fatalf("expensive sampling queries = %d, want 1", got) + } +} + +func TestPostgresDiagnosticTTLDoesNotFollowIngestInvalidation(t *testing.T) { + db := setupTestDB(t) + seedTestData(t, db) + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatal(err) + } + queries := 0 + db.pgStatsQueryHook = func() { + queries++ + if !db.pgStatsMu.TryLock() { + t.Error("expensive sampling ran under the cache lock") + } else { + db.pgStatsMu.Unlock() + } + } + first := db.GetDBSizeStatsTyped() + if first.Rows.Transmissions != 3 || first.SampledAt == "" || first.SampleIntervalSeconds != 30 { + t.Fatalf("initial sample = %+v", first) + } + if _, err := db.conn.Exec(`INSERT INTO transmissions(raw_hex,hash,first_seen,payload_type,decoded_json) VALUES('AA','new-sample-row',$1,5,'{}')`, time.Now().UTC().Format(time.RFC3339)); err != nil { + t.Fatal(err) + } + store.IngestNewFromDB(3, 100) + if got := db.GetDBSizeStatsTyped(); got.Rows.Transmissions != 3 || queries != 1 { + t.Fatalf("ingest invalidated diagnostic sample: rows=%d queries=%d", got.Rows.Transmissions, queries) + } + db.pgStatsMu.Lock() + db.pgStatsExpires = time.Time{} + db.pgStatsMu.Unlock() + if got := db.GetDBSizeStatsTyped(); got.Rows.Transmissions != 4 || queries != 2 { + t.Fatalf("expiry did not refresh: rows=%d queries=%d", got.Rows.Transmissions, queries) + } +} + +func TestPostgresDiagnosticFailureMarksOldSampleStale(t *testing.T) { + db := setupTestDB(t) + seedTestData(t, db) + first, err := db.postgresStats(context.Background()) + if err != nil { + t.Fatal(err) + } + db.pgStatsMu.Lock() + db.pgStatsExpires = time.Time{} + db.pgStatsMu.Unlock() + db.conn.Close() + got := db.GetDBSizeStatsTyped() + if !got.Stale || got.SampledAt != first.SampledAt.UTC().Format(time.RFC3339Nano) || got.Rows.Transmissions != 3 { + t.Fatalf("failed refresh did not disclose stale sample: %+v", got) + } +} + +func TestPostgresDiagnosticCanceledCallerKeepsSharedSample(t *testing.T) { + db := setupTestDB(t) + seedTestData(t, db) + misses, entered, release := make(chan struct{}, 2), make(chan struct{}), make(chan struct{}) + var once sync.Once + var queries atomic.Int64 + db.pgStatsMissHook = func() { misses <- struct{}{} } + db.pgStatsQueryHook = func() { queries.Add(1); once.Do(func() { close(entered) }); <-release } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + first := make(chan error, 1) + go func() { _, err := db.postgresStats(ctx); first <- err }() + <-misses + <-entered + cancel() + if err := <-first; !errors.Is(err, context.Canceled) { + t.Fatalf("caller cancellation: %v", err) + } + second := make(chan error, 1) + go func() { _, err := db.postgresStats(context.Background()); second <- err }() + <-misses + close(release) + if err := <-second; err != nil { + t.Fatalf("shared sample was canceled: %v", err) + } + if queries.Load() != 1 { + t.Fatalf("queries = %d, want 1", queries.Load()) + } +} diff --git a/cmd/server/postgres_test.go b/cmd/server/postgres_test.go new file mode 100644 index 000000000..7aed7edb5 --- /dev/null +++ b/cmd/server/postgres_test.go @@ -0,0 +1,250 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +func TestPostgresCursorStopsAtLimitWithoutSorting(t *testing.T) { + db := setupTestDB(t) + copyTestRows(t, db.conn, "transmissions", []string{"id", "raw_hex", "hash", "first_seen"}, 20000, func(i int) []any { + return []any{i + 1, "AA", fmt.Sprintf("cursor-%d", i), "2026-01-01T00:00:00Z"} + }) + if _, err := db.conn.Exec(`ANALYZE transmissions`); err != nil { + t.Fatal(err) + } + var raw string + if err := db.conn.QueryRow(`EXPLAIN (ANALYZE, FORMAT JSON) `+newTransmissionsSinceSQL, 10000, 100).Scan(&raw); err != nil { + t.Fatal(err) + } + type planNode struct { + Type string `json:"Node Type"` + Rows float64 `json:"Actual Rows"` + Plans []json.RawMessage `json:"Plans"` + } + var plans []struct{ Plan json.RawMessage } + if err := json.Unmarshal([]byte(raw), &plans); err != nil { + t.Fatal(err) + } + pending := []json.RawMessage{plans[0].Plan} + for len(pending) > 0 { + var node planNode + if err := json.Unmarshal(pending[0], &node); err != nil { + t.Fatal(err) + } + pending = append(pending[1:], node.Plans...) + if strings.Contains(node.Type, "Sort") || (strings.Contains(node.Type, "Scan") && node.Rows > 100) { + t.Fatalf("cursor scanned or sorted more than its bounded page: %s rows=%g", node.Type, node.Rows) + } + } + rows, err := db.GetNewTransmissionsSince(10000, 100) + if err != nil || len(rows) != 100 || rows[0]["id"] != 10001 { + t.Fatalf("cursor page: len=%d, %v", len(rows), err) + } +} + +func TestPostgresNegativeOffsetsPreserveFirstPage(t *testing.T) { + db := setupTestDB(t) + seedTestData(t, db) + for name, query := range map[string]func() (*PacketResult, error){ + "packets": func() (*PacketResult, error) { return db.QueryPackets(PacketQuery{Limit: 2, Offset: -1}) }, + "groups": func() (*PacketResult, error) { return db.QueryGroupedPackets(PacketQuery{Limit: 2, Offset: -1}) }, + "multiple nodes": func() (*PacketResult, error) { + return db.QueryMultiNodePackets([]string{"aabbccdd11223344"}, 2, -1, "DESC", "", "") + }, + } { + t.Run(name, func(t *testing.T) { + result, err := query() + if err != nil || len(result.Packets) != 2 { + t.Fatalf("negative offset must return the first page: %+v, %v", result, err) + } + }) + } + nodes, _, _, err := db.GetNodes(NodeQuery{Limit: 2, Offset: -1}) + if err != nil || len(nodes) != 2 { + t.Fatalf("negative node offset must return the first page: %d, %v", len(nodes), err) + } +} + +func TestPostgresCanonicalSchemaAPIs(t *testing.T) { + dsn := pgtest.NewSchema(t) + writer, err := openFixtureSQL(dsn) + if err != nil { + t.Fatal(err) + } + defer writer.Close() + if err := dbschema.Apply(writer, nil); err != nil { + t.Fatal(err) + } + seedTestData(t, &DB{conn: writer}) + if _, err := writer.Exec(`UPDATE transmissions SET last_seen=EXTRACT(EPOCH FROM first_seen::timestamptz)::bigint`); err != nil { + t.Fatal(err) + } + db, err := openFixtureReader(t, dsn) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := dbschema.AssertReady(db.conn); err != nil { + t.Fatal(err) + } + stats, err := db.GetStats() + if err != nil || stats.TotalTransmissions != 3 || stats.TotalObservations != 4 { + t.Fatalf("canonical statistics: %+v, %v", stats, err) + } + pt := 4 + q := PacketQuery{Limit: 5, Type: &pt, Observer: "obs1,obs2", Region: "SJC", Since: "2020-01-01", Until: "2099-01-01", Order: "DESC"} + packets, err := db.QueryPackets(q) + if err != nil || len(packets.Packets) != 2 { + t.Fatalf("filtered packets: %+v, %v", packets, err) + } + grouped, err := db.QueryGroupedPackets(q) + if err != nil || len(grouped.Packets) != 2 { + t.Fatalf("filtered groups: %+v, %v", grouped, err) + } + nodes, count, _, err := db.GetNodes(NodeQuery{Limit: 5, Role: "repeater", Search: "test", RegionPubkeys: []string{"aabbccdd11223344"}}) + if err != nil || count != 1 || len(nodes) != 1 { + t.Fatalf("filtered nodes: %d/%d, %v", len(nodes), count, err) + } + messages, count, err := db.GetChannelMessages("#test", 5, 0, "SJC,SFO") + if err != nil || count != 1 || len(messages) != 1 { + t.Fatalf("channel messages: %d/%d, %v", len(messages), count, err) + } + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatal(err) + } + if len(store.packets) != 3 || store.totalObs != 4 { + t.Fatalf("canonical load: %d transmissions, %d observations", len(store.packets), store.totalObs) + } +} + +func TestPostgresOpenRejectsLegacyFileWithoutLeakingInput(t *testing.T) { + legacy := filepath.Join(t.TempDir(), "private-measurements.db") + db, err := OpenDB(legacy) + if db != nil { + db.Close() + } + if err == nil || !strings.Contains(err.Error(), "PostgreSQL") { + t.Fatalf("expected actionable PostgreSQL migration error, got %v", err) + } + if strings.Contains(err.Error(), legacy) { + t.Fatal("database input leaked in connection error") + } +} + +func TestPostgresReaderRejectsWriterCredential(t *testing.T) { + db, err := OpenDB(pgtest.NewSchema(t)) + if db != nil { + db.Close() + } + if err == nil || !strings.Contains(err.Error(), "reader role") { + t.Fatalf("elevated telemetry credential was not refused: %v", err) + } +} + +func TestPostgresScopeAuditSnapshotKeepsConcurrentCommitOut(t *testing.T) { + dsn := pgtest.NewSchema(t) + writer, err := openFixtureSQL(dsn) + if err != nil { + t.Fatal(err) + } + defer writer.Close() + if _, err := writer.Exec(`CREATE TABLE transmissions(id BIGINT PRIMARY KEY, first_seen TEXT, scope_name TEXT, route_type INTEGER); + CREATE TABLE observations(id BIGINT PRIMARY KEY, transmission_id BIGINT, path_json TEXT); + INSERT INTO transmissions VALUES(1,'2026-01-02T00:00:00Z','scope',1); + INSERT INTO observations VALUES(1,1,'["aabb"]')`); err != nil { + t.Fatal(err) + } + conn, err := pgutil.Open(pgtest.ReadOnly(t, dsn), true) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + db := &DB{conn: conn} + tx, err := db.beginScopeAuditSnapshot(context.Background()) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + meta, err := scopeAuditWindowMeta(tx, "2026-01-01T00:00:00Z") + if err != nil || len(meta) != 1 { + t.Fatalf("initial metadata: %d, %v", len(meta), err) + } + if _, err := writer.Exec(`INSERT INTO transmissions VALUES(2,'2026-01-03T00:00:00Z','scope',1); + INSERT INTO observations VALUES(2,2,'["aabb"]')`); err != nil { + t.Fatal(err) + } + rows, err := tx.Query(scopeAuditForwarderScanQuery, "2026-01-01T00:00:00Z") + if err != nil { + t.Fatal(err) + } + count := 0 + for rows.Next() { + var id int64 + var hop string + if err := rows.Scan(&id, &hop); err != nil { + t.Fatal(err) + } + if id != 1 { + t.Fatal("scope snapshot included a later commit absent from metadata") + } + count++ + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + rows.Close() + if count != 1 { + t.Fatalf("hop rows: %d, want 1", count) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + var visible int + if err := conn.QueryRow(`SELECT COUNT(*) FROM transmissions`).Scan(&visible); err != nil || visible != 2 { + t.Fatalf("fresh snapshot did not see committed row: %d, %v", visible, err) + } +} + +func TestPostgresReaderUsesRestrictedRole(t *testing.T) { + dsn := pgtest.NewSchema(t) + writer, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + defer writer.Close() + _, err = writer.Exec(`CREATE TABLE transmissions (id BIGINT PRIMARY KEY, hash TEXT); + CREATE TABLE observations (id BIGINT PRIMARY KEY, timestamp BIGINT, observer_idx BIGINT); + CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, last_seen TEXT); + CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, inactive INTEGER); + INSERT INTO transmissions VALUES (7, 'stable-hash')`) + if err != nil { + t.Fatal(err) + } + reader, err := OpenDB(pgtest.ReadOnly(t, dsn)) + if err != nil { + t.Fatalf("open PostgreSQL reader: %v", err) + } + defer reader.Close() + if got := reader.GetMaxTransmissionID(); got != 7 { + t.Fatalf("max transmission = %d, want 7", got) + } + if _, err := reader.conn.Exec(`INSERT INTO transmissions VALUES (8, 'forbidden')`); err == nil { + t.Fatal("telemetry reader accepted a write") + } + if _, err := reader.conn.Exec(`SET default_transaction_read_only = off`); err != nil { + t.Fatal(err) + } + if _, err := reader.conn.Exec(`DELETE FROM transmissions`); err == nil { + t.Fatal("reader can write after changing its session flag") + } +} diff --git a/cmd/server/postgres_value_parity_test.go b/cmd/server/postgres_value_parity_test.go new file mode 100644 index 000000000..c70613539 --- /dev/null +++ b/cmd/server/postgres_value_parity_test.go @@ -0,0 +1,199 @@ +package main + +import ( + "database/sql" + "encoding/json" + "fmt" + "net/http/httptest" + "testing" + "time" + + "github.com/gorilla/mux" + "github.com/meshcore-analyzer/dbschema" +) + +func openPostgresValueFixture(t *testing.T) (*DB, *sql.DB) { + t.Helper() + dsn := postgresTestDSN(t) + writer, err := openFixtureSQL(dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { writer.Close() }) + if err := dbschema.Apply(writer, nil); err != nil { + t.Fatal(err) + } + reader, err := openFixtureReader(t, dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { reader.Close() }) + return reader, writer +} + +// SQLite INTEGER affinity permits fractional values, and PostgreSQL float8 +// returns large integral values in a form that NullInt64 cannot scan. Neither +// case may drop an observation or its joined transmission from the server. +func TestPostgresObservationScoreLoadPaths(t *testing.T) { + for _, path := range []string{"full", "chunked", "older", "transmission_poll", "observation_poll"} { + t.Run(path, func(t *testing.T) { + db, writer := openPostgresValueFixture(t) + stamp := time.Now().UTC().Truncate(time.Second).Add(-2 * time.Hour) + if _, err := writer.Exec(`INSERT INTO observers(rowid,id,name,iata) VALUES(1,'score-observer','Score observer','AAA')`); err != nil { + t.Fatal(err) + } + scores := []any{1000000, 3.5, nil} + seedTx := func() { + for i := range scores { + if _, err := writer.Exec(`INSERT INTO transmissions(id,raw_hex,hash,first_seen,last_seen,payload_type) VALUES($1,'AA',$2,$3,$4,0)`, i+1, fmt.Sprintf("score-%d", i), stamp.Format(time.RFC3339), stamp.Unix()); err != nil { + t.Fatal(err) + } + } + } + seedObs := func() { + for i, score := range scores { + if _, err := writer.Exec(`INSERT INTO observations(id,transmission_id,observer_idx,score,snr,rssi,path_json,timestamp) VALUES($1,$1,1,$2,3.5,-90.5,'[]',$3)`, i+1, score, stamp.Unix()); err != nil { + t.Fatal(err) + } + } + } + if path != "transmission_poll" { + seedTx() + if path != "observation_poll" { + seedObs() + } + } + cfg := &PacketStoreConfig{RetentionHours: 24} + if path == "older" { + cfg.HotStartupHours = 1 + } + store := NewPacketStore(db, cfg) + if path == "chunked" { + if err := store.LoadChunked(1); err != nil { + t.Fatal(err) + } + } else if err := store.Load(); err != nil { + t.Fatal(err) + } + switch path { + case "older": + if len(store.packets) != 0 { + t.Fatal("older-history fixture unexpectedly loaded during hot startup") + } + if err := store.loadChunk(stamp.Add(-time.Hour), stamp.Add(time.Hour)); err != nil { + t.Fatal(err) + } + case "transmission_poll": + seedTx() + seedObs() + packets, cursor := store.IngestNewFromDB(0, 10) + if len(packets) != 3 || cursor != 3 { + t.Errorf("poll returned %d transmissions, cursor %d; want 3 and 3", len(packets), cursor) + } + case "observation_poll": + seedObs() + if packets := store.IngestNewObservations(0, 10); len(packets) != 3 { + t.Errorf("observation poll updated %d transmissions; want 3", len(packets)) + } + } + if len(store.packets) != 3 || store.totalObs != 3 { + t.Errorf("loaded %d transmissions/%d observations; want 3/3", len(store.packets), store.totalObs) + } + want := map[string]string{"score-0": "1000000", "score-1": "3.5", "score-2": "null"} + for hash, value := range want { + observations := store.GetObservationsForHash(hash) + if len(observations) != 1 { + t.Errorf("%s loaded %d observations; want 1", hash, len(observations)) + continue + } + raw, err := json.Marshal(observations[0]["score"]) + if err != nil || string(raw) != value { + t.Errorf("%s score JSON=%s, %v; want %s", hash, raw, err, value) + } + } + // Exercise the existing HTTP field that exposes enriched scores. + server := &Server{db: db, store: store} + req := mux.SetURLVars(httptest.NewRequest("GET", "/api/observers/score-observer/analytics?days=1", nil), map[string]string{"id": "score-observer"}) + response := httptest.NewRecorder() + server.handleObserverAnalytics(response, req) + var body struct { + RecentPackets []struct { + Hash string `json:"hash"` + Score json.RawMessage `json:"score"` + } `json:"recentPackets"` + } + if response.Code != 200 || json.Unmarshal(response.Body.Bytes(), &body) != nil { + t.Fatalf("observer analytics response: %d %s", response.Code, response.Body.String()) + } + if len(body.RecentPackets) != len(want) { + t.Errorf("HTTP API returned %d observations; want %d", len(body.RecentPackets), len(want)) + } + for _, packet := range body.RecentPackets { + if expected, ok := want[packet.Hash]; !ok || string(packet.Score) != expected { + t.Errorf("HTTP score %s=%s; want JSON number/null %s", packet.Hash, packet.Score, expected) + } + } + }) + } +} + +func TestPostgresPacketRowScore(t *testing.T) { + db, writer := openPostgresValueFixture(t) + for i, score := range []any{1000000, 3.5, nil} { + if _, err := writer.Exec(`INSERT INTO transmissions(id,raw_hex,hash,first_seen) VALUES($1,'AA',$2,'2026-01-01T00:00:00Z')`, i+1, fmt.Sprintf("score-%d", i)); err != nil { + t.Fatal(err) + } + if _, err := writer.Exec(`INSERT INTO observations(id,transmission_id,score,timestamp) VALUES($1,$1,$2,1)`, i+1, score); err != nil { + t.Fatal(err) + } + } + rows, err := db.conn.Query(`SELECT id,raw_hex,timestamp,observer_id,observer_name,direction,snr,rssi,score,hash,route_type,payload_type,payload_version,path_json,decoded_json,created_at FROM packets_v ORDER BY id`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + want := []string{"1000000", "3.5", "null"} + index := 0 + for rows.Next() { + packet := scanPacketRow(rows) + if packet == nil { + t.Errorf("row %d silently dropped its score", index) + } else if raw, err := json.Marshal(packet["score"]); err != nil || string(raw) != want[index] { + t.Errorf("row %d score=%s, %v; want %s", index, raw, err, want[index]) + } + index++ + } + if rows.Err() != nil || index != len(want) { + t.Fatalf("scanned %d rows, %v", index, rows.Err()) + } +} + +func TestPostgresNodeSearchKeepsLiteralBackslashAndWildcards(t *testing.T) { + db, writer := openPostgresValueFixture(t) + for i, name := range []string{`node\path`, "nodepath", "nodeXpath"} { + if _, err := writer.Exec(`INSERT INTO nodes(public_key,name) VALUES($1,$2)`, fmt.Sprintf("%064x", i+1), name); err != nil { + t.Fatal(err) + } + } + for _, tc := range []struct { + query string + count int + }{{`node\path`, 1}, {`NODE\PATH`, 1}, {`node\%`, 1}, {"node_path", 2}, {"node%path", 3}} { + t.Run(tc.query, func(t *testing.T) { + nodes, total, _, err := db.GetNodes(NodeQuery{Search: tc.query}) + if err != nil || len(nodes) != tc.count || total != tc.count { + t.Errorf("GetNodes(%q): %d rows,total=%d,%v; want %d", tc.query, len(nodes), total, err, tc.count) + } + if tc.count == 1 && len(nodes) == 1 && nodes[0]["name"] != `node\path` { + t.Errorf("GetNodes(%q) matched %v instead of the literal backslash name", tc.query, nodes[0]["name"]) + } + nodes, err = db.SearchNodes(tc.query, 10) + if err != nil || len(nodes) != tc.count { + t.Errorf("SearchNodes(%q): %d rows,%v; want %d", tc.query, len(nodes), err, tc.count) + } + if tc.count == 1 && len(nodes) == 1 && nodes[0]["name"] != `node\path` { + t.Errorf("SearchNodes(%q) matched %v instead of the literal backslash name", tc.query, nodes[0]["name"]) + } + }) + } +} diff --git a/cmd/server/preparable_schema_test.go b/cmd/server/preparable_schema_test.go index 0540091d7..e9b424c94 100644 --- a/cmd/server/preparable_schema_test.go +++ b/cmd/server/preparable_schema_test.go @@ -2,27 +2,17 @@ package main import ( "database/sql" - "strings" "testing" ) -// The statements compiled by prepareStatements reference a set of tables and -// columns. Under modernc.org/sqlite that never mattered for test fixtures: -// newStmt only stored the SQL string and compiled it lazily on first use, so a -// fixture could omit half the schema and still open. github.com/mattn/go-sqlite3 -// calls sqlite3_prepare_v2 inside Prepare, so OpenDB now fails on an incomplete -// schema — which is the production behaviour we want (#1901), but it means -// fixtures have to declare what they are prepared against. -// -// ensurePreparable is the seam. Fixtures that deliberately build a partial -// schema (to exercise schema detection, say) call it on the writable connection -// just before OpenDB, and it fills in whatever is missing without disturbing the -// shapes the test actually cares about. +// Deliberately partial fixtures declare the table shapes they test, then +// ensurePreparable supplies the minimum surface required by OpenDB's prepared +// statements. It never creates or modifies a runtime database. var preparableTables = []string{ - `CREATE TABLE IF NOT EXISTS transmissions (id INTEGER PRIMARY KEY, hash TEXT)`, - `CREATE TABLE IF NOT EXISTS observations (id INTEGER PRIMARY KEY, timestamp TEXT)`, + `CREATE TABLE IF NOT EXISTS transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`, + `CREATE TABLE IF NOT EXISTS observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, timestamp TEXT)`, `CREATE TABLE IF NOT EXISTS nodes (public_key TEXT PRIMARY KEY)`, - `CREATE TABLE IF NOT EXISTS observers (id TEXT)`, + `CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT)`, } // preparableColumns are added when absent. A fixture that already declared the @@ -46,9 +36,9 @@ func ensurePreparable(tb testing.TB, conn *sql.DB) { } } for _, c := range preparableColumns { - _, err := conn.Exec("ALTER TABLE " + c.table + " ADD COLUMN " + c.column + " " + c.decl) + _, err := conn.Exec("ALTER TABLE " + c.table + " ADD COLUMN IF NOT EXISTS " + c.column + " " + c.decl) // "duplicate column name" means the fixture already declared it. - if err != nil && !strings.Contains(err.Error(), "duplicate column name") { + if err != nil { tb.Fatalf("ensurePreparable: add %s.%s: %v", c.table, c.column, err) } } @@ -59,7 +49,7 @@ func ensurePreparable(tb testing.TB, conn *sql.DB) { // create, it fails here rather than as a confusing OpenDB error in whichever // fixture happens to be thinnest. func TestEnsurePreparableMatchesPrepareStatements(t *testing.T) { - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } diff --git a/cmd/server/proposal_handlers_test.go b/cmd/server/proposal_handlers_test.go index 4a4e6ebe4..4e68742af 100644 --- a/cmd/server/proposal_handlers_test.go +++ b/cmd/server/proposal_handlers_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/gorilla/mux" + "github.com/meshcore-analyzer/pgutil/pgtest" "github.com/meshcore-analyzer/users" ) @@ -26,12 +27,12 @@ func newProposalFixture(t *testing.T, ps proposalSettings) *authFixture { func newProposalFixtureWithConfig(t *testing.T, ps proposalSettings, cfg *Config) *authFixture { t.Helper() - a, fake := newTestAuthService(t, "admin@example.org") + a, fake, ownerURL := newTestAuthServiceWithURL(t, pgtest.NewSchema(t), "admin@example.org") a.set.proposals = ps srv := &Server{cfg: cfg, perfStats: NewPerfStats(), auth: a} r := mux.NewRouter() srv.registerAuthRoutes(r) - return &authFixture{srv: srv, router: r, fake: fake, st: a.st} + return &authFixture{srv: srv, router: r, fake: fake, st: a.st, ownerURL: ownerURL} } func proposalPeople(t *testing.T, f *authFixture) (admin, user *client) { diff --git a/cmd/server/readonly_invariant_test.go b/cmd/server/readonly_invariant_test.go index 3e265655e..2df7104f3 100644 --- a/cmd/server/readonly_invariant_test.go +++ b/cmd/server/readonly_invariant_test.go @@ -1,7 +1,6 @@ package main import ( - "database/sql" "fmt" "os" "path/filepath" @@ -9,8 +8,6 @@ import ( "regexp" "strings" "testing" - - _ "github.com/mattn/go-sqlite3" ) // TestServerSourceHasNoCachedRWCalls enforces issue #1287: after the @@ -47,8 +44,8 @@ func TestServerSourceHasNoCachedRWCalls(t *testing.T) { // server publishes via internal/mbcapqueue). // Shapes are normalised before matching (see nodeTableWritePattern): // optional OR-conflict clause, optional quoting, optional alias. - nodeTableWritePattern(`UPDATE(\s+OR\s+\w+)?`, `SET`), - nodeTableWritePattern(`INSERT\s+(OR\s+\w+\s+)?INTO`, ``), + nodeTableWritePattern(`UPDATE(\s+OR\s+\w+)$1`, `SET`), + nodeTableWritePattern(`INSERT\s+(OR\s+\w+\s+)$1INTO`, ``), nodeTableWritePattern(`REPLACE\s+INTO`, ``), nodeTableWritePattern(`DELETE\s+FROM`, ``), regexp.MustCompile(`\bpersistMultibyteCapability\s*\(`), @@ -124,8 +121,7 @@ func TestServerDBHasNoWriteMethods(t *testing.T) { // second RW handle. After the fix, server-side writes are impossible // because there is no helper to open a writable connection. func TestServerDBConnIsReadOnly(t *testing.T) { - dir := t.TempDir() - path := dir + "/ro_invariant.db" + path := postgresTestDSN(t) // Bootstrap a minimal DB with the ingestor-style WAL opener so the // server can attach in read-only mode. @@ -133,7 +129,7 @@ func TestServerDBConnIsReadOnly(t *testing.T) { t.Fatalf("bootstrap: %v", err) } - d, err := OpenDB(path) + d, err := openFixtureReader(t, path) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -150,8 +146,8 @@ func TestServerDBConnIsReadOnly(t *testing.T) { // Kept in *_test.go so it does NOT add any write capability to the // production server binary. func bootstrapMinimalDB(tb testing.TB, path string) error { - dsn := fmt.Sprintf("file:%s?_journal_mode=WAL&_busy_timeout=5000", path) - rw, err := sql.Open("sqlite3", dsn) + dsn := path + rw, err := openFixtureSQL(dsn) if err != nil { return err } @@ -259,7 +255,7 @@ func TestUsersOpenIsTheOnlyServerWritePath(t *testing.T) { t.Fatal(err) } call := regexp.MustCompile(`users\.Open\(`) - want := regexp.MustCompile(`users\.Open\(set\.dbPath, measurementDBPath\)`) + want := regexp.MustCompile(`users\.Open\(set\.databaseURL, measurementDatabaseURL\)`) calls, good := 0, 0 for _, e := range entries { name := e.Name() diff --git a/cmd/server/region_nodes_test.go b/cmd/server/region_nodes_test.go index f8f4a5800..e19ba2beb 100644 --- a/cmd/server/region_nodes_test.go +++ b/cmd/server/region_nodes_test.go @@ -36,7 +36,7 @@ func newRegionTestStore(t testing.TB, observers map[string]string) *PacketStore t.Helper() db := setupTestDB(t) for id, iata := range observers { - if _, err := db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) VALUES (?, ?, ?, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1)`, id, id, iata); err != nil { + if _, err := db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) VALUES ($1, $2, $3, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1)`, id, id, iata); err != nil { t.Fatal(err) } } diff --git a/cmd/server/repeater_liveness_test.go b/cmd/server/repeater_liveness_test.go index 07f108812..e84e1043b 100644 --- a/cmd/server/repeater_liveness_test.go +++ b/cmd/server/repeater_liveness_test.go @@ -13,7 +13,7 @@ func TestRepeaterRelayActivity_Active(t *testing.T) { defer db.conn.Close() pubkey := "aabbccdd11223344" - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", pubkey, "RepActive", "repeater", recentTS(1)) store := NewPacketStore(db, nil) @@ -60,7 +60,7 @@ func seedUnscopedRelayFixture(t *testing.T, hashPrefix string) (*PacketStore, st t.Helper() db := setupCapabilityTestDB(t) pubkey := "aabbccdd11223344" - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", pubkey, "RepUnscoped", "repeater", recentTS(1)) store := NewPacketStore(db, nil) pt := 1 // non-advert (TXT_MSG) @@ -109,7 +109,7 @@ func TestRepeaterRelayActivity_Idle(t *testing.T) { defer db.conn.Close() pubkey := "ccddeeff55667788" - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", pubkey, "RepIdle", "repeater", recentTS(1)) store := NewPacketStore(db, nil) @@ -134,7 +134,7 @@ func TestRepeaterRelayActivity_Stale(t *testing.T) { defer db.conn.Close() pubkey := "1122334455667788" - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", pubkey, "RepStale", "repeater", recentTS(1)) store := NewPacketStore(db, nil) @@ -176,7 +176,7 @@ func TestRepeaterRelayActivity_IgnoresAdverts(t *testing.T) { defer db.conn.Close() pubkey := "deadbeef00000001" - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", pubkey, "RepAdvertOnly", "repeater", recentTS(1)) store := NewPacketStore(db, nil) @@ -227,7 +227,7 @@ func TestRepeaterRelayActivity_PrefixHop(t *testing.T) { defer db.conn.Close() pubkey := "a36a21290d9c25a158130fe7c489541210d5f09f25fab997db5e942fb7680510" - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", pubkey, "RepPrefix", "repeater", recentTS(1)) store := NewPacketStore(db, nil) @@ -274,7 +274,7 @@ func TestRepeaterRelayActivity_DedupAcrossPrefixAndFullKey(t *testing.T) { defer db.conn.Close() pubkey := "a36a21290d9c25a158130fe7c489541210d5f09f25fab997db5e942fb7680510" - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", pubkey, "RepDedup", "repeater", recentTS(1)) store := NewPacketStore(db, nil) diff --git a/cmd/server/resolve_context_test.go b/cmd/server/resolve_context_test.go index c05018503..b6bfed123 100644 --- a/cmd/server/resolve_context_test.go +++ b/cmd/server/resolve_context_test.go @@ -164,7 +164,7 @@ func TestResolveHopsAPI_UniquePrefix(t *testing.T) { _ = srv // Insert a unique node - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + srv.db.conn.Exec("INSERT INTO nodes (public_key, name, lat, lon, role) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", "ff11223344", "UniqueNode", 37.0, -122.0, "repeater") srv.store.InvalidateNodeCache() @@ -189,9 +189,9 @@ func TestResolveHopsAPI_UniquePrefix(t *testing.T) { func TestResolveHopsAPI_AmbiguousNoContext(t *testing.T) { srv, router := setupTestServer(t) - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + srv.db.conn.Exec("INSERT INTO nodes (public_key, name, lat, lon, role) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", "ee1aaaaaaa", "Node-E1", 37.0, -122.0, "repeater") - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + srv.db.conn.Exec("INSERT INTO nodes (public_key, name, lat, lon, role) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", "ee1bbbbbbb", "Node-E2", 38.0, -121.0, "repeater") srv.store.InvalidateNodeCache() @@ -224,11 +224,11 @@ func TestResolveHopsAPI_AmbiguousNoContext(t *testing.T) { func TestResolveHopsAPI_WithAffinityContext(t *testing.T) { srv, router := setupTestServer(t) - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + srv.db.conn.Exec("INSERT INTO nodes (public_key, name, lat, lon, role) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", "dd1aaaaaaa", "Node-D1", 37.0, -122.0, "repeater") - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + srv.db.conn.Exec("INSERT INTO nodes (public_key, name, lat, lon, role) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", "dd1bbbbbbb", "Node-D2", 38.0, -121.0, "repeater") - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + srv.db.conn.Exec("INSERT INTO nodes (public_key, name, lat, lon, role) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", "c0c0c0c0c0", "Context", 37.1, -122.1, "repeater") // Invalidate node cache so the PM includes newly inserted nodes. @@ -279,7 +279,7 @@ func TestResolveHopsAPI_WithAffinityContext(t *testing.T) { func TestResolveHopsAPI_ResponseShape(t *testing.T) { srv, router := setupTestServer(t) - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + srv.db.conn.Exec("INSERT INTO nodes (public_key, name, lat, lon, role) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", "bb1aaaaaaa", "Node-B1", 37.0, -122.0, "repeater") req := httptest.NewRequest("GET", "/api/resolve-hops?hops=bb1a", nil) diff --git a/cmd/server/resolved_index.go b/cmd/server/resolved_index.go index ccc901903..af21be065 100644 --- a/cmd/server/resolved_index.go +++ b/cmd/server/resolved_index.go @@ -158,7 +158,7 @@ func (s *PacketStore) confirmResolvedPathContains(txID int, pubkey string) bool needle := `"` + strings.ToLower(pubkey) + `"` var count int err := s.db.conn.QueryRow( - `SELECT COUNT(*) FROM observations WHERE transmission_id = ? AND INSTR(LOWER(resolved_path), ?) > 0`, + `SELECT COUNT(*) FROM observations WHERE transmission_id = $1 AND strpos(LOWER(resolved_path), $2) > 0`, txID, needle, ).Scan(&count) if err != nil { @@ -167,14 +167,14 @@ func (s *PacketStore) confirmResolvedPathContains(txID int, pubkey string) bool return count > 0 } -// fetchResolvedPathsForTx fetches resolved_path from SQLite for all observations +// fetchResolvedPathsForTx fetches resolved_path from PostgreSQL for all observations // of a transmission. Used for on-demand API responses and eviction cleanup. func (s *PacketStore) fetchResolvedPathsForTx(txID int) map[int][]*string { if s.db == nil || s.db.conn == nil { return nil } rows, err := s.db.conn.Query( - `SELECT id, resolved_path FROM observations WHERE transmission_id = ? AND resolved_path IS NOT NULL`, + `SELECT id, resolved_path FROM observations WHERE transmission_id = $1 AND resolved_path IS NOT NULL`, txID, ) if err != nil { @@ -215,7 +215,7 @@ func (s *PacketStore) fetchResolvedPathForObs(obsID int) []*string { var rpJSON sql.NullString err := s.db.conn.QueryRow( - `SELECT resolved_path FROM observations WHERE id = ?`, obsID, + `SELECT resolved_path FROM observations WHERE id = $1`, obsID, ).Scan(&rpJSON) if err != nil || !rpJSON.Valid { return nil @@ -337,14 +337,14 @@ func (s *PacketStore) lruDelete(obsID int) { // resolvedPubkeysForEvictionBatch fetches resolved pubkeys for multiple txIDs // from SQL in a single batched query. Returns a map from txID to unique pubkeys. // MUST be called WITHOUT holding s.mu — this is the whole point of the batch approach. -// Chunks queries to stay under SQLite's 500-parameter limit. +// Chunks queries to bound each lookup and its result set. func (s *PacketStore) resolvedPubkeysForEvictionBatch(txIDs []int) map[int][]string { result := make(map[int][]string, len(txIDs)) if len(txIDs) == 0 || s.db == nil || s.db.conn == nil { return result } - const chunkSize = 499 // SQLite SQLITE_MAX_VARIABLE_NUMBER default is 999; stay well under + const chunkSize = 499 // Retain the existing bounded lookup batch. for start := 0; start < len(txIDs); start += chunkSize { end := start + chunkSize if end > len(txIDs) { @@ -352,19 +352,12 @@ func (s *PacketStore) resolvedPubkeysForEvictionBatch(txIDs []int) map[int][]str } chunk := txIDs[start:end] - // Build query with placeholders - placeholders := make([]byte, 0, len(chunk)*2) args := make([]interface{}, len(chunk)) for i, id := range chunk { - if i > 0 { - placeholders = append(placeholders, ',') - } - placeholders = append(placeholders, '?') args[i] = id } - query := "SELECT transmission_id, resolved_path FROM observations WHERE transmission_id IN (" + - string(placeholders) + ") AND resolved_path IS NOT NULL" + sqlPlaceholders(len(chunk)) + ") AND resolved_path IS NOT NULL" rows, err := s.db.conn.Query(query, args...) if err != nil { diff --git a/cmd/server/resolved_index_test.go b/cmd/server/resolved_index_test.go index 90c2bfb89..dbde326aa 100644 --- a/cmd/server/resolved_index_test.go +++ b/cmd/server/resolved_index_test.go @@ -1,7 +1,6 @@ package main import ( - "database/sql" "fmt" "reflect" "runtime" @@ -9,8 +8,6 @@ import ( "sync" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // --- Unit tests --- @@ -85,9 +82,9 @@ func TestResolvedPubkeyIndex_HashCollision(t *testing.T) { db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, '', 'h1', '2026-01-01T00:00:00Z')") db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (2, '', 'h2', '2026-01-01T00:00:00Z')") now := time.Now().Unix() - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (1, 1, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (1, 1, 1, $1, $2, $3)", `["aa"]`, now, `["real_match"]`) - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (2, 2, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (2, 2, 1, $1, $2, $3)", `["aa"]`, now, `["false_match"]`) // Simulate collision: manually insert both txIDs under the same hash @@ -308,9 +305,9 @@ func TestEviction_ByNodeCleanup_OnDemandSQL(t *testing.T) { relayPK := "relay_evict_test" // Insert DB data for on-demand SQL fetch during eviction - db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, '', 'h1', ?)", + db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, '', 'h1', $1)", now.Add(-48*time.Hour).Format(time.RFC3339)) - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (1, 1, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (1, 1, 1, $1, $2, $3)", `["aa"]`, now.Add(-48*time.Hour).Unix(), `["`+relayPK+`"]`) tx := &StoreTx{ @@ -384,9 +381,9 @@ func TestRunEviction_NoLockDuringSQLFetch(t *testing.T) { // Create 10 stale packets with resolved_path data in DB for i := 1; i <= 10; i++ { staleTime := now.Add(-48 * time.Hour).Format(time.RFC3339) - db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (?, '', ?, ?)", + db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES ($1, '', $2, $3)", i, fmt.Sprintf("h%d", i), staleTime) - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (?, ?, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES ($1, $2, 1, $3, $4, $5)", i, i, `["aa"]`, now.Add(-48*time.Hour).Unix(), `["pk`+fmt.Sprintf("%d", i)+`"]`) tx := &StoreTx{ @@ -482,7 +479,7 @@ func TestPacketsAPI_OnDemandResolvedPath(t *testing.T) { now := time.Now().Unix() db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, '', 'h1', '2026-01-01')") - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (1, 1, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (1, 1, 1, $1, $2, $3)", `["aa"]`, now, `["aabbccdd"]`) store := &PacketStore{ @@ -507,7 +504,7 @@ func TestPacketsAPI_OnDemandResolvedPath_LRUHit(t *testing.T) { now := time.Now().Unix() db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, '', 'h1', '2026-01-01')") - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (1, 1, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (1, 1, 1, $1, $2, $3)", `["aa"]`, now, `["aabbccdd"]`) store := &PacketStore{ @@ -1096,14 +1093,14 @@ func TestLRU_CapacityAfterBulkDelete(t *testing.T) { // TestConfirmResolvedPathContains_SpecialChars verifies that pubkeys containing // SQL LIKE wildcards (%, _) don't cause false positives with the INSTR approach. func TestConfirmResolvedPathContains_SpecialChars(t *testing.T) { - db, err := sql.Open("sqlite3", ":memory:") + db, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } defer db.Close() _, err = db.Exec(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, resolved_path TEXT )`) @@ -1177,9 +1174,9 @@ func TestResolvedPubkeyIndex_BoundedByEviction(t *testing.T) { pk := fmt.Sprintf("pk_%04d", i) firstSeen := now.Add(-2 * time.Hour).Format(time.RFC3339) - db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (?, '', ?, ?)", + db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES ($1, '', $2, $3)", i, hash, firstSeen) - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (?, ?, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES ($1, $2, 1, $3, $4, $5)", i, i, `["aa"]`, now.Add(-2*time.Hour).Unix(), `["`+pk+`"]`) tx := &StoreTx{ID: i, Hash: hash, FirstSeen: firstSeen} @@ -1299,8 +1296,8 @@ func TestRunEviction_TriggersCompaction(t *testing.T) { // Add one old tx with a large-cap slice in the index pk := "compaction_pk" firstSeen := now.Add(-2 * time.Hour).Format(time.RFC3339) - db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, '', 'h1', ?)", firstSeen) - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (1, 1, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, '', 'h1', $1)", firstSeen) + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (1, 1, 1, $1, $2, $3)", `["aa"]`, now.Add(-2*time.Hour).Unix(), `["`+pk+`"]`) tx := &StoreTx{ID: 1, Hash: "h1", FirstSeen: firstSeen} @@ -1316,8 +1313,8 @@ func TestRunEviction_TriggersCompaction(t *testing.T) { // Add a fresh tx (won't be evicted) with pk shared with old tx pk2 := "shared_pk" freshSeen := now.Format(time.RFC3339) - db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (2, '', 'h2', ?)", freshSeen) - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (2, 2, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (2, '', 'h2', $1)", freshSeen) + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (2, 2, 1, $1, $2, $3)", `["bb"]`, now.Unix(), `["`+pk2+`"]`) tx2 := &StoreTx{ID: 2, Hash: "h2", FirstSeen: freshSeen} @@ -1394,9 +1391,9 @@ func TestResolvedPubkeyIndex_MemoryStableThroughEvictionCycles(t *testing.T) { pk := fmt.Sprintf("pk_%d", txCounter) firstSeen := now.Add(-2 * time.Hour).Format(time.RFC3339) - db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (?, '', ?, ?)", + db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES ($1, '', $2, $3)", txCounter, hash, firstSeen) - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (?, ?, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES ($1, $2, 1, $3, $4, $5)", txCounter, txCounter, `["aa"]`, now.Add(-2*time.Hour).Unix(), `["`+pk+`"]`) tx := &StoreTx{ID: txCounter, Hash: hash, FirstSeen: firstSeen} diff --git a/cmd/server/rf_noise.go b/cmd/server/rf_noise.go index ef49f8f19..f166dbd0c 100644 --- a/cmd/server/rf_noise.go +++ b/cmd/server/rf_noise.go @@ -146,11 +146,11 @@ func (s *Server) requireClientRfSamples(w http.ResponseWriter, r *http.Request) // queryRfNoiseRows returns raw RF-environment samples within a bbox, over a // time window (days; 0 = all time). Read-only (server RO connection). func (s *Server) queryRfNoiseRows(days int, b bbox) ([]rfNoiseRow, error) { - where := []string{"lat BETWEEN ? AND ?", "lon BETWEEN ? AND ?", "noise_floor IS NOT NULL"} + where := []string{"lat BETWEEN $1 AND $2", "lon BETWEEN $3 AND $4", "noise_floor IS NOT NULL"} args := []interface{}{b.MinLat, b.MaxLat, b.MinLon, b.MaxLon} if days > 0 { since := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339) - where = append(where, "sampled_at >= ?") + where = append(where, "sampled_at >= $5") args = append(args, since) } rows, err := s.db.conn.Query(` diff --git a/cmd/server/rf_noise_test.go b/cmd/server/rf_noise_test.go index a3c1f3204..fc20f625b 100644 --- a/cmd/server/rf_noise_test.go +++ b/cmd/server/rf_noise_test.go @@ -188,10 +188,10 @@ func TestRfNoiseRouteGatedOn(t *testing.T) { func seedRfNoiseDB(t *testing.T) *DB { db := setupTestDBv2(t) mustExecDB(t, db, `CREATE TABLE client_rf_samples ( - id INTEGER PRIMARY KEY AUTOINCREMENT, rx_pubkey TEXT, sampled_at TEXT, ingested_at TEXT, - lat REAL, lon REAL, pos_acc_m REAL, stationary INTEGER NOT NULL DEFAULT 0, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, rx_pubkey TEXT, sampled_at TEXT, ingested_at TEXT, + lat DOUBLE PRECISION, lon DOUBLE PRECISION, pos_acc_m DOUBLE PRECISION, stationary INTEGER NOT NULL DEFAULT 0, uptime_secs INTEGER, battery_mv INTEGER, queue_len INTEGER, errors INTEGER, - noise_floor INTEGER, last_rssi INTEGER, last_snr REAL, tx_air_secs INTEGER, + noise_floor INTEGER, last_rssi INTEGER, last_snr DOUBLE PRECISION, tx_air_secs INTEGER, rx_air_secs INTEGER, recv INTEGER, sent INTEGER, flood_rx INTEGER, direct_rx INTEGER, flood_tx INTEGER, direct_tx INTEGER, recv_errors INTEGER)`) return db diff --git a/cmd/server/routes.go b/cmd/server/routes.go index 645d10e3d..a86b94a50 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -273,7 +273,8 @@ func (s *Server) RegisterRoutes(r *mux.Router) { r.HandleFunc("/api/scope-audit", s.handleScopeAudit).Methods("GET") // #1975 r.HandleFunc("/api/perf", s.handlePerf).Methods("GET") r.HandleFunc("/api/perf/io", s.handlePerfIO).Methods("GET") - r.HandleFunc("/api/perf/sqlite", s.handlePerfSqlite).Methods("GET") + r.HandleFunc("/api/perf/postgres", s.handlePerfPostgres).Methods("GET") + r.HandleFunc("/api/perf/sqlite", s.handlePerfPostgres).Methods("GET") // legacy diagnostic route alias r.HandleFunc("/api/perf/write-sources", s.handlePerfWriteSources).Methods("GET") r.HandleFunc("/api/mqtt/status", s.handleMqttStatus).Methods("GET") r.Handle("/api/perf/reset", s.requireAdmin(http.HandlerFunc(s.handlePerfReset))).Methods("POST") @@ -1029,11 +1030,11 @@ func (s *Server) handlePerf(w http.ResponseWriter, r *http.Request) { breakdownNote = memBreakdownNote } - // SQLite stats - var sqliteStats *SqliteStats + // PostgreSQL diagnostics reuse the bounded database sample. + var postgresStats *PostgresStats if s.db != nil { ss := s.db.GetDBSizeStatsTyped() - sqliteStats = &ss + postgresStats = &ss } writeJSON(w, PerfResponse{ @@ -1044,7 +1045,7 @@ func (s *Server) handlePerf(w http.ResponseWriter, r *http.Request) { SlowQueries: slowQueries, Cache: perfCS, PacketStore: pktStoreStats, - Sqlite: sqliteStats, + Postgres: postgresStats, MemoryBreakdown: memBreakdown, MemoryBreakdownNote: breakdownNote, GoRuntime: func() *GoRuntimeStats { @@ -1092,7 +1093,7 @@ func (s *Server) handlePackets(w http.ResponseWriter, r *http.Request) { cleaned = append(cleaned, pk) } } - // Each entry costs one SQLite lookup (resolveNodePubkey) while the + // Each entry costs one database lookup (resolveNodePubkey) while the // packet store's read lock is held. A 1 MB URL fits ~15k pubkeys, // which is seconds of work per request, so cap the list. if len(cleaned) > maxMultiNodePubkeys { @@ -1282,7 +1283,7 @@ func (s *Server) handlePacketDetail(w http.ResponseWriter, r *http.Request) { } } } - // DB fallback: in-memory PacketStore prunes old entries, but the SQLite + // DB fallback: in-memory PacketStore prunes old entries, but the database // DB retains them and is the source for /api/nodes recentAdverts. Without // this fallback, links from node-detail pages 404 once the packet ages out. if packet == nil && s.db != nil { @@ -3666,7 +3667,7 @@ func (s *Server) handleScopeStats(w http.ResponseWriter, r *http.Request) { // geo_filter. Nodes with no GPS fix are always kept. Requires geo_filter to be // configured. // -// Since #1283/#1289 the server opens SQLite read-only, so the actual DELETE is +// Since #1283/#1289 the server opens telemetry read-only, so the actual DELETE is // performed by the ingestor. The server writes a request marker file (see // internal/prunequeue); the ingestor's maintenance loop consumes it and writes a // result marker. The confirm response is 202 Accepted with a request id; @@ -3750,13 +3751,13 @@ func (s *Server) handlePruneGeoFilter(w http.ResponseWriter, r *http.Request) { Reason: "geo-prune", Pubkeys: pubkeys, } - if err := prunequeue.WriteRequest(s.db.path, req); err != nil { + if err := prunequeue.WriteRequest(s.db.statePath(), req); err != nil { log.Printf("[geo-prune] failed to enqueue request %s: %v", id, err) writeError(w, http.StatusInternalServerError, "failed to enqueue prune request") return } log.Printf("[geo-prune] enqueued request %s for %d node(s) (queue dir=%s)", - id, len(pubkeys), prunequeue.QueueDir(s.db.path)) + id, len(pubkeys), prunequeue.QueueDir(s.db.statePath())) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusAccepted) @@ -3782,7 +3783,7 @@ func (s *Server) handlePruneGeoFilterStatus(w http.ResponseWriter, r *http.Reque return } - res, err := prunequeue.ReadResult(s.db.path, id) + res, err := prunequeue.ReadResult(s.db.statePath(), id) if err != nil { if strings.Contains(err.Error(), "invalid prune request id") { writeError(w, http.StatusBadRequest, "invalid id") @@ -3807,7 +3808,7 @@ func (s *Server) handlePruneGeoFilterStatus(w http.ResponseWriter, r *http.Reque return } - pending, err := prunequeue.RequestExists(s.db.path, id) + pending, err := prunequeue.RequestExists(s.db.statePath(), id) if err != nil { if strings.Contains(err.Error(), "invalid prune request id") { writeError(w, http.StatusBadRequest, "invalid id") diff --git a/cmd/server/routes_test.go b/cmd/server/routes_test.go index 572c42219..a89c04205 100644 --- a/cmd/server/routes_test.go +++ b/cmd/server/routes_test.go @@ -552,9 +552,9 @@ func TestPerfEndpoint(t *testing.T) { } // Verify sqlite stats - sqliteStats, ok := body["sqlite"].(map[string]interface{}) + sqliteStats, ok := body["postgres"].(map[string]interface{}) if !ok { - t.Fatal("expected sqlite object in perf response") + t.Fatal("expected postgres object in perf response") } if _, ok := sqliteStats["dbSizeMB"]; !ok { t.Error("expected dbSizeMB in sqlite") @@ -1174,10 +1174,10 @@ func TestChannelMessagesWithRegion(t *testing.T) { db := setupTestDB(t) seedTestData(t, db) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('EEFF', 'chanextra000001', ?, 1, 5, '{"type":"CHAN","channel":"#test","text":"OtherUser: Cross region","sender":"OtherUser"}')`, + VALUES ('EEFF', 'chanextra000001', $1, 1, 5, '{"type":"CHAN","channel":"#test","text":"OtherUser: Cross region","sender":"OtherUser"}')`, time.Now().UTC().Add(-30*time.Minute).Format(time.RFC3339)) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (4, 2, 11.0, -89, '[]', ?)`, time.Now().UTC().Add(-30*time.Minute).Unix()) + VALUES (4, 2, 11.0, -89, '[]', $1)`, time.Now().UTC().Add(-30*time.Minute).Unix()) cfg := &Config{Port: 3000} hub := NewHub() @@ -2030,7 +2030,9 @@ func TestHandlerErrorPaths(t *testing.T) { srv.RegisterRoutes(router) t.Run("stats error", func(t *testing.T) { - db.conn.Exec("DROP TABLE IF EXISTS transmissions") + if _, err := db.conn.Exec("DROP TABLE IF EXISTS transmissions CASCADE"); err != nil { + t.Fatal(err) + } req := httptest.NewRequest("GET", "/api/stats", nil) w := httptest.NewRecorder() router.ServeHTTP(w, req) @@ -2049,7 +2051,9 @@ func TestHandlerErrorChannels(t *testing.T) { router := mux.NewRouter() srv.RegisterRoutes(router) - db.conn.Exec("DROP TABLE IF EXISTS transmissions") + if _, err := db.conn.Exec("DROP TABLE IF EXISTS transmissions CASCADE"); err != nil { + t.Fatal(err) + } req := httptest.NewRequest("GET", "/api/channels", nil) w := httptest.NewRecorder() @@ -2145,7 +2149,9 @@ func TestHandlerErrorPackets(t *testing.T) { srv.RegisterRoutes(router) // Drop transmissions table to trigger error in transmission-centric query - db.conn.Exec("DROP TABLE IF EXISTS transmissions") + if _, err := db.conn.Exec("DROP TABLE IF EXISTS transmissions CASCADE"); err != nil { + t.Fatal(err) + } req := httptest.NewRequest("GET", "/api/packets?limit=10", nil) w := httptest.NewRecorder() @@ -2385,7 +2391,7 @@ func TestGetNodeHashSizeInfoFlipFlop(t *testing.T) { store.WaitIndexesReady(5 * time.Second) pk := "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'TestNode', 'repeater')", pk) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'TestNode', 'repeater') ON CONFLICT DO NOTHING", pk) decoded := `{"name":"TestNode","pubKey":"` + pk + `"}` raw1 := "11" + "01" + "aabb" @@ -2434,7 +2440,7 @@ func TestGetNodeHashSizeInfoDominant(t *testing.T) { store.WaitIndexesReady(5 * time.Second) pk := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'Repeater2B', 'repeater')", pk) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'Repeater2B', 'repeater') ON CONFLICT DO NOTHING", pk) decoded := `{"name":"Repeater2B","pubKey":"` + pk + `"}` raw1byte := "11" + "01" + "aabb" // FLOOD, pathByte=0x01 → hashSize=1 @@ -2478,7 +2484,7 @@ func TestGetNodeHashSizeInfoLatestWins(t *testing.T) { store.WaitIndexesReady(5 * time.Second) pk := "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'LatestWins', 'repeater')", pk) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'LatestWins', 'repeater') ON CONFLICT DO NOTHING", pk) decoded := `{"name":"LatestWins","pubKey":"` + pk + `"}` raw1byte := "11" + "01" + "aabb" // FLOOD, pathByte=0x01 → hashSize=1 @@ -2528,7 +2534,7 @@ func TestGetNodeHashSizeInfoIgnoreDirectZeroHop(t *testing.T) { store.WaitIndexesReady(5 * time.Second) pk := "dddd111122223333444455556666777788889999aaaabbbbccccddddeeee3333" - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'DirIgnore', 'repeater')", pk) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'DirIgnore', 'repeater') ON CONFLICT DO NOTHING", pk) decoded := `{"name":"DirIgnore","pubKey":"` + pk + `"}` rawFlood2B := "11" + "40" + "aabb" // FLOOD advert, hashSize=2 @@ -2576,7 +2582,7 @@ func TestGetNodeHashSizeInfoOnlyDirectZeroHopIgnored(t *testing.T) { store.WaitIndexesReady(5 * time.Second) pk := "eeee111122223333444455556666777788889999aaaabbbbccccddddeeee4444" - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'OnlyDirect', 'repeater')", pk) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'OnlyDirect', 'repeater') ON CONFLICT DO NOTHING", pk) decoded := `{"name":"OnlyDirect","pubKey":"` + pk + `"}` rawDirect0 := "12" + "00" + "aabb" @@ -2611,7 +2617,7 @@ func TestGetNodeHashSizeInfoDirectNonZeroHopCounted(t *testing.T) { store.WaitIndexesReady(5 * time.Second) pk := "ffff111122223333444455556666777788889999aaaabbbbccccddddeeee5555" - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'DirNonZero', 'repeater')", pk) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'DirNonZero', 'repeater') ON CONFLICT DO NOTHING", pk) decoded := `{"name":"DirNonZero","pubKey":"` + pk + `"}` // DIRECT advert (route type 2 = 0x02 in bits 0-1), path byte 0x41: @@ -2651,7 +2657,7 @@ func TestGetNodeHashSizeInfoNoAdverts(t *testing.T) { store.WaitIndexesReady(5 * time.Second) pk := "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'NoAdverts', 'repeater')", pk) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'NoAdverts', 'repeater') ON CONFLICT DO NOTHING", pk) // Add a non-advert packet (payload_type=2 = TXT_MSG) payloadType := 2 @@ -2691,7 +2697,7 @@ func TestHashAnalyticsZeroHopAdvert(t *testing.T) { baseDist1 := baseDist["1"] pk := "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'ZeroHop', 'repeater')", pk) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'ZeroHop', 'repeater') ON CONFLICT DO NOTHING", pk) store.InvalidateNodeCache() decoded := `{"name":"ZeroHop","pubKey":"` + pk + `"}` @@ -2748,8 +2754,8 @@ func TestAnalyticsHashSizeSameNameDifferentPubkey(t *testing.T) { pk2 := "aaaa111122223333444455556666777788889999aaaabbbbccccddddeeee2222" // Insert both nodes as repeaters so they appear in distributionByRepeaters. - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'SameName', 'repeater')", pk1) - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'SameName', 'repeater')", pk2) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'SameName', 'repeater') ON CONFLICT DO NOTHING", pk1) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'SameName', 'repeater') ON CONFLICT DO NOTHING", pk2) store.InvalidateNodeCache() decoded1 := `{"name":"SameName","pubKey":"` + pk1 + `"}` @@ -2827,7 +2833,7 @@ func TestInconsistentNodesExcludesCompanions(t *testing.T) { } for ni, n := range nodes { - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, ?, ?)", n.pk, "Node-"+n.role, n.role) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", n.pk, "Node-"+n.role, n.role) decoded := `{"name":"Node-` + n.role + `","pubKey":"` + n.pk + `"}` // Create flip-flop pattern: 1-byte, 2-byte, 1-byte (transitions=2 → inconsistent) // Use header 0x11 (routeType=FLOOD, payloadType=4) and pathByte 0x41/0x81 @@ -2898,7 +2904,7 @@ func TestHashSizeInfoTimeWindow(t *testing.T) { store.WaitIndexesReady(5 * time.Second) pk := "dd44444444444444444444444444444444444444444444444444444444444444" - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'OldNode', 'repeater')", pk) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'OldNode', 'repeater') ON CONFLICT DO NOTHING", pk) decoded := `{"name":"OldNode","pubKey":"` + pk + `"}` payloadType := 4 @@ -2929,7 +2935,7 @@ func TestHashSizeInfoTimeWindow(t *testing.T) { // Now add recent adverts with consistent hash size — should appear in info pk2 := "ee55555555555555555555555555555555555555555555555555555555555555" - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'NewNode', 'repeater')", pk2) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'NewNode', 'repeater') ON CONFLICT DO NOTHING", pk2) decoded2 := `{"name":"NewNode","pubKey":"` + pk2 + `"}` recentTime := time.Now().UTC().Format("2006-01-02T15:04:05.000Z") for i := 0; i < 3; i++ { @@ -3110,23 +3116,23 @@ func TestQueryGroupedPacketsSortedByLatest(t *testing.T) { oldEpoch := now.Add(-72 * time.Hour).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('sortobs', 'Sort Observer', 'TST', ?, '2026-01-01T00:00:00Z', 1)`, now.Format(time.RFC3339)) + VALUES ('sortobs', 'Sort Observer', 'TST', $1, '2026-01-01T00:00:00Z', 1)`, now.Format(time.RFC3339)) // Packet A: old first_seen, but a very recent observation — should sort first. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('AA01', 'sort_old_first_recent_obs', ?, 1, 2, '{"type":"TXT_MSG","text":"old first"}')`, oldFirst) + VALUES ('AA01', 'sort_old_first_recent_obs', $1, 1, 2, '{"type":"TXT_MSG","text":"old first"}')`, oldFirst) var idA int64 db.conn.QueryRow(`SELECT id FROM transmissions WHERE hash='sort_old_first_recent_obs'`).Scan(&idA) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 10.0, -90, '[]', ?)`, idA, recentEpoch) + VALUES ($1, 1, 10.0, -90, '[]', $2)`, idA, recentEpoch) // Packet B: newer first_seen, but an old observation — should sort second. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('BB02', 'sort_new_first_old_obs', ?, 1, 2, '{"type":"TXT_MSG","text":"new first"}')`, newFirst) + VALUES ('BB02', 'sort_new_first_old_obs', $1, 1, 2, '{"type":"TXT_MSG","text":"new first"}')`, newFirst) var idB int64 db.conn.QueryRow(`SELECT id FROM transmissions WHERE hash='sort_new_first_old_obs'`).Scan(&idB) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 10.0, -90, '[]', ?)`, idB, oldEpoch) + VALUES ($1, 1, 10.0, -90, '[]', $2)`, idB, oldEpoch) store := NewPacketStore(db, nil) if err := store.Load(); err != nil { @@ -3563,9 +3569,9 @@ func TestHashCollisionsWithCollision(t *testing.T) { // Two repeater nodes with same first byte 'CC' and hash_size=1 db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('CC11223344556677', 'Node1', 'repeater', 37.5, -122.0, ?, '2026-01-01T00:00:00Z', 5)`, recent) + VALUES ('CC11223344556677', 'Node1', 'repeater', 37.5, -122.0, $1, '2026-01-01T00:00:00Z', 5)`, recent) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('CC99887766554433', 'Node2', 'repeater', 37.51, -122.01, ?, '2026-01-01T00:00:00Z', 5)`, recent) + VALUES ('CC99887766554433', 'Node2', 'repeater', 37.51, -122.01, $1, '2026-01-01T00:00:00Z', 5)`, recent) cfg := &Config{Port: 3000} hub := NewHub() @@ -3632,7 +3638,7 @@ func TestHashCollisionsShortPublicKey(t *testing.T) { recent := now.Add(-1 * time.Hour).Format(time.RFC3339) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('A', 'ShortKey', 'repeater', 0, 0, ?, '2026-01-01T00:00:00Z', 1)`, recent) + VALUES ('A', 'ShortKey', 'repeater', 0, 0, $1, '2026-01-01T00:00:00Z', 1)`, recent) cfg := &Config{Port: 3000} hub := NewHub() @@ -3663,9 +3669,9 @@ func TestHashCollisionsMissingCoordinates(t *testing.T) { // Two nodes same prefix, no coordinates db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('BB11223344556677', 'NoCoords1', 'repeater', 0, 0, ?, '2026-01-01T00:00:00Z', 1)`, recent) + VALUES ('BB11223344556677', 'NoCoords1', 'repeater', 0, 0, $1, '2026-01-01T00:00:00Z', 1)`, recent) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('BB99887766554433', 'NoCoords2', 'repeater', 0, 0, ?, '2026-01-01T00:00:00Z', 1)`, recent) + VALUES ('BB99887766554433', 'NoCoords2', 'repeater', 0, 0, $1, '2026-01-01T00:00:00Z', 1)`, recent) cfg := &Config{Port: 3000} hub := NewHub() @@ -3715,16 +3721,16 @@ func TestHashCollisionsOnlyRepeaters(t *testing.T) { // 5. sensor with hash_size=1 → should be excluded now := time.Now().Format("2006-01-02 15:04:05") db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES - ('aa11223344556677', 'Repeater1', 'repeater', ?), - ('aa99887766554433', 'UnknownNode', 'repeater', ?), - ('aadeadbeefcafe01', 'Companion1', 'companion', ?), - ('aabbcc1122334455', 'Room1', 'room', ?), - ('aabbcc9988776655', 'Sensor1', 'sensor', ?)`, now, now, now, now, now) + ('aa11223344556677', 'Repeater1', 'repeater', $1), + ('aa99887766554433', 'UnknownNode', 'repeater', $2), + ('aadeadbeefcafe01', 'Companion1', 'companion', $3), + ('aabbcc1122334455', 'Room1', 'room', $4), + ('aabbcc9988776655', 'Sensor1', 'sensor', $5)`, now, now, now, now, now) // We also need a second repeater with hash_size=1 and same prefix to // confirm that genuine collisions ARE still detected. db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES - ('aa00112233445566', 'Repeater2', 'repeater', ?)`, now) + ('aa00112233445566', 'Repeater2', 'repeater', $1)`, now) cfg := &Config{Port: 3000} hub := NewHub() @@ -3799,10 +3805,10 @@ func TestHashCollisionsOneByteIncludesMultiBytePrefixRepeaters(t *testing.T) { // One unrelated repeater with first byte "DD" must NOT appear in CC cell. now := time.Now().Format("2006-01-02 15:04:05") db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES - ('cc11223344556677', 'Rep1B', 'repeater', ?), - ('cc22aabbccddeeff', 'Rep2B', 'repeater', ?), - ('cc33bbddeeff0011', 'Rep3B', 'repeater', ?), - ('dd44556677889900', 'RepDD', 'repeater', ?)`, now, now, now, now) + ('cc11223344556677', 'Rep1B', 'repeater', $1), + ('cc22aabbccddeeff', 'Rep2B', 'repeater', $2), + ('cc33bbddeeff0011', 'Rep3B', 'repeater', $3), + ('dd44556677889900', 'RepDD', 'repeater', $4)`, now, now, now, now) cfg := &Config{Port: 3000} hub := NewHub() @@ -3908,18 +3914,18 @@ func TestNodePathsPrefixCollisionFilter(t *testing.T) { recentEpoch := now.Add(-30 * time.Minute).Unix() // Insert a second node with the same 2-char prefix - srv.db.conn.Exec(`INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) - VALUES ('aacafe0000000000', 'CollisionNode', 'repeater', ?, '2026-01-01T00:00:00Z', 5)`, recent) + srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) + VALUES ('aacafe0000000000', 'CollisionNode', 'repeater', $1, '2026-01-01T00:00:00Z', 5) ON CONFLICT DO NOTHING`, recent) // Insert a transmission with path hop "aa" that resolves to the OTHER node srv.db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('FF01', 'collision_test_hash', ?, 1, 4, '{}')`, recent) + VALUES ('FF01', 'collision_test_hash', $1, 1, 4, '{}')`, recent) // Get its ID var collisionTxID int srv.db.conn.QueryRow(`SELECT id FROM transmissions WHERE hash='collision_test_hash'`).Scan(&collisionTxID) srv.db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) - VALUES (?, 1, 10.0, -90, '["aa","bb"]', ?, '["aacafe0000000000","eeff00112233aabb"]')`, + VALUES ($1, 1, 10.0, -90, '["aa","bb"]', $2, '["aacafe0000000000","eeff00112233aabb"]')`, collisionTxID, recentEpoch) // Reload store to pick up new data @@ -4059,7 +4065,7 @@ func TestMetricsAPIEndpoints(t *testing.T) { now := time.Now().UTC() t1 := now.Add(-1 * time.Hour).Format(time.RFC3339) - srv.db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES (?, ?, ?)", + srv.db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES ($1, $2, $3)", "obs1", t1, -112.0) // Test /api/observers/obs1/metrics @@ -4188,16 +4194,16 @@ func TestPacketDetailFallsBackToDBWhenStoreMisses(t *testing.T) { now := time.Now().UTC().Format(time.RFC3339) if _, err := srv.db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('FFEE', ?, ?, 1, 4, '{"type":"ADVERT"}')`, dbOnlyHash, now); err != nil { + VALUES ('FFEE', $1, $2, 1, 4, '{"type":"ADVERT"}')`, dbOnlyHash, now); err != nil { t.Fatalf("insert: %v", err) } var txID int - if err := srv.db.conn.QueryRow("SELECT id FROM transmissions WHERE hash = ?", dbOnlyHash).Scan(&txID); err != nil { + if err := srv.db.conn.QueryRow("SELECT id FROM transmissions WHERE hash = $1", dbOnlyHash).Scan(&txID); err != nil { t.Fatalf("lookup tx id: %v", err) } if _, err := srv.db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 7.5, -99, '[]', ?)`, txID, time.Now().Unix()); err != nil { + VALUES ($1, 1, 7.5, -99, '[]', $2)`, txID, time.Now().Unix()); err != nil { t.Fatalf("insert obs: %v", err) } @@ -4278,7 +4284,7 @@ func TestHandleScopeStats(t *testing.T) { srv.db.hasScopeName = true // Clear seed transmissions so this test isolates scope-stats math. - if _, err := srv.db.conn.Exec(`DELETE FROM transmissions`); err != nil { + if _, err := srv.db.conn.Exec(`DELETE FROM observations; DELETE FROM transmissions`); err != nil { t.Fatalf("clear transmissions: %v", err) } @@ -4302,7 +4308,7 @@ func TestHandleScopeStats(t *testing.T) { scopeArg = nil // unscoped (NULL) } if _, err := srv.db.conn.Exec( - `INSERT INTO transmissions (raw_hex,hash,first_seen,route_type,payload_type,scope_name) VALUES (?,?,?,?,5,?)`, + `INSERT INTO transmissions (raw_hex,hash,first_seen,route_type,payload_type,scope_name) VALUES ($1,$2,$3,$4,5,$5)`, "aa", r.hash, now, r.route, scopeArg, ); err != nil { t.Fatalf("seed row %d: %v", i, err) @@ -4668,7 +4674,7 @@ func TestPruneGeoFilterEndpoint(t *testing.T) { } // And the marker file must exist on disk. - pending, err := prunequeue.RequestExists(srv.db.path, id) + pending, err := prunequeue.RequestExists(srv.db.statePath(), id) if err != nil { t.Fatalf("RequestExists: %v", err) } @@ -4706,7 +4712,7 @@ func TestPruneGeoFilterEndpoint(t *testing.T) { } // Simulate the ingestor completing the request. - if err := prunequeue.WriteResult(srv.db.path, prunequeue.Result{ + if err := prunequeue.WriteResult(srv.db.statePath(), prunequeue.Result{ ID: id, RequestedAt: time.Now().Add(-1 * time.Second).UTC(), CompletedAt: time.Now().UTC(), diff --git a/cmd/server/runstartup_load_test.go b/cmd/server/runstartup_load_test.go index fa7f22f7c..123411ced 100644 --- a/cmd/server/runstartup_load_test.go +++ b/cmd/server/runstartup_load_test.go @@ -19,7 +19,6 @@ package main import ( "fmt" - "path/filepath" "strings" "sync" "sync/atomic" @@ -31,13 +30,12 @@ import ( // when hotStartupHours == 0 the bg loader has no work to do; healthz // must NOT be stuck on backgroundLoadComplete=false. func TestRunStartupLoad_HotStartupHoursZero_SetsDoneImmediately(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) nowSec := time.Now().UTC().Unix() createTestDBWithLastSeen(t, dbPath, 10, 1, nowSec, 30*time.Minute, 30*time.Minute) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -67,13 +65,12 @@ func TestRunStartupLoad_HotStartupHoursZero_SetsDoneImmediately(t *testing.T) { // implementation left done=false, leaving healthz wedged on the // failure path (dij #1 / adv #7). func TestRunStartupLoad_LoadChunkedError_SetsFailedTerminal(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) nowSec := time.Now().UTC().Unix() createTestDBWithLastSeen(t, dbPath, 5, 1, nowSec, 30*time.Minute, 30*time.Minute) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -107,12 +104,11 @@ func TestRunStartupLoad_LoadChunkedError_SetsFailedTerminal(t *testing.T) { // loadBackgroundChunks must reach its coverage block (totalInDB==0 → // ratio=1.0) and set done=true rather than leaving the store stuck. func TestRunStartupLoad_EmptyDB_SetsDoneTerminal(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) createTestDBWithLastSeen(t, dbPath, 0, 0, time.Now().UTC().Unix(), 30*time.Minute, 30*time.Minute) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -152,13 +148,12 @@ func TestRunStartupLoad_EmptyDB_SetsDoneTerminal(t *testing.T) { // tBgEntry >= tLastChunk monotonically. The runtime invariant in // loadBackgroundChunks remains as the deterministic backstop. func TestRunStartupLoad_BgLoaderRunsAfterLoadChunkedSets_OldestLoaded(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) nowSec := time.Now().UTC().Unix() createTestDBWithLastSeen(t, dbPath, 50, 1, nowSec, 30*time.Minute, 30*time.Minute) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -223,14 +218,13 @@ func TestRunStartupLoad_BgLoaderRunsAfterLoadChunkedSets_OldestLoaded(t *testing // fails and the terminal state is still failed=true,done=true with a // non-empty error. func TestRunStartupLoad_LoadChunkedError_MidLoad(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) nowSec := time.Now().UTC().Unix() // Seed enough rows so a small chunk size yields >1 chunk. createTestDBWithLastSeen(t, dbPath, 200, 1, nowSec, 30*time.Minute, 30*time.Minute) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -270,8 +264,7 @@ func TestRunStartupLoad_LoadChunkedError_MidLoad(t *testing.T) { // must fire so future refactors cannot silently re-introduce the // #1809 race. func TestLoadBackgroundChunks_PanicsOnOldestLoadedEmpty_Invariant(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) // Reuse the existing schema-only fixture helper (0 rows) so this // test does not introduce a new inline CREATE TABLE block (pr-preflight // async-migration gate). The fixture provides exactly the bare schema @@ -279,7 +272,7 @@ func TestLoadBackgroundChunks_PanicsOnOldestLoadedEmpty_Invariant(t *testing.T) createTestDBWithLastSeen(t, dbPath, 0, 0, time.Now().UTC().Unix(), 30*time.Minute, 30*time.Minute) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/rx_coverage.go b/cmd/server/rx_coverage.go index be2080ed3..667a385e3 100644 --- a/cmd/server/rx_coverage.go +++ b/cmd/server/rx_coverage.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "fmt" "net/http" "sort" "strconv" @@ -257,12 +258,18 @@ func prefixOrEmpty(s string, n int) string { return "" } -// sqlPlaceholders returns "?,?,…" with n placeholders (n >= 1). -func sqlPlaceholders(n int) string { - if n <= 1 { - return "?" +// sqlPlaceholders builds native PostgreSQL positional parameters for an IN-list. +// The optional first index supports lists following other bound predicates. +func sqlPlaceholders(n int, first ...int) string { + start := 1 + if len(first) > 0 { + start = first[0] } - return strings.Repeat("?,", n-1) + "?" + parts := make([]string, n) + for i := range parts { + parts[i] = fmt.Sprintf("$%d", start+i) + } + return strings.Join(parts, ",") } // queryCoverageRows returns raw coverage rows where the directly-heard node @@ -279,7 +286,7 @@ func (s *Server) queryCoverageRows(pubkey string, b bbox) ([]coverageRow, error) SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions WHERE heard_key IN (`+sqlPlaceholders(len(cands))+`) - AND lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?`, args...) + AND lat BETWEEN `+fmt.Sprintf("$%d AND $%d AND lon BETWEEN $%d AND $%d", len(cands)+1, len(cands)+2, len(cands)+3, len(cands)+4), args...) if err != nil { return nil, err } diff --git a/cmd/server/rx_coverage_endpoint_test.go b/cmd/server/rx_coverage_endpoint_test.go index f5fa92a83..818af940e 100644 --- a/cmd/server/rx_coverage_endpoint_test.go +++ b/cmd/server/rx_coverage_endpoint_test.go @@ -11,8 +11,8 @@ import ( func seedCoverageDB(t *testing.T) *DB { db := setupTestDBv2(t) mustExecDB(t, db, `CREATE TABLE client_receptions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, rx_pubkey TEXT, heard_key TEXT, heard_keylen INTEGER, - rssi INTEGER, snr REAL, lat REAL, lon REAL, pos_acc_m REAL, rx_at TEXT, ingested_at TEXT, src TEXT)`) + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, rx_pubkey TEXT, heard_key TEXT, heard_keylen INTEGER, + rssi INTEGER, snr DOUBLE PRECISION, lat DOUBLE PRECISION, lon DOUBLE PRECISION, pos_acc_m DOUBLE PRECISION, rx_at TEXT, ingested_at TEXT, src TEXT)`) mustExecDB(t, db, `CREATE TABLE client_observers (pubkey TEXT PRIMARY KEY, name TEXT, last_seen TEXT)`) return db } diff --git a/cmd/server/rx_dashboard.go b/cmd/server/rx_dashboard.go index c4ae3b470..8148fe573 100644 --- a/cmd/server/rx_dashboard.go +++ b/cmd/server/rx_dashboard.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "encoding/json" + "fmt" "log" "net/http" "sort" @@ -99,7 +100,7 @@ func (s *Server) batchResolveHeardKeys(keys []string) map[string][2]string { // so this stays injection-safe regardless of how hexPrefixRe later // evolves. The per-prefix LIMIT 2 lives in a subquery because a bare // LIMIT on a UNION ALL term is a SQLite syntax error. - parts[j] = "SELECT * FROM (SELECT ? AS pfx, public_key, COALESCE(name,'') AS nm FROM nodes WHERE public_key LIKE ? LIMIT 2)" + parts[j] = fmt.Sprintf("SELECT * FROM (SELECT $%d::text AS pfx, public_key, COALESCE(name,'') AS nm FROM nodes WHERE public_key LIKE $%d LIMIT 2) AS matched", len(args)+1, len(args)+2) args = append(args, k, k+"%") } rows, err := s.db.conn.Query(strings.Join(parts, " UNION ALL "), args...) @@ -158,24 +159,24 @@ func (s *Server) resolveHeardKey(heardKey string) (string, string) { // by heard node (prefix/pubkey), contributing client (rx_pubkey), and time window // (days; 0 = all time). Powers the global and per-observer coverage maps. func (s *Server) queryCoverageFiltered(node, rx string, days int, b bbox) ([]coverageRow, error) { - where := []string{"lat BETWEEN ? AND ?", "lon BETWEEN ? AND ?"} + where := []string{"lat BETWEEN $1 AND $2", "lon BETWEEN $3 AND $4"} args := []interface{}{b.MinLat, b.MaxLat, b.MinLon, b.MaxLon} if node != "" { // Sargable heard_key IN-list (see coverageHeardKeyCandidates) so the // (heard_key, …) composite index is used instead of a substr() scan (#5). cands := coverageHeardKeyCandidates(node) - where = append(where, "heard_key IN ("+sqlPlaceholders(len(cands))+")") + where = append(where, "heard_key IN ("+sqlPlaceholders(len(cands), len(args)+1)+")") for _, c := range cands { args = append(args, c) } } if rx != "" { - where = append(where, "rx_pubkey = ?") + where = append(where, fmt.Sprintf("rx_pubkey = $%d", len(args)+1)) args = append(args, strings.ToLower(rx)) } if days > 0 { since := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339) - where = append(where, "rx_at >= ?") + where = append(where, fmt.Sprintf("rx_at >= $%d", len(args)+1)) args = append(args, since) } rows, err := s.db.conn.Query("SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions WHERE "+strings.Join(where, " AND "), args...) @@ -275,9 +276,9 @@ func (s *Server) rxLeaderboard(ctx context.Context, days, limit int) ([]LeaderOb FROM client_receptions cr LEFT JOIN nodes n ON n.public_key = cr.rx_pubkey LEFT JOIN client_observers co ON co.pubkey = cr.rx_pubkey - WHERE cr.rx_at >= ? - ORDER BY cr.rx_at DESC - LIMIT ?`, since, leaderboardScanCap) + WHERE cr.rx_at >= $1 + ORDER BY cr.rx_at DESC NULLS LAST + LIMIT $2`, since, leaderboardScanCap) if err != nil { return nil, err } diff --git a/cmd/server/scope_audit.go b/cmd/server/scope_audit.go index f4694d5e7..b1fe61f8b 100644 --- a/cmd/server/scope_audit.go +++ b/cmd/server/scope_audit.go @@ -20,6 +20,7 @@ package main // #1975 come from a 1179-repeater instance. import ( + "context" "database/sql" "fmt" "log" @@ -104,14 +105,14 @@ func splitRegionsCSV(csv string) []string { // supervisord starts both processes together, so a server that probed first // would ignore every mobile-client region answer until its next restart. // Once seen the answer latches, so the steady state costs nothing; until -// then it is one sqlite_master lookup per caller, and both callers sit +// then it is one schema catalog lookup per caller, and both callers sit // behind 30s caches. func (db *DB) declaredRegionsTablePresent() bool { if db.hasDeclaredRegionsTable || db.declaredRegionsTableLate.Load() { return true } var n int - if err := db.conn.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'node_declared_regions'`).Scan(&n); err != nil || n == 0 { + if err := db.conn.QueryRow(`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='node_declared_regions'`).Scan(&n); err != nil || n == 0 { return false } db.declaredRegionsTableLate.Store(true) @@ -190,7 +191,7 @@ func (db *DB) AllCurrentDeclaredRegions() ([]DeclaredRegionsRow, error) { rows, err := db.conn.Query(` WITH ranked AS ( SELECT target, observed_at, regions_csv, truncated, - ROW_NUMBER() OVER (PARTITION BY target ORDER BY observed_at DESC) AS rn + ROW_NUMBER() OVER (PARTITION BY target ORDER BY observed_at DESC NULLS LAST) AS rn FROM node_declared_regions ) SELECT target, observed_at, regions_csv, truncated @@ -251,7 +252,7 @@ func (db *DB) scopeAuditNodeIdentities(pubkeys []string) map[string]scopeAuditNo placeholders := make([]string, len(pubkeys)) args := make([]interface{}, len(pubkeys)) for i, k := range pubkeys { - placeholders[i] = "?" + placeholders[i] = fmt.Sprintf("$%d", i+1) args[i] = strings.ToLower(k) } rows, err := db.conn.Query( @@ -374,7 +375,7 @@ func scopeAuditPrefixIndex(targets []string) map[int]map[string][]string { // belt-and-braces: one path can carry the same target on several hops. // It applies the SAME three conditions scopeConformanceQuery does — // minForwarderHopHexLen, scopeConformanceForwarderRouteTypesSQL, and the -// explicit json_valid guard against a single malformed path_json row +// explicit JSON-array guard against a single malformed path_json row // erroring the whole query — but does not join against any target list: // attribution to a specific declared target happens in Go // (ScopeAuditForwarding), against the small in-memory prefix index built by @@ -384,12 +385,10 @@ var scopeAuditForwarderScanQuery = ` SELECT t.id, je.value FROM transmissions t JOIN observations o ON o.transmission_id = t.id - JOIN json_each(o.path_json) je - WHERE t.first_seen >= ? + CROSS JOIN LATERAL json_array_elements_text(CASE WHEN o.path_json IS JSON ARRAY THEN o.path_json::json ELSE '[]'::json END) je + WHERE t.first_seen >= $1 AND ` + scopeConformanceForwarderRouteTypesSQL + ` AND o.path_json IS NOT NULL - AND json_valid(o.path_json) - AND json_array_length(o.path_json) > 0 AND LENGTH(je.value) >= ` + fmt.Sprint(minForwarderHopHexLen) + ` ` @@ -456,7 +455,7 @@ var scopeAuditForwarderScanQuery = ` var scopeAuditWindowMetaQuery = ` SELECT t.id, t.scope_name, t.first_seen FROM transmissions t - WHERE t.first_seen >= ? + WHERE t.first_seen >= $1 AND ` + scopeConformanceForwarderRouteTypesSQL + ` ` @@ -566,6 +565,10 @@ type scopeAuditSeenKey struct { // the only thing keeping one transmission from counting several times for the // same target; TestScopeAuditForwardingCountsOneTransmissionOncePerTarget pins // it. +func (db *DB) beginScopeAuditSnapshot(ctx context.Context) (*sql.Tx, error) { + return db.conn.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true}) +} + func (s *PacketStore) ScopeAuditForwarding(sinceISO string, targets []string) (map[string]*scopeAuditTargetAgg, error) { byLen := scopeAuditPrefixIndex(targets) result := make(map[string]*scopeAuditTargetAgg, len(targets)) @@ -575,7 +578,7 @@ func (s *PacketStore) ScopeAuditForwarding(sinceISO string, targets []string) (m // transmission arriving between them would otherwise appear in the hop scan // with no metadata to attribute it by — rare, but the fix is a shared // snapshot rather than a rule about what to do with the leftovers. - tx, err := s.db.conn.Begin() + tx, err := s.db.beginScopeAuditSnapshot(context.Background()) if err != nil { return nil, fmt.Errorf("scope audit forwarder scan begin: %w", err) } diff --git a/cmd/server/scope_audit_test.go b/cmd/server/scope_audit_test.go index 444c34e60..504c537e6 100644 --- a/cmd/server/scope_audit_test.go +++ b/cmd/server/scope_audit_test.go @@ -2,7 +2,6 @@ package main import ( "context" - "database/sql" "encoding/hex" "encoding/json" "fmt" @@ -31,14 +30,14 @@ var ( // fiction, so the join behaves the way it does against a real database. func setupScopeConformanceDB(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } conn.SetMaxOpenConns(1) schema := ` CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT NOT NULL, hash TEXT NOT NULL UNIQUE, first_seen TEXT NOT NULL, @@ -49,7 +48,7 @@ func setupScopeConformanceDB(t *testing.T) *DB { scope_name TEXT ); CREATE TABLE observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER NOT NULL REFERENCES transmissions(id), path_json TEXT, timestamp INTEGER NOT NULL @@ -115,22 +114,23 @@ func seedTransmissionRouteAt(t *testing.T, s *PacketStore, forwarder string, see scopeSeedCounter++ hash := fmt.Sprintf("scopehash%d", scopeSeedCounter) - res, err := s.db.conn.Exec( + var txID int64 + err := s.db.conn.QueryRow( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, code1, code2, scope_name) - VALUES ('AA', ?, ?, ?, 1, ?, '00', ?)`, + VALUES ('AA', $1, $2, $3, 1, $4, '00', $5) RETURNING id`, hash, firstSeen, routeType, seed.code1, seed.scopeName, - ) + ).Scan(&txID) if err != nil { t.Fatalf("seed transmission: %v", err) } - txID, err := res.LastInsertId() + if err != nil { t.Fatalf("seed transmission id: %v", err) } pathJSON := fmt.Sprintf(`["%s"]`, strings.ToUpper(forwarder)) if _, err := s.db.conn.Exec( - `INSERT INTO observations (transmission_id, path_json, timestamp) VALUES (?, ?, ?)`, + `INSERT INTO observations (transmission_id, path_json, timestamp) VALUES ($1, $2, $3)`, txID, pathJSON, time.Now().Unix(), ); err != nil { t.Fatalf("seed observation: %v", err) @@ -257,7 +257,7 @@ func insertDeclared(t *testing.T, srv *Server, target, observedAt, regionsCSV st t.Helper() _ = truncated if _, err := srv.db.conn.Exec( - `INSERT INTO nodes (public_key, configured_scope, configured_scope_at) VALUES (?, ?, ?) + `INSERT INTO nodes (public_key, configured_scope, configured_scope_at) VALUES ($1, $2, $3) ON CONFLICT(public_key) DO UPDATE SET configured_scope = excluded.configured_scope, configured_scope_at = excluded.configured_scope_at`, target, regionsCSV, observedAt); err != nil { @@ -383,7 +383,7 @@ func TestHandleScopeAuditOmitsRepeaterWithNoDeclaredRow(t *testing.T) { declaredPk := testFullPubkeyA neverAskedPk := testFullPubkeyB insertDeclared(t, srv, declaredPk, time.Now().UTC().Format(time.RFC3339), "be", 0) - if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES (?, 'never-asked', 'repeater')`, neverAskedPk); err != nil { + if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES ($1, 'never-asked', 'repeater')`, neverAskedPk); err != nil { t.Fatal(err) } @@ -596,7 +596,7 @@ func TestHandleScopeAuditFiltersHiddenNamePrefix(t *testing.T) { // (configured_scope), so insertDeclared has already created it. The fork // kept them in separate tables and a plain INSERT was safe there. if _, err := srv.db.conn.Exec( - `INSERT INTO nodes (public_key, name, role) VALUES (?, ?, 'repeater') + `INSERT INTO nodes (public_key, name, role) VALUES ($1, $2, 'repeater') ON CONFLICT(public_key) DO UPDATE SET name = excluded.name, role = excluded.role`, hiddenPk, "🚫 ban me"); err != nil { t.Fatal(err) @@ -729,7 +729,7 @@ func createDeclaredRegionsTable(t *testing.T, srv *Server) { t.Helper() if _, err := srv.db.conn.Exec(` CREATE TABLE node_declared_regions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, target TEXT NOT NULL, rx_pubkey TEXT NOT NULL, observed_at TEXT NOT NULL, @@ -761,7 +761,7 @@ func TestDeclaredRegionsTableCreatedAfterStartupIsRead(t *testing.T) { // what happens when the ingestor creates it after the server started. if _, err := srv.db.conn.Exec(` CREATE TABLE node_declared_regions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, target TEXT NOT NULL, rx_pubkey TEXT NOT NULL, observed_at TEXT NOT NULL, @@ -789,7 +789,7 @@ func seedSecondSource(t *testing.T, srv *Server, target, observedAt, regionsCSV t.Helper() if _, err := srv.db.conn.Exec( `INSERT INTO node_declared_regions (target, rx_pubkey, observed_at, ingested_at, regions_csv, truncated) - VALUES (?, 'rxpubkeyhex', ?, ?, ?, ?)`, + VALUES ($1, 'rxpubkeyhex', $2, $3, $4, $5)`, target, observedAt, observedAt, regionsCSV, truncated); err != nil { t.Fatal(err) } @@ -911,15 +911,16 @@ func seedTransmissionPathAt(t *testing.T, s *PacketStore, hops []string, seed sc scopeSeedCounter++ hash := fmt.Sprintf("scopehash%d", scopeSeedCounter) - res, err := s.db.conn.Exec( + var txID int64 + err := s.db.conn.QueryRow( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, code1, code2, scope_name) - VALUES ('AA', ?, ?, ?, 1, ?, '00', ?)`, + VALUES ('AA', $1, $2, $3, 1, $4, '00', $5) RETURNING id`, hash, firstSeen, routeType, seed.code1, seed.scopeName, - ) + ).Scan(&txID) if err != nil { t.Fatalf("seed transmission: %v", err) } - txID, err := res.LastInsertId() + if err != nil { t.Fatalf("seed transmission id: %v", err) } @@ -930,7 +931,7 @@ func seedTransmissionPathAt(t *testing.T, s *PacketStore, hops []string, seed sc } pathJSON := "[" + strings.Join(quoted, ",") + "]" if _, err := s.db.conn.Exec( - `INSERT INTO observations (transmission_id, path_json, timestamp) VALUES (?, ?, ?)`, + `INSERT INTO observations (transmission_id, path_json, timestamp) VALUES ($1, $2, $3)`, txID, pathJSON, time.Now().Unix(), ); err != nil { t.Fatalf("seed observation: %v", err) @@ -1328,19 +1329,20 @@ func seedUnmatchedRawAt(t *testing.T, s *PacketStore, forwarder, rawHex string, t.Helper() scopeSeedCounter++ hash := fmt.Sprintf("scoperaw%d", scopeSeedCounter) - res, err := s.db.conn.Exec( + var txID int64 + err := s.db.conn.QueryRow( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, code1, code2, scope_name) - VALUES (?, ?, ?, ?, 5, '9209', '0000', '')`, - rawHex, hash, firstSeen, routeType) + VALUES ($1, $2, $3, $4, 5, '9209', '0000', '') RETURNING id`, + rawHex, hash, firstSeen, routeType).Scan(&txID) if err != nil { t.Fatal(err) } - txID, err := res.LastInsertId() + if err != nil { t.Fatal(err) } if _, err := s.db.conn.Exec( - `INSERT INTO observations (transmission_id, path_json, timestamp) VALUES (?, ?, 0)`, + `INSERT INTO observations (transmission_id, path_json, timestamp) VALUES ($1, $2, 0)`, txID, `["`+strings.ToUpper(forwarder)+`"]`); err != nil { t.Fatal(err) } diff --git a/cmd/server/scope_config_state_api_test.go b/cmd/server/scope_config_state_api_test.go index c1b5b0ca4..30b9d8ac3 100644 --- a/cmd/server/scope_config_state_api_test.go +++ b/cmd/server/scope_config_state_api_test.go @@ -271,19 +271,20 @@ func TestHandleNodesReportsObservedForUndeclaredForwarder(t *testing.T) { const pk = "a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90" if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'silent-forwarder', 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1)`, pk, + VALUES ($1, 'silent-forwarder', 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1)`, pk, ); err != nil { t.Fatal(err) } - res, err := db.conn.Exec( + var txID int64 + err := db.conn.QueryRow( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, scope_name) - VALUES ('AA', 'observed-scope-hash', ?, 0, 1, 1, '{}', '#be')`, + VALUES ('AA', 'observed-scope-hash', $1, 0, 1, 1, '{}', '#be') RETURNING id`, time.Now().UTC().Format(time.RFC3339), - ) + ).Scan(&txID) if err != nil { t.Fatal(err) } - txID, err := res.LastInsertId() + if err != nil { t.Fatal(err) } @@ -296,7 +297,7 @@ func TestHandleNodesReportsObservedForUndeclaredForwarder(t *testing.T) { // too, which also pins the lower-casing in addTxToPathHopIndex. if _, err := db.conn.Exec( `INSERT INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp) - VALUES (?, 0, 'rx', 1.0, -100, 0, ?, ?)`, + VALUES ($1, 0, 'rx', 1.0, -100, 0, $2, $3)`, txID, `["`+strings.ToUpper(pk)+`"]`, time.Now().Unix(), ); err != nil { t.Fatal(err) @@ -358,7 +359,7 @@ func TestScopeAuditAndNodesAgreeEndToEnd(t *testing.T) { for pk, csv := range cases { if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, configured_scope, configured_scope_at) - VALUES (?, ?, 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, ?, '2026-01-01T00:00:00Z')`, + VALUES ($1, $2, 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, $3, '2026-01-01T00:00:00Z')`, pk, "rp-"+pk, csv, ); err != nil { t.Fatal(err) diff --git a/cmd/server/scope_verify.go b/cmd/server/scope_verify.go index 3988df440..834651573 100644 --- a/cmd/server/scope_verify.go +++ b/cmd/server/scope_verify.go @@ -184,11 +184,11 @@ func (s *PacketStore) unmatchedTransmissionsInWindow(sinceISO string) ([]unmatch rows, err := s.db.conn.Query(` SELECT t.id, t.raw_hex FROM transmissions t - WHERE t.first_seen >= ? + WHERE t.first_seen >= $1 AND t.scope_name = '' AND `+scopeConformanceForwarderRouteTypesSQL+` ORDER BY t.first_seen DESC - LIMIT ?`, sinceISO, scopeVerifyMaxWindowPackets) + LIMIT $2`, sinceISO, scopeVerifyMaxWindowPackets) if err != nil { return nil, false, fmt.Errorf("unmatched transmissions scan: %w", err) } diff --git a/cmd/server/state_dir.go b/cmd/server/state_dir.go new file mode 100644 index 000000000..0620a10e0 --- /dev/null +++ b/cmd/server/state_dir.go @@ -0,0 +1,15 @@ +package main + +import "path/filepath" + +// runtimeStateDir is set once before the server's background workers start. +var runtimeStateDir = "data" + +// Queue helpers use an anchor filename; it is unrelated to the database URL. +func (db *DB) statePath() string { + dir := db.stateDir + if dir == "" { + dir = runtimeStateDir + } + return filepath.Join(dir, "meshcore") +} diff --git a/cmd/server/store.go b/cmd/server/store.go index e59194188..e94792f59 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -82,7 +82,7 @@ type StoreObs struct { Direction string SNR *float64 RSSI *float64 - Score *int + Score *float64 PathJSON string RawHex string Timestamp string @@ -450,9 +450,9 @@ type PacketStore struct { // Hot startup atomic gates — see contract below. // // Contract / ordering invariant (PR #1187): - // * hashMigrationComplete (set by migrateContentHashesAsync) gates - // content-hash–dependent code paths (e.g. dedup correctness on the - // write side). Set true ONLY after the migration loop finishes. + // * hashMigrationComplete (set by verifyContentHashesAsync) gates + // the diagnostic that all loaded content hashes match the current + // formula. Only the ingestor or offline migration can repair hashes. // * backgroundLoadDone is set true exactly once, after // loadBackgroundChunks finishes its loop AND its post-load index // rebuild. It gates "hot startup has finished filling @@ -501,7 +501,7 @@ type PacketStore struct { // perf payload for prod observability. loadCoverageRatio float64 - // Async hash migration state: set after migrateContentHashesAsync completes. + // Loaded content-hash verification state; retained API field name. hashMigrationComplete atomic.Bool // Chunked startup load state (#1009). LoadChunked closes @@ -773,7 +773,7 @@ func NewPacketStore(db *DB, cfg *PacketStoreConfig, cacheTTLs ...map[string]inte return ps } -// Load reads transmissions + observations from SQLite into memory. +// Load reads transmissions + observations from PostgreSQL into memory. // When maxMemoryMB > 0, loads only the newest N transmissions that fit // within the memory budget, avoiding OOM on large databases. func (s *PacketStore) Load() error { @@ -876,7 +876,7 @@ func (s *PacketStore) Load() error { loadSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, t.payload_type, t.payload_version, t.decoded_json, o.id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, - o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')` + obsRawHexCol + rpCol + scopeNameCol + ` + o.snr, o.rssi, o.score, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')` + obsRawHexCol + rpCol + scopeNameCol + ` FROM transmissions t LEFT JOIN observations o ON o.transmission_id = t.id LEFT JOIN observers obs ON obs.rowid = o.observer_idx` + filterClause + ` @@ -900,7 +900,7 @@ func (s *PacketStore) Load() error { // which would silently mis-attribute hops). Fetched BEFORE opening // the rows cursor below: getCachedNodesAndPM issues its own DB query, // which would deadlock against the still-open cursor on a single- - // connection SQLite pool. Load holds s.mu.Lock(), so calling + // connection database pool. Load holds s.mu.Lock(), so calling // getCachedNodesAndPM directly is safe. _, relayPM := s.getCachedNodesAndPM() var coldLoadAmbiguousHopsSkipped int @@ -920,7 +920,7 @@ func (s *PacketStore) Load() error { var obsID sql.NullInt64 var observerID, observerName, observerIATA, direction, pathJSON, obsTimestamp sql.NullString var snr, rssi sql.NullFloat64 - var score sql.NullInt64 + var score sql.NullFloat64 var obsRawHex sql.NullString var resolvedPathStr sql.NullString var scopeName sql.NullString @@ -994,7 +994,7 @@ func (s *PacketStore) Load() error { Direction: nullStrVal(direction), SNR: nullFloatPtr(snr), RSSI: nullFloatPtr(rssi), - Score: nullIntPtr(score), + Score: nullFloatPtr(score), PathJSON: obsPJ, Timestamp: normalizeTimestamp(nullStrVal(obsTimestamp)), } @@ -1125,7 +1125,7 @@ func (s *PacketStore) Load() error { return nil } -// loadChunk queries a [from, to) time window from SQLite without holding the +// loadChunk queries a [from, to) time window from PostgreSQL without holding the // write lock, builds local data structures, then merges them into the store // under s.mu.Lock(). It is the building block for the background loader. // @@ -1186,10 +1186,10 @@ func (s *PacketStore) loadChunk(from, to time.Time) error { var filterClause string var qArgs []interface{} if s.db.hasLastSeen { - filterClause = "\n\t\t\tWHERE t.last_seen >= ? AND t.last_seen < ?" + filterClause = "\n\t\t\tWHERE t.last_seen >= $1 AND t.last_seen < $2" qArgs = []interface{}{fromUnix, toUnix} } else { - filterClause = "\n\t\t\tWHERE t.first_seen >= ? AND t.first_seen < ?" + filterClause = "\n\t\t\tWHERE t.first_seen >= $1 AND t.first_seen < $2" qArgs = []interface{}{fromStr, toStr} } @@ -1198,7 +1198,7 @@ func (s *PacketStore) loadChunk(from, to time.Time) error { chunkSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, t.payload_type, t.payload_version, t.decoded_json, o.id, obs.id, obs.name, o.direction, - o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')` + obsRawHexCol + rpCol + scopeNameCol + ` + o.snr, o.rssi, o.score, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')` + obsRawHexCol + rpCol + scopeNameCol + ` FROM transmissions t LEFT JOIN observations o ON o.transmission_id = t.id LEFT JOIN observers obs ON obs.rowid = o.observer_idx` + filterClause + ` @@ -1222,7 +1222,7 @@ func (s *PacketStore) loadChunk(from, to time.Time) error { // (affinity-tier resolution against ≤168h-old observations would // silently mis-attribute hops). Fetched BEFORE opening the rows // cursor below: getCachedNodesAndPM issues its own DB query, which - // would deadlock against the open cursor on a single-connection SQLite + // would deadlock against the open cursor on a single-connection database // pool. s.mu.RLock() _, relayPM := s.getCachedNodesAndPM() @@ -1264,7 +1264,7 @@ func (s *PacketStore) loadChunk(from, to time.Time) error { var obsID sql.NullInt64 var observerID, observerName, direction, pathJSON, obsTimestamp sql.NullString var snr, rssi sql.NullFloat64 - var score sql.NullInt64 + var score sql.NullFloat64 var obsRawHex sql.NullString var resolvedPathStr sql.NullString var scopeName sql.NullString @@ -1330,7 +1330,7 @@ func (s *PacketStore) loadChunk(from, to time.Time) error { Direction: nullStrVal(direction), SNR: nullFloatPtr(snr), RSSI: nullFloatPtr(rssi), - Score: nullIntPtr(score), + Score: nullFloatPtr(score), PathJSON: obsPJ, Timestamp: normalizeTimestamp(nullStrVal(obsTimestamp)), } @@ -1602,10 +1602,10 @@ func (s *PacketStore) loadBackgroundChunks() { var countQuery string var countArg interface{} if s.db.hasLastSeen { - countQuery = `SELECT COUNT(*) FROM transmissions WHERE last_seen >= ?` + countQuery = `SELECT COUNT(*) FROM transmissions WHERE last_seen >= $1` countArg = retentionFloor } else { - countQuery = `SELECT COUNT(*) FROM transmissions WHERE first_seen >= ?` + countQuery = `SELECT COUNT(*) FROM transmissions WHERE first_seen >= $1` countArg = retentionFloorStr } if err := s.db.conn.QueryRow(countQuery, countArg).Scan(&totalInDB); err != nil { @@ -1897,7 +1897,7 @@ func (s *PacketStore) untrackAdvertPubkey(tx *StoreTx) { // QueryPackets returns filtered, paginated packets from memory. func (s *PacketStore) QueryPackets(q PacketQuery) *PacketResult { // SQL fallback: if the query window predates the in-memory window, delegate - // to the DB layer which covers the full SQLite retention period. + // to the DB layer which covers the full database retention period. s.mu.RLock() oldest := s.oldestLoaded s.mu.RUnlock() @@ -2177,7 +2177,7 @@ func (s *PacketStore) GetStoreStats() (*Stats, error) { defer wg.Done() nodeErr = s.db.conn.QueryRow( `SELECT - (SELECT COUNT(*) FROM nodes WHERE last_seen > ?) AS active_nodes, + (SELECT COUNT(*) FROM nodes WHERE last_seen > $1) AS active_nodes, (SELECT COUNT(*) FROM nodes) AS all_nodes, (SELECT COUNT(*) FROM observers WHERE inactive IS NULL OR inactive = 0) AS observers`, sevenDaysAgo, @@ -2221,9 +2221,9 @@ func (s *PacketStore) refreshObsCounts(oneHourAgo, oneDayAgo int64) (int, int, e var lastHour, last24h int if qErr := s.db.conn.QueryRow( `SELECT - COALESCE(SUM(CASE WHEN timestamp > ? THEN 1 ELSE 0 END), 0), - COALESCE(SUM(CASE WHEN timestamp > ? THEN 1 ELSE 0 END), 0) - FROM observations WHERE timestamp > ?`, + COALESCE(SUM(CASE WHEN timestamp > $1 THEN 1 ELSE 0 END), 0), + COALESCE(SUM(CASE WHEN timestamp > $2 THEN 1 ELSE 0 END), 0) + FROM observations WHERE timestamp > $3`, oneHourAgo, oneDayAgo, oneDayAgo, ).Scan(&lastHour, &last24h); qErr != nil { return nil, qErr @@ -2708,7 +2708,7 @@ func (s *PacketStore) QueryMultiNodePackets(pubkeys []string, limit, offset int, return &PacketResult{Packets: packets, Total: total} } -// IngestNewFromDB loads new transmissions from SQLite into memory and returns +// IngestNewFromDB loads new transmissions from PostgreSQL into memory and returns // broadcast-ready maps plus the new max transmission ID. func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interface{}, int) { if limit <= 0 { @@ -2732,11 +2732,11 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac querySQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, t.payload_type, t.payload_version, t.decoded_json, o.id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, - o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')` + obsRHCol + scopeNameCol + ` + o.snr, o.rssi, o.score, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')` + obsRHCol + scopeNameCol + ` FROM transmissions t LEFT JOIN observations o ON o.transmission_id = t.id LEFT JOIN observers obs ON obs.rowid = o.observer_idx - WHERE t.id > ? + WHERE t.id > $1 ORDER BY t.id ASC, o.timestamp DESC` } else { querySQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, @@ -2746,7 +2746,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac FROM transmissions t LEFT JOIN observations o ON o.transmission_id = t.id LEFT JOIN observers obs ON obs.id = o.observer_id - WHERE t.id > ? + WHERE t.id > $1 ORDER BY t.id ASC, o.timestamp DESC` } @@ -2767,7 +2767,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac obsRawHex string scopeName *string snr, rssi *float64 - score *int + score *float64 } var tempRows []tempRow @@ -2781,7 +2781,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac var obsIDVal sql.NullInt64 var observerID, observerName, observerIATA, direction, pathJSON, obsTimestamp sql.NullString var snrVal, rssiVal sql.NullFloat64 - var scoreVal sql.NullInt64 + var scoreVal sql.NullFloat64 var obsRawHex sql.NullString var scopeName sql.NullString @@ -2825,7 +2825,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac scopeName: nullStrPtr(scopeName), snr: nullFloatPtr(snrVal), rssi: nullFloatPtr(rssiVal), - score: nullIntPtr(scoreVal), + score: nullFloatPtr(scoreVal), } if obsIDVal.Valid { oid := int(obsIDVal.Int64) @@ -3155,20 +3155,20 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string] } if s.db.isV3 { querySQL = `SELECT o.id, o.transmission_id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, - o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')` + obsRHCol2 + ` + o.snr, o.rssi, o.score, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')` + obsRHCol2 + ` FROM observations o LEFT JOIN observers obs ON obs.rowid = o.observer_idx - WHERE o.id > ? + WHERE o.id > $1 ORDER BY o.id ASC - LIMIT ?` + LIMIT $2` } else { querySQL = `SELECT o.id, o.transmission_id, o.observer_id, o.observer_name, COALESCE(obs.iata, ''), o.direction, o.snr, o.rssi, o.score, o.path_json, o.timestamp` + obsRHCol2 + ` FROM observations o LEFT JOIN observers obs ON obs.id = o.observer_id - WHERE o.id > ? + WHERE o.id > $1 ORDER BY o.id ASC - LIMIT ?` + LIMIT $2` } rows, err := s.db.conn.Query(querySQL, sinceObsID, limit) @@ -3186,7 +3186,7 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string] observerIATA string direction string snr, rssi *float64 - score *int + score *float64 pathJSON string rawHex string timestamp string @@ -3197,7 +3197,7 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string] var oid, txID int var observerID, observerName, observerIATA, direction, pathJSON, ts sql.NullString var snr, rssi sql.NullFloat64 - var score sql.NullInt64 + var score sql.NullFloat64 var obsRawHex sql.NullString scanArgs3 := []interface{}{&oid, &txID, &observerID, &observerName, &observerIATA, &direction, @@ -3218,7 +3218,7 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string] direction: nullStrVal(direction), snr: nullFloatPtr(snr), rssi: nullFloatPtr(rssi), - score: nullIntPtr(score), + score: nullFloatPtr(score), pathJSON: nullStrVal(pathJSON), rawHex: nullStrVal(obsRawHex), timestamp: nullStrVal(ts), @@ -3950,7 +3950,7 @@ func (s *PacketStore) enrichObsWithTx(obs *StoreObs, tx *StoreTx) map[string]int "direction": strOrNil(obs.Direction), "snr": floatPtrOrNil(obs.SNR), "rssi": floatPtrOrNil(obs.RSSI), - "score": intPtrOrNil(obs.Score), + "score": floatPtrOrNil(obs.Score), "path_json": strOrNil(obs.PathJSON), } // On-demand SQL fetch for resolved_path @@ -9287,12 +9287,12 @@ func (s *PacketStore) loadMultibyteCapFromDB() { // RunMultibyteCapPersist consumes the file and writes confirmed / // suspected entries to the DB. // -// INVARIANT (#1289/#1324): the server is the read path and opens -// SQLite mode=ro. It MUST NOT execute any UPDATE on +// INVARIANT (#1289/#1324): the server is the read path and uses a +// restricted PostgreSQL reader. It MUST NOT execute any UPDATE on // nodes.multibyte_* — see readonly_invariant_test.go. This helper is // the only side-effect path for capability data leaving the server. func (s *PacketStore) publishMultibyteCapSnapshot(entries []MultiByteCapEntry) { - if s.db == nil || s.db.path == "" { + if s.db == nil || s.db.stateDir == "" { return } out := make([]mbcapqueue.Entry, 0, len(entries)) @@ -9303,7 +9303,7 @@ func (s *PacketStore) publishMultibyteCapSnapshot(entries []MultiByteCapEntry) { Evidence: e.Evidence, }) } - if err := mbcapqueue.WriteSnapshot(s.db.path, mbcapqueue.Snapshot{Entries: out}); err != nil { + if err := mbcapqueue.WriteSnapshot(s.db.statePath(), mbcapqueue.Snapshot{Entries: out}); err != nil { log.Printf("[multibyte] publish snapshot: %v", err) } } @@ -9549,7 +9549,7 @@ func (s *PacketStore) GetBulkHealth(limit int, region, area string) []map[string if regionNodeKeys != nil || areaNodes != nil { queryLimit = 10000 } - rows, err := s.db.conn.Query("SELECT public_key, name, role, lat, lon FROM nodes ORDER BY last_seen DESC LIMIT ?", queryLimit) + rows, err := s.db.conn.Query("SELECT public_key, name, role, lat, lon FROM nodes ORDER BY last_seen DESC NULLS LAST LIMIT $1", queryLimit) if err != nil { return []map[string]interface{}{} } diff --git a/cmd/server/store_memory_diagnostics_test.go b/cmd/server/store_memory_diagnostics_test.go index e1253d2bf..9edf9719f 100644 --- a/cmd/server/store_memory_diagnostics_test.go +++ b/cmd/server/store_memory_diagnostics_test.go @@ -1,9 +1,7 @@ package main import ( - "database/sql" "math" - "path/filepath" "strings" "testing" "time" @@ -89,30 +87,29 @@ func TestStoreMemoryBreakdown_Empty(t *testing.T) { // falling back to the parent transmission. If this regresses, obs raw_hex would // be silently lost — so this is the safety gate for dropping obs.RawHex. func TestObsRawHexNotRetainedOnLoad(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "obsrawhex.db") + dbPath := postgresTestDSN(t) const txHex = "deadbeefcafe" const obsHex = "c0ffee0102" // distinct from txHex: proves we DON'T keep it - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } stmts := []string{ `CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT )`, `CREATE TABLE observations ( - id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, - observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, - path_json TEXT, timestamp TEXT, raw_hex TEXT + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, + observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp BIGINT, raw_hex TEXT )`, - `CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, + `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, first_seen TEXT, frequency REAL + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, + last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION )`, `CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`, } @@ -123,19 +120,19 @@ func TestObsRawHexNotRetainedOnLoad(t *testing.T) { } now := time.Now().UTC().Truncate(time.Second) if _, err := conn.Exec( - `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, 1, txHex, "hashobs", now.Add(-time.Minute).Format(time.RFC3339), 1, 5, 0, `{"type":"CHAN"}`); err != nil { t.Fatalf("insert tx: %v", err) } // Observation carries its OWN non-empty raw_hex in the DB. if _, err := conn.Exec( - `INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp, raw_hex) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + `INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp, raw_hex) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, 1, 1, "obs1", "Obs1", "rx", 5.0, -95.0, 0, `["AA"]`, now.Add(-time.Minute).Unix(), obsHex); err != nil { t.Fatalf("insert obs: %v", err) } conn.Close() - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/subpaths_window_test.go b/cmd/server/subpaths_window_test.go index 933dbff64..d2dad06fb 100644 --- a/cmd/server/subpaths_window_test.go +++ b/cmd/server/subpaths_window_test.go @@ -36,19 +36,19 @@ func setupSubpathWindowDB(t *testing.T) *DB { // Observer db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Observer One', 'SJC', ?, '2025-01-01T00:00:00Z', 100)`, recent) + VALUES ('obs1', 'Observer One', 'SJC', $1, '2025-01-01T00:00:00Z', 100)`, recent) // Recent transmission with path ["aa","bb"] db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('01', 'recent_hash_window_001', ?, 1, 4, '{}')`, recent) + VALUES ('01', 'recent_hash_window_001', $1, 1, 4, '{}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '["aa","bb"]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -90, '["aa","bb"]', $1)`, recentEpoch) // Old transmission (30d ago) with disjoint path ["cc","dd"] db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('02', 'old_hash_window_002', ?, 1, 4, '{}')`, old) + VALUES ('02', 'old_hash_window_002', $1, 1, 4, '{}')`, old) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 10.0, -90, '["cc","dd"]', ?)`, oldEpoch) + VALUES (2, 1, 10.0, -90, '["cc","dd"]', $1)`, oldEpoch) return db } diff --git a/cmd/server/testdata/golden/shapes.json b/cmd/server/testdata/golden/shapes.json index 4556ddd55..b6d514126 100644 --- a/cmd/server/testdata/golden/shapes.json +++ b/cmd/server/testdata/golden/shapes.json @@ -1461,32 +1461,15 @@ } } }, - "sqlite": { + "postgres": { "type": "object", "keys": { - "dbSizeMB": { - "type": "number" - }, - "walSizeMB": { - "type": "number" + "engine": { + "type": "string" }, - "freelistMB": { + "dbSizeMB": { "type": "number" }, - "walPages": { - "type": "object", - "keys": { - "total": { - "type": "number" - }, - "checkpointed": { - "type": "number" - }, - "busy": { - "type": "number" - } - } - }, "rows": { "type": "object", "keys": { @@ -1503,6 +1486,15 @@ "type": "number" } } + }, + "sampledAt": { + "type": "string" + }, + "sampleIntervalSeconds": { + "type": "number" + }, + "stale": { + "type": "boolean" } } }, diff --git a/cmd/server/topology_dedup_test.go b/cmd/server/topology_dedup_test.go index b875eb016..b8e2980b0 100644 --- a/cmd/server/topology_dedup_test.go +++ b/cmd/server/topology_dedup_test.go @@ -1,21 +1,16 @@ package main import ( - "database/sql" "fmt" - "path/filepath" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // TestTopologyDedup_RepeatersMergeByPubkey verifies that topRepeaters // merges entries whose hop prefixes resolve unambiguously to the same node. func TestTopologyDedup_RepeatersMergeByPubkey(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + dbPath := postgresTestDSN(t) + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -27,17 +22,17 @@ func TestTopologyDedup_RepeatersMergeByPubkey(t *testing.T) { } } exec(`CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT )`) exec(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`) - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + exec(`CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) exec(`CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, frequency REAL + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, + last_seen TEXT, frequency DOUBLE PRECISION )`) exec(`CREATE TABLE schema_version (version INTEGER)`) exec(`INSERT INTO schema_version (version) VALUES (1)`) @@ -61,9 +56,9 @@ func TestTopologyDedup_RepeatersMergeByPubkey(t *testing.T) { for i := 0; i < count; i++ { ts := base.Add(time.Duration(txID) * time.Minute).Format(time.RFC3339) hash := fmt.Sprintf("h%04d", txID) - conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, 0, 4, 1, ?)", + conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES ($1, $2, $3, $4, 0, 4, 1, $5)", txID, "aabb", hash, ts, fmt.Sprintf(`{"pubKey":"pk%04d"}`, txID)) - conn.Exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + conn.Exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", obsID, txID, "obs1", "Obs1", "RX", -10.0, -80.0, 5, path, ts) txID++ obsID++ @@ -78,7 +73,7 @@ func TestTopologyDedup_RepeatersMergeByPubkey(t *testing.T) { // Total: BETA appears as "99aa" (10×), "99" (5×), "99aabb" (3×) = 18 total // After dedup, each should appear ONCE with count=18. - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -137,9 +132,8 @@ func TestTopologyDedup_RepeatersMergeByPubkey(t *testing.T) { // TestTopologyDedup_AmbiguousPrefixNotMerged verifies that ambiguous short // prefixes (matching multiple nodes) are NOT merged — they stay separate. func TestTopologyDedup_AmbiguousPrefixNotMerged(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + dbPath := postgresTestDSN(t) + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -151,17 +145,17 @@ func TestTopologyDedup_AmbiguousPrefixNotMerged(t *testing.T) { } } exec(`CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT )`) exec(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`) - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + exec(`CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) exec(`CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, frequency REAL + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, + last_seen TEXT, frequency DOUBLE PRECISION )`) exec(`CREATE TABLE schema_version (version INTEGER)`) exec(`INSERT INTO schema_version (version) VALUES (1)`) @@ -179,9 +173,9 @@ func TestTopologyDedup_AmbiguousPrefixNotMerged(t *testing.T) { for i := 0; i < 10; i++ { ts := base.Add(time.Duration(txID) * time.Minute).Format(time.RFC3339) hash := fmt.Sprintf("h%04d", txID) - conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, 0, 4, 1, ?)", + conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES ($1, $2, $3, $4, 0, 4, 1, $5)", txID, "aabb", hash, ts, fmt.Sprintf(`{"pubKey":"pk%04d"}`, txID)) - conn.Exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + conn.Exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", obsID, txID, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `["ab"]`, ts) txID++ obsID++ @@ -190,15 +184,15 @@ func TestTopologyDedup_AmbiguousPrefixNotMerged(t *testing.T) { for i := 0; i < 5; i++ { ts := base.Add(time.Duration(txID) * time.Minute).Format(time.RFC3339) hash := fmt.Sprintf("h%04d", txID) - conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, 0, 4, 1, ?)", + conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES ($1, $2, $3, $4, 0, 4, 1, $5)", txID, "aabb", hash, ts, fmt.Sprintf(`{"pubKey":"pk%04d"}`, txID)) - conn.Exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + conn.Exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", obsID, txID, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `["ab1122"]`, ts) txID++ obsID++ } - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -243,9 +237,8 @@ func TestTopologyDedup_AmbiguousPrefixNotMerged(t *testing.T) { // TestTopologyDedup_PairsMergeByPubkey verifies that topPairs merges // pair entries whose hops resolve unambiguously to the same node pair. func TestTopologyDedup_PairsMergeByPubkey(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + dbPath := postgresTestDSN(t) + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -257,17 +250,17 @@ func TestTopologyDedup_PairsMergeByPubkey(t *testing.T) { } } exec(`CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT )`) exec(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`) - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + exec(`CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) exec(`CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, frequency REAL + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, + last_seen TEXT, frequency DOUBLE PRECISION )`) exec(`CREATE TABLE schema_version (version INTEGER)`) exec(`INSERT INTO schema_version (version) VALUES (1)`) @@ -283,9 +276,9 @@ func TestTopologyDedup_PairsMergeByPubkey(t *testing.T) { for i := 0; i < count; i++ { ts := base.Add(time.Duration(txID) * time.Minute).Format(time.RFC3339) hash := fmt.Sprintf("h%04d", txID) - conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, 0, 4, 1, ?)", + conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES ($1, $2, $3, $4, 0, 4, 1, $5)", txID, "aabb", hash, ts, fmt.Sprintf(`{"pubKey":"pk%04d"}`, txID)) - conn.Exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + conn.Exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", obsID, txID, "obs1", "Obs1", "RX", -10.0, -80.0, 5, path, ts) txID++ obsID++ @@ -299,7 +292,7 @@ func TestTopologyDedup_PairsMergeByPubkey(t *testing.T) { insertTx(`["07","99aa"]`, 10) insertTx(`["0735bc","99"]`, 5) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } diff --git a/cmd/server/traffic_share_score_test.go b/cmd/server/traffic_share_score_test.go index 7610336de..f936faa09 100644 --- a/cmd/server/traffic_share_score_test.go +++ b/cmd/server/traffic_share_score_test.go @@ -34,7 +34,7 @@ func TestTrafficShareScore_HandleNodesSurface(t *testing.T) { for _, p := range []string{pk, left, right} { if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'rpt', 'repeater', 37.5, -122.0, ?, ?, 10)`, + VALUES ($1, 'rpt', 'repeater', 37.5, -122.0, $2, $3, 10)`, p, recent, recent); err != nil { t.Fatal(err) } @@ -126,7 +126,7 @@ func TestTrafficShareScore_NodeDetail(t *testing.T) { recent := time.Now().UTC().Format("2006-01-02T15:04:05.000Z") if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'rpt', 'repeater', 37.5, -122.0, ?, ?, 10)`, + VALUES ($1, 'rpt', 'repeater', 37.5, -122.0, $2, $3, 10)`, pk, recent, recent); err != nil { t.Fatal(err) } diff --git a/cmd/server/types.go b/cmd/server/types.go index 3b9cffd22..b6fa5fedd 100644 --- a/cmd/server/types.go +++ b/cmd/server/types.go @@ -261,25 +261,21 @@ type PerfPacketStoreStats struct { LoadCoverageRatio float64 `json:"loadCoverageRatio"` } -type WalPages struct { - Total int `json:"total"` - Checkpointed int `json:"checkpointed"` - Busy int `json:"busy"` -} - -type SqliteRowCounts struct { +type DatabaseRowCounts struct { Transmissions int `json:"transmissions"` Observations int `json:"observations"` Nodes int `json:"nodes"` Observers int `json:"observers"` } -type SqliteStats struct { - DbSizeMB float64 `json:"dbSizeMB"` - WalSizeMB float64 `json:"walSizeMB"` - FreelistMB float64 `json:"freelistMB"` - WalPages *WalPages `json:"walPages"` - Rows *SqliteRowCounts `json:"rows"` +type PostgresStats struct { + Engine string `json:"engine"` + DbSizeMB *float64 `json:"dbSizeMB"` + Rows *DatabaseRowCounts `json:"rows"` + SampledAt string `json:"sampledAt,omitempty"` + SampleIntervalSeconds int `json:"sampleIntervalSeconds"` + Stale bool `json:"stale"` + Error string `json:"error,omitempty"` } type PerfResponse struct { @@ -290,7 +286,7 @@ type PerfResponse struct { SlowQueries []SlowQuery `json:"slowQueries"` Cache PerfCacheStats `json:"cache"` PacketStore *PerfPacketStoreStats `json:"packetStore"` - Sqlite *SqliteStats `json:"sqlite"` + Postgres *PostgresStats `json:"postgres"` GoRuntime *GoRuntimeStats `json:"goRuntime,omitempty"` // MemoryBreakdown is populated only for /api/perf?mem=1 (an O(tx+obs) // walk, opt-in so the normal hot endpoint stays cheap). It sizes the diff --git a/cmd/server/usefulness_axes_handle_nodes_test.go b/cmd/server/usefulness_axes_handle_nodes_test.go index 9628525cb..fb5f15e47 100644 --- a/cmd/server/usefulness_axes_handle_nodes_test.go +++ b/cmd/server/usefulness_axes_handle_nodes_test.go @@ -31,7 +31,7 @@ func TestUsefulnessAxes_HandleNodesSurface(t *testing.T) { for _, pk := range pks { if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, 'repeater', 37.5, -122.0, ?, ?, 10)`, + VALUES ($1, $2, 'repeater', 37.5, -122.0, $3, $4, 10)`, pk, "node-"+pk[:4], recent, recent); err != nil { t.Fatal(err) } @@ -43,7 +43,7 @@ func TestUsefulnessAxes_HandleNodesSurface(t *testing.T) { clientPK := "eeee000000000000000000000000000000000000000000000000000000000000" if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'client-eeee', 'client', 37.5, -122.0, ?, ?, 10)`, + VALUES ($1, 'client-eeee', 'client', 37.5, -122.0, $2, $3, 10)`, clientPK, recent, recent); err != nil { t.Fatal(err) } diff --git a/cmd/server/user_mgmt_config.go b/cmd/server/user_mgmt_config.go index 6683685d0..2a899a384 100644 --- a/cmd/server/user_mgmt_config.go +++ b/cmd/server/user_mgmt_config.go @@ -15,6 +15,7 @@ import ( // UserManagementConfig is the "userManagement" block of config.json. type UserManagementConfig struct { Enabled bool `json:"enabled"` + DatabaseURL string `json:"databaseURL,omitempty"` DBPath string `json:"dbPath,omitempty"` AdminEmails []string `json:"adminEmails,omitempty"` PublicBaseURL string `json:"publicBaseUrl,omitempty"` @@ -134,19 +135,19 @@ type UsersBackupConfig struct { // backupSettings is the resolved form; the zero value means off. type backupSettings struct { enabled bool - dir string // relative paths resolve against the working directory, like dbPath + dir string // relative paths resolve against the working directory keep int // snapshots kept after each run } const defaultBackupKeep = 7 -// resolveBackup fills the defaults: dir "backups" next to users.db, keep 7 +// resolveBackup defaults to "backups" under the state directory, keep 7 // for an absent, zero or negative value. Only "enabled": false turns it off. -func resolveBackup(c *UsersBackupConfig, dbPath string) backupSettings { +func resolveBackup(c *UsersBackupConfig, stateDir string) backupSettings { if c != nil && c.Enabled != nil && !*c.Enabled { return backupSettings{} } - b := backupSettings{enabled: true, dir: filepath.Join(filepath.Dir(dbPath), "backups"), keep: defaultBackupKeep} + b := backupSettings{enabled: true, dir: filepath.Join(stateDir, "backups"), keep: defaultBackupKeep} if c == nil { return b } @@ -165,7 +166,7 @@ func (c *Config) UserManagementEnabled() bool { // userMgmtSettings is the validated, resolved form the auth service runs on. type userMgmtSettings struct { - dbPath string + databaseURL string adminEmails map[string]bool baseURL *url.URL // no trailing slash, no query or fragment secureCookie bool @@ -189,12 +190,15 @@ var fakeMailerAllowed bool // resolveUserManagement validates the block and fills defaults. It refuses // configurations where nobody could activate an account, so the server // fails at startup instead of running half-working. -func resolveUserManagement(u *UserManagementConfig, measurementDBPath string, getenv func(string) string) (*userMgmtSettings, error) { +func resolveUserManagement(u *UserManagementConfig, stateDir string, getenv func(string) string) (*userMgmtSettings, error) { set := &userMgmtSettings{adminEmails: map[string]bool{}} - set.dbPath = strings.TrimSpace(u.DBPath) - if set.dbPath == "" { - set.dbPath = filepath.Join(filepath.Dir(measurementDBPath), "users.db") + if u.DBPath != "" { + return nil, errors.New("userManagement.dbPath is no longer supported; import the accounts database and configure userManagement.databaseURL") + } + set.databaseURL = envOrValue(getenv, "CORESCOPE_USERS_DATABASE_URL", u.DatabaseURL) + if set.databaseURL == "" { + return nil, errors.New("userManagement.databaseURL or CORESCOPE_USERS_DATABASE_URL is required for the separate PostgreSQL accounts database") } for _, raw := range u.AdminEmails { e, err := users.NormalizeEmail(raw) @@ -256,7 +260,7 @@ func resolveUserManagement(u *UserManagementConfig, measurementDBPath string, ge } set.proposals = resolveProposals(u.ChannelProposals) set.notify = resolveNotifications(u.Notifications) - set.backup = resolveBackup(u.Backup, set.dbPath) + set.backup = resolveBackup(u.Backup, stateDir) return set, nil } diff --git a/cmd/server/user_mgmt_config_test.go b/cmd/server/user_mgmt_config_test.go index 09ccb6dcf..a06cef3f8 100644 --- a/cmd/server/user_mgmt_config_test.go +++ b/cmd/server/user_mgmt_config_test.go @@ -10,6 +10,7 @@ import ( func validUM() *UserManagementConfig { return &UserManagementConfig{ Enabled: true, + DatabaseURL: "postgresql://accounts@localhost/users", AdminEmails: []string{" Boss@Example.org "}, PublicBaseURL: "https://scope.example.org/", Mail: UserMailConfig{BrevoAPIKey: "xkeysib-abc", FromEmail: "noreply@example.org"}, @@ -19,12 +20,12 @@ func validUM() *UserManagementConfig { func noEnv(string) string { return "" } func TestResolveUserManagementDefaults(t *testing.T) { - set, err := resolveUserManagement(validUM(), filepath.Join("data", "meshcore.db"), noEnv) + set, err := resolveUserManagement(validUM(), "data", noEnv) if err != nil { t.Fatal(err) } - if set.dbPath != filepath.Join("data", "users.db") { - t.Errorf("dbPath = %q", set.dbPath) + if set.databaseURL != validUM().DatabaseURL { + t.Errorf("dbPath = %q", set.databaseURL) } if !set.adminEmails["boss@example.org"] { t.Errorf("adminEmails not normalized: %v", set.adminEmails) @@ -38,11 +39,14 @@ func TestResolveUserManagementDefaults(t *testing.T) { } func TestResolveUserManagementEnvWins(t *testing.T) { - env := map[string]string{"CORESCOPE_BREVO_API_KEY": "from-env", "CORESCOPE_BREVO_WEBHOOK_SECRET": "env-secret-0123456789"} + env := map[string]string{"CORESCOPE_USERS_DATABASE_URL": "postgresql://accounts@localhost/from_env", "CORESCOPE_BREVO_API_KEY": "from-env", "CORESCOPE_BREVO_WEBHOOK_SECRET": "env-secret-0123456789"} set, err := resolveUserManagement(validUM(), "meshcore.db", func(k string) string { return env[k] }) if err != nil { t.Fatal(err) } + if set.databaseURL != env["CORESCOPE_USERS_DATABASE_URL"] { + t.Fatal("account database environment override not applied") + } if set.brevoAPIKey != "from-env" || set.webhookSecret != "env-secret-0123456789" { t.Fatalf("env not applied: %q %q", set.brevoAPIKey, set.webhookSecret) } @@ -143,7 +147,7 @@ func TestResolveNotifications(t *testing.T) { func TestResolveUsersBackup(t *testing.T) { u := validUM() - meas := filepath.Join("data", "meshcore.db") + meas := "data" defaults := backupSettings{enabled: true, dir: filepath.Join("data", "backups"), keep: 7} set, err := resolveUserManagement(u, meas, noEnv) if err != nil { @@ -152,9 +156,9 @@ func TestResolveUsersBackup(t *testing.T) { if set.backup != defaults { t.Fatalf("absent block = %+v; want %+v", set.backup, defaults) } - u.DBPath = filepath.Join("accounts", "users.db") - if set, _ = resolveUserManagement(u, meas, noEnv); set.backup.dir != filepath.Join("accounts", "backups") { - t.Fatalf("dir follows dbPath: %q", set.backup.dir) + u.DBPath = "private-users.db" + if _, err := resolveUserManagement(u, meas, noEnv); err == nil || strings.Contains(err.Error(), u.DBPath) { + t.Fatalf("legacy account path must be refused: %v", err) } u.DBPath = "" off, on := false, true diff --git a/cmd/server/user_mgmt_off_test.go b/cmd/server/user_mgmt_off_test.go index 2b78f9bf3..adcbc8887 100644 --- a/cmd/server/user_mgmt_off_test.go +++ b/cmd/server/user_mgmt_off_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/gorilla/mux" + "github.com/meshcore-analyzer/pgutil/pgtest" ) func TestUserManagementOffIsUnchanged(t *testing.T) { @@ -73,30 +74,29 @@ func TestAuthRoutesThroughRealRouterAreNoStore(t *testing.T) { } func TestInitUserManagementRefusesMeasurementDB(t *testing.T) { - dir := t.TempDir() - measurement := filepath.Join(dir, "meshcore.db") - os.WriteFile(measurement, nil, 0o644) - srv := &Server{cfg: &Config{UserManagement: &UserManagementConfig{ - Enabled: true, DBPath: measurement, PublicBaseURL: testBase, - Mail: UserMailConfig{BrevoAPIKey: "k", FromEmail: "noreply@example.org"}, - }}} - err := srv.initUserManagement(measurement) - if err == nil || !strings.Contains(err.Error(), "measurement database") { + a, _ := newTestAuthService(t) + u := validUM() + u.DatabaseURL = a.set.databaseURL + srv := &Server{cfg: &Config{UserManagement: u}} + if err := srv.initUserManagement(u.DatabaseURL); err == nil || !strings.Contains(err.Error(), "measurement database") { t.Fatalf("err = %v", err) } } -func TestInitUserManagementCreatesUsersDB(t *testing.T) { - dir := t.TempDir() - srv := &Server{cfg: &Config{UserManagement: &UserManagementConfig{ - Enabled: true, PublicBaseURL: testBase, - Mail: UserMailConfig{BrevoAPIKey: "k", FromEmail: "noreply@example.org"}, - }}} - if err := srv.initUserManagement(filepath.Join(dir, "meshcore.db")); err != nil { +func TestInitUserManagementRequiresBootstrappedDatabase(t *testing.T) { + u := validUM() + u.DatabaseURL = pgtest.NewDatabase(t) + srv := &Server{cfg: &Config{UserManagement: u}} + if err := srv.initUserManagement(pgtest.NewDatabase(t)); err == nil { + t.Fatal("runtime created missing account schema") + } + db, err := openFixtureSQL(u.DatabaseURL) + if err != nil { t.Fatal(err) } - defer srv.closeUserManagement() - if _, err := os.Stat(filepath.Join(dir, "users.db")); err != nil { - t.Fatalf("users.db not created next to the measurement DB: %v", err) + defer db.Close() + var tables int + if err := db.QueryRow(`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='public'`).Scan(&tables); err != nil || tables != 0 { + t.Fatalf("runtime created tables: %d, %v", tables, err) } } diff --git a/cmd/server/users_backup.go b/cmd/server/users_backup.go index bebe788da..f920e7793 100644 --- a/cmd/server/users_backup.go +++ b/cmd/server/users_backup.go @@ -26,17 +26,19 @@ const ( // usersBackupName matches the snapshot files this code writes and rotates. // Apart from orphaned temporary files (usersBackupTemp), nothing else in // the directory is ever touched. -var usersBackupName = regexp.MustCompile(`^users-\d{8}-\d{6}\.db$`) +var usersBackupName = regexp.MustCompile(`^users-\d{8}-\d{6}\.dump$`) // usersBackupTemp matches the temporary name a snapshot is written under. // One is only left behind when the process died mid-write. -var usersBackupTemp = regexp.MustCompile(`^users-\d{8}-\d{6}\.db\.tmp$`) +var usersBackupTemp = regexp.MustCompile(`^users-\d{8}-\d{6}\.dump\.tmp$`) // usersBackupTempMaxAge is how old an orphaned temporary snapshot must be // before the sweep removes it; no snapshot write takes this long. const usersBackupTempMaxAge = 24 * time.Hour -func usersBackupFile(t time.Time) string { return "users-" + t.UTC().Format(usersBackupLayout) + ".db" } +func usersBackupFile(t time.Time) string { + return "users-" + t.UTC().Format(usersBackupLayout) + ".dump" +} // listUsersBackups returns the snapshot file names in dir, oldest first // (the UTC timestamp sorts by name). A missing dir has none. @@ -64,7 +66,7 @@ func listUsersBackups(dir string) ([]string, error) { // back) are skipped so they cannot stop the backups. func usersBackupDue(names []string, now time.Time) bool { for i := len(names) - 1; i >= 0; i-- { - t, err := time.Parse(usersBackupLayout, strings.TrimSuffix(strings.TrimPrefix(names[i], "users-"), ".db")) + t, err := time.Parse(usersBackupLayout, strings.TrimSuffix(strings.TrimPrefix(names[i], "users-"), ".dump")) if err != nil || t.After(now) { continue } @@ -187,7 +189,7 @@ func (a *authService) maybeBackup(now time.Time) { // handleAdminUsersBackup streams a fresh users.db snapshot from a temp // directory, removed afterwards. The file holds password hashes and // addresses: admin only, audited as user.backup. -func (s *Server) handleAdminUsersBackup(w http.ResponseWriter, _ *http.Request, admin *users.User, _ *users.Session) { +func (s *Server) handleAdminUsersBackup(w http.ResponseWriter, r *http.Request, admin *users.User, _ *users.Session) { a := s.auth tmpDir, err := os.MkdirTemp("", "corescope-users-") if err != nil { @@ -200,9 +202,9 @@ func (s *Server) handleAdminUsersBackup(w http.ResponseWriter, _ *http.Request, log.Printf("[users] backup download cleanup: %v", err) } }() - name := "corescope-users-" + time.Now().UTC().Format(usersBackupLayout) + ".db" + name := "corescope-users-" + time.Now().UTC().Format(usersBackupLayout) + ".dump" path := filepath.Join(tmpDir, name) - if err := a.st.Snapshot(path); err != nil { + if err := a.st.SnapshotContext(r.Context(), path); err != nil { log.Printf("[users] backup download: %v", err) writeError(w, http.StatusInternalServerError, "internal error") return diff --git a/cmd/server/users_backup_test.go b/cmd/server/users_backup_test.go index c57117d25..72b841fd6 100644 --- a/cmd/server/users_backup_test.go +++ b/cmd/server/users_backup_test.go @@ -2,7 +2,6 @@ package main import ( "bytes" - "database/sql" "os" "path/filepath" "reflect" @@ -12,6 +11,7 @@ import ( "testing" "time" + "github.com/meshcore-analyzer/pgutil/pgtest" "github.com/meshcore-analyzer/users" ) @@ -21,7 +21,7 @@ var backupNow = time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) // created directory. func backupService(t *testing.T, keep int) (*authService, string) { t.Helper() - a, _ := newTestAuthService(t) + a, _ := newTestBackupAuthService(t) dir := filepath.Join(t.TempDir(), "backups") a.set.backup = backupSettings{enabled: true, dir: dir, keep: keep} return a, dir @@ -53,13 +53,10 @@ func TestUsersBackupWrittenWhenNoneExists(t *testing.T) { } a.maybeBackup(backupNow) names := snapshotNames(t, dir) - if len(names) != 1 || names[0] != usersBackupFile(backupNow) || names[0] != "users-20261008-120000.db" { + if len(names) != 1 || names[0] != usersBackupFile(backupNow) || names[0] != "users-20261008-120000.dump" { t.Fatalf("snapshots = %v", names) } - db, err := sql.Open("sqlite", filepath.Join(dir, names[0])) - if err != nil { - t.Fatal(err) - } + db := restorePostgresTestBackup(t, filepath.Join(dir, names[0])) defer db.Close() var n int if err := db.QueryRow(`SELECT COUNT(*) FROM users WHERE email = 'a@example.org'`).Scan(&n); err != nil || n != 1 { @@ -229,25 +226,27 @@ func TestUsersBackupDisabled(t *testing.T) { } func TestInitUserManagementBacksUpAtStartup(t *testing.T) { - dir := t.TempDir() - srv := &Server{cfg: &Config{UserManagement: &UserManagementConfig{ - Enabled: true, PublicBaseURL: testBase, - Mail: UserMailConfig{BrevoAPIKey: "k", FromEmail: "noreply@example.org"}, - }}} - if err := srv.initUserManagement(filepath.Join(dir, "meshcore.db")); err != nil { + a, _ := newTestBackupAuthService(t) + dir := filepath.Join(t.TempDir(), "backups") + u := validUM() + u.DatabaseURL = a.set.databaseURL + u.Backup = &UsersBackupConfig{Dir: dir} + srv := &Server{cfg: &Config{UserManagement: u}} + if err := srv.initUserManagement(pgtest.NewDatabase(t)); err != nil { t.Fatal(err) } - // The janitor's first pass (prune, then backup) runs before it sees stop. srv.closeUserManagement() - if names := snapshotNames(t, filepath.Join(dir, "backups")); len(names) != 1 { + if names := snapshotNames(t, dir); len(names) != 1 { t.Fatalf("snapshots after startup = %v; want 1", names) } } -var usersBackupFilenameRE = regexp.MustCompile(`^attachment; filename="corescope-users-\d{8}-\d{6}\.db"$`) +var usersBackupFilenameRE = regexp.MustCompile(`^attachment; filename="corescope-users-\d{8}-\d{6}\.dump"$`) func TestAdminUsersBackupDownload(t *testing.T) { - f, boss, uma := adminFixture(t) + f := newBackupAuthFixture(t, "boss@example.org") + boss := f.registerAndActivate(t, "boss@example.org", "Boss", pw) + uma := f.registerAndActivate(t, "uma@example.org", "Uma", pw) tmp := t.TempDir() for _, k := range []string{"TMPDIR", "TMP", "TEMP"} { t.Setenv(k, tmp) @@ -267,20 +266,17 @@ func TestAdminUsersBackupDownload(t *testing.T) { t.Errorf("Content-Disposition = %q", cd) } body := w.Body.Bytes() - if !bytes.HasPrefix(body, []byte("SQLite format 3\x00")) { - t.Fatalf("body is not a SQLite file (%d bytes)", len(body)) + if !bytes.HasPrefix(body, []byte("PGDMP")) { + t.Fatalf("body is not a PostgreSQL custom archive (%d bytes)", len(body)) } if cl := w.Header().Get("Content-Length"); cl != strconv.Itoa(len(body)) { t.Errorf("Content-Length = %q; body is %d bytes", cl, len(body)) } - path := filepath.Join(t.TempDir(), "download.db") + path := filepath.Join(t.TempDir(), "download.dump") if err := os.WriteFile(path, body, 0o600); err != nil { t.Fatal(err) } - db, err := sql.Open("sqlite", path) - if err != nil { - t.Fatal(err) - } + db := restorePostgresTestBackup(t, path) defer db.Close() var n int if err := db.QueryRow(`SELECT COUNT(*) FROM users`).Scan(&n); err != nil || n != 2 { diff --git a/cmd/server/websocket.go b/cmd/server/websocket.go index 14befa64c..121906cc0 100644 --- a/cmd/server/websocket.go +++ b/cmd/server/websocket.go @@ -287,7 +287,7 @@ func (c *Client) writePump(pingInterval time.Duration) { } } -// Poller watches for new transmissions in SQLite and broadcasts them. +// Poller watches for new transmissions in PostgreSQL and broadcasts them. type Poller struct { db *DB hub *Hub @@ -331,9 +331,9 @@ func (p *Poller) Start() { // Ingest new observations for existing transmissions (fixes #174) nextObsID := lastObsID if err := p.db.conn.QueryRow(` - SELECT COALESCE(MAX(id), ?) FROM ( + SELECT COALESCE(MAX(id), $1) FROM ( SELECT id FROM observations - WHERE id > ? + WHERE id > $2 ORDER BY id ASC LIMIT 500 )`, lastObsID, lastObsID).Scan(&nextObsID); err != nil { diff --git a/cmd/server/websocket_test.go b/cmd/server/websocket_test.go index fbdd62106..68e0cf6be 100644 --- a/cmd/server/websocket_test.go +++ b/cmd/server/websocket_test.go @@ -285,7 +285,7 @@ func TestPollerBroadcastsMultipleObservations(t *testing.T) { now := time.Now().UTC().Format(time.RFC3339) if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('FACE', 'starbursthash237a', ?, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now); err != nil { + VALUES ('FACE', 'starbursthash237a', $1, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now); err != nil { t.Fatalf("insert tx failed: %v", err) } var txID int @@ -294,9 +294,9 @@ func TestPollerBroadcastsMultipleObservations(t *testing.T) { } ts := time.Now().Unix() if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 14.0, -82, '["aa"]', ?), - (?, 2, 10.5, -90, '["aa","bb"]', ?), - (?, 1, 7.0, -96, '["aa","bb","cc"]', ?)`, + VALUES ($1, 1, 14.0, -82, '["aa"]', $2), + ($3, 2, 10.5, -90, '["aa","bb"]', $4), + ($5, 1, 7.0, -96, '["aa","bb","cc"]', $6)`, txID, ts, txID, ts+1, txID, ts+2); err != nil { t.Fatalf("insert observations failed: %v", err) } @@ -367,8 +367,8 @@ func TestIngestNewObservationsBroadcast(t *testing.T) { maxObs := db.GetMaxObservationID() now := time.Now().Unix() if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 2, 6.0, -100, '["aa","zz"]', ?), - (1, 1, 5.0, -101, '["aa","yy"]', ?)`, now, now+1); err != nil { + VALUES (1, 2, 6.0, -100, '["aa","zz"]', $1), + (1, 1, 5.0, -101, '["aa","yy"]', $2)`, now, now+1); err != nil { t.Fatalf("insert new observations failed: %v", err) } diff --git a/cmd/server/zerohop_hashsize_test.go b/cmd/server/zerohop_hashsize_test.go index 6509989e6..f40a2dd39 100644 --- a/cmd/server/zerohop_hashsize_test.go +++ b/cmd/server/zerohop_hashsize_test.go @@ -25,7 +25,7 @@ func insertAdvert(t *testing.T, db *DB, id int, rawHex, hash, pubKey, name strin t.Helper() if _, err := db.conn.Exec( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES (?, ?, ?, ?, 4, ?)`, + VALUES ($1, $2, $3, $4, 4, $5)`, rawHex, hash, seen, routeType, fmt.Sprintf(`{"pubKey":"%s","name":"%s","type":"ADVERT"}`, pubKey, name), ); err != nil { @@ -33,7 +33,7 @@ func insertAdvert(t *testing.T, db *DB, id int, rawHex, hash, pubKey, name strin } if _, err := db.conn.Exec( `INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 10.0, -90, '[]', ?)`, id, seenEpoch, + VALUES ($1, 1, 10.0, -90, '[]', $2)`, id, seenEpoch, ); err != nil { t.Fatalf("insert observation %s: %v", hash, err) } @@ -45,7 +45,7 @@ func zeroHopStore(t *testing.T) (*DB, string, int64) { now := time.Now().UTC().Add(-1 * time.Hour) recent := now.Format(time.RFC3339) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Obs', 'PRG', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs1', 'Obs', 'PRG', $1, '2026-01-01T00:00:00Z', 10)`, recent) return db, recent, now.Unix() } @@ -161,7 +161,7 @@ func TestZeroHopDeclaredSizeReachesMultiByteCapability(t *testing.T) { const pk = "5361aa08929b6248bb61cb1fab0fdb97d6dba8ae44f53a44427f63e7798a6b6b" db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) - VALUES (?, 'tth-hrebecna', 'repeater', ?, ?, 1)`, pk, recent, recent) + VALUES ($1, 'tth-hrebecna', 'repeater', $2, $3, 1)`, pk, recent, recent) insertAdvert(t, db, 1, "1240"+pk+"cafe", "zh_cap", pk, "tth-hrebecna", 2, recent, epoch) store := NewPacketStore(db, nil) diff --git a/config.example.json b/config.example.json index 012a35ec6..0b372d260 100644 --- a/config.example.json +++ b/config.example.json @@ -1,6 +1,10 @@ { "port": 3000, "apiKey": "your-secret-api-key-here", + "databaseURL": "", + "stateDir": "data", + "_comment_databaseURL": "PostgreSQL telemetry URL. Prefer environment credentials: CORESCOPE_READER_DATABASE_URL for the server, CORESCOPE_WRITER_DATABASE_URL for the ingestor, or CORESCOPE_DATABASE_URL when running a single process. Runtime roles are separate from the bootstrap owner. SQLite dbPath is no longer a runtime setting; upgrade existing databases with the offline importer described in docs/postgresql-upgrade.md.", + "_comment_stateDir": "Directory shared by the server and ingestor for file queues, statistics and account backups; CORESCOPE_STATE_DIR overrides it. This is a filesystem path, never a database URL. The default is data; containers use /app/data.", "nodeBlacklist": [], "_comment_nodeBlacklist": "Public keys of nodes to hide from all API responses. Use for trolls, offensive names, or nodes reporting false data that operators refuse to fix.", "hiddenNamePrefixes": ["🚫"], @@ -19,14 +23,12 @@ "_comment": "nodeDays: nodes not seen in N days moved to inactive_nodes (default 7). observerDays: observers not sending data in N days are removed (-1 = keep forever, default 14). observerPurgeDays: observers already marked inactive AND not seen in N days are hard-deleted, but only when no observation, observer_metrics or dropped_packets row still references them (0 = disabled, the default). Set it above both observerDays and packetDays — below those the reference guards keep every row anyway. packetDays: transmissions older than N days are deleted (0 = disabled). clientRxDays: mobile client-RX coverage rows (client_receptions/client_observers) older than N days are deleted (0 = disabled) — bounds the opt-in coverage tables (#1727). clientRxObsDays: diagnostic client_rx_observations rows (by rx_at) older than N days are deleted (0 = disabled) — bounds the opt-in clientRxObservations table. clientRfDays: RF environment sample rows (client_rf_samples, by sampled_at) older than N days are deleted (0 = disabled) — bounds the opt-in clientRfSamples table; unset means unbounded growth at roughly 240 rows/hour per driver. clientRegionsDays: region-discovery answer rows (node_declared_regions, by observed_at) older than N days are deleted (0 = disabled) — bounds the opt-in clientRegions table. NOTE (#1283): all retention fields are consumed by the INGESTOR process. The server is read-only and never prunes." }, "db": { - "vacuumOnStartup": false, - "incrementalVacuumPages": 1024, "load": { "chunkSize": 10000, "_comment": "chunkSize: rows fetched per chunk by PacketStore.LoadChunked during startup (#1009). Default 10000. Lower values surface the early-HTTP-readiness signal sooner (the listener binds after the first chunk) at the cost of more SQL round-trips. Higher values reduce per-chunk overhead but delay first-chunk readiness. The X-CoreScope-Load-Status response header reports loading|ready and progress= until the load completes." }, - "_comment": "vacuumOnStartup: run one-time full VACUUM to enable incremental auto-vacuum on existing DBs. Executed by the INGESTOR at startup, BEFORE the MQTT subscriber starts (#1283), so there is no contention with concurrent writes. Blocks ingestor startup for minutes on large DBs; requires 2x DB file size in free disk space. incrementalVacuumPages: free pages returned to OS after each retention reaper cycle (default 1024). See #919. load.chunkSize: see nested _comment (#1009).", - "_comment_slowWriterMs": "#1340 — SQLite writer-lock log threshold (default 500). Any wrapped writer call (tagged neighbor_builder, mqtt_handler, prune_packets, prune_observers, prune_metrics, vacuum) whose hold_ms exceeds this emits a single [db-slow-writer] log line. Configured per-process via the CORESCOPE_DB_SLOW_WRITER_MS environment variable on the INGESTOR (e.g. CORESCOPE_DB_SLOW_WRITER_MS=200 for tighter alerting). Per-component wait_ms / hold_ms / contention_total histograms are surfaced via /api/perf/write-sources under .writer_perf regardless of this threshold." + "_comment": "PostgreSQL manages VACUUM, ANALYZE and checkpoints. The former SQLite vacuumOnStartup, incrementalVacuumPages and analysisLimit controls do not apply. Keep autovacuum enabled in the database service. load.chunkSize controls application startup loading; see the nested comment.", + "_comment_slowWriterMs": "Writer-lock log threshold (default 500 ms). Set CORESCOPE_DB_SLOW_WRITER_MS on the ingestor to adjust logging. Per-component wait/hold time and contention counters are available at /api/perf/write-sources. These counters describe the application writer, not PostgreSQL cluster-wide commits or WAL flushes." }, "listLimits": { "packetsMax": 10000, @@ -36,7 +38,7 @@ "bulkHealthMax": 200, "_comment": "Maximum row counts returned by list API endpoints. These enforce a DoS-bounded ceiling for both UI and external requests. Operators with small/embedded deployments can tighten these; operators running large regional meshes can raise them. bulkHealthMax is intentionally separate from nodesMax: /api/nodes/bulk-health is per-row much heavier than /api/nodes (it joins per-node observer health + recent-packet stats) so its ceiling stays low (default 200) even if nodesMax is raised." }, - "_comment_ingestorStats": "Ingestor publishes a 1-Hz stats snapshot consumed by the server's /api/perf/io and /api/perf/write-sources endpoints (#1120). Path is configured via the CORESCOPE_INGESTOR_STATS environment variable on the INGESTOR process. Default: /tmp/corescope-ingestor-stats.json. The writer uses O_NOFOLLOW + 0o600, so a pre-planted symlink in /tmp cannot be used to clobber an arbitrary file. SECURITY: in shared-tmp environments (multi-tenant hosts), point CORESCOPE_INGESTOR_STATS at a private directory like /var/lib/corescope/ingestor-stats.json that only the corescope user can write to.", + "_comment_ingestorStats": "Ingestor publishes a 1-Hz snapshot consumed by /api/perf/io and /api/perf/write-sources. The default is ingestor-stats.json under stateDir. CORESCOPE_INGESTOR_STATS overrides it and must select the same file for both processes. Keep the state directory writable only by the instance operator.", "corsAllowedOrigins": [], "_comment_corsAllowedOrigins": "Cross-origin allowlist for embed scenarios (#1369) AND for /ws WebSocket upgrades (#1793 — CSWSH defense per OWASP WebSocket Security Cheat Sheet). Exact-match origins, e.g. [\"https://blog.example.com\", \"https://embed.example.com\"]. Same-origin requests (Origin host == request Host) and non-browser clients (no Origin header) are always allowed for /ws. When empty (default), no Access-Control-* headers are sent and browsers enforce same-origin; cross-origin /ws upgrades are rejected. When non-empty, only the listed origins receive CORS headers, and Access-Control-Allow-Methods is limited to GET, HEAD, OPTIONS (the cross-domain surface is read-only — same-origin admin writes are unaffected). Use [\"*\"] to allow any origin for CORS XHR (NOT recommended for write-capable deployments); note that \"*\" is deliberately NOT honored for /ws upgrades — list explicit origins to permit cross-origin WebSocket clients (OWASP guidance: avoid wildcards in CSWSH defense). Operators can override per-deployment with the CORS_ALLOWED_ORIGINS environment variable (comma-separated). No credentialed CORS is enabled. To embed the map or channels pages cross-domain, add the embedding origin here and use the URL pattern '/#/map?embed=1' or '/#/channels?embed=1' — embed mode hides the top-nav, bottom-nav, and side drawer for full-bleed iframe rendering.", "webSocket": { @@ -431,7 +433,8 @@ "_comment_analytics": "Issue #1240 + #1256 + #1265. Each analytics endpoint (topology, rf, distance, channels, hashCollisions, hashSizes, roles, observersClockSkew, nodesClockSkew) is recomputed in the background on the configured interval and served from an atomic-pointer cache. Reads never block on compute. Default 300s (5 min) per endpoint reflects the operator principle: serving slightly stale data quickly beats real-time data slowly. Lower values = fresher data at higher CPU cost. Only the default query (no region/window) is precomputed; region- and window-filtered requests fall back to the legacy on-request compute + 60s TTL cache.", "userManagement": { "enabled": false, - "dbPath": "", + "databaseURL": "", + "approvedChannelsDatabaseURL": "", "adminEmails": ["operator@example.org"], "publicBaseUrl": "https://corescope.example.org", "sessionDays": 30, @@ -462,10 +465,10 @@ "keep": 7 } }, - "_comment_userManagement": "Optional accounts (off by default; see docs/user-guide/accounts.md). When enabled: visitors can register with email + password and activate through a mailed link; addresses in adminEmails become admins; admins manage users at #/admin/users and can use the operator endpoints without the apiKey. The dashboard stays public. dbPath defaults to users.db next to the analyzer database: a separate file, the server never writes measurement data. publicBaseUrl is required and is used for every link in every mail. Mail goes through Brevo's transactional API (a free account is enough); brevoApiKey and webhookSecret may instead come from CORESCOPE_BREVO_API_KEY and CORESCOPE_BREVO_WEBHOOK_SECRET. Set webhookSecret (16+ chars) and point a Brevo transactional webhook at /api/mail/brevo/webhook with bearer auth to see delivery status per user. trustedProxies (CIDRs) lets login rate limits use X-Forwarded-For behind a reverse proxy. Startup fails if enabled with an incomplete mail setup.", - "_comment_channelProposals": "Optional, needs userManagement.enabled (docs/user-guide/accounts.md, Channel proposals). Logged-in users propose hashtag channels; admins approve, reject or revoke them on the Proposals tab. An approved channel is decrypted by the ingestor from its next refresh (once a minute, no restart) and listed for every visitor on the Channels page. maxApproved bounds the decryption work (every approved key is tried on every group message); perUserPerDay and maxPending limit proposals. The ingestor reads users.db read-only from userManagement.dbPath, or next to the analyzer database; set dbPath explicitly when the server and the ingestor are not given the same analyzer database path.", + "_comment_userManagement": "Optional accounts (off by default; see docs/user-guide/accounts.md). When enabled: visitors can register with email + password and activate through a mailed link; addresses in adminEmails become admins; admins manage users at #/admin/users and can use the operator endpoints without the apiKey. The dashboard stays public. databaseURL (or CORESCOPE_USERS_DATABASE_URL) selects a separate PostgreSQL account database with its own restricted writer role. The server never writes telemetry. Initialize schemas with the migration owner before starting either service. publicBaseUrl is required and is used for every link in every mail. Mail goes through Brevo's transactional API (a free account is enough); brevoApiKey and webhookSecret may instead come from CORESCOPE_BREVO_API_KEY and CORESCOPE_BREVO_WEBHOOK_SECRET. Set webhookSecret (16+ chars) and point a Brevo transactional webhook at /api/mail/brevo/webhook with bearer auth to see delivery status per user. trustedProxies (CIDRs) lets login rate limits use X-Forwarded-For behind a reverse proxy. Startup fails if enabled with an incomplete mail setup.", + "_comment_channelProposals": "Optional, needs userManagement.enabled (docs/user-guide/accounts.md, Channel proposals). Logged-in users propose hashtag channels; admins approve, reject or revoke them on the Proposals tab. An approved channel is decrypted by the ingestor from its next refresh (once a minute, no restart) and listed for every visitor on the Channels page. maxApproved bounds the decryption work (every approved key is tried on every group message); perUserPerDay and maxPending limit proposals. The ingestor reads only the approved_channels view using userManagement.approvedChannelsDatabaseURL or CORESCOPE_APPROVED_CHANNELS_DATABASE_URL. This reader must have no access to account tables or write privileges.", "_comment_notifications": "Optional, needs userManagement.enabled (docs/user-guide/accounts.md, Node notifications). Logged-in users watch nodes and get one mail per check when a watched node goes offline, comes back or reports a low battery; admins can also watch for new foreign nodes and observers going offline. Thresholds are healthThresholds and batteryThresholds. A check runs every intervalMinutes; at most perUserPerDay mails per user and maxMailsPerDay in total go out per 24 hours (over a limit the change is recorded and dropped, never queued). maxMailsPerDay counts notification mail only: keep it below the mail provider's daily quota so activation, password-reset and address-change mail still go out (the default 100 leaves 200 of Brevo's free 300 a day); maxWatchesPerUser bounds each watch list. Every mail has a one-click unsubscribe.", - "_comment_usersBackup": "Part of userManagement (docs/user-guide/accounts.md, Backups). With accounts on, the server snapshots users.db at startup when the newest snapshot is older than 24 hours, then about every 24 hours, to users-.db (UTC, mode 0600) in dir (empty: backups/ next to users.db), and deletes the oldest beyond keep (default 7), never the one just written. On unless enabled is false. Snapshots hold password hashes, and deleted accounts stay in them for up to keep days: protect the directory like users.db. A dir that already exists is not tightened to 0700: set that yourself. Admins download a fresh snapshot at /api/admin/users-backup. A restore brings back accounts deleted after the snapshot (also self-deletions), old passwords, revoked sessions and used links: follow the Restore steps in the doc (note the deletions from the current audit log first, then disable those accounts and clear sessions and tokens before starting, and delete them again in the admin area).", + "_comment_usersBackup": "With accounts enabled, the server creates a consistent PostgreSQL custom-format archive at startup when the newest snapshot is older than 24 hours, then daily. Files are users-.dump under backup.dir (default: backups under stateDir). keep limits retained snapshots (default 7). pg_dump 18 must be available. Archives contain authentication state: protect the directory and follow the account restore procedure before restarting, because restoring also restores old passwords, sessions, used tokens and deleted accounts.", "customizer": { "disabledTabs": [], "_comment_disabledTabs": "Issue #1508. List of customizer-modal tab ids to hide from end users. Useful when operators want to expose only daily-use viewer controls and keep one-shot admin chrome out of the way. Recognized ids: branding, theme, nodes, home, display, geofilter, export. Example: [\"branding\", \"geofilter\", \"export\"] hides the admin tabs and leaves theme/colors/home/display visible. Default [] keeps the legacy behavior (all tabs visible). Operators edit this in config.json directly — there is no in-app UI for this list (by design)." diff --git a/docker-compose.example.yml b/docker-compose.example.yml index e600becbb..d5bf80136 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -1,5 +1,5 @@ -# CoreScope — simple single-container deployment using the pre-built -# image from GHCR. +# CoreScope — application and PostgreSQL deployment using the pre-built +# application image from GHCR. # # AUDIENCE: third-party operators running a self-hosted CoreScope on # ONE host. Ships with the built-in Mosquitto broker DISABLED by @@ -14,14 +14,37 @@ # Usage: docker compose -f docker-compose.example.yml up -d # Docs: https://github.com/Kpa-clawbot/CoreScope/blob/master/DEPLOY.md +name: corescope + services: + postgres: + extends: + file: docker/postgres.compose.yml + service: postgres + volumes: + - ${POSTGRES_DATA_DIR:-./postgres-data}:/var/lib/postgresql + bootstrap: + extends: + file: docker/postgres.compose.yml + service: bootstrap + image: ${CORESCOPE_IMAGE:-ghcr.io/kpa-clawbot/corescope:latest} + volumes: + - ${DATA_DIR:-./data}:/app/data:ro corescope: - image: ghcr.io/kpa-clawbot/corescope:latest + depends_on: + bootstrap: + condition: service_completed_successfully + image: ${CORESCOPE_IMAGE:-ghcr.io/kpa-clawbot/corescope:latest} ports: - "${HTTP_PORT:-80}:80" volumes: - ${DATA_DIR:-./data}:/app/data environment: + - CORESCOPE_STATE_DIR=/app/data + - CORESCOPE_READER_DATABASE_URL=postgresql://corescope_reader:${CORESCOPE_READER_PASSWORD:?Set CORESCOPE_READER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable + - CORESCOPE_WRITER_DATABASE_URL=postgresql://corescope_writer:${CORESCOPE_WRITER_PASSWORD:?Set CORESCOPE_WRITER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable + - CORESCOPE_USERS_DATABASE_URL=postgresql://corescope_accounts:${CORESCOPE_ACCOUNTS_PASSWORD:?Set CORESCOPE_ACCOUNTS_PASSWORD}@postgres/corescope_accounts?sslmode=disable + - CORESCOPE_APPROVED_CHANNELS_DATABASE_URL=postgresql://corescope_channels:${CORESCOPE_CHANNELS_PASSWORD:?Set CORESCOPE_CHANNELS_PASSWORD}@postgres/corescope_accounts?sslmode=disable - DISABLE_CADDY=${DISABLE_CADDY:-true} - DISABLE_MOSQUITTO=${DISABLE_MOSQUITTO:-false} restart: unless-stopped diff --git a/docker-compose.no-mosquitto.yml b/docker-compose.no-mosquitto.yml index d8b114080..ebc0f055f 100644 --- a/docker-compose.no-mosquitto.yml +++ b/docker-compose.no-mosquitto.yml @@ -13,8 +13,26 @@ # deploy that keeps the built-in broker, see # `docker-compose.example.yml`. +name: corescope + services: + postgres: + extends: + file: docker/postgres.compose.yml + service: postgres + volumes: + - ${PROD_POSTGRES_DATA_DIR:-~/corescope-postgres}:/var/lib/postgresql + bootstrap: + extends: + file: docker/postgres.compose.yml + service: bootstrap + image: corescope:latest + volumes: + - ${PROD_DATA_DIR:-~/meshcore-data}:/app/data:ro prod: + depends_on: + bootstrap: + condition: service_completed_successfully build: context: . args: @@ -36,6 +54,11 @@ services: - ${PROD_DATA_DIR:-~/meshcore-data}:/app/data - caddy-data:/data/caddy environment: + - CORESCOPE_STATE_DIR=/app/data + - CORESCOPE_READER_DATABASE_URL=postgresql://corescope_reader:${CORESCOPE_READER_PASSWORD:?Set CORESCOPE_READER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable + - CORESCOPE_WRITER_DATABASE_URL=postgresql://corescope_writer:${CORESCOPE_WRITER_PASSWORD:?Set CORESCOPE_WRITER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable + - CORESCOPE_USERS_DATABASE_URL=postgresql://corescope_accounts:${CORESCOPE_ACCOUNTS_PASSWORD:?Set CORESCOPE_ACCOUNTS_PASSWORD}@postgres/corescope_accounts?sslmode=disable + - CORESCOPE_APPROVED_CHANNELS_DATABASE_URL=postgresql://corescope_channels:${CORESCOPE_CHANNELS_PASSWORD:?Set CORESCOPE_CHANNELS_PASSWORD}@postgres/corescope_accounts?sslmode=disable - NODE_ENV=production - DISABLE_MOSQUITTO=true healthcheck: diff --git a/docker-compose.staging.no-mosquitto.yml b/docker-compose.staging.no-mosquitto.yml index 281164d19..a1eb0e6ca 100644 --- a/docker-compose.staging.no-mosquitto.yml +++ b/docker-compose.staging.no-mosquitto.yml @@ -12,8 +12,26 @@ # `docker-compose.example.yml` (or `docker-compose.no-mosquitto.yml` # for a prod-flavoured variant with an external broker). +name: corescope-staging + services: + postgres: + extends: + file: docker/postgres.compose.yml + service: postgres + volumes: + - ${STAGING_POSTGRES_DATA_DIR:-~/corescope-staging-postgres}:/var/lib/postgresql + bootstrap: + extends: + file: docker/postgres.compose.yml + service: bootstrap + image: corescope-go:latest + volumes: + - ${STAGING_DATA_DIR:-~/meshcore-staging-data}:/app/data:ro staging-go: + depends_on: + bootstrap: + condition: service_completed_successfully build: context: . dockerfile: Dockerfile @@ -40,6 +58,11 @@ services: - ${STAGING_DATA_DIR:-~/meshcore-staging-data}:/app/data - caddy-data-staging-go:/data/caddy environment: + - CORESCOPE_STATE_DIR=/app/data + - CORESCOPE_READER_DATABASE_URL=postgresql://corescope_reader:${CORESCOPE_READER_PASSWORD:?Set CORESCOPE_READER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable + - CORESCOPE_WRITER_DATABASE_URL=postgresql://corescope_writer:${CORESCOPE_WRITER_PASSWORD:?Set CORESCOPE_WRITER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable + - CORESCOPE_USERS_DATABASE_URL=postgresql://corescope_accounts:${CORESCOPE_ACCOUNTS_PASSWORD:?Set CORESCOPE_ACCOUNTS_PASSWORD}@postgres/corescope_accounts?sslmode=disable + - CORESCOPE_APPROVED_CHANNELS_DATABASE_URL=postgresql://corescope_channels:${CORESCOPE_CHANNELS_PASSWORD:?Set CORESCOPE_CHANNELS_PASSWORD}@postgres/corescope_accounts?sslmode=disable - NODE_ENV=staging - ENABLE_PPROF=true - DISABLE_MOSQUITTO=true diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml index 6002fb916..7bb91ea1c 100644 --- a/docker-compose.staging.yml +++ b/docker-compose.staging.yml @@ -20,8 +20,26 @@ # # Override defaults via .env or environment variables. +name: corescope-staging + services: + postgres: + extends: + file: docker/postgres.compose.yml + service: postgres + volumes: + - ${STAGING_POSTGRES_DATA_DIR:-~/corescope-staging-postgres}:/var/lib/postgresql + bootstrap: + extends: + file: docker/postgres.compose.yml + service: bootstrap + image: corescope-go:latest + volumes: + - ${STAGING_DATA_DIR:-~/meshcore-staging-data}:/app/data:ro staging-go: + depends_on: + bootstrap: + condition: service_completed_successfully build: context: . dockerfile: Dockerfile @@ -55,6 +73,11 @@ services: - ${STAGING_DATA_DIR:-~/meshcore-staging-data}:/app/data - caddy-data-staging-go:/data/caddy environment: + - CORESCOPE_STATE_DIR=/app/data + - CORESCOPE_READER_DATABASE_URL=postgresql://corescope_reader:${CORESCOPE_READER_PASSWORD:?Set CORESCOPE_READER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable + - CORESCOPE_WRITER_DATABASE_URL=postgresql://corescope_writer:${CORESCOPE_WRITER_PASSWORD:?Set CORESCOPE_WRITER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable + - CORESCOPE_USERS_DATABASE_URL=postgresql://corescope_accounts:${CORESCOPE_ACCOUNTS_PASSWORD:?Set CORESCOPE_ACCOUNTS_PASSWORD}@postgres/corescope_accounts?sslmode=disable + - CORESCOPE_APPROVED_CHANNELS_DATABASE_URL=postgresql://corescope_channels:${CORESCOPE_CHANNELS_PASSWORD:?Set CORESCOPE_CHANNELS_PASSWORD}@postgres/corescope_accounts?sslmode=disable - NODE_ENV=staging - ENABLE_PPROF=true # Default TRUE: the standalone mqtt-broker container owns MQTT on diff --git a/docker-compose.yml b/docker-compose.yml index 3fb4b8bbd..a229d18d7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,8 +3,26 @@ # CRITICAL: All data mounts use bind mounts (~/path), NOT named volumes. # This ensures the DB and theme are visible on the host filesystem for backup. +name: corescope + services: + postgres: + extends: + file: docker/postgres.compose.yml + service: postgres + volumes: + - ${PROD_POSTGRES_DATA_DIR:-~/corescope-postgres}:/var/lib/postgresql + bootstrap: + extends: + file: docker/postgres.compose.yml + service: bootstrap + image: corescope:latest + volumes: + - ${PROD_DATA_DIR:-~/meshcore-data}:/app/data:ro prod: + depends_on: + bootstrap: + condition: service_completed_successfully build: context: . args: @@ -27,6 +45,11 @@ services: - ${PROD_DATA_DIR:-~/meshcore-data}:/app/data - caddy-data:/data/caddy environment: + - CORESCOPE_STATE_DIR=/app/data + - CORESCOPE_READER_DATABASE_URL=postgresql://corescope_reader:${CORESCOPE_READER_PASSWORD:?Set CORESCOPE_READER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable + - CORESCOPE_WRITER_DATABASE_URL=postgresql://corescope_writer:${CORESCOPE_WRITER_PASSWORD:?Set CORESCOPE_WRITER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable + - CORESCOPE_USERS_DATABASE_URL=postgresql://corescope_accounts:${CORESCOPE_ACCOUNTS_PASSWORD:?Set CORESCOPE_ACCOUNTS_PASSWORD}@postgres/corescope_accounts?sslmode=disable + - CORESCOPE_APPROVED_CHANNELS_DATABASE_URL=postgresql://corescope_channels:${CORESCOPE_CHANNELS_PASSWORD:?Set CORESCOPE_CHANNELS_PASSWORD}@postgres/corescope_accounts?sslmode=disable - NODE_ENV=production - DISABLE_MOSQUITTO=${DISABLE_MOSQUITTO:-false} - DISABLE_CADDY=${DISABLE_CADDY:-false} diff --git a/docker/postgres-bootstrap.sh b/docker/postgres-bootstrap.sh new file mode 100755 index 000000000..8f5386bae --- /dev/null +++ b/docker/postgres-bootstrap.sh @@ -0,0 +1,26 @@ +#!/bin/sh +set -eu + +# A legacy instance must go through the explicit offline importer. Retained +# SQLite files are harmless once both PostgreSQL stores have verified readiness. +if [ -f /app/data/meshcore.db ]; then + /app/corescope-migrate -check-import-kind=telemetry -from-sqlite=/app/data/meshcore.db || { + echo 'SQLite telemetry exists. Follow docs/postgresql-upgrade.md before starting this instance.' >&2 + exit 1 + } +fi +if [ -f /app/data/users.db ]; then + /app/corescope-migrate -check-import-kind=accounts -users-from-sqlite=/app/data/users.db || { + echo 'SQLite account data exists. Follow docs/postgresql-upgrade.md before starting this instance.' >&2 + exit 1 + } +fi +/app/corescope-migrate + +# libpq uses these environment settings; URLs/passwords never appear in argv. +export PGHOST="${PGHOST:-postgres}" PGPORT="${PGPORT:-5432}" PGSSLMODE=disable +export PGUSER=corescope_owner PGPASSWORD="${CORESCOPE_OWNER_PASSWORD:?set CORESCOPE_OWNER_PASSWORD}" +export PGDATABASE=corescope_telemetry +psql -X -v ON_ERROR_STOP=1 -f /app/postgres-grants.sql +export PGDATABASE=corescope_accounts +psql -X -v ON_ERROR_STOP=1 -f /app/postgres-grants.sql diff --git a/docker/postgres-grants.sql b/docker/postgres-grants.sql new file mode 100644 index 000000000..177a20a9a --- /dev/null +++ b/docker/postgres-grants.sql @@ -0,0 +1,22 @@ +\set ON_ERROR_STOP on +-- Execute as the migration owner after schema installation/import, once in +-- each database. Runtime credentials never own objects or change readiness. +SELECT kind FROM corescope_schema \gset +SELECT :'kind' = 'telemetry' AS telemetry \gset +REVOKE CREATE ON SCHEMA public FROM PUBLIC; +\if :telemetry +GRANT USAGE ON SCHEMA public TO corescope_reader, corescope_writer; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO corescope_reader; +GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO corescope_reader; +GRANT SELECT ON corescope_schema TO corescope_writer; +SELECT format('GRANT SELECT,INSERT,UPDATE,DELETE ON TABLE %I.%I TO corescope_writer',schemaname,tablename) +FROM pg_tables WHERE schemaname='public' AND tablename NOT LIKE 'corescope\_%' ESCAPE '\' \gexec +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO corescope_writer; +\else +GRANT USAGE ON SCHEMA public TO corescope_accounts, corescope_channels; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO corescope_accounts; +SELECT format('GRANT SELECT,INSERT,UPDATE,DELETE ON TABLE %I.%I TO corescope_accounts',schemaname,tablename) +FROM pg_tables WHERE schemaname='public' AND tablename NOT LIKE 'corescope\_%' ESCAPE '\' AND tablename <> 'schema_version' \gexec +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO corescope_accounts; +GRANT SELECT ON approved_channels,corescope_schema TO corescope_channels; +\endif diff --git a/docker/postgres-init.sh b/docker/postgres-init.sh new file mode 100755 index 000000000..5d855ed0d --- /dev/null +++ b/docker/postgres-init.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# Runs once, against an empty PostgreSQL data directory. Passwords stay in the +# environment; psql reads them without putting them in process arguments. +set -eu + +for password in "${CORESCOPE_OWNER_PASSWORD:?set CORESCOPE_OWNER_PASSWORD}" \ + "${CORESCOPE_READER_PASSWORD:?set CORESCOPE_READER_PASSWORD}" \ + "${CORESCOPE_WRITER_PASSWORD:?set CORESCOPE_WRITER_PASSWORD}" \ + "${CORESCOPE_ACCOUNTS_PASSWORD:?set CORESCOPE_ACCOUNTS_PASSWORD}" \ + "${CORESCOPE_CHANNELS_PASSWORD:?set CORESCOPE_CHANNELS_PASSWORD}"; do + case "$password" in + *[!a-fA-F0-9]*|'') echo 'Database role passwords must be at least 32 hexadecimal characters; use openssl rand -hex 32.' >&2; exit 1 ;; + esac + if [ "${#password}" -lt 32 ]; then + echo 'Database role passwords must be at least 32 hexadecimal characters.' >&2 + exit 1 + fi +done +unset password + +psql -X -v ON_ERROR_STOP=1 --username "${POSTGRES_USER:-postgres}" --dbname postgres <<'SQL' +\getenv owner_password CORESCOPE_OWNER_PASSWORD +\getenv reader_password CORESCOPE_READER_PASSWORD +\getenv writer_password CORESCOPE_WRITER_PASSWORD +\getenv accounts_password CORESCOPE_ACCOUNTS_PASSWORD +\getenv channels_password CORESCOPE_CHANNELS_PASSWORD +CREATE ROLE corescope_owner LOGIN PASSWORD :'owner_password'; +CREATE ROLE corescope_reader LOGIN PASSWORD :'reader_password'; +CREATE ROLE corescope_writer LOGIN PASSWORD :'writer_password'; +CREATE ROLE corescope_accounts LOGIN PASSWORD :'accounts_password'; +CREATE ROLE corescope_channels LOGIN PASSWORD :'channels_password'; +CREATE DATABASE corescope_telemetry OWNER corescope_owner TEMPLATE template0 ENCODING 'UTF8' LOCALE 'C'; +CREATE DATABASE corescope_accounts OWNER corescope_owner TEMPLATE template0 ENCODING 'UTF8' LOCALE 'C'; +REVOKE ALL ON DATABASE corescope_telemetry FROM PUBLIC; +REVOKE ALL ON DATABASE corescope_accounts FROM PUBLIC; +GRANT CONNECT ON DATABASE corescope_telemetry TO corescope_reader, corescope_writer; +GRANT CONNECT ON DATABASE corescope_accounts TO corescope_accounts, corescope_channels; +ALTER ROLE corescope_reader SET default_transaction_read_only = on; +ALTER ROLE corescope_channels SET default_transaction_read_only = on; +SQL diff --git a/docker/postgres.compose.yml b/docker/postgres.compose.yml new file mode 100644 index 000000000..32304de06 --- /dev/null +++ b/docker/postgres.compose.yml @@ -0,0 +1,34 @@ +# Shared services extended by the deployment variants. Relative bind sources +# here resolve beside this file. Runtime services receive no owner password. +services: + postgres: + image: postgres:18.6-alpine3.24 + restart: unless-stopped + stop_grace_period: 60s + shm_size: 128mb + environment: + POSTGRES_PASSWORD: ${POSTGRES_ADMIN_PASSWORD:?Set POSTGRES_ADMIN_PASSWORD} + CORESCOPE_OWNER_PASSWORD: ${CORESCOPE_OWNER_PASSWORD:?Set CORESCOPE_OWNER_PASSWORD} + CORESCOPE_READER_PASSWORD: ${CORESCOPE_READER_PASSWORD:?Set CORESCOPE_READER_PASSWORD} + CORESCOPE_WRITER_PASSWORD: ${CORESCOPE_WRITER_PASSWORD:?Set CORESCOPE_WRITER_PASSWORD} + CORESCOPE_ACCOUNTS_PASSWORD: ${CORESCOPE_ACCOUNTS_PASSWORD:?Set CORESCOPE_ACCOUNTS_PASSWORD} + CORESCOPE_CHANNELS_PASSWORD: ${CORESCOPE_CHANNELS_PASSWORD:?Set CORESCOPE_CHANNELS_PASSWORD} + volumes: + - ${POSTGRES_DATA_DIR:-../postgres-data}:/var/lib/postgresql + - ./postgres-init.sh:/docker-entrypoint-initdb.d/10-corescope.sh:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U postgres -d postgres"] + interval: 5s + timeout: 5s + retries: 20 + bootstrap: + image: ${CORESCOPE_IMAGE:-corescope:latest} + entrypoint: ["/bin/sh", "/app/postgres-bootstrap.sh"] + restart: "no" + depends_on: + postgres: + condition: service_healthy + environment: + CORESCOPE_DATABASE_URL: postgresql://corescope_owner:${CORESCOPE_OWNER_PASSWORD:?Set CORESCOPE_OWNER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable + CORESCOPE_USERS_DATABASE_URL: postgresql://corescope_owner:${CORESCOPE_OWNER_PASSWORD:?Set CORESCOPE_OWNER_PASSWORD}@postgres/corescope_accounts?sslmode=disable + CORESCOPE_OWNER_PASSWORD: ${CORESCOPE_OWNER_PASSWORD:?Set CORESCOPE_OWNER_PASSWORD} diff --git a/docker/supervisord-go-no-caddy.conf b/docker/supervisord-go-no-caddy.conf index 3ca0af00f..a89179a1a 100644 --- a/docker/supervisord-go-no-caddy.conf +++ b/docker/supervisord-go-no-caddy.conf @@ -29,7 +29,7 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 [program:corescope-server] -command=/app/corescope-server -config-dir /app -db /app/data/meshcore.db -public /app/public -port 3000 +command=/app/corescope-server -config-dir /app -state-dir /app/data -public /app/public -port 3000 directory=/app autostart=true autorestart=true diff --git a/docker/supervisord-go-no-mosquitto-no-caddy.conf b/docker/supervisord-go-no-mosquitto-no-caddy.conf index 661d77196..3d2386740 100644 --- a/docker/supervisord-go-no-mosquitto-no-caddy.conf +++ b/docker/supervisord-go-no-mosquitto-no-caddy.conf @@ -20,7 +20,7 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 [program:corescope-server] -command=/app/corescope-server -config-dir /app -db /app/data/meshcore.db -public /app/public -port 3000 +command=/app/corescope-server -config-dir /app -state-dir /app/data -public /app/public -port 3000 directory=/app autostart=true autorestart=true diff --git a/docker/supervisord-go-no-mosquitto.conf b/docker/supervisord-go-no-mosquitto.conf index fc98edb19..1acd90427 100644 --- a/docker/supervisord-go-no-mosquitto.conf +++ b/docker/supervisord-go-no-mosquitto.conf @@ -20,7 +20,7 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 [program:corescope-server] -command=/app/corescope-server -config-dir /app -db /app/data/meshcore.db -public /app/public -port 3000 +command=/app/corescope-server -config-dir /app -state-dir /app/data -public /app/public -port 3000 directory=/app autostart=true autorestart=true diff --git a/docker/supervisord-go.conf b/docker/supervisord-go.conf index 315b10ab6..697405cad 100644 --- a/docker/supervisord-go.conf +++ b/docker/supervisord-go.conf @@ -29,7 +29,7 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 [program:corescope-server] -command=/app/corescope-server -config-dir /app -db /app/data/meshcore.db -public /app/public -port 3000 +command=/app/corescope-server -config-dir /app -state-dir /app/data -public /app/public -port 3000 directory=/app autostart=true autorestart=true diff --git a/docs/deployment.md b/docs/deployment.md index 4b1142bca..557cd2894 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -18,129 +18,47 @@ Comprehensive guide to deploying and operating CoreScope. For a quick start, see ## System Requirements -| Resource | Minimum | Recommended | -|----------|---------|-------------| -| RAM | 256 MB | 512 MB+ | -| Disk | 500 MB (image + DB) | 2 GB+ for long-term data | -| CPU | 1 core | 2+ cores | -| Architecture | `linux/amd64`, `linux/arm64` | — | -| Docker | 20.10+ | Latest stable | - -CoreScope runs well on Raspberry Pi 4/5 (ARM64). The Go server uses ~300 MB RAM for 56K+ packets. +The supplied images support `linux/amd64` and `linux/arm64`. Use Docker with the Compose plugin and PostgreSQL 18.6. Budget for the application and database together: earlier SQLite memory figures do not size a PostgreSQL installation. Measure the retained dataset, packet cache, database indexes/WAL and backup storage before choosing capacity. ---- +For migration, retain the original data plus recovery and normalized snapshots while PostgreSQL builds its own data and indexes. Required disk space and downtime must be measured on a representative copy. See [offline upgrade and recovery](postgresql-upgrade.md). ## Docker Deployment -### Quick Start (one command) - -```bash -docker run -d --name corescope \ - -p 80:80 \ - -v corescope-data:/app/data \ - ghcr.io/kpa-clawbot/corescope:latest -``` - -Open `http://localhost` — you'll see an empty dashboard ready to receive packets. - -No `config.json` is required. The server starts with sensible defaults: -- HTTP on port 3000 (Caddy proxies port 80 → 3000 internally) -- Internal Mosquitto MQTT broker on port 1883 -- Ingestor connects to `mqtt://localhost:1883` automatically -- SQLite database at `/app/data/meshcore.db` - -### Full `docker run` Reference (recommended) - -The bare `docker run` command is the primary deployment method. One image, documented parameters — run it however you want. - -```bash -docker run -d --name corescope \ - --restart=unless-stopped \ - -p 80:80 -p 443:443 -p 1883:1883 \ - -e DISABLE_MOSQUITTO=false \ - -e DISABLE_CADDY=false \ - -v /your/data:/app/data \ - -v /your/Caddyfile:/etc/caddy/Caddyfile:ro \ - -v /your/caddy-data:/data/caddy \ - ghcr.io/kpa-clawbot/corescope:latest -``` - -#### Parameters - -| Parameter | Required | Description | -|-----------|----------|-------------| -| `-p 80:80` | Yes | HTTP web UI | -| `-p 443:443` | No | HTTPS (only if using built-in Caddy with a domain) | -| `-p 1883:1883` | No | MQTT broker (expose if external gateways connect directly) | -| `-v /your/data:/app/data` | Yes | Persistent data: SQLite DB, config.json, theme.json | -| `-v /your/Caddyfile:/etc/caddy/Caddyfile:ro` | No | Custom Caddyfile for HTTPS | -| `-v /your/caddy-data:/data/caddy` | No | Caddy TLS certificate storage | -| `-e DISABLE_MOSQUITTO=true` | No | Skip the internal Mosquitto broker (use your own) | -| `-e DISABLE_CADDY=true` | No | Skip the built-in Caddy reverse proxy | -| `-e MQTT_BROKER=mqtt://host:1883` | No | Override MQTT broker URL | - -#### `/app/data/.env` convenience file - -Instead of passing `-e` flags, you can drop a `.env` file in your data volume: - -```bash -# /your/data/.env -DISABLE_MOSQUITTO=true -DISABLE_CADDY=true -MQTT_BROKER=mqtt://my-broker:1883 -``` +### Complete Compose checkout -The entrypoint sources this file before starting services. This works with any launch method (`docker run`, compose, or manage.sh). +Use [DEPLOY.md](../DEPLOY.md#quick-start) to clone and pin the complete repository, populate the private `.env` and select a matching reviewed image. A downloaded YAML alone is insufficient: it extends `docker/postgres.compose.yml` and mounts the committed initialization script. -### Docker Compose (legacy alternative) - -Docker Compose files are maintained for backward compatibility but are no longer the recommended approach. - -```bash -curl -sL https://raw.githubusercontent.com/Kpa-clawbot/CoreScope/master/docker-compose.example.yml \ - -o docker-compose.yml -docker compose up -d +```sh +docker compose -f docker-compose.example.yml up -d +docker compose -f docker-compose.example.yml ps ``` -#### Compose environment variables +The PostgreSQL service creates separate telemetry and account databases. The one-shot bootstrap installs schemas and grants before CoreScope starts. Existing SQLite files require a completed offline import; startup refuses a manually initialized empty replacement. -| Variable | Default | Description | -|----------|---------|-------------| -| `HTTP_PORT` | `80` | Host port for the web UI | -| `DATA_DIR` | `./data` | Host path for persistent data | -| `DISABLE_MOSQUITTO` | `false` | Set `true` to use an external MQTT broker | -| `DISABLE_CADDY` | `false` | Set `true` to skip the built-in Caddy proxy | +The application container runs the server and ingestor, optional Mosquitto, and optional Caddy. PostgreSQL runs separately. Runtime services receive restricted roles; the schema owner belongs only to bootstrap/import. -### manage.sh (legacy alternative) +| Setting | Purpose | +|---|---| +| `HTTP_PORT` | Example deployment's host HTTP port (default 80) | +| `DATA_DIR` | Application state: config, theme, queues, statistics and account backups | +| `POSTGRES_DATA_DIR` | Separate PostgreSQL storage; never copy live files as a logical backup | +| `CORESCOPE_IMAGE` | Reviewed application image tag or digest matching the checkout | +| `DISABLE_MOSQUITTO` | Use an external broker when true | +| `DISABLE_CADDY` | Use an external reverse proxy when true | +| `CORESCOPE_*_PASSWORD` | Distinct owner and runtime credentials; populate every field in `.env.example` | -The `manage.sh` wrapper script provides a setup wizard and convenience commands. It uses Docker Compose internally. See [DEPLOY.md](../DEPLOY.md) for usage. New deployments should prefer bare `docker run`. - -### Image tags - -| Tag | Use case | -|-----|----------| -| `v3.4.1` | Pinned release — recommended for production | -| `v3.4` | Latest patch in the v3.4.x series | -| `v3` | Latest minor+patch in v3.x | -| `latest` | Latest release tag | -| `edge` | Built from master on every push — unstable | +The production and staging variants use their own directory/port variables listed in `.env.example`. `manage.sh` operates those variants; `./manage.sh start --with-staging` clones telemetry only into an empty staging database and keeps staging accounts independent. ### Updating -```bash -docker compose pull -docker compose up -d -``` +Preserve a native backup and private configuration, pin the new checkout and matching image, then use the same Compose variant: -For `docker run` users: - -```bash -docker pull ghcr.io/kpa-clawbot/corescope:latest -docker stop corescope && docker rm corescope -docker run -d --name corescope ... # same flags as before +```sh +docker compose -f docker-compose.example.yml pull +docker compose -f docker-compose.example.yml up -d ``` -Data is preserved in the volume — updates are non-destructive. +A SQLite-to-PostgreSQL upgrade requires the [offline cutover](postgresql-upgrade.md), not only an image update. Keep the previous application and recovery files until validation is complete. --- @@ -148,9 +66,9 @@ Data is preserved in the volume — updates are non-destructive. CoreScope uses a layered configuration system (highest priority wins): -1. **Environment variables** — `MQTT_BROKER`, `DB_PATH`, etc. +1. **Environment variables** — `MQTT_BROKER`, `CORESCOPE_READER_DATABASE_URL`, etc. 2. **`/app/data/config.json`** — full config file (volume-mounted) -3. **Built-in defaults** — work out of the box with no config +3. **Built-in defaults** — database URLs and credentials are still required ### Environment variable overrides @@ -158,21 +76,17 @@ CoreScope uses a layered configuration system (highest priority wins): |----------|---------|-------------| | `MQTT_BROKER` | `mqtt://localhost:1883` | MQTT broker URL (overrides config file) | | `MQTT_TOPIC` | `meshcore/#` | MQTT topic subscription pattern | -| `DB_PATH` | `data/meshcore.db` | SQLite database path | +| `CORESCOPE_READER_DATABASE_URL` | Required by server | Restricted telemetry reader URL | +| `CORESCOPE_WRITER_DATABASE_URL` | Required by ingestor | Restricted telemetry writer URL | +| `CORESCOPE_USERS_DATABASE_URL` | Required when accounts are enabled | Separate account database writer URL | +| `CORESCOPE_APPROVED_CHANNELS_DATABASE_URL` | Required when proposals are enabled | Restricted approved-channel view reader URL | +| `CORESCOPE_STATE_DIR` | `data` for native binaries | Shared filesystem state, separate from database URLs | | `DISABLE_MOSQUITTO` | `false` | Skip the internal Mosquitto broker | | `DISABLE_CADDY` | `false` | Skip the built-in Caddy reverse proxy | ### config.json -For advanced configuration, create a `config.json` and mount it at `/app/data/config.json`: - -```bash -docker run -d --name corescope \ - -p 80:80 \ - -v corescope-data:/app/data \ - -v ./config.json:/app/data/config.json:ro \ - ghcr.io/kpa-clawbot/corescope:latest -``` +Place `config.json` in the mounted state directory (`DATA_DIR` in the example deployment). Use private environment configuration for database passwords; do not publish rendered Compose configuration or URLs. Supported native keys are `databaseURL`, `userManagement.databaseURL` and `stateDir`; legacy `dbPath`/`DB_PATH` settings are refused. See `config.example.json` in the repository for all available options including: - MQTT sources (multiple brokers) @@ -236,13 +150,7 @@ CoreScope receives MeshCore packets via MQTT. The container ships with an intern The built-in Mosquitto broker listens on port 1883 inside the container. Point your MeshCore gateways at it: -```bash -# Expose MQTT port for external gateways -docker run -d --name corescope \ - -p 80:80 -p 1883:1883 \ - -v corescope-data:/app/data \ - ghcr.io/kpa-clawbot/corescope:latest -``` +Publish `1883:1883` on the application service in your Compose override when external gateways need to reach the built-in broker. Keep `DISABLE_MOSQUITTO=false`. The database service must remain on the internal network. ### External broker @@ -250,12 +158,12 @@ To use your own MQTT broker (Mosquitto, EMQX, HiveMQ, etc.): 1. Disable the internal broker: ```bash - -e DISABLE_MOSQUITTO=true + DISABLE_MOSQUITTO=true ``` 2. Point the ingestor at your broker: ```bash - -e MQTT_BROKER=mqtt://your-broker:1883 + MQTT_BROKER=mqtt://your-broker:1883 ``` Or via `config.json`: @@ -340,15 +248,7 @@ The container includes Caddy for automatic Let's Encrypt certificates: } ``` -2. Mount it and expose TLS ports: - ```bash - docker run -d --name corescope \ - -p 80:80 -p 443:443 \ - -v corescope-data:/app/data \ - -v caddy-certs:/data/caddy \ - -v ./Caddyfile:/etc/caddy/Caddyfile:ro \ - ghcr.io/kpa-clawbot/corescope:latest - ``` +2. Set `DISABLE_CADDY=false` for the application service, publish ports 80 and 443, mount the Caddyfile at `/etc/caddy/Caddyfile:ro` and persist certificate storage at `/data/caddy`. Make these changes in the same Compose deployment so the PostgreSQL dependency and role configuration remain intact. Caddy handles certificate issuance and renewal automatically. @@ -544,41 +444,19 @@ docker stats corescope ## Backup & Restore -### Backup - -All persistent data lives in `/app/data`. The critical file is the SQLite database: +Use native custom-format PostgreSQL archives. The authenticated `GET /api/backup` and admin `GET /api/admin/users-backup` endpoints download `.dump` files. The server needs PostgreSQL 18 `pg_dump` on `PATH`; the supplied image includes it. -```bash -# Copy from the Docker volume -docker cp corescope:/app/data/meshcore.db ./backup-$(date +%Y%m%d).db +For the production variant managed by `manage.sh`: -# Or if using a bind mount -cp ./data/meshcore.db ./backup-$(date +%Y%m%d).db +```sh +./manage.sh backup ./backups/instance ``` -Optional files to back up: -- `config.json` — custom configuration -- `theme.json` — custom theme/branding - -### Restore - -```bash -# Stop the container -docker stop corescope - -# Replace the database -docker cp ./backup.db corescope:/app/data/meshcore.db - -# Restart -docker start corescope -``` +This saves `telemetry.dump`, `accounts.dump` and available config/theme/Caddy files. Each archive is consistent, but two sequential dumps are not a shared cross-database snapshot; stop both writers for a coordinated pair. Preserve credentials, queues/state and TLS material separately. Encrypt private backups off-host. -### Automated backups +`./manage.sh restore ` requires the application stopped and both target databases empty. It reapplies runtime grants and leaves the application stopped for validation. It never replaces a nonempty PostgreSQL database or accepts a SQLite file. Use a fresh PostgreSQL storage location while retaining the previous cluster for recovery. -```bash -# cron: daily backup at 3 AM, keep 7 days -0 3 * * * docker cp corescope:/app/data/meshcore.db /backups/corescope-$(date +\%Y\%m\%d).db && find /backups -name "corescope-*.db" -mtime +7 -delete -``` +Account backups contain password hashes, addresses, sessions and tokens. Restoration can revive deleted accounts or revoked credentials. Review the [account recovery procedure](user-guide/accounts.md#backups) before reopening the service. See [native backup and cutover boundaries](postgresql-upgrade.md#native-backups-and-restores) for downloaded archives and rollback. --- @@ -627,16 +505,16 @@ The in-memory packet store grows with retained packets. Configure retention limi `packetStore.maxMemoryMB` bounds the store **and the caches that belong to it** — the decoded-packet cache and per-packet index entries, not just the stored rows. It is enforced in two places: the startup load stops at the budget, and the store evicts oldest-first when it exceeds it. Leaving it unset means no limit. Actual usage is on `/api/perf` as `packetStore.trackedMB`, next to `maxMB`. -### Database locked errors +### Database readiness or privilege errors -SQLite doesn't support concurrent writers well. Ensure only one CoreScope instance accesses the database file. If running multiple containers, each needs its own database. +Confirm the PostgreSQL service is healthy, the owner bootstrap/import finished, and each process uses its intended runtime role. An incomplete import stays closed until resume verifies every requested store. Do not grant owner privileges to runtime processes or manually flip readiness markers. The ingestor owns telemetry writes and retention; multiple active ingestors targeting one store are refused. ### Container unhealthy Check logs: `docker compose logs --tail 50`. Common causes: - Port 3000 already in use inside the container -- Database file permissions (must be writable by the container user) -- Corrupted database — restore from backup +- Missing/wrong runtime credentials, privilege grants or schema readiness +- Database failure — investigate logs and restore a verified archive into a fresh target ### ARM / Raspberry Pi issues diff --git a/docs/postgresql-migration-plan.md b/docs/postgresql-migration-plan.md new file mode 100644 index 000000000..afc7f281f --- /dev/null +++ b/docs/postgresql-migration-plan.md @@ -0,0 +1,55 @@ +# PostgreSQL conversion plan + +Status: approved for implementation; application conversion, migration and benchmark work in progress. + +## Goal and baseline + +Deliver one CoreScope PR that makes PostgreSQL the runtime database, supplies a verified upgrade path for existing SQLite instances, and includes reproducible performance results against unmodified upstream commit `9dbc287579a237ffa744dd0c91fa7227d09763ac` (v3.14.0). + +The user has approved a planned downtime window. This plan covers telemetry and the optional account store, not just the main database. The inventory found 18 telemetry tables plus `packets_v`, 12 account tables, four Go commands, and database calls across 50 production Go files. + +Issue #1442 asks for optional PostgreSQL support and contains objections to previous unmeasured engine changes. Reference that discussion without claiming this full replacement satisfies its dual-backend scope. Open a focused conversion/upgrade/benchmark issue before the implementation PR. Acceptance of the proposal by maintainers is not assumed. + +## Design commitments + +- Target supported PostgreSQL 18, pinning the exact tested patch/image in deployment and benchmark evidence. Use pgx through the existing `database/sql` boundaries and explicit PostgreSQL SQL. Preserve the Go services, PacketStore caches, static frontend and polling architecture. +- Keep telemetry and accounts in separate logical databases on one PostgreSQL service by default. Keep separate credentials and handles: telemetry writer, telemetry reader, account writer, and a restricted approved-channel reader for the ingestor. Migration/bootstrap privileges are separate. Test that runtime roles cannot cross these boundaries. +- Retain SQLite only for legacy upgrade/import support and importer test fixtures. Convert server, ingestor, decrypt/migrate commands, active maintenance tools, both backup paths, account export, diagnostics, deployment and CI. +- Preserve public JSON/WebSocket behavior. Explicitly document the necessary changes to database diagnostics and downloadable native backup formats. Introduce a state-directory setting for the existing small file queues/sidecars; a database URL must never become a filesystem path or appear unredacted in logs. +- Preserve observation identity, null behavior, raw values, observer identities, snapshot consistency and committed feed ordering. Do not increase write concurrency merely because PostgreSQL permits it. Replace SQLite-supplied serialization with explicit transactional/locking rules where required, including account compare-and-set and one-time transitions. + +## Milestones and validation + +### 1. PostgreSQL runtime and schema + +Port schema creation/upgrades, connection setup and SQL to PostgreSQL. Replace hidden observer row IDs with explicit stable keys; use returned IDs rather than SQLite insertion APIs. Keep legacy migration history separate from PostgreSQL schema readiness. Move or retire the server's existing attempted telemetry hash writes rather than enabling them through broad credentials. + +Write failing tests before implementation. Validate fresh/repeated/concurrent startup, role-denied writes, deduplication and nulls, time and ordering behavior, delayed commits/ID gaps, snapshot consistency, retention and API/WebSocket parity. Exercise accounts both enabled and disabled, including login/session preservation, single-use tokens, concurrent settings/proposal changes and deletion/anonymization. + +### 2. Existing-instance upgrade + +Provide an explicit offline import command and operator cutover procedure. Stop telemetry and account writers, retain a WAL-aware consistent SQLite recovery snapshot, normalize supported legacy shapes on a working copy, and load an empty PostgreSQL destination in bounded batches. The supported source-layout/version matrix must be documented and covered by fixtures; unknown/newer schemas get an actionable refusal. + +Preserve all supported application data, observer identity links, IDs, sequence high-water marks, password hashes, sessions, preferences, proposals and notifications. Validate counts, stable logical digests and relationships before recording completion. Partial imports must not be accepted by running services; interruption/resume must be deterministic and bound to the same source/target. Never silently drop incompatible records. + +Test source immutability, current and supported older snapshots, accounts absent, null/orphan links, gaps/deleted high IDs, malformed/unsupported data, cancellation, rerun/resume and completion gating. Restore both native PostgreSQL backup types into fresh databases and verify data plus API behavior. Document rollback before PostgreSQL accepts new writes; do not promise lossless reverse conversion after cutover. Document the planned ingestion gap and preservation of existing configuration/key material. + +### 3. Reproducible performance comparison + +Use an isolated Linux runner, running both exact revisions serially on the same machine with the same compiler, workload, logical data, cache policy, retention, resource limits and durability. Keep SQLite WAL/FULL synchronous settings and PostgreSQL durable settings enabled. Count verified database effects and errors; handler returns and the existing WALCommits counter are not proof of durable success. + +The committed 499-transmission fixture is only a correctness input. Add deterministic synthetic corpora with declared distributions: 30K/90K transmissions/observations, 128K/2.048M for the primary comparison, and 1M/16M for larger-data qualification where runner capacity permits. Record measured sizes, not inferred GB labels. Use five paired repetitions for primary comparisons and report resource/cold-cache limitations explicitly. + +Measure durable ingestion, first/full startup readiness, SQL-backed API latency separately from warm memory/cache controls, concurrent ingest/API load, WebSocket visibility lag, bounded retention, whole-stack CPU/memory/storage/WAL, migration duration and disk headroom. Publish absolute values, p50/p95/p99 where sample counts support them, paired variation, errors, regressions, query plans, manifests and raw results. Do not assume or promise a speedup. If the required comparison cannot execute, the performance deliverable remains incomplete. + +The current Windows host has limited free RAM and its existing Podman configuration fails before listing containers. Do not repair or disturb that global environment for this work; use isolated CI resources and bounded local checks. + +### 4. Operational integration and PR review + +Update configuration examples and comments, environment/secret handling, Compose/supervisor variants, native operation, build packaging, migration/backup instructions and database-specific UI labels. Credentials remain deployment configuration. Document pool/timeout controls; track any genuinely user-facing new tuning control for appropriate admin/customizer exposure. + +Run all Go modules, the authoritative standalone frontend suite, PostgreSQL integration tests, imported-fixture browser E2E, account-on/off journeys, backup/restore tests and real-browser API/UI checks. Retain assertion-based red-to-green CI evidence. Perform independent correctness, database/schema, security and TDD reviews; resolve findings, pass exact-head CI and include the benchmark report before marking the single PR ready for maintainer review. + +## Approval boundary + +The implementation plan has received explicit sign-off under repository `AGENTS.md` rule 5. Approval authorizes the implementation and testing above; it does not authorize merging or deploying the PR to live instances. diff --git a/docs/postgresql-performance.md b/docs/postgresql-performance.md new file mode 100644 index 000000000..0a2247168 --- /dev/null +++ b/docs/postgresql-performance.md @@ -0,0 +1,51 @@ +# PostgreSQL conversion performance + +**Status: not measured.** No same-host, source-pinned paired comparison has been +executed for the final candidate. A PostgreSQL speedup, regression, capacity +increase or equivalent resource cost has not been established. + +The comparison baseline is unmodified upstream commit +`9dbc287579a237ffa744dd0c91fa7227d09763ac`. The final candidate must be identified by +its full commit SHA. The [benchmark harness](../scripts/postgres-benchmark/README.md) +archives both revisions and adds test-only overlays. Production source hashes +before and after measurement prove that a later documentation-only commit did +not change the application being measured. + +The primary corpus is synthetic B: 128,000 transmissions and 2,048,000 +observations, plus the documented nodes, observers, metrics and feature tables. +Five alternating paired runs use one Linux host, Go 1.27.2, PostgreSQL 18.6, +durable settings, equal cache policy, and one shared 3-CPU/6-GiB application plus +database envelope. The actual corpus size, configuration, binaries, settings and +resource enforcement must be present in the result manifest. + +| Acceptance evidence | Current result | +| --- | --- | +| Durable ingestion by event class and full-handler control | Not measured | +| First/full startup, including hot startup and loaded counts | Not measured | +| SQL misses separately from cached/mixed HTTP requests | Not measured | +| Concurrent ingest/API and WebSocket visibility | Not measured | +| Bounded retention under ingestion and empty maintenance pass | Not measured | +| Application + PostgreSQL CPU, cgroup memory, PSS/RSS, I/O, storage and WAL | Not measured | +| Offline migration duration, source immutability, parity and disk headroom | Not measured | +| Five paired repetitions, absolute p50/p95 and adequately sampled p99 | Not measured | +| Actual query plans and raw artifacts | Not measured | +| OS-cold, long soak, L and multi-GiB qualification | Not measured; optional capacity/extended runs | + +The generator and harness controls have local correctness checks. These checks +are preparation evidence, not a database performance comparison. Existing unit +timing assertions likewise do not establish matched upstream/candidate results. + +When the Linux run completes, replace this placeholder with links to the +sanitized manifest and raw artifacts; report absolute measurements, paired +variation, errors, gains and regressions together. Keep p99 unavailable for +classes with fewer than 1,000 successful samples. Preserve a failed or overloaded +run when selecting a lower common offered rate for a separately labeled run. + +The conversion changes more than the driver: packet writes become atomic and +serialized through commit; observer identity becomes explicit; the native SQL +and indexes differ; repeated neighbor-edge writes are aggregated before bounded +upserts; PostgreSQL owns vacuum and checkpoint maintenance. Results therefore +describe this conversion as shipped, not an isolated engine substitution. + +No production data or endpoint is a benchmark input. Publish only the harness's +`public/` bundle, and retain the private disposable state for diagnosing failures. diff --git a/docs/postgresql-upgrade.md b/docs/postgresql-upgrade.md new file mode 100644 index 000000000..99a4e9d6d --- /dev/null +++ b/docs/postgresql-upgrade.md @@ -0,0 +1,116 @@ +# PostgreSQL installation and offline upgrade + +CoreScope uses PostgreSQL for telemetry and optional accounts. The tested database version is PostgreSQL 18.6. Keep the application checkout, image, shared Compose file, bootstrap scripts and configuration example from the same reviewed revision. + +Existing SQLite instances need a planned outage. Stop **both** the ingestor and server: the account janitor, sessions, proposals and notification evaluator also write data. MQTT messages arriving during this window may be lost unless the broker and publishers provide a separately verified replay mechanism. + +## Databases and credentials + +The supplied deployment creates `corescope_telemetry` and `corescope_accounts` on one PostgreSQL service. They are separate logical databases, with `C` text collation. Runtime URLs must name one host and database; use the default `public` schema for native backups. + +| Role | Purpose | +|---|---| +| `corescope_owner` | Offline import, schema bootstrap and privilege grants. Keep it out of runtime processes. | +| `corescope_reader` | Server telemetry reads and native telemetry backups. | +| `corescope_writer` | Ingestor telemetry writes and retention; no schema ownership. | +| `corescope_accounts` | Account data writes; no schema or readiness-marker writes. | +| `corescope_channels` | Ingestor reads of the approved-channel view; no access to passwords or sessions. | + +Copy `.env.example` to a private `.env` and fill every PostgreSQL password field with a different value from `openssl rand -hex 32`. Keep file permissions at `0600`. The Compose files supply runtime URLs; credentials do not belong in Git, screenshots, issue reports or command arguments. On native installations, use private environment configuration with `CORESCOPE_READER_DATABASE_URL`, `CORESCOPE_WRITER_DATABASE_URL`, `CORESCOPE_USERS_DATABASE_URL` and `CORESCOPE_APPROVED_CHANNELS_DATABASE_URL`. + +`databaseURL` replaces `dbPath`; `CORESCOPE_DATABASE_URL` is the per-process fallback. Remove legacy `dbPath`, `userManagement.dbPath` and `DB_PATH` settings after preserving their old values for recovery. `stateDir` / `CORESCOPE_STATE_DIR` is a separate filesystem directory for queues, statistics and account backups. A database URL is never a directory name. + +Main runtime pool limits are four server telemetry connections, one ingestor connection and one account connection. Bootstrap, identity checks and backups open additional connections. Connection attempts default to a ten-second upper bound; a shorter URL `connect_timeout` is retained. Native application dumps also have a ten-minute deadline. Backend credentials and pool limits are not theme/customizer controls. + +## Supported SQLite sources + +| Source | Handling | +|---|---| +| Telemetry v3 with `observations.observer_idx` and the known optional columns | Supported. Missing supported columns/tables are added only to a working copy. | +| Telemetry v2 with `observer_id`, or an older Node-era layout | Refused. Upgrade a recovery copy using the last SQLite release, review the result, then import it. | +| Account `schema_version` 0–5 | Supported, including empty version 0. Each declared version must have its complete historical shape. | +| Accounts absent/disabled | Omit `-users-from-sqlite`; an explicitly configured account destination is initialized empty. | +| Unknown/newer columns, tables, views, triggers or virtual tables | Refused, with no silent field or record loss. | +| Duplicate observation identity, empty required transmission fields or case-colliding node keys | Refused. Perform an explicit, reviewed repair on a disposable recovery copy with the last SQLite release. | + +Malformed JSON strings, empty strings, nulls, timestamp text, fractional scores, mixed-case values, ID zero and ID gaps are preserved when PostgreSQL can represent them. Invalid UTF-8, embedded NUL text, incompatible numeric values and records over the native 64 MiB SQLite length limit are refused. The importer never silently coerces incompatible records into zero or null. + +Identity sequences are reseeded above imported IDs, available `sqlite_sequence` high-water marks and historical references, including orphan observer references and deleted users mentioned by audit records. SQLite integer primary keys without `AUTOINCREMENT` do not retain a deleted high-water mark once all references disappear; that information cannot be recovered or invented. + +## Preflight and recovery + +1. Record the exact application revision/image, broker settings, channel keys, account mail settings, config/theme files and state-directory contents. Keep the private `.env` and PostgreSQL credentials separately recoverable. +2. Arrange the ingestion gap and stop every instance writer. For Compose, stop the application service (`prod`, `staging-go` or `corescope`); leave the new PostgreSQL service available to the importer. +3. Retain the original SQLite directories, including any `-wal` and `-shm` siblings. Prefer a clean writer shutdown. Mount complete source directories, not just a `.db` file, so committed WAL data is available. +4. Use new, empty PostgreSQL destinations. The importer refuses occupied destinations, future schemas and a different source or destination on resume. +5. Allocate a private migration-state directory. It holds a WAL-aware immutable recovery snapshot, a normalized working snapshot and a source/destination manifest for each store. Snapshot creation uses SQLite's backup API so hidden observer/mail-event row IDs do not change. +6. Measure available disk space against the actual source, both snapshot copies, PostgreSQL data/indexes, WAL and temporary work. The required space and duration depend on the data; no fixed multiplier or downtime estimate is guaranteed. + +If an uncheckpointed WAL cannot be opened from a read-only mount, keep the original untouched and make a quiescent filesystem recovery copy of the whole source directory in private writable storage. Import that identical copy. Never omit a nonempty WAL or run `VACUUM` on the only recovery copy. + +## Compose cutover + +Clone and pin the complete repository. A downloaded Compose YAML alone is insufficient: the variants extend `docker/postgres.compose.yml` and mount `docker/postgres-init.sh`. Select the matching image through `CORESCOPE_IMAGE` when using `docker-compose.example.yml`. + +The following uses the example deployment. Substitute the production/staging Compose variant and the actual source filenames for your instance. Preserve the original data directories first. + +```sh +docker compose -f docker-compose.example.yml stop corescope +docker compose -f docker-compose.example.yml up -d --wait postgres +install -d -m 700 migration-state + +docker compose -f docker-compose.example.yml run --rm \ + --entrypoint /app/corescope-migrate \ + -v "$PWD/migration-state:/migration" bootstrap \ + -offline -from-sqlite /app/data/meshcore.db \ + -users-from-sqlite /app/data/users.db -state-dir /migration +``` + +Omit the users source flag only when there is no account source to preserve. For sources outside `/app/data`, add a read-only directory mount and pass its exact file path. The bootstrap service supplies owner URLs through its environment. + +The importer copies in bounded batches, commits progress with each batch, verifies counts and logical digests in primary-key order, reseeds identities and analyzes only the imported tables. Normalization restores original values only in the columns historical migrations can change; it does not rewrite the observation corpus. Both requested stores must verify before cutover finalization. Until then their readiness markers remain false and runtime services refuse them. + +For an interruption, keep the same source files, PostgreSQL destinations and migration-state directory, then repeat the command with `-resume`. A changed source, changed snapshot, unexpected target rows or corrupted progress refuses resume. Do not delete the state directory or manually set readiness true. + +After a successful import: + +```sh +docker compose -f docker-compose.example.yml run --rm \ + --entrypoint /app/corescope-migrate bootstrap \ + -check-import-kind=telemetry -from-sqlite /app/data/meshcore.db + +docker compose -f docker-compose.example.yml run --rm \ + --entrypoint /app/corescope-migrate bootstrap \ + -check-import-kind=accounts -users-from-sqlite /app/data/users.db + +docker compose -f docker-compose.example.yml run --rm bootstrap +docker compose -f docker-compose.example.yml up -d corescope +``` + +Skip the account-source check when accounts were absent. The normal bootstrap verifies any retained SQLite files against the completed import marker and source fingerprint, then grants runtime privileges. A manually initialized empty PostgreSQL database does not satisfy this upgrade guard. Absent and zero-length WAL sidecars have the same fingerprint; every nonempty WAL byte remains significant. + +For native installations, invoke `corescope-migrate` with owner URLs in `CORESCOPE_DATABASE_URL` and, when applicable, `CORESCOPE_USERS_DATABASE_URL`. The same `-offline`, source, state-directory and resume flags apply. `-check-ready` performs read-only schema checks. Run `docker/postgres-grants.sql` as the owner once in each database after bootstrap/import, then start the services with their restricted runtime URLs and a shared state directory. + +## Acceptance and rollback + +Verify packet and observation counts, representative raw/null values, observers and paths, live ingestion/WebSocket visibility, retention, account login, existing sessions/CSRF, proposals, notifications and account export. Test accounts enabled and disabled as applicable. Restore a native backup into fresh isolated databases and exercise the API/login there before declaring recovery proven. + +Rollback is straightforward **before PostgreSQL accepts new writes**: stop the new services and restart the pinned SQLite application with the retained original files, configuration and keys. Once PostgreSQL has accepted new telemetry or account changes, the old SQLite copy is stale. There is no automatic lossless reverse converter; stop and decide explicitly how to handle the new data before reverting. + +Keep at least one verified recovery copy. If you later archive the old SQLite files outside the active state directory, move their WAL siblings with them and preserve the import report and original fingerprint evidence. Do not remove the only rollback copy to reclaim space. + +## Native backups and restores + +`GET /api/backup` and `/api/admin/users-backup` download PostgreSQL custom-format `.dump` archives (`application/octet-stream`). Account snapshots remain subject to their configured schedule and rotation. `pg_dump` 18 must be available to the application; the supplied image includes it. + +`./manage.sh backup ` creates `telemetry.dump`, `accounts.dump` and available config/theme/Caddy files. Each database archive is transactionally consistent; two sequential dumps are not one cross-database snapshot. Stop both writers when you need a coordinated pair. Preserve private credentials, queues/state and TLS material separately as part of instance recovery. Never copy a live PostgreSQL data directory as a logical backup. + +`./manage.sh restore ` requires both databases to be empty and the application stopped. It refuses SQLite files and nonempty destinations, uses transactional `pg_restore`, reapplies runtime grants and retains previous configuration copies. Services remain stopped for validation. For a single downloaded archive, use `pg_restore --no-owner --no-privileges --exit-on-error --single-transaction --dbname=` with connection credentials supplied through private libpq environment settings; then apply the appropriate grants and validation. + +Account restoration can revive deleted accounts, old passwords, revoked sessions and already-used links. Record relevant deletions from the current audit log before replacement. With the restored server still stopped, disable those accounts and invalidate sessions/tokens as required, then repeat account deletion through the application so mail addresses are anonymized and the deletion is audited. See [account recovery](user-guide/accounts.md#backups). + +## Test fixtures and benchmark evidence + +The committed historical fixture contains a duplicate observation identity group. CI explicitly repairs a disposable copy with `scripts/prepare-postgres-fixture.go`, reports before/after counts, freshens/seeds that copy, then runs `scripts/migrate-fixture-hashes.go` using the shared runtime hash implementation. These are test-fixture tools, not production repair flags. The strict importer still refuses the unreviewed original fixture. + +Run the PostgreSQL integration suites with `CORESCOPE_TEST_POSTGRES_URL` pointing to a disposable PostgreSQL 18 administrator connection and native 18.x dump/restore clients on `PATH`. Missing database configuration fails the tests. Performance reports must compare the pinned SQLite baseline and exact candidate with matched compiler, workload, durability and resource limits; migration time includes snapshotting, normalization, verification and `ANALYZE`. No speedup is assumed. diff --git a/docs/user-guide/accounts.md b/docs/user-guide/accounts.md index 2be2eff77..9514198c5 100644 --- a/docs/user-guide/accounts.md +++ b/docs/user-guide/accounts.md @@ -45,7 +45,7 @@ incomplete, and the log says what is missing. |---|---| | `adminEmails` | Addresses that become admin on activation. They cannot be demoted, disabled or deleted from the UI. Remove an address here first. | | `publicBaseUrl` | The address visitors use. Every mail link is built from it, never from the request. It must match the browser origin, because state-changing requests from another origin are refused. | -| `dbPath` | Where accounts are stored. Default: `users.db` next to the analyzer database. | +| `databaseURL` | Required separate PostgreSQL account database, using the restricted account writer. Prefer `CORESCOPE_USERS_DATABASE_URL` in private environment configuration. | | `sessionDays` | Login lifetime, extended while in use. Default 30, maximum 365. | | `trustedProxies` | CIDRs of your reverse proxy, so the per-IP login limits see real client IPs. Without it, behind a proxy every client shares the proxy's IP for the per-IP limits (they are switched off when that IP is loopback or private). Per-address and per-account limits apply either way. | | `mail.webhookSecret` | Enables delivery status (below). At least 16 characters. | @@ -130,11 +130,7 @@ on the *Proposals* tab. decision. - `maxApproved` bounds the decryption work: every approved key is tried on every group message. `perUserPerDay` and `maxPending` limit proposals (HTTP 429). -- The ingestor reads `users.db` read-only, from `userManagement.dbPath` or next to the - analyzer database. Set `dbPath` explicitly when the server and the ingestor are not - given the same analyzer database path (for example `DB_PATH` set for one of them). - Both log the absolute path they use at startup (server: `[users] user management - enabled: db=...`, ingestor: `[proposals] reading approved channels from ...`). +- The ingestor reads only the `approved_channels` view through `CORESCOPE_APPROVED_CHANNELS_DATABASE_URL`. This role can check account readiness and read approved channels, but cannot read users, password hashes, sessions or tokens. It connects to the same account database as the server with a separate restricted credential. - Names: at most 31 bytes including the `#` (MeshCore stores 32 with the terminator), no invisible or control characters (blank fillers such as U+3164 and spaces other than the plain space count as invisible), case-sensitive, not Public. Emoji work, except @@ -163,7 +159,7 @@ watched node changes state. Admins can also watch the instance. - A check runs every `intervalMinutes` (the first one an interval after startup, and only after the startup load). All changes for one user in one check go into one mail. The first check of a newly watched node stores its state without a mail; states - are kept in `users.db`, so a restart does not mail again. + are kept in the account database, so a restart does not mail again. - While ingest is stale (the newest packet in the packet store is older than 30 minutes, for example when the MQTT broker or the ingestor is down), the offline checks for nodes and observers pause: their states stay as they were and nothing is mailed for @@ -191,12 +187,12 @@ watched node changes state. Admins can also watch the instance. ### Backups -`users.db` holds password hashes and addresses. The server keeps its own snapshots of +The separate PostgreSQL account database holds password hashes and addresses. The server keeps its own snapshots of it: at startup when the newest snapshot is older than 24 hours (or there is none), then -about every 24 hours (the check runs hourly). A snapshot is a complete copy named `users-.db` (UTC), -readable by the server's user only, in `backups/` next to `users.db`. After each new +about every 24 hours (the check runs hourly). A snapshot is a native PostgreSQL custom-format archive named `users-.dump` (UTC), +readable by the server's user only, in `backups/` under the configured `stateDir`. After each new snapshot the oldest ones beyond `keep` are deleted, never the one just written. -Temporary files of an interrupted snapshot (`users-.db.tmp`) are +Temporary files of an interrupted snapshot (`users-.dump.tmp`) are deleted once they are older than 24 hours; other files in that directory are never touched. A failed snapshot is logged (`[users] backup failed: ...`) and the next run tries again. @@ -218,16 +214,16 @@ delete that copy. | Key | Meaning | |---|---| | `backup.enabled` | Default `true`, also when the block is absent. `false` turns the snapshots off. | -| `backup.dir` | Where snapshots go. Empty: `backups/` next to `users.db`. A relative path is relative to the server's working directory. | +| `backup.dir` | Where snapshots go. Empty: `backups/` under `stateDir`. A relative path is relative to the server's working directory. | | `backup.keep` | How many snapshots are kept. Default 7, also for 0 or less. | Snapshots on the same disk are lost with that disk. To keep a copy elsewhere, log in as an admin and open `/api/admin/users-backup` in the same browser: it downloads a fresh -snapshot (`corescope-users-.db`) and records it in the audit log +snapshot (`corescope-users-.dump`) and records it in the audit log (`user.backup`). Store the download encrypted: it holds every password hash and address. The analyzer database has its own backup route, `GET /api/backup`. -**What a restore brings back.** `users.db` returns to the moment of the snapshot: +**What a restore brings back.** The account database returns to the moment of the snapshot: - accounts deleted after it, including accounts users deleted themselves; - old passwords of users who changed them since; @@ -237,36 +233,39 @@ The analyzer database has its own backup route, `GET /api/backup`. The audit log is replaced too, so note the deletions before you overwrite it. -**Restore** (not automated). The commands use the `sqlite3` command-line tool; replace -`2026-10-08 12:00:00` with the snapshot's time from its file name (UTC). +**Restore** into a fresh account database while the server and every account writer are stopped. Keep the old database intact for recovery and audit review. Supply PostgreSQL connection credentials through private libpq environment settings (`PGHOST`, `PGUSER`, `PGDATABASE` and a protected password source), never command arguments. -1. Stop the server. -2. List the accounts deleted since the snapshot, from the current `users.db`: +1. On the current database, record accounts deleted since the snapshot. Replace the timestamp with the snapshot's UTC time: - ```sh - sqlite3 users.db "SELECT target_user_id, action, datetime(at, 'unixepoch') FROM audit_log - WHERE action IN ('user.delete', 'user.delete.self') - AND at >= CAST(strftime('%s', '2026-10-08 12:00:00') AS INTEGER);" + ```sql + SELECT target_user_id, action, to_timestamp(at) + FROM audit_log + WHERE action IN ('user.delete', 'user.delete.self') + AND at >= EXTRACT(EPOCH FROM TIMESTAMPTZ '2026-10-08 12:00:00+00'); ``` -3. Copy the snapshot over `users.db`, and delete `users.db-wal` and `users.db-shm` if - they exist. -4. Still with the server stopped, disable the accounts from step 2 (replace `12, 34` - with their ids) and clear all sessions and links, so everyone logs in again: +2. Point the owner connection at an empty account database and restore: ```sh - sqlite3 users.db "UPDATE users SET status = 'disabled' WHERE id IN (12, 34); - DELETE FROM sessions; DELETE FROM tokens;" + pg_restore --no-owner --no-privileges --exit-on-error --single-transaction --dbname="$PGDATABASE" account-backup.dump + psql -X -v ON_ERROR_STOP=1 -f docker/postgres-grants.sql + psql -X -v ON_ERROR_STOP=1 -c 'ANALYZE' + ``` + +3. Still with the server stopped, disable the previously deleted accounts and clear sessions and one-time links. Replace the example IDs with the recorded IDs: + + ```sql + BEGIN; + UPDATE users SET status = 'disabled' WHERE id IN (12, 34); + DELETE FROM sessions; + DELETE FROM tokens; + COMMIT; ``` -5. Start the server. -6. Delete the accounts from step 2 again in the admin area. Deleting there, not in - `sqlite3`, also replaces the addresses in the mail history with a hash and records - the deletion in the audit log. +4. Check schema readiness with the matching `corescope-migrate -check-ready` binary and the owner account URL. Update the restricted runtime URLs to the restored destination and start the server. +5. Delete those accounts again through the admin area. Application deletion also replaces addresses in mail history with a hash and records the deletion in the audit log. -A snapshot made by a newer CoreScope version is refused at startup ("database schema -version N is newer than this binary supports"): run that version or newer. Deleting -`users.db` removes all accounts and nothing else. +Keep notification delivery paused until its restored state has been reviewed; old notification preferences and unsubscribe links also return. A newer schema is refused by older binaries. Never delete or overwrite the current database just to make restoration succeed. See [native backups and restores](../postgresql-upgrade.md#native-backups-and-restores) for the two-database recovery procedure. ## For users diff --git a/internal/dbconfig/dbconfig.go b/internal/dbconfig/dbconfig.go index 21ac79416..ccb0b0555 100644 --- a/internal/dbconfig/dbconfig.go +++ b/internal/dbconfig/dbconfig.go @@ -1,12 +1,13 @@ // Package dbconfig provides the shared DBConfig struct used by both the server -// and ingestor binaries for SQLite vacuum and maintenance settings (#919, #921). +// and ingestor binaries for startup loading and legacy maintenance settings. package dbconfig -// DBConfig controls SQLite vacuum and maintenance behavior (#919). +// DBConfig controls startup loading. The three former SQLite controls remain +// decodable for old config files; PostgreSQL maintenance does not use them. type DBConfig struct { - VacuumOnStartup bool `json:"vacuumOnStartup"` // one-time full VACUUM on startup if auto_vacuum is not INCREMENTAL - IncrementalVacuumPages int `json:"incrementalVacuumPages"` // pages returned to OS per reaper cycle (default 1024) - AnalysisLimit int `json:"analysisLimit"` // index rows ANALYZE visits per index (default 10000); negative disables the planner stats refresh (#2058) + VacuumOnStartup bool `json:"vacuumOnStartup"` // Deprecated: PostgreSQL autovacuum owns maintenance. + IncrementalVacuumPages int `json:"incrementalVacuumPages"` // Deprecated: retained for legacy configuration decoding. + AnalysisLimit int `json:"analysisLimit"` // Deprecated: PostgreSQL owns planner statistics. // Load controls chunked startup loading (#1009). Load *LoadConfig `json:"load,omitempty"` diff --git a/internal/dbschema/collation_test.go b/internal/dbschema/collation_test.go new file mode 100644 index 000000000..29faacf6a --- /dev/null +++ b/internal/dbschema/collation_test.go @@ -0,0 +1,31 @@ +package dbschema + +import ( + "testing" + + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +func TestTelemetryTextUsesBinaryCollation(t *testing.T) { + db, err := pgutil.Open(pgtest.NewSchema(t), false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + if err := Apply(db, t.Logf); err != nil { + t.Fatal(err) + } + var inherited int + err = db.QueryRow(`SELECT COUNT(*) FROM pg_attribute a + JOIN pg_class c ON c.oid=a.attrelid JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname=current_schema() AND c.relkind='r' AND c.relname<>'corescope_schema' + AND a.attnum>0 AND NOT a.attisdropped AND a.atttypid='text'::regtype + AND a.attcollation<>'"C"'::regcollation`).Scan(&inherited) + if err != nil { + t.Fatal(err) + } + if inherited != 0 { + t.Fatalf("%d telemetry text columns inherit host-specific collation instead of SQLite byte order", inherited) + } +} diff --git a/internal/dbschema/dbschema.go b/internal/dbschema/dbschema.go index ba0533ae7..46cccfdd6 100644 --- a/internal/dbschema/dbschema.go +++ b/internal/dbschema/dbschema.go @@ -1,753 +1,267 @@ -// Package dbschema centralizes schema migrations and read-side schema -// assertions for the CoreScope SQLite DB. Per issue #1287 the writer -// (cmd/ingestor) owns ALL CREATE/ALTER/INSERT/UPDATE/DELETE on schema -// objects; the server (cmd/server) only ASSERTS that the schema is in -// the expected shape and refuses to start otherwise. -// -// Apply(rw, log) runs from the ingestor at startup BEFORE subscribing to -// MQTT. AssertReady(ro) runs from the server at startup and returns an -// error listing every missing column/index/table. -// -// INVARIANT (#1321): Any optional column the SERVER detects via PRAGMA -// (e.g. cmd/server/db.go detectSchema → hasScopeName, hasDefaultScope, -// hasObsRawHex, hasResolvedPath) MUST be added here, in Apply, AND -// asserted in AssertReady. Adding it only to cmd/ingestor/db.go -// applySchema reintroduces the startup-race bug from #1321: the -// server's PRAGMA can run before the ingestor finishes applySchema, -// the boolean caches `false`, and feature endpoints permanently 500 -// until the server restarts. Source of truth lives here — full stop. +// Package dbschema owns the canonical PostgreSQL telemetry schema. Only the +// offline migration command uses Apply; both runtime processes assert readiness. package dbschema import ( + "context" "database/sql" - "errors" + _ "embed" "fmt" "strings" ) -// Logger is the minimal logging surface used by Apply. Both cmd/server -// and cmd/ingestor satisfy this with the stdlib `log` package's Printf -// (passed as a closure to avoid an indirect log dependency here). -type Logger func(format string, args ...interface{}) +type Logger func(string, ...interface{}) -// Apply runs every server-side ensure_* migration against the given -// read-write SQLite connection. Each operation is idempotent -// (IF NOT EXISTS / column-probe-before-ALTER). Safe to call repeatedly. +// SchemaSQL is inspectable native PostgreSQL DDL, shared with the importer. // -// Called by the ingestor at startup. The server MUST NOT call this — -// it only calls AssertReady. -func Apply(rw *sql.DB, logf Logger) error { - if logf == nil { - logf = func(string, ...interface{}) {} - } - if err := ensureServerIndexes(rw); err != nil { - return fmt.Errorf("ensure server indexes: %w", err) - } - if err := ensureObservationsDedupIndex(rw, logf); err != nil { - return fmt.Errorf("ensure observations dedup index: %w", err) - } - if err := ensureNeighborEdgesTable(rw); err != nil { - return fmt.Errorf("ensure neighbor_edges: %w", err) - } - if err := ensureInactiveNodesTable(rw); err != nil { - return fmt.Errorf("ensure inactive_nodes: %w", err) - } - if err := ensureResolvedPathColumn(rw, logf); err != nil { - return fmt.Errorf("ensure resolved_path: %w", err) - } - if err := ensureObserverInactiveColumn(rw, logf); err != nil { - return fmt.Errorf("ensure observers.inactive: %w", err) - } - if err := ensureLastPacketAtColumn(rw, logf); err != nil { - return fmt.Errorf("ensure observers.last_packet_at: %w", err) - } - if err := ensureObserverIATAColumn(rw, logf); err != nil { - return fmt.Errorf("ensure observers.iata: %w", err) - } - // Must run after ensureObserverIATAColumn: the expression index below - // is on observers.iata, which that step guarantees exists. - if err := ensureChannelIndexes(rw); err != nil { - return fmt.Errorf("ensure channel indexes: %w", err) - } - if err := ensureForeignAdvertColumn(rw, logf); err != nil { - return fmt.Errorf("ensure foreign_advert: %w", err) - } - if err := ensureFromPubkeyColumn(rw, logf); err != nil { - return fmt.Errorf("ensure from_pubkey: %w", err) - } - if err := ensureScopeNameColumn(rw, logf); err != nil { - return fmt.Errorf("ensure scope_name: %w", err) - } - if err := ensureDefaultScopeColumns(rw, logf); err != nil { - return fmt.Errorf("ensure default_scope: %w", err) - } - if err := ensureConfiguredScopeColumns(rw, logf); err != nil { - return fmt.Errorf("ensure configured_scope: %w", err) - } - if err := ensureObservationsRawHexColumn(rw, logf); err != nil { - return fmt.Errorf("ensure observations.raw_hex: %w", err) - } - if err := ensureMultibyteCapColumns(rw, logf); err != nil { - return fmt.Errorf("ensure multibyte_cap columns: %w", err) - } - if err := ensureObserverNaiveClockColumns(rw, logf); err != nil { - return fmt.Errorf("ensure observers naive-clock columns: %w", err) - } - if err := ensureObserverCanRelayColumn(rw, logf); err != nil { - return fmt.Errorf("ensure observers.can_relay: %w", err) - } - if err := ensureObserverCanRelaySeenColumn(rw, logf); err != nil { - return fmt.Errorf("ensure observers.can_relay_seen: %w", err) - } - // #1690: denormalized last_seen on transmissions so cold-load filters - // on effective recency rather than first-ever first_seen. The column - // add + index creation are cheap (single ALTER, indexed INTEGER - // column); the *backfill* from observations is potentially expensive - // and runs via RunAsyncMigration from the ingestor (see - // cmd/ingestor/db.go OpenStore). - if err := ensureTransmissionsLastSeenColumn(rw, logf); err != nil { - return fmt.Errorf("ensure transmissions.last_seen: %w", err) - } - if err := ensureAdvertEvidence(rw); err != nil { - return fmt.Errorf("ensure advert route evidence: %w", err) - } - // Advert evidence is intentionally not required by AssertReady: a legacy - // read-only server reports unknown until this table/backfill is available, - // and re-probes absence instead of caching the startup race permanently. - return nil -} - -// AssertReady verifies the schema is in the expected shape. The server -// calls this at startup against a read-only connection; if it returns -// non-nil, the server MUST fatal-log and exit so the operator restarts -// the ingestor (which owns migrations). -func AssertReady(ro *sql.DB) error { - var missing []string - - mustCol := func(table, col string) { - has, err := TableHasColumn(ro, table, col) - if err != nil { - missing = append(missing, fmt.Sprintf("%s.%s (probe error: %v)", table, col, err)) - return - } - if !has { - missing = append(missing, fmt.Sprintf("%s.%s", table, col)) - } - } - mustTable := func(name string) { - var n int - err := ro.QueryRow(`SELECT 1 FROM sqlite_master WHERE type='table' AND name=?`, name).Scan(&n) - if errors.Is(err, sql.ErrNoRows) { - missing = append(missing, "table:"+name) - } else if err != nil { - missing = append(missing, fmt.Sprintf("table:%s (probe error: %v)", name, err)) - } - } +//go:embed schema.sql +var SchemaSQL string - mustTable("neighbor_edges") - mustCol("observations", "resolved_path") - mustCol("observers", "inactive") - mustCol("observers", "last_packet_at") - mustCol("observers", "iata") - mustCol("nodes", "foreign_advert") - mustCol("inactive_nodes", "foreign_advert") - mustCol("transmissions", "from_pubkey") - // #1690: denormalized recency axis for cold-load. Owned by ingestor - // (cmd/ingestor/db.go OpenStore: ALTER + index + async backfill); - // server reads it to filter the hot-window query. - mustCol("transmissions", "last_seen") - // #1321: server's detectSchema PRAGMA-detects these and caches a - // boolean. To kill the startup race they're owned + asserted here. - mustCol("transmissions", "scope_name") - mustCol("nodes", "default_scope") - mustCol("inactive_nodes", "default_scope") - // #1865: confirmed region scopes from the observer /neighbors report. - mustCol("nodes", "configured_scope") - mustCol("inactive_nodes", "configured_scope") - mustCol("observations", "raw_hex") - // Multi-byte capability cache (#1324 follow-up; PR #903 surface). - // Owned by ingestor — server reads these for O(1) /api/nodes - // enrichment, ingestor's RunMultibyteCapPersist is the only writer. - mustCol("nodes", "multibyte_sup") - mustCol("nodes", "multibyte_evidence") - mustCol("inactive_nodes", "multibyte_sup") - mustCol("inactive_nodes", "multibyte_evidence") - // Issue #1478: per-observer naive-clock skew tracking. Server reads - // all three to populate ObserverResp.ClockNaive / ClockSkew* fields. - mustCol("observers", "clock_skew_seconds") - mustCol("observers", "clock_skew_count_24h") - mustCol("observers", "clock_last_naive_at") - // Issue #1290: firmware 1.16 publishes a `repeat: on|off` flag in - // the MQTT /status JSON. Ingestor persists it as can_relay; server - // reads it to filter listener-only observers out of the path-hop - // disambiguator candidate set. Default 1 preserves prior behavior - // for legacy observers that never sent the field. - mustCol("observers", "can_relay") - // Issue #1290 follow-up (PR #1624 MAJOR-2): tri-state badge. The - // can_relay column defaults to 1 at INSERT, so we cannot distinguish - // "firmware confirmed repeater" from "legacy observer that never - // sent a repeat field". can_relay_seen=1 means the ingestor wrote - // an explicit value; can_relay_seen=0 means leave the UI badge - // unset (unknown state). - mustCol("observers", "can_relay_seen") +const Version = 1 - if len(missing) > 0 { - return fmt.Errorf("schema not migrated by ingestor; restart ingestor first. missing: %s", - strings.Join(missing, ", ")) - } - return nil -} +// WriterLockKey serializes ID allocation through commit, including backfills, +// across ingestor processes. The server's ID cursor relies on this ordering. +const WriterLockKey int64 = 18950410699801933 -// Querier is the read surface shared by *sql.DB and *sql.Tx. -// -// Not *sql.Conn: it exposes only QueryContext/QueryRowContext, so it does not -// satisfy this. Widen the interface to the Context variants if a caller ever -// needs one. -// -// It exists so schema probes can run on whichever handle the caller already -// holds. Taking *sql.DB unconditionally is a deadlock waiting to happen: a -// caller inside a transaction has the connection checked out, and on a pool -// capped at one connection — which is what cmd/ingestor runs — a probe against -// the pool waits forever for the connection its own transaction is holding. type Querier interface { - Query(query string, args ...any) (*sql.Rows, error) - QueryRow(query string, args ...any) *sql.Row + Query(string, ...any) (*sql.Rows, error) + QueryRow(string, ...any) *sql.Row } -// TableHasColumn reports whether the given table has the given column. -// Exported because tests and the read-side need it without re-implementing. -// -// Pass the transaction, not the pool, when you are inside one. See Querier. -func TableHasColumn(db Querier, table, column string) (bool, error) { - rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%s)", table)) - if err != nil { - return false, err - } - defer rows.Close() - for rows.Next() { - var cid int - var name string - var ctype sql.NullString - var notnull, pk int - var dflt sql.NullString - if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil { - return false, err - } - if name == column { - return true, nil - } - } - return false, rows.Err() -} +func Apply(db *sql.DB, logf Logger) error { return apply(db, logf, true) } -// ─── ensure_* helpers (writer side) ──────────────────────────────────────── +// ApplyForImport atomically creates an unready target; runtime cannot see a +// temporary ready-empty schema while the importer starts its first batch. +func ApplyForImport(db *sql.DB, logf Logger) error { return apply(db, logf, false) } -func ensureServerIndexes(rw *sql.DB) error { - stmts := []string{ - `CREATE INDEX IF NOT EXISTS idx_transmissions_first_seen ON transmissions(first_seen)`, - `CREATE INDEX IF NOT EXISTS idx_transmissions_hash ON transmissions(hash)`, - `CREATE INDEX IF NOT EXISTS idx_transmissions_payload_type ON transmissions(payload_type)`, - `CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp)`, - `CREATE INDEX IF NOT EXISTS idx_observations_transmission_id ON observations(transmission_id)`, - // Composite covers GetChannelMessages' grouped MAX(timestamp) per - // transmission_id (issue #1366 / PR #1368). With this index sqlite can - // satisfy the aggregate index-only without touching the heap. - `CREATE INDEX IF NOT EXISTS idx_observations_tx_ts ON observations(transmission_id, timestamp)`, - } - for _, s := range stmts { - if _, err := rw.Exec(s); err != nil { - return fmt.Errorf("ensure index %q: %w", s, err) - } - } - // observer_idx (v3) vs observer_id (v2) — probe + index the matching one. - hasIdx, err := TableHasColumn(rw, "observations", "observer_idx") +// ApplyForImportTx lets the importer commit schema and source/progress metadata +// in one transaction. The caller owns commit/rollback; no ready window exists. +func ApplyForImportTx(tx *sql.Tx, _ Logger) error { return applyTx(tx, false) } +func apply(db *sql.DB, logf Logger, initialReady bool) error { + tx, err := db.BeginTx(context.Background(), nil) if err != nil { return err } - if hasIdx { - if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_observations_observer_idx ON observations(observer_idx)`); err != nil { - return err - } - } - hasID, err := TableHasColumn(rw, "observations", "observer_id") - if err != nil { + defer tx.Rollback() + if err := applyTx(tx, initialReady); err != nil { return err } - if hasID { - if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_observations_observer_id ON observations(observer_id)`); err != nil { - return err - } - } - return nil -} - -// ensureChannelIndexes speeds up /api/channels and -// /api/channels/{hash}/messages. -// -// - idx_transmissions_channel_hash_payload adds first_seen to the -// existing channel_hash index (cmd/ingestor/db.go migration -// 'channel_hash_v1', idx_tx_channel_hash) so GetChannels' "latest -// message" correlated subquery (channel_hash + payload_type, ORDER BY -// first_seen DESC LIMIT 1) and GetChannelMessages' count/page queries -// resolve as index-order scans instead of sorting per channel. -// - idx_observers_iata_norm indexes the UPPER(TRIM(iata)) expression -// that GetChannels/GetEncryptedChannels/GetChannelMessages use for -// region filtering; a plain index on iata can't be used under that -// expression, so the expression itself is indexed. -// -// transmissions.channel_hash itself is added by the ingestor's legacy -// 'channel_hash_v1' migration (cmd/ingestor/db.go), not by this package, -// so it isn't guaranteed to exist yet on every DB Apply runs against — -// probe before indexing, same pattern as the observer_idx/observer_id -// probe in ensureServerIndexes above. -func ensureChannelIndexes(rw *sql.DB) error { - hasChannelHash, err := TableHasColumn(rw, "transmissions", "channel_hash") - if err != nil { + if err := tx.Commit(); err != nil { return err } - if hasChannelHash { - if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_transmissions_channel_hash_payload ON transmissions(channel_hash, payload_type, first_seen)`); err != nil { - return fmt.Errorf("ensure idx_transmissions_channel_hash_payload: %w", err) + if initialReady { + if err := AssertReady(db); err != nil { + return err } } - if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_observers_iata_norm ON observers(UPPER(TRIM(iata)))`); err != nil { - return fmt.Errorf("ensure idx_observers_iata_norm: %w", err) + if logf != nil { + logf("[dbschema] PostgreSQL telemetry schema %d initialized", Version) } return nil } - -func ensureNeighborEdgesTable(rw *sql.DB) error { - _, err := rw.Exec(`CREATE TABLE IF NOT EXISTS neighbor_edges ( - node_a TEXT NOT NULL, - node_b TEXT NOT NULL, - count INTEGER DEFAULT 1, - last_seen TEXT, - PRIMARY KEY (node_a, node_b) - )`) - return err -} - -// ensureInactiveNodesTable creates the inactive_nodes table if missing. -// The ingestor's applySchema also creates this table — duplicating it -// here makes dbschema.Apply self-sufficient when called against a -// fixture DB that pre-dates the soft-delete feature (e.g. CI's -// test-fixtures/e2e-fixture.db, which never had any inactive rows). -// Schema kept in sync with cmd/ingestor/db.go:applySchema. -func ensureInactiveNodesTable(rw *sql.DB) error { - _, err := rw.Exec(`CREATE TABLE IF NOT EXISTS inactive_nodes ( - public_key TEXT PRIMARY KEY, - name TEXT, - role TEXT, - lat REAL, - lon REAL, - last_seen TEXT, - first_seen TEXT, - advert_count INTEGER DEFAULT 0, - battery_mv INTEGER, - temperature_c REAL, - foreign_advert INTEGER DEFAULT 0 - )`) - if err != nil { +func applyTx(tx *sql.Tx, initialReady bool) error { + var err error + if _, err = tx.Exec(`SELECT pg_advisory_xact_lock(hashtextextended(current_schema(),$1))`, WriterLockKey); err != nil { return err } - _, err = rw.Exec(`CREATE INDEX IF NOT EXISTS idx_inactive_nodes_last_seen ON inactive_nodes(last_seen)`) - return err -} - -func ensureResolvedPathColumn(rw *sql.DB, logf Logger) error { - has, err := TableHasColumn(rw, "observations", "resolved_path") - if err != nil { - return err + if _, err = tx.Exec(`CREATE TABLE IF NOT EXISTS corescope_schema (version INTEGER NOT NULL, kind TEXT PRIMARY KEY, ready BOOLEAN NOT NULL)`); err != nil { + return fmt.Errorf("create readiness marker: %w", err) } - if has { + var version int + var ready bool + err = tx.QueryRow(`SELECT version,ready FROM corescope_schema WHERE kind='telemetry'`).Scan(&version, &ready) + if err == nil { + if version != Version { + return fmt.Errorf("unsupported telemetry schema version %d (binary supports %d)", version, Version) + } + if !ready { + return fmt.Errorf("telemetry import is incomplete; finish or restore the offline import") + } + if !initialReady { + return fmt.Errorf("import target is already initialized") + } return nil } - if _, err := rw.Exec("ALTER TABLE observations ADD COLUMN resolved_path TEXT"); err != nil { + if err != sql.ErrNoRows { return err } - logf("[dbschema] added resolved_path column to observations") - return nil -} - -func ensureObserverInactiveColumn(rw *sql.DB, logf Logger) error { - has, err := TableHasColumn(rw, "observers", "inactive") - if err != nil { + var count int + if err = tx.QueryRow(`SELECT count(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name <> 'corescope_schema'`).Scan(&count); err != nil { return err } - if has { - return nil + if count != 0 { + return fmt.Errorf("refusing to initialize a non-empty unversioned schema") + } + if _, err = tx.Exec(SchemaSQL); err != nil { + return fmt.Errorf("create telemetry schema: %w", err) } - if _, err := rw.Exec("ALTER TABLE observers ADD COLUMN inactive INTEGER DEFAULT 0"); err != nil { + if _, err = tx.Exec(`INSERT INTO corescope_schema(version,kind,ready) VALUES($1,'telemetry',$2)`, Version, initialReady); err != nil { return err } - logf("[dbschema] added inactive column to observers") + return nil } -func ensureLastPacketAtColumn(rw *sql.DB, logf Logger) error { - has, err := TableHasColumn(rw, "observers", "last_packet_at") - if err != nil { - return err +func AssertReady(db *sql.DB) error { + var version int + var ready bool + if err := db.QueryRow(`SELECT version,ready FROM corescope_schema WHERE kind='telemetry'`).Scan(&version, &ready); err != nil { + return fmt.Errorf("telemetry schema is not initialized; run the offline migration command: %w", err) } - if has { - return nil + if version != Version { + return fmt.Errorf("unsupported telemetry schema version %d (binary supports %d)", version, Version) } - if _, err := rw.Exec("ALTER TABLE observers ADD COLUMN last_packet_at TEXT"); err != nil { - return err + if !ready { + return fmt.Errorf("telemetry import is incomplete; finish or restore the offline import") } - logf("[dbschema] added last_packet_at column to observers") - return nil -} - -func ensureObserverIATAColumn(rw *sql.DB, logf Logger) error { - has, err := TableHasColumn(rw, "observers", "iata") + rows, err := db.Query(`SELECT table_name,column_name,data_type,is_identity FROM information_schema.columns WHERE table_schema=current_schema()`) if err != nil { return err } - if has { - return nil - } - if _, err := rw.Exec("ALTER TABLE observers ADD COLUMN iata TEXT"); err != nil { - return err - } - logf("[dbschema] added iata column to observers") - return nil -} - -func ensureForeignAdvertColumn(rw *sql.DB, logf Logger) error { - for _, table := range []string{"nodes", "inactive_nodes"} { - has, err := TableHasColumn(rw, table, "foreign_advert") - if err != nil { - return fmt.Errorf("inspect %s: %w", table, err) - } - if has { - continue - } - if _, err := rw.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN foreign_advert INTEGER DEFAULT 0", table)); err != nil { + defer rows.Close() + have := make(map[string]bool) + types := make(map[string]string) + identities := make(map[string]bool) + for rows.Next() { + var table, col, dataType, identity string + if err := rows.Scan(&table, &col, &dataType, &identity); err != nil { return err } - logf("[dbschema] added foreign_advert column to %s", table) + have[table+"."+col] = true + types[table+"."+col] = strings.ToUpper(dataType) + identities[table+"."+col] = identity == "YES" } - return nil -} - -func ensureFromPubkeyColumn(rw *sql.DB, logf Logger) error { - has, err := TableHasColumn(rw, "transmissions", "from_pubkey") - if err != nil { + if err := rows.Err(); err != nil { return err } - if !has { - if _, err := rw.Exec("ALTER TABLE transmissions ADD COLUMN from_pubkey TEXT"); err != nil { - return err + var missing []string + for _, table := range Tables { + for _, col := range table.Columns { + key := table.Name + "." + col.Name + if !have[key] { + missing = append(missing, key) + } else if types[key] != col.Type || identities[key] != col.Identity { + missing = append(missing, key+" (unexpected type or identity)") + } } - logf("[dbschema] added from_pubkey column to transmissions (#1143)") } - if _, err := rw.Exec("CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey)"); err != nil { - return err - } - return nil -} - -// ensureScopeNameColumn adds transmissions.scope_name (+ partial index) and -// records the legacy `_migrations` marker so the ingestor's old gated path -// stays idempotent on existing DBs. Moved here from cmd/ingestor/db.go in -// #1321 to be the canonical source of truth for the optional column the -// server PRAGMA-detects as hasScopeName. -func ensureScopeNameColumn(rw *sql.DB, logf Logger) error { - if err := ensureMigrationsTable(rw); err != nil { - return err + if !have["packets_v.id"] { + missing = append(missing, "packets_v") } - has, err := TableHasColumn(rw, "transmissions", "scope_name") + indexRows, err := db.Query(`SELECT indexname FROM pg_indexes WHERE schemaname=current_schema()`) if err != nil { return err } - if !has { - if _, err := rw.Exec(`ALTER TABLE transmissions ADD COLUMN scope_name TEXT DEFAULT NULL`); err != nil { - return fmt.Errorf("alter transmissions add scope_name: %w", err) + indexes := make(map[string]bool) + for indexRows.Next() { + var name string + if err := indexRows.Scan(&name); err != nil { + indexRows.Close() + return err } - logf("[dbschema] added scope_name column to transmissions (#899)") - } - if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_tx_scope_name ON transmissions(scope_name) WHERE scope_name IS NOT NULL`); err != nil { - return fmt.Errorf("create idx_tx_scope_name: %w", err) + indexes[name] = true } - if _, err := rw.Exec(`INSERT OR IGNORE INTO _migrations (name) VALUES ('scope_name_v1')`); err != nil { - return fmt.Errorf("record scope_name_v1: %w", err) - } - return nil -} - -// ensureDefaultScopeColumns adds default_scope to nodes + inactive_nodes -// and records the `_migrations` marker. Source of truth lives here per -// #1321 (was previously cmd/ingestor/db.go only). -func ensureDefaultScopeColumns(rw *sql.DB, logf Logger) error { - if err := ensureMigrationsTable(rw); err != nil { + if err := indexRows.Err(); err != nil { + indexRows.Close() return err } - for _, table := range []string{"nodes", "inactive_nodes"} { - has, err := TableHasColumn(rw, table, "default_scope") - if err != nil { - return fmt.Errorf("inspect %s.default_scope: %w", table, err) - } - if has { - continue - } - if _, err := rw.Exec(fmt.Sprintf(`ALTER TABLE %s ADD COLUMN default_scope TEXT DEFAULT NULL`, table)); err != nil { - return fmt.Errorf("alter %s add default_scope: %w", table, err) + indexRows.Close() + for _, name := range Indexes { + if !indexes[name] { + missing = append(missing, "index:"+name) } - logf("[dbschema] added default_scope column to %s (#899)", table) } - if _, err := rw.Exec(`INSERT OR IGNORE INTO _migrations (name) VALUES ('nodes_default_scope_v1')`); err != nil { - return fmt.Errorf("record nodes_default_scope_v1: %w", err) + if len(missing) > 0 { + return fmt.Errorf("incomplete telemetry schema: %s", strings.Join(missing, ", ")) } return nil } -// ensureConfiguredScopeColumns adds nodes.configured_scope + -// nodes.configured_scope_at (and mirrors on inactive_nodes) for #1865. -// Unlike default_scope (inferred from observed advert transport scope, and -// overwritten on every observation), configured_scope holds the region scopes -// a node has CONFIGURED, taken as concrete evidence from the observer -// /neighbors report — written only for the observer's own `self` scopes and -// for neighbors whose OTA scope query returned status="responded". The -// server PRAGMA-detects configured_scope as hasConfiguredScope. -func ensureConfiguredScopeColumns(rw *sql.DB, logf Logger) error { - if err := ensureMigrationsTable(rw); err != nil { - return err - } - for _, table := range []string{"nodes", "inactive_nodes"} { - for _, col := range []string{"configured_scope", "configured_scope_at"} { - has, err := TableHasColumn(rw, table, col) - if err != nil { - return fmt.Errorf("inspect %s.%s: %w", table, col, err) - } - if has { - continue - } - if _, err := rw.Exec(fmt.Sprintf(`ALTER TABLE %s ADD COLUMN %s TEXT DEFAULT NULL`, table, col)); err != nil { - return fmt.Errorf("alter %s add %s: %w", table, col, err) - } - logf("[dbschema] added %s column to %s (#1865)", col, table) - } - } - if _, err := rw.Exec(`INSERT OR IGNORE INTO _migrations (name) VALUES ('nodes_configured_scope_v1')`); err != nil { - return fmt.Errorf("record nodes_configured_scope_v1: %w", err) - } - return nil +func TableHasColumn(db Querier, table, column string) (bool, error) { + var found bool + err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name=$1 AND column_name=$2)`, table, column).Scan(&found) + return found, err } -// ensureObservationsRawHexColumn adds observations.raw_hex (#881). -// Source of truth lives here per #1321 (was previously cmd/ingestor/db.go only): -// the server PRAGMA-detects this column as hasObsRawHex. -func ensureObservationsRawHexColumn(rw *sql.DB, logf Logger) error { - if err := ensureMigrationsTable(rw); err != nil { - return err - } - has, err := TableHasColumn(rw, "observations", "raw_hex") +// AssertWriter refuses migration-owner credentials in the runtime process. +func AssertWriter(db *sql.DB) error { + var elevated, createSchema, ownsSchema bool + err := db.QueryRow(`SELECT r.rolsuper OR r.rolcreatedb OR r.rolcreaterole, has_schema_privilege(current_schema(),'CREATE') OR has_database_privilege(current_database(),'CREATE'), pg_has_role(current_user,n.nspowner,'MEMBER') OR EXISTS(SELECT 1 FROM pg_class c WHERE c.relnamespace=n.oid AND pg_has_role(current_user,c.relowner,'MEMBER')) OR has_table_privilege('corescope_schema','INSERT,UPDATE,DELETE,TRUNCATE,TRIGGER') FROM pg_roles r JOIN pg_namespace n ON n.nspname=current_schema() WHERE r.rolname=current_user`).Scan(&elevated, &createSchema, &ownsSchema) if err != nil { return err } - if !has { - if _, err := rw.Exec(`ALTER TABLE observations ADD COLUMN raw_hex TEXT`); err != nil { - return fmt.Errorf("alter observations add raw_hex: %w", err) - } - logf("[dbschema] added raw_hex column to observations (#881)") - } - if _, err := rw.Exec(`INSERT OR IGNORE INTO _migrations (name) VALUES ('observations_raw_hex_v1')`); err != nil { - return fmt.Errorf("record observations_raw_hex_v1: %w", err) + if elevated || createSchema || ownsSchema { + return fmt.Errorf("telemetry runtime requires a restricted writer role without schema ownership, CREATE, or administrative privileges") } - return nil -} - -// ensureMigrationsTable is idempotent — the ingestor's applySchema also -// creates this table on legacy paths. Keeping it here lets the gated -// `INSERT OR IGNORE` markers above stay self-sufficient even when Apply -// runs against a brand-new fixture DB (e.g. dbschema_test.go). -func ensureMigrationsTable(rw *sql.DB) error { - _, err := rw.Exec(`CREATE TABLE IF NOT EXISTS _migrations (name TEXT PRIMARY KEY)`) - return err -} - -// SoftDeleteBlacklistedObservers marks the given observer IDs as -// inactive=1 (case-insensitive match). Returns count affected. -// Writer-side helper; ingestor calls it at startup with the operator -// blacklist (read from config). -func SoftDeleteBlacklistedObservers(rw *sql.DB, blacklist []string) (int64, error) { - placeholders := make([]string, 0, len(blacklist)) - args := make([]interface{}, 0, len(blacklist)) - for _, pk := range blacklist { - t := strings.TrimSpace(pk) - if t == "" { - continue + for _, table := range Tables { + var allowed bool + if err := db.QueryRow(`SELECT has_table_privilege($1,'SELECT') AND has_table_privilege($1,'INSERT') AND has_table_privilege($1,'UPDATE') AND has_table_privilege($1,'DELETE')`, table.Name).Scan(&allowed); err != nil { + return err } - placeholders = append(placeholders, "LOWER(?)") - args = append(args, t) - } - if len(placeholders) == 0 { - return 0, nil - } - q := "UPDATE observers SET inactive = 1 WHERE LOWER(id) IN (" + - strings.Join(placeholders, ",") + ") AND (inactive IS NULL OR inactive = 0)" - res, err := rw.Exec(q, args...) - if err != nil { - return 0, err - } - n, _ := res.RowsAffected() - return n, nil -} - -// ensureMultibyteCapColumns adds the multi-byte capability cache columns -// to nodes / inactive_nodes (PR #903, canonical owner per #1324 -// follow-up). These columns are populated by the ingestor's -// RunMultibyteCapPersist from snapshot files written by the server's -// analytics cycle; the server is read-only since #1289 and MUST NOT -// write here. The schema itself lives here in dbschema (the writer -// owns migrations, the read-only server merely AssertReady's them). -func ensureMultibyteCapColumns(rw *sql.DB, logf Logger) error { - for _, table := range []string{"nodes", "inactive_nodes"} { - hasSup, err := TableHasColumn(rw, table, "multibyte_sup") - if err != nil { - return fmt.Errorf("inspect %s.multibyte_sup: %w", table, err) + if !allowed { + return fmt.Errorf("telemetry writer lacks data privileges on %s", table.Name) } - if !hasSup { - if _, err := rw.Exec(fmt.Sprintf( - "ALTER TABLE %s ADD COLUMN multibyte_sup INTEGER NOT NULL DEFAULT 0", table)); err != nil { - return fmt.Errorf("add %s.multibyte_sup: %w", table, err) + for _, column := range table.Columns { + if !column.Identity { + continue } - logf("[dbschema] added multibyte_sup column to %s", table) - } - hasEvid, err := TableHasColumn(rw, table, "multibyte_evidence") - if err != nil { - return fmt.Errorf("inspect %s.multibyte_evidence: %w", table, err) - } - if !hasEvid { - if _, err := rw.Exec(fmt.Sprintf( - "ALTER TABLE %s ADD COLUMN multibyte_evidence TEXT", table)); err != nil { - return fmt.Errorf("add %s.multibyte_evidence: %w", table, err) + if err := db.QueryRow(`SELECT has_sequence_privilege(pg_get_serial_sequence($1,$2),'USAGE')`, table.Name, column.Name).Scan(&allowed); err != nil { + return err + } + if !allowed { + return fmt.Errorf("telemetry writer lacks sequence USAGE on %s.%s", table.Name, column.Name) } - logf("[dbschema] added multibyte_evidence column to %s", table) } } return nil } -// ensureObserverNaiveClockColumns adds the three per-observer naive-clock -// skew tracking columns (#1478). Server reads them to populate the -// clock_naive / clock_skew_seconds / clock_skew_count_24h / -// clock_last_naive_at fields in /api/observers responses; ingestor writes -// them from resolveRxTime via Store.RecordNaiveSkew on each clamp event. -// Owned here per the #1321 source-of-truth invariant — the cmd/ingestor -// copy migrates real prod DBs, dbschema migrates fixture/staging DBs via -// cmd/migrate. -func ensureObserverNaiveClockColumns(rw *sql.DB, logf Logger) error { - type col struct{ name, ddl string } - cols := []col{ - {"clock_skew_seconds", "ALTER TABLE observers ADD COLUMN clock_skew_seconds INTEGER DEFAULT NULL"}, - {"clock_skew_count_24h", "ALTER TABLE observers ADD COLUMN clock_skew_count_24h INTEGER DEFAULT 0"}, - {"clock_last_naive_at", "ALTER TABLE observers ADD COLUMN clock_last_naive_at TEXT DEFAULT NULL"}, - } - for _, c := range cols { - has, err := TableHasColumn(rw, "observers", c.name) - if err != nil { - return fmt.Errorf("inspect observers.%s: %w", c.name, err) - } - if has { - continue - } - if _, err := rw.Exec(c.ddl); err != nil { - return fmt.Errorf("add observers.%s: %w", c.name, err) +// ReseedIdentities preserves imported IDs and legacy deleted high-water marks. +// The caller runs this in its import transaction before marking ready. +func ReseedIdentities(tx *sql.Tx, highWater map[string]int64) error { + for _, table := range Tables { + for _, column := range table.Columns { + if !column.Identity { + continue + } + key := table.Name + "." + column.Name + var high int64 + if err := tx.QueryRow(`SELECT COALESCE(MAX("` + column.Name + `"),0) FROM "` + table.Name + `"`).Scan(&high); err != nil { + return err + } + if highWater[key] > high { + high = highWater[key] + } + if high < 1 { + high = 1 + if _, err := tx.Exec(`SELECT setval(pg_get_serial_sequence($1,$2),$3,false)`, table.Name, column.Name, high); err != nil { + return err + } + } else { + if _, err := tx.Exec(`SELECT setval(pg_get_serial_sequence($1,$2),$3,true)`, table.Name, column.Name, high); err != nil { + return err + } + } } - logf("[dbschema] added %s column to observers", c.name) - } - return nil -} - -// ensureObserverCanRelayColumn adds the can_relay column to observers. -// Firmware 1.16 publishes a `repeat: on|off` flag in the MQTT /status -// JSON (#1290); the ingestor parses it and writes 0/1 here. The server's -// path-hop disambiguator (cmd/server/store.go pm.resolveWithContext) -// excludes observers with can_relay=0 from the candidate set. Default 1 -// preserves prior behavior for legacy observers (no repeat field). -func ensureObserverCanRelayColumn(rw *sql.DB, logf Logger) error { - has, err := TableHasColumn(rw, "observers", "can_relay") - if err != nil { - return err - } - if has { - return nil - } - // PREFLIGHT: async=true reason="single-column ALTER on observers (low-cardinality, ~1k rows in prod); DEFAULT 1 is a constant so SQLite does the rewrite as a metadata-only schema update, no row scan" - if _, err := rw.Exec("ALTER TABLE observers ADD COLUMN can_relay INTEGER DEFAULT 1"); err != nil { - return err } - logf("[dbschema] added can_relay column to observers") return nil } -// ensureObserverCanRelaySeenColumn adds the can_relay_seen tracking column. -// Issue #1290 follow-up (PR #1624 MAJOR-2): can_relay defaults to 1 at -// INSERT, which conflates "confirmed repeater" with "legacy observer -// that never sent the repeat field". can_relay_seen=1 is written by -// the ingestor whenever the firmware actually provided the field; the -// server's read layer returns CanRelay=nil whenever seen=0 so the UI -// can render the tri-state badge (no badge for unknown). -func ensureObserverCanRelaySeenColumn(rw *sql.DB, logf Logger) error { - has, err := TableHasColumn(rw, "observers", "can_relay_seen") - if err != nil { - return err - } - if has { - return nil - } - // PREFLIGHT: async=true reason="single-column ALTER on observers (low-cardinality, ~1k rows in prod); DEFAULT 0 is a constant so SQLite does the rewrite as a metadata-only schema update, no row scan" - if _, err := rw.Exec("ALTER TABLE observers ADD COLUMN can_relay_seen INTEGER DEFAULT 0"); err != nil { - return err +func SoftDeleteBlacklistedObservers(db *sql.DB, blacklist []string) (int64, error) { + var slots []string + var args []any + for _, id := range blacklist { + if id = strings.TrimSpace(id); id != "" { + args = append(args, id) + slots = append(slots, fmt.Sprintf("LOWER($%d)", len(args))) + } } - logf("[dbschema] added can_relay_seen column to observers") - return nil -} - -// ensureTransmissionsLastSeenColumn adds transmissions.last_seen (#1690), -// the denormalized "most recent observation timestamp" column the cold-load -// path filters on. Without it, hot-window queries that match -// `first_seen >= cutoff` exclude long-lived hashes whose hashes were first -// inserted weeks ago but are still active — see the issue for the full -// 0.3%-of-DB-loaded post-mortem. -// -// The backfill from `MAX(observations.timestamp) GROUP BY transmission_id` -// is potentially expensive (1.9M+ obs rows in prod) and runs as a separate -// async migration scheduled by cmd/ingestor/db.go::OpenStore via -// Store.RunAsyncMigration. -func ensureTransmissionsLastSeenColumn(rw *sql.DB, logf Logger) error { - if err := ensureMigrationsTable(rw); err != nil { - return err + if len(args) == 0 { + return 0, nil } - has, err := TableHasColumn(rw, "transmissions", "last_seen") + res, err := db.Exec(`UPDATE observers SET inactive=1 WHERE LOWER(id) IN (`+strings.Join(slots, ",")+`) AND (inactive IS NULL OR inactive=0)`, args...) if err != nil { - return err - } - if !has { - // PREFLIGHT: async=true reason="single-column ALTER TABLE on transmissions; DEFAULT 0 is a constant so SQLite does a metadata-only schema rewrite (no row scan) — cheap at any scale" - if _, err := rw.Exec(`ALTER TABLE transmissions ADD COLUMN last_seen INTEGER NOT NULL DEFAULT 0`); err != nil { - return fmt.Errorf("alter transmissions add last_seen: %w", err) - } - logf("[dbschema] added last_seen column to transmissions (#1690)") - } - // #1740 step (a): create the PARTIAL index first. Scans only rows with - // last_seen=0 — i.e. the un-backfilled hot subset the chunked backfill - // MAX(id) lookup walks. Degenerates to near-empty once the backfill - // converges, so it stops competing for page cache. - // - // PREFLIGHT: async=false reason="partial index on `WHERE last_seen=0`; subset is bounded by un-backfilled rows (≤0 in steady state, ≤ingest-rate during ops) — cheap at any scale even on a 1.9M-row prod table" - if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_tx_last_seen_zero ON transmissions(id) WHERE last_seen=0`); err != nil { - return fmt.Errorf("create idx_tx_last_seen_zero: %w", err) - } - // #1740 step (b): gated DROP of the legacy full index. Only runs AFTER - // step (a) succeeded (sequential — we're past the CREATE above). The - // DROP is a metadata-only schema rewrite in SQLite; no row scan. - // - // PREFLIGHT: async=false reason="DROP INDEX is metadata-only in SQLite (no row scan, no rewrite); safe inline at any DB size" - if _, err := rw.Exec(`DROP INDEX IF EXISTS idx_tx_last_seen`); err != nil { - return fmt.Errorf("drop legacy idx_tx_last_seen: %w", err) + return 0, err } - return nil + return res.RowsAffected() } diff --git a/internal/dbschema/go.mod b/internal/dbschema/go.mod index 3dca3dc9d..5996cad5c 100644 --- a/internal/dbschema/go.mod +++ b/internal/dbschema/go.mod @@ -1,5 +1,20 @@ module github.com/meshcore-analyzer/dbschema -go 1.22 +go 1.25.0 -require github.com/mattn/go-sqlite3 v1.14.52 +require ( + github.com/mattn/go-sqlite3 v1.14.52 + github.com/meshcore-analyzer/pgutil v0.0.0 +) + +require ( + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.7.6 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + golang.org/x/crypto v0.37.0 // indirect + golang.org/x/sync v0.13.0 // indirect + golang.org/x/text v0.24.0 // indirect +) + +replace github.com/meshcore-analyzer/pgutil => ../pgutil diff --git a/internal/dbschema/go.sum b/internal/dbschema/go.sum index 75bf6eb53..c83dde8cc 100644 --- a/internal/dbschema/go.sum +++ b/internal/dbschema/go.sum @@ -1,2 +1,30 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/dbschema/advert_evidence.go b/internal/dbschema/legacy/advert_evidence.go similarity index 98% rename from internal/dbschema/advert_evidence.go rename to internal/dbschema/legacy/advert_evidence.go index dc6e2bacc..62648f0b4 100644 --- a/internal/dbschema/advert_evidence.go +++ b/internal/dbschema/legacy/advert_evidence.go @@ -1,4 +1,4 @@ -package dbschema +package legacy import "database/sql" diff --git a/internal/dbschema/legacy/base.go b/internal/dbschema/legacy/base.go new file mode 100644 index 000000000..e26ab9a3c --- /dev/null +++ b/internal/dbschema/legacy/base.go @@ -0,0 +1,707 @@ +package legacy + +import ( + "database/sql" + "fmt" + "log" + "strings" +) + +func ApplyBase(db *sql.DB) error { + // auto_vacuum=INCREMENTAL is set via DSN pragma (must be before journal_mode). + // Logging of current mode is handled by CheckAutoVacuum — no duplicate log here. + + schema := ` + CREATE TABLE IF NOT EXISTS nodes ( + public_key TEXT PRIMARY KEY, + name TEXT, + role TEXT, + lat REAL, + lon REAL, + last_seen TEXT, + first_seen TEXT, + advert_count INTEGER DEFAULT 0, + battery_mv INTEGER, + temperature_c REAL, + foreign_advert INTEGER DEFAULT 0 + ); + + CREATE TABLE IF NOT EXISTS scope_match_totals ( + id INTEGER PRIMARY KEY CHECK (id = 1), + since_unix INTEGER NOT NULL, + unique_matches INTEGER NOT NULL, + explicit_over_derived INTEGER NOT NULL, + ambiguous INTEGER NOT NULL, + none_matches INTEGER NOT NULL, + updated_unix INTEGER NOT NULL + ); + + CREATE TABLE IF NOT EXISTS observers ( + id TEXT PRIMARY KEY, + name TEXT, + iata TEXT, + last_seen TEXT, + first_seen TEXT, + packet_count INTEGER DEFAULT 0, + model TEXT, + firmware TEXT, + client_version TEXT, + radio TEXT, + battery_mv INTEGER, + uptime_secs INTEGER, + noise_floor REAL, + inactive INTEGER DEFAULT 0, + last_packet_at TEXT DEFAULT NULL, + clock_skew_seconds INTEGER DEFAULT NULL, + clock_skew_count_24h INTEGER DEFAULT 0, + clock_last_naive_at TEXT DEFAULT NULL, + can_relay INTEGER DEFAULT 1, + can_relay_seen INTEGER DEFAULT 0 + ); + + CREATE INDEX IF NOT EXISTS idx_nodes_last_seen ON nodes(last_seen); + CREATE INDEX IF NOT EXISTS idx_observers_last_seen ON observers(last_seen); + + CREATE TABLE IF NOT EXISTS inactive_nodes ( + public_key TEXT PRIMARY KEY, + name TEXT, + role TEXT, + lat REAL, + lon REAL, + last_seen TEXT, + first_seen TEXT, + advert_count INTEGER DEFAULT 0, + battery_mv INTEGER, + temperature_c REAL, + foreign_advert INTEGER DEFAULT 0 + ); + + CREATE INDEX IF NOT EXISTS idx_inactive_nodes_last_seen ON inactive_nodes(last_seen); + + CREATE TABLE IF NOT EXISTS transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_hex TEXT NOT NULL, + hash TEXT NOT NULL UNIQUE, + first_seen TEXT NOT NULL, + route_type INTEGER, + payload_type INTEGER, + payload_version INTEGER, + decoded_json TEXT, + from_pubkey TEXT, + last_seen INTEGER NOT NULL DEFAULT 0, + created_at TEXT DEFAULT (datetime('now')), + code1 TEXT, + code2 TEXT + ); + + CREATE INDEX IF NOT EXISTS idx_transmissions_hash ON transmissions(hash); + CREATE INDEX IF NOT EXISTS idx_transmissions_first_seen ON transmissions(first_seen); + CREATE INDEX IF NOT EXISTS idx_transmissions_payload_type ON transmissions(payload_type); + -- idx_tx_code1 is created by the transport_codes_v1 migration below, + -- after the ALTER runs — same reasoning as idx_transmissions_from_pubkey + -- and idx_tx_last_seen_zero above: a legacy DB (table-exists, + -- column-missing) would trip on this CREATE INDEX referencing code1 + -- before the column exists, since CREATE TABLE IF NOT EXISTS is a + -- no-op against a pre-existing table. + -- idx_transmissions_from_pubkey is created by the from_pubkey_v1 + -- migration after the column is added on legacy DBs (#1143). + -- idx_tx_last_seen_zero (partial, WHERE last_seen=0) is created by + -- dbschema.Apply after ensuring the last_seen column exists (#1690, + -- partial-index swap #1740) — keep it OUT of this base schema block + -- so legacy DBs (table-exists, column-missing) don't trip on the + -- CREATE INDEX before the ALTER runs. + + -- Mobile client RX coverage: a roaming companion = a mobile observer + -- with a moving GPS position, so it gets its own table rather than + -- observations (which assumes a fixed observer/location). + CREATE TABLE IF NOT EXISTS client_receptions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + rx_pubkey TEXT NOT NULL, + heard_key TEXT NOT NULL, + heard_keylen INTEGER NOT NULL, + rssi INTEGER, + snr REAL, + lat REAL NOT NULL, + lon REAL NOT NULL, + pos_acc_m REAL, + rx_at TEXT NOT NULL, + ingested_at TEXT NOT NULL, + src TEXT NOT NULL, + UNIQUE(rx_pubkey, heard_key, rx_at) + ); + -- Coverage queries filter by bbox AND match the heard node either by full + -- key (heard_keylen=32 AND heard_key=?) or by 2-3 byte prefix. The composite + -- (heard_key, heard_keylen, lat, lon) serves the heard_key-equality seek and + -- carries lat/lon so the bbox range is satisfied from the index; it also + -- supersedes the old single-column heard_key index. idx_client_recept_latlon + -- lets the planner instead drive from a selective bbox. (#5, #18) + CREATE INDEX IF NOT EXISTS idx_client_recept_heard_geo ON client_receptions(heard_key, heard_keylen, lat, lon); + CREATE INDEX IF NOT EXISTS idx_client_recept_latlon ON client_receptions(lat, lon); + -- rx_at backs both the retention reaper (DELETE WHERE rx_at < ?) and the + -- leaderboard, which range-scans WHERE rx_at >= ? and aggregates per + -- rx_pubkey in Go (see rxLeaderboard's frontier-weighted scoring). Without + -- this index either would full-scan the table under the writer lock + -- (verified by an EXPLAIN test). A dedicated rx_pubkey index stays + -- redundant — the leaderboard no longer groups by rx_pubkey in SQL. + CREATE INDEX IF NOT EXISTS idx_client_recept_rxat ON client_receptions(rx_at); + DROP INDEX IF EXISTS idx_client_recept_rxpk; + + -- Self-reported name of each mobile client (companion), from the SELF_INFO + -- name the app sends as "origin". Lets the leaderboard show a name even + -- when the companion never advertised (so it isn't in the nodes table). + CREATE TABLE IF NOT EXISTS client_observers ( + pubkey TEXT PRIMARY KEY, + name TEXT, + last_seen TEXT + ); + + -- Diagnostic RF observations from mobile clients. Unlike client_receptions + -- this holds EVERY decodable packet, attributable or not, so it must never + -- be used for coverage. pkt_hash is ComputeContentHash() — identical to + -- transmissions.hash — so dark-traffic queries are a plain equality join. + CREATE TABLE IF NOT EXISTS client_rx_observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + rx_pubkey TEXT NOT NULL, + rx_at TEXT NOT NULL, + ingested_at TEXT NOT NULL, + pkt_hash TEXT NOT NULL, + route_type INTEGER NOT NULL, + payload_type INTEGER NOT NULL, + code1 TEXT, + code2 TEXT, + scope_name TEXT, + hash_size INTEGER NOT NULL, + hop_count INTEGER NOT NULL, + path_json TEXT, + forwarder TEXT, + snr REAL, + rssi INTEGER, + lat REAL NOT NULL, + lon REAL NOT NULL, + pos_acc_m REAL, + UNIQUE(rx_pubkey, pkt_hash, rx_at) + ); + CREATE INDEX IF NOT EXISTS idx_cro_prune ON client_rx_observations(rx_at); + CREATE INDEX IF NOT EXISTS idx_cro_hash ON client_rx_observations(pkt_hash, rx_at); + CREATE INDEX IF NOT EXISTS idx_cro_forwarder ON client_rx_observations(forwarder, rx_at); + CREATE INDEX IF NOT EXISTS idx_cro_scope ON client_rx_observations(scope_name, rx_at); + + -- RF environment samples from mobile clients: radio counters paired with + -- a GPS point. Absolutes only — deltas are computed at query time, and a + -- decrease in uptime_secs (reboot) or any counter (wrap) breaks the chain. + CREATE TABLE IF NOT EXISTS client_rf_samples ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + rx_pubkey TEXT NOT NULL, + sampled_at TEXT NOT NULL, + ingested_at TEXT NOT NULL, + lat REAL NOT NULL, + lon REAL NOT NULL, + pos_acc_m REAL, + stationary INTEGER NOT NULL DEFAULT 0, + uptime_secs INTEGER NOT NULL, + battery_mv INTEGER, + queue_len INTEGER, + errors INTEGER, + noise_floor INTEGER, + last_rssi INTEGER, + last_snr REAL, + tx_air_secs INTEGER, + rx_air_secs INTEGER, + recv INTEGER, + sent INTEGER, + flood_rx INTEGER, + direct_rx INTEGER, + flood_tx INTEGER, + direct_tx INTEGER, + recv_errors INTEGER, + UNIQUE(rx_pubkey, sampled_at) + ); + CREATE INDEX IF NOT EXISTS idx_crf_prune ON client_rf_samples(sampled_at); + CREATE INDEX IF NOT EXISTS idx_crf_track ON client_rf_samples(rx_pubkey, sampled_at); + + -- Declared region lists reported by repeaters via ANON_REQ_TYPE_REGIONS. + -- Observations, never state: "current" is the greatest observed_at for a + -- target, NOT the greatest ingested_at — a drive buffered offline can + -- arrive days late and must not overwrite a fresher reading. + CREATE TABLE IF NOT EXISTS node_declared_regions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + target TEXT NOT NULL, + rx_pubkey TEXT NOT NULL, + observed_at TEXT NOT NULL, + ingested_at TEXT NOT NULL, + regions_csv TEXT NOT NULL, + truncated INTEGER NOT NULL DEFAULT 0, + lat REAL, + lon REAL, + pos_acc_m REAL, + repeater_clock INTEGER, + UNIQUE(target, rx_pubkey, observed_at) + ); + CREATE INDEX IF NOT EXISTS idx_ndr_target ON node_declared_regions(target, observed_at); + CREATE INDEX IF NOT EXISTS idx_ndr_prune ON node_declared_regions(observed_at); + ` + if _, err := db.Exec(schema); err != nil { + return fmt.Errorf("base schema: %w", err) + } + + // Create observations table (v3 schema) + obsExists := false + row := db.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name='observations'") + var dummy string + if row.Scan(&dummy) == nil { + obsExists = true + } + + if !obsExists { + obs := ` + CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER NOT NULL REFERENCES transmissions(id), + observer_idx INTEGER, + direction TEXT, + snr REAL, + rssi REAL, + score INTEGER, + path_json TEXT, + timestamp INTEGER NOT NULL + ); + CREATE INDEX idx_observations_transmission_id ON observations(transmission_id); + CREATE INDEX idx_observations_observer_idx ON observations(observer_idx); + CREATE INDEX idx_observations_timestamp ON observations(timestamp); + CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ON observations(transmission_id, observer_idx, COALESCE(path_json, '')); + ` + if _, err := db.Exec(obs); err != nil { + return fmt.Errorf("observations schema: %w", err) + } + } + + // Create/rebuild packets_v view (v3 schema: observer_idx → observers.rowid) + // The Go server reads this view; without it fresh installs get "no such table: packets_v". + db.Exec(`DROP VIEW IF EXISTS packets_v`) + _, vErr := db.Exec(` + CREATE VIEW packets_v AS + SELECT o.id, COALESCE(o.raw_hex, t.raw_hex) AS raw_hex, + datetime(o.timestamp, 'unixepoch') AS timestamp, + obs.id AS observer_id, obs.name AS observer_name, + o.direction, o.snr, o.rssi, o.score, t.hash, t.route_type, + t.payload_type, t.payload_version, o.path_json, t.decoded_json, + t.created_at + FROM observations o + JOIN transmissions t ON t.id = o.transmission_id + LEFT JOIN observers obs ON obs.rowid = o.observer_idx AND (obs.inactive IS NULL OR obs.inactive = 0) + `) + if vErr != nil { + return fmt.Errorf("packets_v view: %w", vErr) + } + + // One-time migration: recalculate advert_count to count unique transmissions only + db.Exec(`CREATE TABLE IF NOT EXISTS _migrations (name TEXT PRIMARY KEY)`) + var migDone int + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'advert_count_unique_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Recalculating advert_count (unique transmissions only)...") + // Note: this migration is gated on a one-shot _migrations row, so it + // runs at most once per DB. The historical version used a LIKE-on-JSON + // substring match (#1143). Switching to from_pubkey here is safe even + // though the column may not yet be backfilled on legacy DBs: the + // migration is already marked done on those DBs and won't re-run. + db.Exec(` + UPDATE nodes SET advert_count = ( + SELECT COUNT(*) FROM transmissions t + WHERE t.payload_type = 4 + AND t.from_pubkey = nodes.public_key + ) + `) + db.Exec(`INSERT INTO _migrations (name) VALUES ('advert_count_unique_v1')`) + log.Println("[migration] advert_count recalculated") + } + + // One-time migration: change noise_floor from INTEGER to REAL affinity. + // SQLite doesn't support ALTER COLUMN, but existing float values are stored + // as REAL regardless of column affinity. New table definition already uses REAL. + // This migration casts any integer-stored noise_floor values to real. + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'noise_floor_real_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Ensuring noise_floor values are stored as REAL...") + db.Exec(`UPDATE observers SET noise_floor = CAST(noise_floor AS REAL) WHERE noise_floor IS NOT NULL AND typeof(noise_floor) = 'integer'`) + db.Exec(`INSERT INTO _migrations (name) VALUES ('noise_floor_real_v1')`) + log.Println("[migration] noise_floor migration complete") + } + + // One-time migration: add telemetry columns to nodes and inactive_nodes tables. + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'node_telemetry_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Adding telemetry columns to nodes/inactive_nodes...") + + // checkAndAddColumn checks whether `column` already exists in `table` + // using PRAGMA table_info, and adds it if missing. All call sites pass + // hardcoded table/column/type literals so there is no SQL injection risk. + checkAndAddColumn := func(table, column, colType string) error { + rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%s)", table)) + if err != nil { + return fmt.Errorf("querying table info for %s: %w", table, err) + } + defer rows.Close() + + exists := false + for rows.Next() { + var cid int + var name, ctype string + var notnull, pk int + var dfltValue sql.NullString + if err := rows.Scan(&cid, &name, &ctype, ¬null, &dfltValue, &pk); err != nil { + return fmt.Errorf("scanning table info for %s: %w", table, err) + } + if name == column { + exists = true + break + } + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterating table info for %s: %w", table, err) + } + if exists { + return nil + } + if _, err := db.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, column, colType)); err != nil { + return fmt.Errorf("adding column %s to %s: %w", column, table, err) + } + return nil + } + + if err := checkAndAddColumn("nodes", "battery_mv", "INTEGER"); err != nil { + return err + } + if err := checkAndAddColumn("nodes", "temperature_c", "REAL"); err != nil { + return err + } + if err := checkAndAddColumn("inactive_nodes", "battery_mv", "INTEGER"); err != nil { + return err + } + if err := checkAndAddColumn("inactive_nodes", "temperature_c", "REAL"); err != nil { + return err + } + if _, err := db.Exec(`INSERT INTO _migrations (name) VALUES ('node_telemetry_v1')`); err != nil { + return fmt.Errorf("recording node_telemetry_v1 migration: %w", err) + } + log.Println("[migration] node telemetry columns added") + } + + // One-time migration: add timestamp index on observations for fast stats queries. + // Older databases created before this index was added suffer from full table scans + // on COUNT(*) WHERE timestamp > ?, causing /api/stats to take 30s+. + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'obs_timestamp_index_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Adding timestamp index on observations...") + db.Exec(`CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp)`) + db.Exec(`INSERT INTO _migrations (name) VALUES ('obs_timestamp_index_v1')`) + log.Println("[migration] observations timestamp index created") + } + + // #1481 P0-3: covering index for GetObserverPacketCounts. The query + // joins observations → observers and GROUP BYs observer_idx with a + // timestamp WHERE filter; a composite (observer_idx, timestamp) + // index lets SQLite resolve the grouping + range filter from the + // index alone instead of a 1.9M-row scan. + // + // CONVERTED TO ASYNC (preflight-async-migration-gate). Scheduling + // happens in OpenStore() once the real *Store exists so the + // backfill WaitGroup is shared with the rest of the ingestor. + // The legacy `_migrations` gate is preserved by the async fn so + // DBs that already completed the sync build stay no-op. + + // #1483: normalize nodes.public_key to lowercase. The server's + // GetNodeLocationsByKeys lookup dropped LOWER(public_key) for perf + // (#1481 P0-3) and now relies on stored keys being lowercase. The + // decoder writes lowercase today, but legacy/admin/API inserts may + // have left mixed-case rows. Idempotent: counts and lowers any + // non-lowercase rows on every boot, runs once via _migrations gate + // for the bulk fix. Re-running stays cheap because subsequent + // passes match zero rows. + if r := db.QueryRow("SELECT COUNT(*) FROM nodes WHERE public_key != lower(public_key)"); r != nil { + var n int64 + _ = r.Scan(&n) + if n > 0 { + log.Printf("[migration] Normalizing %d nodes.public_key row(s) to lowercase (#1483)...", n) + if _, err := db.Exec(`UPDATE nodes SET public_key = lower(public_key) WHERE public_key != lower(public_key)`); err != nil { + log.Printf("[migration] public_key lowercase normalize failed: %v", err) + } else { + log.Printf("[migration] public_key lowercase normalize complete (%d rows)", n) + } + } + } + + // observer_metrics table for RF health dashboard + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observer_metrics_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Creating observer_metrics table...") + _, err := db.Exec(` + CREATE TABLE IF NOT EXISTS observer_metrics ( + observer_id TEXT NOT NULL, + timestamp TEXT NOT NULL, + noise_floor REAL, + tx_air_secs INTEGER, + rx_air_secs INTEGER, + recv_errors INTEGER, + battery_mv INTEGER, + PRIMARY KEY (observer_id, timestamp) + ) + `) + if err != nil { + return fmt.Errorf("observer_metrics schema: %w", err) + } + db.Exec(`INSERT INTO _migrations (name) VALUES ('observer_metrics_v1')`) + log.Println("[migration] observer_metrics table created") + } + + // Migration: add timestamp index for cross-observer time-range queries + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observer_metrics_ts_idx'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Creating observer_metrics timestamp index...") + _, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_observer_metrics_timestamp ON observer_metrics(timestamp)`) + if err != nil { + return fmt.Errorf("observer_metrics timestamp index: %w", err) + } + db.Exec(`INSERT INTO _migrations (name) VALUES ('observer_metrics_ts_idx')`) + log.Println("[migration] observer_metrics timestamp index created") + } + + // Migration: add inactive column to observers for soft-delete retention + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observers_inactive_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Adding inactive column to observers...") + _, err := db.Exec(`ALTER TABLE observers ADD COLUMN inactive INTEGER DEFAULT 0`) + if err != nil { + // Column may already exist (e.g. fresh install with schema above) + log.Printf("[migration] observers.inactive: %v (may already exist)", err) + } + db.Exec(`INSERT INTO _migrations (name) VALUES ('observers_inactive_v1')`) + log.Println("[migration] observers.inactive column added") + } + + // Migration: add packets_sent and packets_recv columns to observer_metrics + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observer_metrics_packets_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Adding packets_sent/packets_recv columns to observer_metrics...") + db.Exec(`ALTER TABLE observer_metrics ADD COLUMN packets_sent INTEGER`) + db.Exec(`ALTER TABLE observer_metrics ADD COLUMN packets_recv INTEGER`) + db.Exec(`INSERT INTO _migrations (name) VALUES ('observer_metrics_packets_v1')`) + log.Println("[migration] packets_sent/packets_recv columns added") + } + + // Migration: add channel_hash column for fast channel queries (#762) + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'channel_hash_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Adding channel_hash column to transmissions...") + db.Exec(`ALTER TABLE transmissions ADD COLUMN channel_hash TEXT DEFAULT NULL`) + db.Exec(`CREATE INDEX IF NOT EXISTS idx_tx_channel_hash ON transmissions(channel_hash) WHERE payload_type = 5`) + // Backfill: extract channel name for decrypted (CHAN) packets + res, err := db.Exec(`UPDATE transmissions SET channel_hash = json_extract(decoded_json, '$.channel') WHERE payload_type = 5 AND channel_hash IS NULL AND json_extract(decoded_json, '$.type') = 'CHAN'`) + if err == nil { + n, _ := res.RowsAffected() + log.Printf("[migration] Backfilled channel_hash for %d CHAN packets", n) + } + // Backfill: extract channelHashHex for encrypted (GRP_TXT) packets, prefixed with 'enc_' + res, err = db.Exec(`UPDATE transmissions SET channel_hash = 'enc_' || json_extract(decoded_json, '$.channelHashHex') WHERE payload_type = 5 AND channel_hash IS NULL AND json_extract(decoded_json, '$.type') = 'GRP_TXT'`) + if err == nil { + n, _ := res.RowsAffected() + log.Printf("[migration] Backfilled channel_hash for %d GRP_TXT packets", n) + } + db.Exec(`INSERT INTO _migrations (name) VALUES ('channel_hash_v1')`) + log.Println("[migration] channel_hash column added and backfilled") + } + + // Migration: dropped_packets table for signature validation failures (#793) + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'dropped_packets_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Creating dropped_packets table...") + _, err := db.Exec(` + CREATE TABLE IF NOT EXISTS dropped_packets ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + hash TEXT, + raw_hex TEXT, + reason TEXT NOT NULL, + observer_id TEXT, + observer_name TEXT, + node_pubkey TEXT, + node_name TEXT, + dropped_at DATETIME DEFAULT CURRENT_TIMESTAMP + ); + CREATE INDEX IF NOT EXISTS idx_dropped_observer ON dropped_packets(observer_id); + CREATE INDEX IF NOT EXISTS idx_dropped_node ON dropped_packets(node_pubkey); + `) + if err != nil { + return fmt.Errorf("dropped_packets schema: %w", err) + } + db.Exec(`INSERT INTO _migrations (name) VALUES ('dropped_packets_v1')`) + log.Println("[migration] dropped_packets table created") + } + + // Migration: observations.raw_hex (#881) is now owned by + // internal/dbschema/dbschema.go (#1321). The server PRAGMA-detects + // this column as hasObsRawHex; keeping a single canonical Apply + // path closes the startup race where the server's detector ran + // before this ALTER finished. + + // Migration: transmissions.scope_name (#899) is now owned by + // internal/dbschema/dbschema.go (#1321). See above. + + // Migration: add last_packet_at column to observers (#last-packet-at) + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observers_last_packet_at_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Adding last_packet_at column to observers...") + _, alterErr := db.Exec(`ALTER TABLE observers ADD COLUMN last_packet_at TEXT DEFAULT NULL`) + if alterErr != nil && !strings.Contains(alterErr.Error(), "duplicate column") { + return fmt.Errorf("observers last_packet_at ALTER: %w", alterErr) + } + // Backfill: set last_packet_at = last_seen only for observers that actually have + // observation rows (packet_count alone is unreliable — UpsertObserver sets it to 1 + // on INSERT even for status-only observers). + res, err := db.Exec(`UPDATE observers SET last_packet_at = last_seen + WHERE last_packet_at IS NULL + AND rowid IN (SELECT DISTINCT observer_idx FROM observations WHERE observer_idx IS NOT NULL)`) + if err == nil { + n, _ := res.RowsAffected() + log.Printf("[migration] Backfilled last_packet_at for %d observers with packets", n) + } + db.Exec(`INSERT INTO _migrations (name) VALUES ('observers_last_packet_at_v1')`) + log.Println("[migration] observers.last_packet_at column added") + } + + // Migration: per-observer naive-clock skew tracking (#1478). + // When the ingestor clamps a packet's envelope timestamp because the + // observer emitted a zone-less local-time string off from UTC by >15min + // (resolveRxTime in main.go), we record the event here so the UI can + // surface a ⚠️ chip + banner. Decays after 24h via server-side read sweep. + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observers_clock_naive_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Adding clock-naive columns to observers (#1478)...") + // Each ALTER is independent — ignore "duplicate column" so reruns are safe. + for _, stmt := range []string{ + `ALTER TABLE observers ADD COLUMN clock_skew_seconds INTEGER DEFAULT NULL`, + `ALTER TABLE observers ADD COLUMN clock_skew_count_24h INTEGER DEFAULT 0`, + `ALTER TABLE observers ADD COLUMN clock_last_naive_at TEXT DEFAULT NULL`, + } { + if _, err := db.Exec(stmt); err != nil && !strings.Contains(err.Error(), "duplicate column") { + return fmt.Errorf("clock_naive migration: %w", err) + } + } + db.Exec(`INSERT INTO _migrations (name) VALUES ('observers_clock_naive_v1')`) + log.Println("[migration] observers.clock_naive columns added") + } + + // Migration: backfill observations.path_json from raw_hex (#888) + // NOTE: This runs ASYNC via BackfillPathJSONAsync() to avoid blocking MQTT startup. + // See staging outage where ~502K rows blocked ingest for 15+ hours. + + // One-time cleanup: delete legacy packets with empty hash or empty first_seen (#994) + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'cleanup_legacy_null_hash_ts'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Cleaning up legacy packets with empty hash/timestamp...") + db.Exec(`DELETE FROM observations WHERE transmission_id IN (SELECT id FROM transmissions WHERE hash = '' OR first_seen = '')`) + res, err := db.Exec(`DELETE FROM transmissions WHERE hash = '' OR first_seen = ''`) + if err == nil { + deleted, _ := res.RowsAffected() + log.Printf("[migration] deleted %d legacy packets with empty hash/timestamp", deleted) + } + db.Exec(`INSERT INTO _migrations (name) VALUES ('cleanup_legacy_null_hash_ts')`) + } + + // Migration: foreign_advert column on nodes/inactive_nodes (#730) + // Marks nodes whose ADVERT GPS lies outside the configured geofilter polygon. + // Default 0; set to 1 by the ingestor when GeoFilter is configured and + // PassesFilter() returns false. Allows operators to surface bridged/leaked + // adverts without silently dropping them. + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'foreign_advert_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Adding foreign_advert column to nodes/inactive_nodes...") + if _, err := db.Exec(`ALTER TABLE nodes ADD COLUMN foreign_advert INTEGER DEFAULT 0`); err != nil { + log.Printf("[migration] nodes.foreign_advert: %v (may already exist)", err) + } + if _, err := db.Exec(`ALTER TABLE inactive_nodes ADD COLUMN foreign_advert INTEGER DEFAULT 0`); err != nil { + log.Printf("[migration] inactive_nodes.foreign_advert: %v (may already exist)", err) + } + db.Exec(`CREATE INDEX IF NOT EXISTS idx_nodes_foreign_advert ON nodes(foreign_advert) WHERE foreign_advert = 1`) + db.Exec(`INSERT INTO _migrations (name) VALUES ('foreign_advert_v1')`) + log.Println("[migration] foreign_advert column added") + } + + // Migration: from_pubkey column on transmissions (#1143). + // Replaces the unsound `decoded_json LIKE '%pubkey%'` attribution path with + // an exact-match indexed column. Synchronously adds the column + index; + // row-level backfill is run by the SERVER asynchronously + // (cmd/server/from_pubkey_migration.go) so we don't block ingestor boot. + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'from_pubkey_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Adding from_pubkey column + index to transmissions (#1143)...") + if _, err := db.Exec(`ALTER TABLE transmissions ADD COLUMN from_pubkey TEXT`); err != nil { + log.Printf("[migration] transmissions.from_pubkey: %v (may already exist)", err) + } + if _, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey)`); err != nil { + log.Printf("[migration] idx_transmissions_from_pubkey: %v", err) + } + db.Exec(`INSERT INTO _migrations (name) VALUES ('from_pubkey_v1')`) + log.Println("[migration] from_pubkey column + index added") + } + + // Migration: nodes.default_scope (#899 Feature 3) is now owned by + // internal/dbschema/dbschema.go (#1321). The server PRAGMA-detects + // this column as hasDefaultScope; keeping a single canonical Apply + // path closes the startup race that #1321 documented. + + // Migration: normalize known channel_hash values for existing rows. + // Before this PR, config key "public" was stored as channel_hash="public". + // After this PR, new rows use channel_hash="Public". Without backfill, + // channel grouping queries split into two buckets across the upgrade boundary. + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'channel_hash_casing_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Normalizing known channel_hash values...") + res, err := db.Exec(`UPDATE transmissions SET channel_hash = 'Public' WHERE channel_hash = 'public' AND payload_type = 5`) + if err != nil { + log.Printf("[migration] ERROR: failed to normalize channel_hash: %v", err) + return fmt.Errorf("migration channel_hash_casing_v1 UPDATE failed: %w", err) + } + n, _ := res.RowsAffected() + log.Printf("[migration] Normalized %d channel_hash rows from 'public' to 'Public'", n) + if _, err := db.Exec(`INSERT OR IGNORE INTO _migrations (name) VALUES ('channel_hash_casing_v1')`); err != nil { + log.Printf("[migration] WARNING: failed to record migration: %v", err) + } + log.Println("[migration] channel_hash casing normalization complete") + } + + // Migration: transmissions.code1/code2 — the transport codes were decoded + // and discarded; storing them makes scope forwarding queryable per repeater. + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'transport_codes_v1'") + if row.Scan(new(int)) != nil { + log.Println("[migration] Adding code1/code2 columns to transmissions...") + // Each ALTER is independent — ignore "duplicate column" so reruns (and + // the fresh-database path, where the base schema already created the + // columns) are safe. Any other failure must stop before the guard row + // is inserted, or prepareStatements' code1/code2 INSERT fails forever + // on every subsequent restart with no way to retry the migration. + for _, stmt := range []string{ + `ALTER TABLE transmissions ADD COLUMN code1 TEXT DEFAULT NULL`, + `ALTER TABLE transmissions ADD COLUMN code2 TEXT DEFAULT NULL`, + } { + if _, err := db.Exec(stmt); err != nil && !strings.Contains(err.Error(), "duplicate column") { + return fmt.Errorf("transport_codes_v1 migration: %w", err) + } + } + db.Exec(`CREATE INDEX IF NOT EXISTS idx_tx_code1 ON transmissions(code1) WHERE code1 IS NOT NULL`) + db.Exec(`INSERT INTO _migrations (name) VALUES ('transport_codes_v1')`) + log.Println("[migration] code1/code2 columns added") + } + + return nil +} + +// Normalize upgrades an offline SQLite working copy before import. +func Normalize(db *sql.DB, logf Logger) error { + if err := CheckLegacySource(db); err != nil { + return err + } + if err := ApplyBase(db); err != nil { + return err + } + return Apply(db, logf) +} diff --git a/internal/dbschema/legacy/check.go b/internal/dbschema/legacy/check.go new file mode 100644 index 000000000..636006fd2 --- /dev/null +++ b/internal/dbschema/legacy/check.go @@ -0,0 +1,50 @@ +package legacy + +import ( + "database/sql" + "fmt" +) + +// CheckLegacySource rejects layouts or data the historical normalizer would +// discard. The importer also rejects unknown tables/columns before copying. +// Supported layouts are v3 observations with the known optional columns absent. +func CheckLegacySource(db *sql.DB) error { + var integrity string + if err := db.QueryRow(`PRAGMA quick_check`).Scan(&integrity); err != nil { + return err + } + if integrity != "ok" { + return fmt.Errorf("legacy SQLite integrity check failed") + } + for _, table := range []string{"nodes", "observers", "transmissions", "observations"} { + var n int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='table' AND name=?`, table).Scan(&n); err != nil { + return err + } + if n != 1 { + return fmt.Errorf("unsupported legacy schema: missing %s", table) + } + } + has, err := TableHasColumn(db, "observations", "observer_idx") + if err != nil { + return err + } + if !has { + return fmt.Errorf("legacy v2 observer_id layout is unsupported; upgrade an offline copy with the last SQLite release before importing") + } + checks := []struct{ name, query string }{ + {"empty transmission hash/timestamp", `SELECT count(*) FROM transmissions WHERE hash IS NULL OR hash='' OR first_seen IS NULL OR first_seen=''`}, + {"case-colliding node public keys", `SELECT count(*) FROM (SELECT lower(public_key) FROM nodes GROUP BY lower(public_key) HAVING count(*)>1)`}, + {"duplicate observation identity", `SELECT count(*) FROM (SELECT transmission_id,observer_idx,coalesce(path_json,'') FROM observations WHERE observer_idx IS NOT NULL GROUP BY transmission_id,observer_idx,coalesce(path_json,'') HAVING count(*)>1)`}, + } + for _, check := range checks { + var n int + if err := db.QueryRow(check.query).Scan(&n); err != nil { + return err + } + if n > 0 { + return fmt.Errorf("legacy source contains %s (%d groups/rows); resolve explicitly before import", check.name, n) + } + } + return nil +} diff --git a/internal/dbschema/legacy/dbschema.go b/internal/dbschema/legacy/dbschema.go new file mode 100644 index 000000000..e915f67ef --- /dev/null +++ b/internal/dbschema/legacy/dbschema.go @@ -0,0 +1,753 @@ +// Package dbschema centralizes schema migrations and read-side schema +// assertions for the CoreScope SQLite DB. Per issue #1287 the writer +// (cmd/ingestor) owns ALL CREATE/ALTER/INSERT/UPDATE/DELETE on schema +// objects; the server (cmd/server) only ASSERTS that the schema is in +// the expected shape and refuses to start otherwise. +// +// Apply(rw, log) runs from the ingestor at startup BEFORE subscribing to +// MQTT. AssertReady(ro) runs from the server at startup and returns an +// error listing every missing column/index/table. +// +// INVARIANT (#1321): Any optional column the SERVER detects via PRAGMA +// (e.g. cmd/server/db.go detectSchema → hasScopeName, hasDefaultScope, +// hasObsRawHex, hasResolvedPath) MUST be added here, in Apply, AND +// asserted in AssertReady. Adding it only to cmd/ingestor/db.go +// applySchema reintroduces the startup-race bug from #1321: the +// server's PRAGMA can run before the ingestor finishes applySchema, +// the boolean caches `false`, and feature endpoints permanently 500 +// until the server restarts. Source of truth lives here — full stop. +package legacy + +import ( + "database/sql" + "errors" + "fmt" + "strings" +) + +// Logger is the minimal logging surface used by Apply. Both cmd/server +// and cmd/ingestor satisfy this with the stdlib `log` package's Printf +// (passed as a closure to avoid an indirect log dependency here). +type Logger func(format string, args ...interface{}) + +// Apply runs every server-side ensure_* migration against the given +// read-write SQLite connection. Each operation is idempotent +// (IF NOT EXISTS / column-probe-before-ALTER). Safe to call repeatedly. +// +// Called by the ingestor at startup. The server MUST NOT call this — +// it only calls AssertReady. +func Apply(rw *sql.DB, logf Logger) error { + if logf == nil { + logf = func(string, ...interface{}) {} + } + if err := ensureServerIndexes(rw); err != nil { + return fmt.Errorf("ensure server indexes: %w", err) + } + if err := ensureObservationsDedupIndex(rw, logf); err != nil { + return fmt.Errorf("ensure observations dedup index: %w", err) + } + if err := ensureNeighborEdgesTable(rw); err != nil { + return fmt.Errorf("ensure neighbor_edges: %w", err) + } + if err := ensureInactiveNodesTable(rw); err != nil { + return fmt.Errorf("ensure inactive_nodes: %w", err) + } + if err := ensureResolvedPathColumn(rw, logf); err != nil { + return fmt.Errorf("ensure resolved_path: %w", err) + } + if err := ensureObserverInactiveColumn(rw, logf); err != nil { + return fmt.Errorf("ensure observers.inactive: %w", err) + } + if err := ensureLastPacketAtColumn(rw, logf); err != nil { + return fmt.Errorf("ensure observers.last_packet_at: %w", err) + } + if err := ensureObserverIATAColumn(rw, logf); err != nil { + return fmt.Errorf("ensure observers.iata: %w", err) + } + // Must run after ensureObserverIATAColumn: the expression index below + // is on observers.iata, which that step guarantees exists. + if err := ensureChannelIndexes(rw); err != nil { + return fmt.Errorf("ensure channel indexes: %w", err) + } + if err := ensureForeignAdvertColumn(rw, logf); err != nil { + return fmt.Errorf("ensure foreign_advert: %w", err) + } + if err := ensureFromPubkeyColumn(rw, logf); err != nil { + return fmt.Errorf("ensure from_pubkey: %w", err) + } + if err := ensureScopeNameColumn(rw, logf); err != nil { + return fmt.Errorf("ensure scope_name: %w", err) + } + if err := ensureDefaultScopeColumns(rw, logf); err != nil { + return fmt.Errorf("ensure default_scope: %w", err) + } + if err := ensureConfiguredScopeColumns(rw, logf); err != nil { + return fmt.Errorf("ensure configured_scope: %w", err) + } + if err := ensureObservationsRawHexColumn(rw, logf); err != nil { + return fmt.Errorf("ensure observations.raw_hex: %w", err) + } + if err := ensureMultibyteCapColumns(rw, logf); err != nil { + return fmt.Errorf("ensure multibyte_cap columns: %w", err) + } + if err := ensureObserverNaiveClockColumns(rw, logf); err != nil { + return fmt.Errorf("ensure observers naive-clock columns: %w", err) + } + if err := ensureObserverCanRelayColumn(rw, logf); err != nil { + return fmt.Errorf("ensure observers.can_relay: %w", err) + } + if err := ensureObserverCanRelaySeenColumn(rw, logf); err != nil { + return fmt.Errorf("ensure observers.can_relay_seen: %w", err) + } + // #1690: denormalized last_seen on transmissions so cold-load filters + // on effective recency rather than first-ever first_seen. The column + // add + index creation are cheap (single ALTER, indexed INTEGER + // column); the *backfill* from observations is potentially expensive + // and runs via RunAsyncMigration from the ingestor (see + // cmd/ingestor/db.go OpenStore). + if err := ensureTransmissionsLastSeenColumn(rw, logf); err != nil { + return fmt.Errorf("ensure transmissions.last_seen: %w", err) + } + if err := ensureAdvertEvidence(rw); err != nil { + return fmt.Errorf("ensure advert route evidence: %w", err) + } + // Advert evidence is intentionally not required by AssertReady: a legacy + // read-only server reports unknown until this table/backfill is available, + // and re-probes absence instead of caching the startup race permanently. + return nil +} + +// AssertReady verifies the schema is in the expected shape. The server +// calls this at startup against a read-only connection; if it returns +// non-nil, the server MUST fatal-log and exit so the operator restarts +// the ingestor (which owns migrations). +func AssertReady(ro *sql.DB) error { + var missing []string + + mustCol := func(table, col string) { + has, err := TableHasColumn(ro, table, col) + if err != nil { + missing = append(missing, fmt.Sprintf("%s.%s (probe error: %v)", table, col, err)) + return + } + if !has { + missing = append(missing, fmt.Sprintf("%s.%s", table, col)) + } + } + mustTable := func(name string) { + var n int + err := ro.QueryRow(`SELECT 1 FROM sqlite_master WHERE type='table' AND name=?`, name).Scan(&n) + if errors.Is(err, sql.ErrNoRows) { + missing = append(missing, "table:"+name) + } else if err != nil { + missing = append(missing, fmt.Sprintf("table:%s (probe error: %v)", name, err)) + } + } + + mustTable("neighbor_edges") + mustCol("observations", "resolved_path") + mustCol("observers", "inactive") + mustCol("observers", "last_packet_at") + mustCol("observers", "iata") + mustCol("nodes", "foreign_advert") + mustCol("inactive_nodes", "foreign_advert") + mustCol("transmissions", "from_pubkey") + // #1690: denormalized recency axis for cold-load. Owned by ingestor + // (cmd/ingestor/db.go OpenStore: ALTER + index + async backfill); + // server reads it to filter the hot-window query. + mustCol("transmissions", "last_seen") + // #1321: server's detectSchema PRAGMA-detects these and caches a + // boolean. To kill the startup race they're owned + asserted here. + mustCol("transmissions", "scope_name") + mustCol("nodes", "default_scope") + mustCol("inactive_nodes", "default_scope") + // #1865: confirmed region scopes from the observer /neighbors report. + mustCol("nodes", "configured_scope") + mustCol("inactive_nodes", "configured_scope") + mustCol("observations", "raw_hex") + // Multi-byte capability cache (#1324 follow-up; PR #903 surface). + // Owned by ingestor — server reads these for O(1) /api/nodes + // enrichment, ingestor's RunMultibyteCapPersist is the only writer. + mustCol("nodes", "multibyte_sup") + mustCol("nodes", "multibyte_evidence") + mustCol("inactive_nodes", "multibyte_sup") + mustCol("inactive_nodes", "multibyte_evidence") + // Issue #1478: per-observer naive-clock skew tracking. Server reads + // all three to populate ObserverResp.ClockNaive / ClockSkew* fields. + mustCol("observers", "clock_skew_seconds") + mustCol("observers", "clock_skew_count_24h") + mustCol("observers", "clock_last_naive_at") + // Issue #1290: firmware 1.16 publishes a `repeat: on|off` flag in + // the MQTT /status JSON. Ingestor persists it as can_relay; server + // reads it to filter listener-only observers out of the path-hop + // disambiguator candidate set. Default 1 preserves prior behavior + // for legacy observers that never sent the field. + mustCol("observers", "can_relay") + // Issue #1290 follow-up (PR #1624 MAJOR-2): tri-state badge. The + // can_relay column defaults to 1 at INSERT, so we cannot distinguish + // "firmware confirmed repeater" from "legacy observer that never + // sent a repeat field". can_relay_seen=1 means the ingestor wrote + // an explicit value; can_relay_seen=0 means leave the UI badge + // unset (unknown state). + mustCol("observers", "can_relay_seen") + + if len(missing) > 0 { + return fmt.Errorf("schema not migrated by ingestor; restart ingestor first. missing: %s", + strings.Join(missing, ", ")) + } + return nil +} + +// Querier is the read surface shared by *sql.DB and *sql.Tx. +// +// Not *sql.Conn: it exposes only QueryContext/QueryRowContext, so it does not +// satisfy this. Widen the interface to the Context variants if a caller ever +// needs one. +// +// It exists so schema probes can run on whichever handle the caller already +// holds. Taking *sql.DB unconditionally is a deadlock waiting to happen: a +// caller inside a transaction has the connection checked out, and on a pool +// capped at one connection — which is what cmd/ingestor runs — a probe against +// the pool waits forever for the connection its own transaction is holding. +type Querier interface { + Query(query string, args ...any) (*sql.Rows, error) + QueryRow(query string, args ...any) *sql.Row +} + +// TableHasColumn reports whether the given table has the given column. +// Exported because tests and the read-side need it without re-implementing. +// +// Pass the transaction, not the pool, when you are inside one. See Querier. +func TableHasColumn(db Querier, table, column string) (bool, error) { + rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%s)", table)) + if err != nil { + return false, err + } + defer rows.Close() + for rows.Next() { + var cid int + var name string + var ctype sql.NullString + var notnull, pk int + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil { + return false, err + } + if name == column { + return true, nil + } + } + return false, rows.Err() +} + +// ─── ensure_* helpers (writer side) ──────────────────────────────────────── + +func ensureServerIndexes(rw *sql.DB) error { + stmts := []string{ + `CREATE INDEX IF NOT EXISTS idx_transmissions_first_seen ON transmissions(first_seen)`, + `CREATE INDEX IF NOT EXISTS idx_transmissions_hash ON transmissions(hash)`, + `CREATE INDEX IF NOT EXISTS idx_transmissions_payload_type ON transmissions(payload_type)`, + `CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp)`, + `CREATE INDEX IF NOT EXISTS idx_observations_transmission_id ON observations(transmission_id)`, + // Composite covers GetChannelMessages' grouped MAX(timestamp) per + // transmission_id (issue #1366 / PR #1368). With this index sqlite can + // satisfy the aggregate index-only without touching the heap. + `CREATE INDEX IF NOT EXISTS idx_observations_tx_ts ON observations(transmission_id, timestamp)`, + } + for _, s := range stmts { + if _, err := rw.Exec(s); err != nil { + return fmt.Errorf("ensure index %q: %w", s, err) + } + } + // observer_idx (v3) vs observer_id (v2) — probe + index the matching one. + hasIdx, err := TableHasColumn(rw, "observations", "observer_idx") + if err != nil { + return err + } + if hasIdx { + if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_observations_observer_idx ON observations(observer_idx)`); err != nil { + return err + } + } + hasID, err := TableHasColumn(rw, "observations", "observer_id") + if err != nil { + return err + } + if hasID { + if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_observations_observer_id ON observations(observer_id)`); err != nil { + return err + } + } + return nil +} + +// ensureChannelIndexes speeds up /api/channels and +// /api/channels/{hash}/messages. +// +// - idx_transmissions_channel_hash_payload adds first_seen to the +// existing channel_hash index (cmd/ingestor/db.go migration +// 'channel_hash_v1', idx_tx_channel_hash) so GetChannels' "latest +// message" correlated subquery (channel_hash + payload_type, ORDER BY +// first_seen DESC LIMIT 1) and GetChannelMessages' count/page queries +// resolve as index-order scans instead of sorting per channel. +// - idx_observers_iata_norm indexes the UPPER(TRIM(iata)) expression +// that GetChannels/GetEncryptedChannels/GetChannelMessages use for +// region filtering; a plain index on iata can't be used under that +// expression, so the expression itself is indexed. +// +// transmissions.channel_hash itself is added by the ingestor's legacy +// 'channel_hash_v1' migration (cmd/ingestor/db.go), not by this package, +// so it isn't guaranteed to exist yet on every DB Apply runs against — +// probe before indexing, same pattern as the observer_idx/observer_id +// probe in ensureServerIndexes above. +func ensureChannelIndexes(rw *sql.DB) error { + hasChannelHash, err := TableHasColumn(rw, "transmissions", "channel_hash") + if err != nil { + return err + } + if hasChannelHash { + if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_transmissions_channel_hash_payload ON transmissions(channel_hash, payload_type, first_seen)`); err != nil { + return fmt.Errorf("ensure idx_transmissions_channel_hash_payload: %w", err) + } + } + if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_observers_iata_norm ON observers(UPPER(TRIM(iata)))`); err != nil { + return fmt.Errorf("ensure idx_observers_iata_norm: %w", err) + } + return nil +} + +func ensureNeighborEdgesTable(rw *sql.DB) error { + _, err := rw.Exec(`CREATE TABLE IF NOT EXISTS neighbor_edges ( + node_a TEXT NOT NULL, + node_b TEXT NOT NULL, + count INTEGER DEFAULT 1, + last_seen TEXT, + PRIMARY KEY (node_a, node_b) + )`) + return err +} + +// ensureInactiveNodesTable creates the inactive_nodes table if missing. +// The ingestor's applySchema also creates this table — duplicating it +// here makes dbschema.Apply self-sufficient when called against a +// fixture DB that pre-dates the soft-delete feature (e.g. CI's +// test-fixtures/e2e-fixture.db, which never had any inactive rows). +// Schema kept in sync with cmd/ingestor/db.go:applySchema. +func ensureInactiveNodesTable(rw *sql.DB) error { + _, err := rw.Exec(`CREATE TABLE IF NOT EXISTS inactive_nodes ( + public_key TEXT PRIMARY KEY, + name TEXT, + role TEXT, + lat REAL, + lon REAL, + last_seen TEXT, + first_seen TEXT, + advert_count INTEGER DEFAULT 0, + battery_mv INTEGER, + temperature_c REAL, + foreign_advert INTEGER DEFAULT 0 + )`) + if err != nil { + return err + } + _, err = rw.Exec(`CREATE INDEX IF NOT EXISTS idx_inactive_nodes_last_seen ON inactive_nodes(last_seen)`) + return err +} + +func ensureResolvedPathColumn(rw *sql.DB, logf Logger) error { + has, err := TableHasColumn(rw, "observations", "resolved_path") + if err != nil { + return err + } + if has { + return nil + } + if _, err := rw.Exec("ALTER TABLE observations ADD COLUMN resolved_path TEXT"); err != nil { + return err + } + logf("[dbschema] added resolved_path column to observations") + return nil +} + +func ensureObserverInactiveColumn(rw *sql.DB, logf Logger) error { + has, err := TableHasColumn(rw, "observers", "inactive") + if err != nil { + return err + } + if has { + return nil + } + if _, err := rw.Exec("ALTER TABLE observers ADD COLUMN inactive INTEGER DEFAULT 0"); err != nil { + return err + } + logf("[dbschema] added inactive column to observers") + return nil +} + +func ensureLastPacketAtColumn(rw *sql.DB, logf Logger) error { + has, err := TableHasColumn(rw, "observers", "last_packet_at") + if err != nil { + return err + } + if has { + return nil + } + if _, err := rw.Exec("ALTER TABLE observers ADD COLUMN last_packet_at TEXT"); err != nil { + return err + } + logf("[dbschema] added last_packet_at column to observers") + return nil +} + +func ensureObserverIATAColumn(rw *sql.DB, logf Logger) error { + has, err := TableHasColumn(rw, "observers", "iata") + if err != nil { + return err + } + if has { + return nil + } + if _, err := rw.Exec("ALTER TABLE observers ADD COLUMN iata TEXT"); err != nil { + return err + } + logf("[dbschema] added iata column to observers") + return nil +} + +func ensureForeignAdvertColumn(rw *sql.DB, logf Logger) error { + for _, table := range []string{"nodes", "inactive_nodes"} { + has, err := TableHasColumn(rw, table, "foreign_advert") + if err != nil { + return fmt.Errorf("inspect %s: %w", table, err) + } + if has { + continue + } + if _, err := rw.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN foreign_advert INTEGER DEFAULT 0", table)); err != nil { + return err + } + logf("[dbschema] added foreign_advert column to %s", table) + } + return nil +} + +func ensureFromPubkeyColumn(rw *sql.DB, logf Logger) error { + has, err := TableHasColumn(rw, "transmissions", "from_pubkey") + if err != nil { + return err + } + if !has { + if _, err := rw.Exec("ALTER TABLE transmissions ADD COLUMN from_pubkey TEXT"); err != nil { + return err + } + logf("[dbschema] added from_pubkey column to transmissions (#1143)") + } + if _, err := rw.Exec("CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey)"); err != nil { + return err + } + return nil +} + +// ensureScopeNameColumn adds transmissions.scope_name (+ partial index) and +// records the legacy `_migrations` marker so the ingestor's old gated path +// stays idempotent on existing DBs. Moved here from cmd/ingestor/db.go in +// #1321 to be the canonical source of truth for the optional column the +// server PRAGMA-detects as hasScopeName. +func ensureScopeNameColumn(rw *sql.DB, logf Logger) error { + if err := ensureMigrationsTable(rw); err != nil { + return err + } + has, err := TableHasColumn(rw, "transmissions", "scope_name") + if err != nil { + return err + } + if !has { + if _, err := rw.Exec(`ALTER TABLE transmissions ADD COLUMN scope_name TEXT DEFAULT NULL`); err != nil { + return fmt.Errorf("alter transmissions add scope_name: %w", err) + } + logf("[dbschema] added scope_name column to transmissions (#899)") + } + if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_tx_scope_name ON transmissions(scope_name) WHERE scope_name IS NOT NULL`); err != nil { + return fmt.Errorf("create idx_tx_scope_name: %w", err) + } + if _, err := rw.Exec(`INSERT OR IGNORE INTO _migrations (name) VALUES ('scope_name_v1')`); err != nil { + return fmt.Errorf("record scope_name_v1: %w", err) + } + return nil +} + +// ensureDefaultScopeColumns adds default_scope to nodes + inactive_nodes +// and records the `_migrations` marker. Source of truth lives here per +// #1321 (was previously cmd/ingestor/db.go only). +func ensureDefaultScopeColumns(rw *sql.DB, logf Logger) error { + if err := ensureMigrationsTable(rw); err != nil { + return err + } + for _, table := range []string{"nodes", "inactive_nodes"} { + has, err := TableHasColumn(rw, table, "default_scope") + if err != nil { + return fmt.Errorf("inspect %s.default_scope: %w", table, err) + } + if has { + continue + } + if _, err := rw.Exec(fmt.Sprintf(`ALTER TABLE %s ADD COLUMN default_scope TEXT DEFAULT NULL`, table)); err != nil { + return fmt.Errorf("alter %s add default_scope: %w", table, err) + } + logf("[dbschema] added default_scope column to %s (#899)", table) + } + if _, err := rw.Exec(`INSERT OR IGNORE INTO _migrations (name) VALUES ('nodes_default_scope_v1')`); err != nil { + return fmt.Errorf("record nodes_default_scope_v1: %w", err) + } + return nil +} + +// ensureConfiguredScopeColumns adds nodes.configured_scope + +// nodes.configured_scope_at (and mirrors on inactive_nodes) for #1865. +// Unlike default_scope (inferred from observed advert transport scope, and +// overwritten on every observation), configured_scope holds the region scopes +// a node has CONFIGURED, taken as concrete evidence from the observer +// /neighbors report — written only for the observer's own `self` scopes and +// for neighbors whose OTA scope query returned status="responded". The +// server PRAGMA-detects configured_scope as hasConfiguredScope. +func ensureConfiguredScopeColumns(rw *sql.DB, logf Logger) error { + if err := ensureMigrationsTable(rw); err != nil { + return err + } + for _, table := range []string{"nodes", "inactive_nodes"} { + for _, col := range []string{"configured_scope", "configured_scope_at"} { + has, err := TableHasColumn(rw, table, col) + if err != nil { + return fmt.Errorf("inspect %s.%s: %w", table, col, err) + } + if has { + continue + } + if _, err := rw.Exec(fmt.Sprintf(`ALTER TABLE %s ADD COLUMN %s TEXT DEFAULT NULL`, table, col)); err != nil { + return fmt.Errorf("alter %s add %s: %w", table, col, err) + } + logf("[dbschema] added %s column to %s (#1865)", col, table) + } + } + if _, err := rw.Exec(`INSERT OR IGNORE INTO _migrations (name) VALUES ('nodes_configured_scope_v1')`); err != nil { + return fmt.Errorf("record nodes_configured_scope_v1: %w", err) + } + return nil +} + +// ensureObservationsRawHexColumn adds observations.raw_hex (#881). +// Source of truth lives here per #1321 (was previously cmd/ingestor/db.go only): +// the server PRAGMA-detects this column as hasObsRawHex. +func ensureObservationsRawHexColumn(rw *sql.DB, logf Logger) error { + if err := ensureMigrationsTable(rw); err != nil { + return err + } + has, err := TableHasColumn(rw, "observations", "raw_hex") + if err != nil { + return err + } + if !has { + if _, err := rw.Exec(`ALTER TABLE observations ADD COLUMN raw_hex TEXT`); err != nil { + return fmt.Errorf("alter observations add raw_hex: %w", err) + } + logf("[dbschema] added raw_hex column to observations (#881)") + } + if _, err := rw.Exec(`INSERT OR IGNORE INTO _migrations (name) VALUES ('observations_raw_hex_v1')`); err != nil { + return fmt.Errorf("record observations_raw_hex_v1: %w", err) + } + return nil +} + +// ensureMigrationsTable is idempotent — the ingestor's applySchema also +// creates this table on legacy paths. Keeping it here lets the gated +// `INSERT OR IGNORE` markers above stay self-sufficient even when Apply +// runs against a brand-new fixture DB (e.g. dbschema_test.go). +func ensureMigrationsTable(rw *sql.DB) error { + _, err := rw.Exec(`CREATE TABLE IF NOT EXISTS _migrations (name TEXT PRIMARY KEY)`) + return err +} + +// SoftDeleteBlacklistedObservers marks the given observer IDs as +// inactive=1 (case-insensitive match). Returns count affected. +// Writer-side helper; ingestor calls it at startup with the operator +// blacklist (read from config). +func SoftDeleteBlacklistedObservers(rw *sql.DB, blacklist []string) (int64, error) { + placeholders := make([]string, 0, len(blacklist)) + args := make([]interface{}, 0, len(blacklist)) + for _, pk := range blacklist { + t := strings.TrimSpace(pk) + if t == "" { + continue + } + placeholders = append(placeholders, "LOWER(?)") + args = append(args, t) + } + if len(placeholders) == 0 { + return 0, nil + } + q := "UPDATE observers SET inactive = 1 WHERE LOWER(id) IN (" + + strings.Join(placeholders, ",") + ") AND (inactive IS NULL OR inactive = 0)" + res, err := rw.Exec(q, args...) + if err != nil { + return 0, err + } + n, _ := res.RowsAffected() + return n, nil +} + +// ensureMultibyteCapColumns adds the multi-byte capability cache columns +// to nodes / inactive_nodes (PR #903, canonical owner per #1324 +// follow-up). These columns are populated by the ingestor's +// RunMultibyteCapPersist from snapshot files written by the server's +// analytics cycle; the server is read-only since #1289 and MUST NOT +// write here. The schema itself lives here in dbschema (the writer +// owns migrations, the read-only server merely AssertReady's them). +func ensureMultibyteCapColumns(rw *sql.DB, logf Logger) error { + for _, table := range []string{"nodes", "inactive_nodes"} { + hasSup, err := TableHasColumn(rw, table, "multibyte_sup") + if err != nil { + return fmt.Errorf("inspect %s.multibyte_sup: %w", table, err) + } + if !hasSup { + if _, err := rw.Exec(fmt.Sprintf( + "ALTER TABLE %s ADD COLUMN multibyte_sup INTEGER NOT NULL DEFAULT 0", table)); err != nil { + return fmt.Errorf("add %s.multibyte_sup: %w", table, err) + } + logf("[dbschema] added multibyte_sup column to %s", table) + } + hasEvid, err := TableHasColumn(rw, table, "multibyte_evidence") + if err != nil { + return fmt.Errorf("inspect %s.multibyte_evidence: %w", table, err) + } + if !hasEvid { + if _, err := rw.Exec(fmt.Sprintf( + "ALTER TABLE %s ADD COLUMN multibyte_evidence TEXT", table)); err != nil { + return fmt.Errorf("add %s.multibyte_evidence: %w", table, err) + } + logf("[dbschema] added multibyte_evidence column to %s", table) + } + } + return nil +} + +// ensureObserverNaiveClockColumns adds the three per-observer naive-clock +// skew tracking columns (#1478). Server reads them to populate the +// clock_naive / clock_skew_seconds / clock_skew_count_24h / +// clock_last_naive_at fields in /api/observers responses; ingestor writes +// them from resolveRxTime via Store.RecordNaiveSkew on each clamp event. +// Owned here per the #1321 source-of-truth invariant — the cmd/ingestor +// copy migrates real prod DBs, dbschema migrates fixture/staging DBs via +// cmd/migrate. +func ensureObserverNaiveClockColumns(rw *sql.DB, logf Logger) error { + type col struct{ name, ddl string } + cols := []col{ + {"clock_skew_seconds", "ALTER TABLE observers ADD COLUMN clock_skew_seconds INTEGER DEFAULT NULL"}, + {"clock_skew_count_24h", "ALTER TABLE observers ADD COLUMN clock_skew_count_24h INTEGER DEFAULT 0"}, + {"clock_last_naive_at", "ALTER TABLE observers ADD COLUMN clock_last_naive_at TEXT DEFAULT NULL"}, + } + for _, c := range cols { + has, err := TableHasColumn(rw, "observers", c.name) + if err != nil { + return fmt.Errorf("inspect observers.%s: %w", c.name, err) + } + if has { + continue + } + if _, err := rw.Exec(c.ddl); err != nil { + return fmt.Errorf("add observers.%s: %w", c.name, err) + } + logf("[dbschema] added %s column to observers", c.name) + } + return nil +} + +// ensureObserverCanRelayColumn adds the can_relay column to observers. +// Firmware 1.16 publishes a `repeat: on|off` flag in the MQTT /status +// JSON (#1290); the ingestor parses it and writes 0/1 here. The server's +// path-hop disambiguator (cmd/server/store.go pm.resolveWithContext) +// excludes observers with can_relay=0 from the candidate set. Default 1 +// preserves prior behavior for legacy observers (no repeat field). +func ensureObserverCanRelayColumn(rw *sql.DB, logf Logger) error { + has, err := TableHasColumn(rw, "observers", "can_relay") + if err != nil { + return err + } + if has { + return nil + } + // PREFLIGHT: async=true reason="single-column ALTER on observers (low-cardinality, ~1k rows in prod); DEFAULT 1 is a constant so SQLite does the rewrite as a metadata-only schema update, no row scan" + if _, err := rw.Exec("ALTER TABLE observers ADD COLUMN can_relay INTEGER DEFAULT 1"); err != nil { + return err + } + logf("[dbschema] added can_relay column to observers") + return nil +} + +// ensureObserverCanRelaySeenColumn adds the can_relay_seen tracking column. +// Issue #1290 follow-up (PR #1624 MAJOR-2): can_relay defaults to 1 at +// INSERT, which conflates "confirmed repeater" with "legacy observer +// that never sent the repeat field". can_relay_seen=1 is written by +// the ingestor whenever the firmware actually provided the field; the +// server's read layer returns CanRelay=nil whenever seen=0 so the UI +// can render the tri-state badge (no badge for unknown). +func ensureObserverCanRelaySeenColumn(rw *sql.DB, logf Logger) error { + has, err := TableHasColumn(rw, "observers", "can_relay_seen") + if err != nil { + return err + } + if has { + return nil + } + // PREFLIGHT: async=true reason="single-column ALTER on observers (low-cardinality, ~1k rows in prod); DEFAULT 0 is a constant so SQLite does the rewrite as a metadata-only schema update, no row scan" + if _, err := rw.Exec("ALTER TABLE observers ADD COLUMN can_relay_seen INTEGER DEFAULT 0"); err != nil { + return err + } + logf("[dbschema] added can_relay_seen column to observers") + return nil +} + +// ensureTransmissionsLastSeenColumn adds transmissions.last_seen (#1690), +// the denormalized "most recent observation timestamp" column the cold-load +// path filters on. Without it, hot-window queries that match +// `first_seen >= cutoff` exclude long-lived hashes whose hashes were first +// inserted weeks ago but are still active — see the issue for the full +// 0.3%-of-DB-loaded post-mortem. +// +// The backfill from `MAX(observations.timestamp) GROUP BY transmission_id` +// is potentially expensive (1.9M+ obs rows in prod) and runs as a separate +// async migration scheduled by cmd/ingestor/db.go::OpenStore via +// Store.RunAsyncMigration. +func ensureTransmissionsLastSeenColumn(rw *sql.DB, logf Logger) error { + if err := ensureMigrationsTable(rw); err != nil { + return err + } + has, err := TableHasColumn(rw, "transmissions", "last_seen") + if err != nil { + return err + } + if !has { + // PREFLIGHT: async=true reason="single-column ALTER TABLE on transmissions; DEFAULT 0 is a constant so SQLite does a metadata-only schema rewrite (no row scan) — cheap at any scale" + if _, err := rw.Exec(`ALTER TABLE transmissions ADD COLUMN last_seen INTEGER NOT NULL DEFAULT 0`); err != nil { + return fmt.Errorf("alter transmissions add last_seen: %w", err) + } + logf("[dbschema] added last_seen column to transmissions (#1690)") + } + // #1740 step (a): create the PARTIAL index first. Scans only rows with + // last_seen=0 — i.e. the un-backfilled hot subset the chunked backfill + // MAX(id) lookup walks. Degenerates to near-empty once the backfill + // converges, so it stops competing for page cache. + // + // PREFLIGHT: async=false reason="partial index on `WHERE last_seen=0`; subset is bounded by un-backfilled rows (≤0 in steady state, ≤ingest-rate during ops) — cheap at any scale even on a 1.9M-row prod table" + if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_tx_last_seen_zero ON transmissions(id) WHERE last_seen=0`); err != nil { + return fmt.Errorf("create idx_tx_last_seen_zero: %w", err) + } + // #1740 step (b): gated DROP of the legacy full index. Only runs AFTER + // step (a) succeeded (sequential — we're past the CREATE above). The + // DROP is a metadata-only schema rewrite in SQLite; no row scan. + // + // PREFLIGHT: async=false reason="DROP INDEX is metadata-only in SQLite (no row scan, no rewrite); safe inline at any DB size" + if _, err := rw.Exec(`DROP INDEX IF EXISTS idx_tx_last_seen`); err != nil { + return fmt.Errorf("drop legacy idx_tx_last_seen: %w", err) + } + return nil +} diff --git a/internal/dbschema/dbschema_test.go b/internal/dbschema/legacy/dbschema_test.go similarity index 99% rename from internal/dbschema/dbschema_test.go rename to internal/dbschema/legacy/dbschema_test.go index 5145777b5..3723df9ef 100644 --- a/internal/dbschema/dbschema_test.go +++ b/internal/dbschema/legacy/dbschema_test.go @@ -1,4 +1,4 @@ -package dbschema +package legacy import ( "database/sql" diff --git a/internal/dbschema/dedup_index.go b/internal/dbschema/legacy/dedup_index.go similarity index 99% rename from internal/dbschema/dedup_index.go rename to internal/dbschema/legacy/dedup_index.go index fb0f728fc..080e97fbb 100644 --- a/internal/dbschema/dedup_index.go +++ b/internal/dbschema/legacy/dedup_index.go @@ -1,4 +1,4 @@ -package dbschema +package legacy import ( "database/sql" diff --git a/internal/dbschema/dedup_index_test.go b/internal/dbschema/legacy/dedup_index_test.go similarity index 99% rename from internal/dbschema/dedup_index_test.go rename to internal/dbschema/legacy/dedup_index_test.go index 550e328d9..fc4c6d36e 100644 --- a/internal/dbschema/dedup_index_test.go +++ b/internal/dbschema/legacy/dedup_index_test.go @@ -1,4 +1,4 @@ -package dbschema +package legacy import ( "database/sql" diff --git a/internal/dbschema/legacy/ingestor_migration_test.go b/internal/dbschema/legacy/ingestor_migration_test.go new file mode 100644 index 000000000..d277fc934 --- /dev/null +++ b/internal/dbschema/legacy/ingestor_migration_test.go @@ -0,0 +1,67 @@ +package legacy + +import ( + "database/sql" + "path/filepath" + "testing" +) + +func TestNormalizeRejectsDestructiveLegacyCleanup(t *testing.T) { + path := filepath.Join(t.TempDir(), "legacy.db") + + // Create a bare-bones DB with legacy bad data + db, err := sql.Open("sqlite3", path+"?_journal_mode=WAL&_busy_timeout=5000") + if err != nil { + t.Fatal(err) + } + db.Exec(`CREATE TABLE IF NOT EXISTS transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_hex TEXT NOT NULL, + hash TEXT NOT NULL, + first_seen TEXT NOT NULL, + route_type INTEGER, + payload_type INTEGER, + payload_version INTEGER, + decoded_json TEXT, + created_at TEXT DEFAULT (datetime('now')), + channel_hash TEXT DEFAULT NULL + )`) + db.Exec(`CREATE TABLE IF NOT EXISTS observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER NOT NULL REFERENCES transmissions(id), + observer_idx INTEGER, + direction TEXT, + snr REAL, + rssi REAL, + score INTEGER, + path_json TEXT, + timestamp INTEGER NOT NULL + )`) + db.Exec(`CREATE TABLE IF NOT EXISTS _migrations (name TEXT PRIMARY KEY)`) + db.Exec(`CREATE TABLE IF NOT EXISTS nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL)`) + db.Exec(`CREATE TABLE IF NOT EXISTS observers (id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT, packet_count INTEGER DEFAULT 0, model TEXT, firmware TEXT, client_version TEXT, radio TEXT, battery_mv INTEGER, uptime_secs INTEGER, noise_floor REAL, inactive INTEGER DEFAULT 0, last_packet_at TEXT DEFAULT NULL)`) + + // Insert good transmission + db.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, 'aabb', 'abc123', '2024-01-01T00:00:00Z')`) + db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (1, 1, 1704067200)`) + + // Insert bad: empty hash + db.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (2, 'ccdd', '', '2024-01-01T00:00:00Z')`) + db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (2, 1, 1704067200)`) + + // Insert bad: empty first_seen + db.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (3, 'eeff', 'def456', '')`) + db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (3, 2, 1704067200)`) + + defer db.Close() + if err := Normalize(db, nil); err == nil { + t.Fatal("lossy legacy cleanup accepted") + } + var count int + if err := db.QueryRow(`SELECT count(*) FROM transmissions`).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 3 { + t.Fatalf("source rows changed on refusal: %d", count) + } +} diff --git a/cmd/ingestor/public_key_lowercase_migration_test.go b/internal/dbschema/legacy/public_key_test.go similarity index 84% rename from cmd/ingestor/public_key_lowercase_migration_test.go rename to internal/dbschema/legacy/public_key_test.go index 160ab4aae..8c6e66e32 100644 --- a/cmd/ingestor/public_key_lowercase_migration_test.go +++ b/internal/dbschema/legacy/public_key_test.go @@ -1,7 +1,8 @@ -package main +package legacy import ( "database/sql" + "path/filepath" "strings" "testing" ) @@ -11,8 +12,12 @@ import ( // The ingestor must normalize any legacy uppercase rows on boot so // the lookup remains correct. func TestPublicKeyLowercaseNormalizationMigration(t *testing.T) { - dbPath := tempDBPath(t) - s, err := OpenStore(dbPath) + dbPath := filepath.Join(t.TempDir(), "legacy.db") + db, err := sql.Open("sqlite3", dbPath) + s := struct{ db *sql.DB }{db} + if err == nil { + err = ApplyBase(db) + } if err != nil { t.Fatalf("first OpenStore: %v", err) } @@ -31,14 +36,18 @@ func TestPublicKeyLowercaseNormalizationMigration(t *testing.T) { if pk != "AABBCCDDEEFF11223344" { t.Fatalf("pre-check: expected uppercase, got %s", pk) } - s.Close() + s.db.Close() // Reopen — the boot-time migration should normalize the row. - s2, err := OpenStore(dbPath) + db2, err := sql.Open("sqlite3", dbPath) + s2 := struct{ db *sql.DB }{db2} + if err == nil { + err = Normalize(db2, nil) + } if err != nil { t.Fatalf("reopen: %v", err) } - defer s2.Close() + defer s2.db.Close() // The uppercase row should be gone. var still int diff --git a/internal/dbschema/writer_dsn.go b/internal/dbschema/legacy/writer_dsn.go similarity index 98% rename from internal/dbschema/writer_dsn.go rename to internal/dbschema/legacy/writer_dsn.go index fb0f7acd2..829f6b9b5 100644 --- a/internal/dbschema/writer_dsn.go +++ b/internal/dbschema/legacy/writer_dsn.go @@ -1,4 +1,4 @@ -package dbschema +package legacy // WriterDSN builds the DSN every writer must open the database with. // diff --git a/internal/dbschema/writer_dsn_test.go b/internal/dbschema/legacy/writer_dsn_test.go similarity index 98% rename from internal/dbschema/writer_dsn_test.go rename to internal/dbschema/legacy/writer_dsn_test.go index 1e83fa020..79f2d157f 100644 --- a/internal/dbschema/writer_dsn_test.go +++ b/internal/dbschema/legacy/writer_dsn_test.go @@ -1,4 +1,4 @@ -package dbschema +package legacy import ( "database/sql" diff --git a/internal/dbschema/manifest.go b/internal/dbschema/manifest.go new file mode 100644 index 000000000..f7a6f8816 --- /dev/null +++ b/internal/dbschema/manifest.go @@ -0,0 +1,36 @@ +package dbschema + +// Column describes the lossless import representation. Identity columns are reseeded after import. +type Column struct { + Name string + Type string + Identity bool +} +type Table struct { + Name string + Columns []Column +} + +// Tables is the complete telemetry import manifest, in dependency order. +var Tables = []Table{ + {Name: "nodes", Columns: []Column{{Name: "public_key", Type: "TEXT", Identity: false}, {Name: "name", Type: "TEXT", Identity: false}, {Name: "role", Type: "TEXT", Identity: false}, {Name: "lat", Type: "DOUBLE PRECISION", Identity: false}, {Name: "lon", Type: "DOUBLE PRECISION", Identity: false}, {Name: "last_seen", Type: "TEXT", Identity: false}, {Name: "first_seen", Type: "TEXT", Identity: false}, {Name: "advert_count", Type: "BIGINT", Identity: false}, {Name: "battery_mv", Type: "BIGINT", Identity: false}, {Name: "temperature_c", Type: "DOUBLE PRECISION", Identity: false}, {Name: "foreign_advert", Type: "BIGINT", Identity: false}, {Name: "default_scope", Type: "TEXT", Identity: false}, {Name: "configured_scope", Type: "TEXT", Identity: false}, {Name: "configured_scope_at", Type: "TEXT", Identity: false}, {Name: "multibyte_sup", Type: "BIGINT", Identity: false}, {Name: "multibyte_evidence", Type: "TEXT", Identity: false}}}, + {Name: "inactive_nodes", Columns: []Column{{Name: "public_key", Type: "TEXT", Identity: false}, {Name: "name", Type: "TEXT", Identity: false}, {Name: "role", Type: "TEXT", Identity: false}, {Name: "lat", Type: "DOUBLE PRECISION", Identity: false}, {Name: "lon", Type: "DOUBLE PRECISION", Identity: false}, {Name: "last_seen", Type: "TEXT", Identity: false}, {Name: "first_seen", Type: "TEXT", Identity: false}, {Name: "advert_count", Type: "BIGINT", Identity: false}, {Name: "battery_mv", Type: "BIGINT", Identity: false}, {Name: "temperature_c", Type: "DOUBLE PRECISION", Identity: false}, {Name: "foreign_advert", Type: "BIGINT", Identity: false}, {Name: "default_scope", Type: "TEXT", Identity: false}, {Name: "configured_scope", Type: "TEXT", Identity: false}, {Name: "configured_scope_at", Type: "TEXT", Identity: false}, {Name: "multibyte_sup", Type: "BIGINT", Identity: false}, {Name: "multibyte_evidence", Type: "TEXT", Identity: false}}}, + {Name: "observers", Columns: []Column{{Name: "rowid", Type: "BIGINT", Identity: true}, {Name: "id", Type: "TEXT", Identity: false}, {Name: "name", Type: "TEXT", Identity: false}, {Name: "iata", Type: "TEXT", Identity: false}, {Name: "last_seen", Type: "TEXT", Identity: false}, {Name: "first_seen", Type: "TEXT", Identity: false}, {Name: "packet_count", Type: "BIGINT", Identity: false}, {Name: "model", Type: "TEXT", Identity: false}, {Name: "firmware", Type: "TEXT", Identity: false}, {Name: "client_version", Type: "TEXT", Identity: false}, {Name: "radio", Type: "TEXT", Identity: false}, {Name: "battery_mv", Type: "BIGINT", Identity: false}, {Name: "uptime_secs", Type: "BIGINT", Identity: false}, {Name: "noise_floor", Type: "DOUBLE PRECISION", Identity: false}, {Name: "inactive", Type: "BIGINT", Identity: false}, {Name: "last_packet_at", Type: "TEXT", Identity: false}, {Name: "clock_skew_seconds", Type: "BIGINT", Identity: false}, {Name: "clock_skew_count_24h", Type: "BIGINT", Identity: false}, {Name: "clock_last_naive_at", Type: "TEXT", Identity: false}, {Name: "can_relay", Type: "BIGINT", Identity: false}, {Name: "can_relay_seen", Type: "BIGINT", Identity: false}}}, + {Name: "transmissions", Columns: []Column{{Name: "id", Type: "BIGINT", Identity: true}, {Name: "raw_hex", Type: "TEXT", Identity: false}, {Name: "hash", Type: "TEXT", Identity: false}, {Name: "first_seen", Type: "TEXT", Identity: false}, {Name: "route_type", Type: "BIGINT", Identity: false}, {Name: "payload_type", Type: "BIGINT", Identity: false}, {Name: "payload_version", Type: "BIGINT", Identity: false}, {Name: "decoded_json", Type: "TEXT", Identity: false}, {Name: "from_pubkey", Type: "TEXT", Identity: false}, {Name: "last_seen", Type: "BIGINT", Identity: false}, {Name: "created_at", Type: "TEXT", Identity: false}, {Name: "code1", Type: "TEXT", Identity: false}, {Name: "code2", Type: "TEXT", Identity: false}, {Name: "channel_hash", Type: "TEXT", Identity: false}, {Name: "scope_name", Type: "TEXT", Identity: false}}}, + {Name: "observations", Columns: []Column{{Name: "id", Type: "BIGINT", Identity: true}, {Name: "transmission_id", Type: "BIGINT", Identity: false}, {Name: "observer_idx", Type: "BIGINT", Identity: false}, {Name: "direction", Type: "TEXT", Identity: false}, {Name: "snr", Type: "DOUBLE PRECISION", Identity: false}, {Name: "rssi", Type: "DOUBLE PRECISION", Identity: false}, {Name: "score", Type: "DOUBLE PRECISION", Identity: false}, {Name: "path_json", Type: "TEXT", Identity: false}, {Name: "timestamp", Type: "BIGINT", Identity: false}, {Name: "raw_hex", Type: "TEXT", Identity: false}, {Name: "resolved_path", Type: "TEXT", Identity: false}}}, + {Name: "observer_metrics", Columns: []Column{{Name: "observer_id", Type: "TEXT", Identity: false}, {Name: "timestamp", Type: "TEXT", Identity: false}, {Name: "noise_floor", Type: "DOUBLE PRECISION", Identity: false}, {Name: "tx_air_secs", Type: "BIGINT", Identity: false}, {Name: "rx_air_secs", Type: "BIGINT", Identity: false}, {Name: "recv_errors", Type: "BIGINT", Identity: false}, {Name: "battery_mv", Type: "BIGINT", Identity: false}, {Name: "packets_sent", Type: "BIGINT", Identity: false}, {Name: "packets_recv", Type: "BIGINT", Identity: false}}}, + {Name: "neighbor_edges", Columns: []Column{{Name: "node_a", Type: "TEXT", Identity: false}, {Name: "node_b", Type: "TEXT", Identity: false}, {Name: "count", Type: "BIGINT", Identity: false}, {Name: "last_seen", Type: "TEXT", Identity: false}}}, + {Name: "dropped_packets", Columns: []Column{{Name: "id", Type: "BIGINT", Identity: true}, {Name: "hash", Type: "TEXT", Identity: false}, {Name: "raw_hex", Type: "TEXT", Identity: false}, {Name: "reason", Type: "TEXT", Identity: false}, {Name: "observer_id", Type: "TEXT", Identity: false}, {Name: "observer_name", Type: "TEXT", Identity: false}, {Name: "node_pubkey", Type: "TEXT", Identity: false}, {Name: "node_name", Type: "TEXT", Identity: false}, {Name: "dropped_at", Type: "TEXT", Identity: false}}}, + {Name: "client_receptions", Columns: []Column{{Name: "id", Type: "BIGINT", Identity: true}, {Name: "rx_pubkey", Type: "TEXT", Identity: false}, {Name: "heard_key", Type: "TEXT", Identity: false}, {Name: "heard_keylen", Type: "BIGINT", Identity: false}, {Name: "rssi", Type: "BIGINT", Identity: false}, {Name: "snr", Type: "DOUBLE PRECISION", Identity: false}, {Name: "lat", Type: "DOUBLE PRECISION", Identity: false}, {Name: "lon", Type: "DOUBLE PRECISION", Identity: false}, {Name: "pos_acc_m", Type: "DOUBLE PRECISION", Identity: false}, {Name: "rx_at", Type: "TEXT", Identity: false}, {Name: "ingested_at", Type: "TEXT", Identity: false}, {Name: "src", Type: "TEXT", Identity: false}}}, + {Name: "client_observers", Columns: []Column{{Name: "pubkey", Type: "TEXT", Identity: false}, {Name: "name", Type: "TEXT", Identity: false}, {Name: "last_seen", Type: "TEXT", Identity: false}}}, + {Name: "client_rx_observations", Columns: []Column{{Name: "id", Type: "BIGINT", Identity: true}, {Name: "rx_pubkey", Type: "TEXT", Identity: false}, {Name: "rx_at", Type: "TEXT", Identity: false}, {Name: "ingested_at", Type: "TEXT", Identity: false}, {Name: "pkt_hash", Type: "TEXT", Identity: false}, {Name: "route_type", Type: "BIGINT", Identity: false}, {Name: "payload_type", Type: "BIGINT", Identity: false}, {Name: "code1", Type: "TEXT", Identity: false}, {Name: "code2", Type: "TEXT", Identity: false}, {Name: "scope_name", Type: "TEXT", Identity: false}, {Name: "hash_size", Type: "BIGINT", Identity: false}, {Name: "hop_count", Type: "BIGINT", Identity: false}, {Name: "path_json", Type: "TEXT", Identity: false}, {Name: "forwarder", Type: "TEXT", Identity: false}, {Name: "snr", Type: "DOUBLE PRECISION", Identity: false}, {Name: "rssi", Type: "BIGINT", Identity: false}, {Name: "lat", Type: "DOUBLE PRECISION", Identity: false}, {Name: "lon", Type: "DOUBLE PRECISION", Identity: false}, {Name: "pos_acc_m", Type: "DOUBLE PRECISION", Identity: false}}}, + {Name: "client_rf_samples", Columns: []Column{{Name: "id", Type: "BIGINT", Identity: true}, {Name: "rx_pubkey", Type: "TEXT", Identity: false}, {Name: "sampled_at", Type: "TEXT", Identity: false}, {Name: "ingested_at", Type: "TEXT", Identity: false}, {Name: "lat", Type: "DOUBLE PRECISION", Identity: false}, {Name: "lon", Type: "DOUBLE PRECISION", Identity: false}, {Name: "pos_acc_m", Type: "DOUBLE PRECISION", Identity: false}, {Name: "stationary", Type: "BIGINT", Identity: false}, {Name: "uptime_secs", Type: "BIGINT", Identity: false}, {Name: "battery_mv", Type: "BIGINT", Identity: false}, {Name: "queue_len", Type: "BIGINT", Identity: false}, {Name: "errors", Type: "BIGINT", Identity: false}, {Name: "noise_floor", Type: "BIGINT", Identity: false}, {Name: "last_rssi", Type: "BIGINT", Identity: false}, {Name: "last_snr", Type: "DOUBLE PRECISION", Identity: false}, {Name: "tx_air_secs", Type: "BIGINT", Identity: false}, {Name: "rx_air_secs", Type: "BIGINT", Identity: false}, {Name: "recv", Type: "BIGINT", Identity: false}, {Name: "sent", Type: "BIGINT", Identity: false}, {Name: "flood_rx", Type: "BIGINT", Identity: false}, {Name: "direct_rx", Type: "BIGINT", Identity: false}, {Name: "flood_tx", Type: "BIGINT", Identity: false}, {Name: "direct_tx", Type: "BIGINT", Identity: false}, {Name: "recv_errors", Type: "BIGINT", Identity: false}}}, + {Name: "node_declared_regions", Columns: []Column{{Name: "id", Type: "BIGINT", Identity: true}, {Name: "target", Type: "TEXT", Identity: false}, {Name: "rx_pubkey", Type: "TEXT", Identity: false}, {Name: "observed_at", Type: "TEXT", Identity: false}, {Name: "ingested_at", Type: "TEXT", Identity: false}, {Name: "regions_csv", Type: "TEXT", Identity: false}, {Name: "truncated", Type: "BIGINT", Identity: false}, {Name: "lat", Type: "DOUBLE PRECISION", Identity: false}, {Name: "lon", Type: "DOUBLE PRECISION", Identity: false}, {Name: "pos_acc_m", Type: "DOUBLE PRECISION", Identity: false}, {Name: "repeater_clock", Type: "BIGINT", Identity: false}}}, + {Name: "scope_match_totals", Columns: []Column{{Name: "id", Type: "BIGINT", Identity: false}, {Name: "since_unix", Type: "BIGINT", Identity: false}, {Name: "unique_matches", Type: "BIGINT", Identity: false}, {Name: "explicit_over_derived", Type: "BIGINT", Identity: false}, {Name: "ambiguous", Type: "BIGINT", Identity: false}, {Name: "none_matches", Type: "BIGINT", Identity: false}, {Name: "updated_unix", Type: "BIGINT", Identity: false}}}, + {Name: "_migrations", Columns: []Column{{Name: "name", Type: "TEXT", Identity: false}}}, + {Name: "_async_migrations", Columns: []Column{{Name: "name", Type: "TEXT", Identity: false}, {Name: "status", Type: "TEXT", Identity: false}, {Name: "started_at", Type: "TEXT", Identity: false}, {Name: "ended_at", Type: "TEXT", Identity: false}, {Name: "error", Type: "TEXT", Identity: false}}}, + {Name: "advert_route_evidence", Columns: []Column{{Name: "id", Type: "BIGINT", Identity: true}, {Name: "tx_id", Type: "BIGINT", Identity: false}, {Name: "bit", Type: "BIGINT", Identity: false}}}, + {Name: "advert_evidence_backfill", Columns: []Column{{Name: "id", Type: "BIGINT", Identity: false}, {Name: "tx_cursor", Type: "BIGINT", Identity: false}, {Name: "obs_cursor", Type: "BIGINT", Identity: false}}}, +} + +var Indexes = []string{"idx_observations_observer_idx_timestamp", "idx_nodes_last_seen", "idx_observers_last_seen", "idx_inactive_nodes_last_seen", "idx_transmissions_hash", "idx_transmissions_first_seen", "idx_transmissions_payload_type", "idx_client_recept_heard_geo", "idx_client_recept_latlon", "idx_client_recept_rxat", "idx_cro_prune", "idx_cro_hash", "idx_cro_forwarder", "idx_cro_scope", "idx_crf_prune", "idx_crf_track", "idx_ndr_target", "idx_ndr_prune", "idx_observations_transmission_id", "idx_observations_observer_idx", "idx_observations_timestamp", "idx_observations_dedup", "idx_observer_metrics_timestamp", "idx_tx_channel_hash", "idx_dropped_observer", "idx_dropped_node", "idx_nodes_foreign_advert", "idx_transmissions_from_pubkey", "idx_tx_code1", "idx_observations_tx_ts", "idx_transmissions_channel_hash_payload", "idx_observers_iata_norm", "idx_tx_scope_name", "idx_tx_last_seen_zero"} diff --git a/internal/dbschema/postgres_integration_test.go b/internal/dbschema/postgres_integration_test.go new file mode 100644 index 000000000..179841084 --- /dev/null +++ b/internal/dbschema/postgres_integration_test.go @@ -0,0 +1,179 @@ +package dbschema + +import ( + "database/sql" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" + "sync" + "testing" +) + +func TestPostgresImportInitializationNeverReady(t *testing.T) { + db, err := pgutil.Open(pgtest.NewSchema(t), false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := ApplyForImport(db, nil); err != nil { + t.Fatal(err) + } + if err := AssertReady(db); err == nil { + t.Fatal("empty import target accepted before verification") + } + if err := Apply(db, nil); err == nil { + t.Fatal("normal initialization bypassed incomplete import") + } +} + +func TestPostgresConcurrentInitialization(t *testing.T) { + db, err := pgutil.Open(pgtest.NewSchema(t), false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + var wg sync.WaitGroup + for i := 0; i < 3; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if err := Apply(db, nil); err != nil { + t.Error(err) + } + }() + } + wg.Wait() + if err := AssertReady(db); err != nil { + t.Fatal(err) + } +} + +func TestPostgresIdentityRawValuesAndNullDedup(t *testing.T) { + db, err := pgutil.Open(pgtest.NewSchema(t), false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := Apply(db, nil); err != nil { + t.Fatal(err) + } + for _, q := range []string{ + `INSERT INTO observers(rowid,id) VALUES(0,'zero'),(77,'stable')`, + `INSERT INTO transmissions(id,raw_hex,hash,first_seen,decoded_json) VALUES(0,'aa','zero','2026-01-01T01:02:03.123456Z','{malformed')`, + `INSERT INTO observations(id,transmission_id,observer_idx,path_json,timestamp,score) VALUES(0,0,77,NULL,1,3.5),(50,0,NULL,NULL,2,NULL),(60,0,NULL,'',3,NULL)`, + } { + if _, err := db.Exec(q); err != nil { + t.Fatal(err) + } + } + if _, err := db.Exec(`INSERT INTO observations(transmission_id,observer_idx,path_json,timestamp) VALUES(0,77,'',4)`); err == nil { + t.Fatal("null/empty path identity diverged") + } + var score float64 + var raw, stamp string + if err := db.QueryRow(`SELECT o.score,t.decoded_json,t.first_seen FROM observations o JOIN transmissions t ON t.id=o.transmission_id WHERE o.id=0`).Scan(&score, &raw, &stamp); err != nil { + t.Fatal(err) + } + if score != 3.5 || raw != "{malformed" || stamp != "2026-01-01T01:02:03.123456Z" { + t.Fatal("raw values changed") + } + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + if err := ReseedIdentities(tx, map[string]int64{"observations.id": 500}); err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + var id int64 + if err := db.QueryRow(`INSERT INTO observations(transmission_id,timestamp) VALUES(0,5) RETURNING id`).Scan(&id); err != nil { + t.Fatal(err) + } + if id != 501 { + t.Fatalf("deleted identity high-water lost: %d", id) + } +} + +func TestPostgresWriterRejectsOwnerAndReader(t *testing.T) { + dsn := pgtest.NewSchema(t) + owner, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + defer owner.Close() + if err := Apply(owner, nil); err != nil { + t.Fatal(err) + } + if err := AssertWriter(owner); err == nil { + t.Fatal("owner accepted as runtime writer") + } + reader, err := pgutil.Open(pgtest.ReadOnly(t, dsn), true) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + if err := AssertReady(reader); err != nil { + t.Fatal(err) + } + if err := AssertWriter(reader); err == nil { + t.Fatal("SELECT-only role accepted as writer") + } + for _, q := range []string{`INSERT INTO nodes(public_key) VALUES('x')`, `UPDATE nodes SET name='x'`, `DELETE FROM nodes`, `CREATE TABLE forbidden(id int)`} { + if _, err := reader.Exec(q); err == nil { + t.Fatalf("reader permitted %s", q) + } + } +} + +func TestPostgresInitializationRejectsUnversionedData(t *testing.T) { + db, err := pgutil.Open(pgtest.NewSchema(t), false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`CREATE TABLE existing(id int)`); err != nil { + t.Fatal(err) + } + if err := Apply(db, nil); err == nil { + t.Fatal("unversioned nonempty schema accepted") + } + var marker sql.NullString + if err := db.QueryRow(`SELECT to_regclass('corescope_schema')::text`).Scan(&marker); err != nil { + t.Fatal(err) + } + if marker.Valid { + t.Fatal("failed initialization left readiness metadata behind") + } +} + +func TestPostgresSchemaFreshRepeatedAndReadiness(t *testing.T) { + dsn := pgtest.NewSchema(t) + db, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + apply := Apply + for i := 0; i < 2; i++ { + if err := apply(db, nil); err != nil { + t.Fatalf("initialize PostgreSQL schema: %v", err) + } + } + if err := AssertReady(db); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`UPDATE corescope_schema SET ready=false`); err != nil { + t.Fatal(err) + } + if err := AssertReady(db); err == nil { + t.Fatal("incomplete import accepted") + } + if _, err := db.Exec(`UPDATE corescope_schema SET ready=true,version=999`); err != nil { + t.Fatal(err) + } + if err := AssertReady(db); err == nil { + t.Fatal("newer schema accepted") + } +} diff --git a/internal/dbschema/postgres_test.go b/internal/dbschema/postgres_test.go new file mode 100644 index 000000000..4ffad1fab --- /dev/null +++ b/internal/dbschema/postgres_test.go @@ -0,0 +1,29 @@ +package dbschema + +import ( + "strings" + "testing" +) + +func TestPostgresManifestPreservesTelemetry(t *testing.T) { + if len(Tables) != 18 { + t.Fatalf("telemetry manifest has %d tables, want 18", len(Tables)) + } + for _, name := range []string{"nodes", "inactive_nodes", "observers", "transmissions", "observations", "observer_metrics", "neighbor_edges", "dropped_packets", "client_receptions", "client_observers", "client_rx_observations", "client_rf_samples", "node_declared_regions", "scope_match_totals", "_migrations", "_async_migrations", "advert_route_evidence", "advert_evidence_backfill"} { + found := false + for _, table := range Tables { + if table.Name == name { + found = true + } + } + if !found { + t.Errorf("manifest missing %s", name) + } + } + if !strings.Contains(SchemaSQL, "rowid BIGINT GENERATED BY DEFAULT AS IDENTITY") { + t.Error("observer identity must be explicit and importable") + } + if strings.Contains(strings.ToUpper(SchemaSQL), "JSONB") { + t.Error("raw JSON must remain text") + } +} diff --git a/internal/dbschema/schema.sql b/internal/dbschema/schema.sql new file mode 100644 index 000000000..3bfcd1995 --- /dev/null +++ b/internal/dbschema/schema.sql @@ -0,0 +1,325 @@ +-- Canonical PostgreSQL telemetry schema. Raw JSON and timestamp strings remain TEXT COLLATE "C". +CREATE TABLE nodes ( + public_key TEXT COLLATE "C" PRIMARY KEY, + name TEXT COLLATE "C", + role TEXT COLLATE "C", + lat DOUBLE PRECISION, + lon DOUBLE PRECISION, + last_seen TEXT COLLATE "C", + first_seen TEXT COLLATE "C", + advert_count BIGINT DEFAULT 0, + battery_mv BIGINT, + temperature_c DOUBLE PRECISION, + foreign_advert BIGINT DEFAULT 0, + default_scope TEXT COLLATE "C", + configured_scope TEXT COLLATE "C", + configured_scope_at TEXT COLLATE "C", + multibyte_sup BIGINT NOT NULL DEFAULT 0, + multibyte_evidence TEXT COLLATE "C" +); + +CREATE TABLE inactive_nodes ( + public_key TEXT COLLATE "C" PRIMARY KEY, + name TEXT COLLATE "C", + role TEXT COLLATE "C", + lat DOUBLE PRECISION, + lon DOUBLE PRECISION, + last_seen TEXT COLLATE "C", + first_seen TEXT COLLATE "C", + advert_count BIGINT DEFAULT 0, + battery_mv BIGINT, + temperature_c DOUBLE PRECISION, + foreign_advert BIGINT DEFAULT 0, + default_scope TEXT COLLATE "C", + configured_scope TEXT COLLATE "C", + configured_scope_at TEXT COLLATE "C", + multibyte_sup BIGINT NOT NULL DEFAULT 0, + multibyte_evidence TEXT COLLATE "C" +); + +CREATE TABLE observers ( + rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, + id TEXT COLLATE "C" PRIMARY KEY, + name TEXT COLLATE "C", + iata TEXT COLLATE "C", + last_seen TEXT COLLATE "C", + first_seen TEXT COLLATE "C", + packet_count BIGINT DEFAULT 0, + model TEXT COLLATE "C", + firmware TEXT COLLATE "C", + client_version TEXT COLLATE "C", + radio TEXT COLLATE "C", + battery_mv BIGINT, + uptime_secs BIGINT, + noise_floor DOUBLE PRECISION, + inactive BIGINT DEFAULT 0, + last_packet_at TEXT COLLATE "C" DEFAULT NULL, + clock_skew_seconds BIGINT DEFAULT NULL, + clock_skew_count_24h BIGINT DEFAULT 0, + clock_last_naive_at TEXT COLLATE "C" DEFAULT NULL, + can_relay BIGINT DEFAULT 1, + can_relay_seen BIGINT DEFAULT 0 +); + +CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT COLLATE "C" NOT NULL, + hash TEXT COLLATE "C" NOT NULL UNIQUE, + first_seen TEXT COLLATE "C" NOT NULL, + route_type BIGINT, + payload_type BIGINT, + payload_version BIGINT, + decoded_json TEXT COLLATE "C", + from_pubkey TEXT COLLATE "C", + last_seen BIGINT NOT NULL DEFAULT 0, + created_at TEXT COLLATE "C" DEFAULT (to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS')), + code1 TEXT COLLATE "C", + code2 TEXT COLLATE "C", + channel_hash TEXT COLLATE "C", + scope_name TEXT COLLATE "C" +); + +CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + transmission_id BIGINT NOT NULL REFERENCES transmissions(id), + observer_idx BIGINT, + direction TEXT COLLATE "C", + snr DOUBLE PRECISION, + rssi DOUBLE PRECISION, + score DOUBLE PRECISION, + path_json TEXT COLLATE "C", + timestamp BIGINT NOT NULL, + raw_hex TEXT COLLATE "C", + resolved_path TEXT COLLATE "C" +); + +CREATE TABLE observer_metrics ( + observer_id TEXT COLLATE "C" NOT NULL, + timestamp TEXT COLLATE "C" NOT NULL, + noise_floor DOUBLE PRECISION, + tx_air_secs BIGINT, + rx_air_secs BIGINT, + recv_errors BIGINT, + battery_mv BIGINT, + packets_sent BIGINT, + packets_recv BIGINT, + PRIMARY KEY (observer_id, timestamp) +); + +CREATE TABLE neighbor_edges ( + node_a TEXT COLLATE "C" NOT NULL, + node_b TEXT COLLATE "C" NOT NULL, + count BIGINT DEFAULT 1, + last_seen TEXT COLLATE "C", + PRIMARY KEY (node_a, node_b) +); + +CREATE TABLE dropped_packets ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + hash TEXT COLLATE "C", + raw_hex TEXT COLLATE "C", + reason TEXT COLLATE "C" NOT NULL, + observer_id TEXT COLLATE "C", + observer_name TEXT COLLATE "C", + node_pubkey TEXT COLLATE "C", + node_name TEXT COLLATE "C", + dropped_at TEXT COLLATE "C" DEFAULT (to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS')) +); + +CREATE TABLE client_receptions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + rx_pubkey TEXT COLLATE "C" NOT NULL, + heard_key TEXT COLLATE "C" NOT NULL, + heard_keylen BIGINT NOT NULL, + rssi BIGINT, + snr DOUBLE PRECISION, + lat DOUBLE PRECISION NOT NULL, + lon DOUBLE PRECISION NOT NULL, + pos_acc_m DOUBLE PRECISION, + rx_at TEXT COLLATE "C" NOT NULL, + ingested_at TEXT COLLATE "C" NOT NULL, + src TEXT COLLATE "C" NOT NULL, + UNIQUE(rx_pubkey, heard_key, rx_at) +); + +CREATE TABLE client_observers ( + pubkey TEXT COLLATE "C" PRIMARY KEY, + name TEXT COLLATE "C", + last_seen TEXT COLLATE "C" +); + +CREATE TABLE client_rx_observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + rx_pubkey TEXT COLLATE "C" NOT NULL, + rx_at TEXT COLLATE "C" NOT NULL, + ingested_at TEXT COLLATE "C" NOT NULL, + pkt_hash TEXT COLLATE "C" NOT NULL, + route_type BIGINT NOT NULL, + payload_type BIGINT NOT NULL, + code1 TEXT COLLATE "C", + code2 TEXT COLLATE "C", + scope_name TEXT COLLATE "C", + hash_size BIGINT NOT NULL, + hop_count BIGINT NOT NULL, + path_json TEXT COLLATE "C", + forwarder TEXT COLLATE "C", + snr DOUBLE PRECISION, + rssi BIGINT, + lat DOUBLE PRECISION NOT NULL, + lon DOUBLE PRECISION NOT NULL, + pos_acc_m DOUBLE PRECISION, + UNIQUE(rx_pubkey, pkt_hash, rx_at) +); + +CREATE TABLE client_rf_samples ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + rx_pubkey TEXT COLLATE "C" NOT NULL, + sampled_at TEXT COLLATE "C" NOT NULL, + ingested_at TEXT COLLATE "C" NOT NULL, + lat DOUBLE PRECISION NOT NULL, + lon DOUBLE PRECISION NOT NULL, + pos_acc_m DOUBLE PRECISION, + stationary BIGINT NOT NULL DEFAULT 0, + uptime_secs BIGINT NOT NULL, + battery_mv BIGINT, + queue_len BIGINT, + errors BIGINT, + noise_floor BIGINT, + last_rssi BIGINT, + last_snr DOUBLE PRECISION, + tx_air_secs BIGINT, + rx_air_secs BIGINT, + recv BIGINT, + sent BIGINT, + flood_rx BIGINT, + direct_rx BIGINT, + flood_tx BIGINT, + direct_tx BIGINT, + recv_errors BIGINT, + UNIQUE(rx_pubkey, sampled_at) +); + +CREATE TABLE node_declared_regions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + target TEXT COLLATE "C" NOT NULL, + rx_pubkey TEXT COLLATE "C" NOT NULL, + observed_at TEXT COLLATE "C" NOT NULL, + ingested_at TEXT COLLATE "C" NOT NULL, + regions_csv TEXT COLLATE "C" NOT NULL, + truncated BIGINT NOT NULL DEFAULT 0, + lat DOUBLE PRECISION, + lon DOUBLE PRECISION, + pos_acc_m DOUBLE PRECISION, + repeater_clock BIGINT, + UNIQUE(target, rx_pubkey, observed_at) +); + +CREATE TABLE scope_match_totals ( + id BIGINT PRIMARY KEY CHECK (id = 1), + since_unix BIGINT NOT NULL, + unique_matches BIGINT NOT NULL, + explicit_over_derived BIGINT NOT NULL, + ambiguous BIGINT NOT NULL, + none_matches BIGINT NOT NULL, + updated_unix BIGINT NOT NULL +); + +CREATE TABLE _migrations ( + name TEXT COLLATE "C" PRIMARY KEY +); + +CREATE TABLE _async_migrations ( + name TEXT COLLATE "C" PRIMARY KEY, + status TEXT COLLATE "C" NOT NULL, + started_at TEXT COLLATE "C" NOT NULL DEFAULT (to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS')), + ended_at TEXT COLLATE "C", + error TEXT COLLATE "C" +); + +CREATE TABLE advert_route_evidence ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + tx_id BIGINT NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, + bit BIGINT NOT NULL CHECK(bit IN (1,2)), + UNIQUE(tx_id,bit) +); + +CREATE TABLE advert_evidence_backfill ( + id BIGINT PRIMARY KEY CHECK(id=1), + tx_cursor BIGINT NOT NULL DEFAULT 0, + obs_cursor BIGINT NOT NULL DEFAULT 0 +); + +CREATE INDEX IF NOT EXISTS idx_observations_observer_idx_timestamp ON observations(observer_idx,timestamp); + +CREATE INDEX IF NOT EXISTS idx_nodes_last_seen ON nodes(last_seen); + +CREATE INDEX IF NOT EXISTS idx_observers_last_seen ON observers(last_seen); + +CREATE INDEX IF NOT EXISTS idx_inactive_nodes_last_seen ON inactive_nodes(last_seen); + +CREATE INDEX IF NOT EXISTS idx_transmissions_hash ON transmissions(hash); + +CREATE INDEX IF NOT EXISTS idx_transmissions_first_seen ON transmissions(first_seen,id); + +CREATE INDEX IF NOT EXISTS idx_transmissions_payload_type ON transmissions(payload_type); + +CREATE INDEX IF NOT EXISTS idx_client_recept_heard_geo ON client_receptions(heard_key, heard_keylen, lat, lon); + +CREATE INDEX IF NOT EXISTS idx_client_recept_latlon ON client_receptions(lat, lon); + +CREATE INDEX IF NOT EXISTS idx_client_recept_rxat ON client_receptions(rx_at); + +CREATE INDEX IF NOT EXISTS idx_cro_prune ON client_rx_observations(rx_at); + +CREATE INDEX IF NOT EXISTS idx_cro_hash ON client_rx_observations(pkt_hash, rx_at); + +CREATE INDEX IF NOT EXISTS idx_cro_forwarder ON client_rx_observations(forwarder, rx_at); + +CREATE INDEX IF NOT EXISTS idx_cro_scope ON client_rx_observations(scope_name, rx_at); + +CREATE INDEX IF NOT EXISTS idx_crf_prune ON client_rf_samples(sampled_at); + +CREATE INDEX IF NOT EXISTS idx_crf_track ON client_rf_samples(rx_pubkey, sampled_at); + +CREATE INDEX IF NOT EXISTS idx_ndr_target ON node_declared_regions(target, observed_at); + +CREATE INDEX IF NOT EXISTS idx_ndr_prune ON node_declared_regions(observed_at); + +CREATE INDEX IF NOT EXISTS idx_observations_transmission_id ON observations(transmission_id); + +CREATE INDEX IF NOT EXISTS idx_observations_observer_idx ON observations(observer_idx); + +CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ON observations(transmission_id,observer_idx,COALESCE(path_json,'')); + +CREATE INDEX IF NOT EXISTS idx_observer_metrics_timestamp ON observer_metrics(timestamp); + +CREATE INDEX IF NOT EXISTS idx_tx_channel_hash ON transmissions(channel_hash) WHERE payload_type = 5; + +CREATE INDEX IF NOT EXISTS idx_dropped_observer ON dropped_packets(observer_id); + +CREATE INDEX IF NOT EXISTS idx_dropped_node ON dropped_packets(node_pubkey); + +CREATE INDEX IF NOT EXISTS idx_nodes_foreign_advert ON nodes(foreign_advert) WHERE foreign_advert = 1; + +CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey); + +CREATE INDEX IF NOT EXISTS idx_tx_code1 ON transmissions(code1) WHERE code1 IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_observations_tx_ts ON observations(transmission_id, timestamp); + +CREATE INDEX IF NOT EXISTS idx_transmissions_channel_hash_payload ON transmissions(channel_hash, payload_type, first_seen); + +CREATE INDEX IF NOT EXISTS idx_observers_iata_norm ON observers(UPPER(TRIM(iata))); + +CREATE INDEX IF NOT EXISTS idx_tx_scope_name ON transmissions(scope_name) WHERE scope_name IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_tx_last_seen_zero ON transmissions(id) WHERE last_seen=0; + +CREATE VIEW packets_v AS SELECT o.id,COALESCE(o.raw_hex,t.raw_hex) AS raw_hex, +to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC','YYYY-MM-DD HH24:MI:SS') AS timestamp, +obs.id AS observer_id,obs.name AS observer_name,o.direction,o.snr,o.rssi,o.score,t.hash, +t.route_type,t.payload_type,t.payload_version,o.path_json,t.decoded_json,t.created_at +FROM observations o JOIN transmissions t ON t.id=o.transmission_id +LEFT JOIN observers obs ON obs.rowid=o.observer_idx AND (obs.inactive IS NULL OR obs.inactive=0); diff --git a/internal/packetpath/hash.go b/internal/packetpath/hash.go new file mode 100644 index 000000000..9d4c4b051 --- /dev/null +++ b/internal/packetpath/hash.go @@ -0,0 +1,61 @@ +package packetpath + +import ( + "crypto/sha256" + "encoding/hex" +) + +// ContentHash preserves CoreScope packet identity. This is the existing server +// algorithm shared by runtime and offline fixture preparation. Transport routes +// and TRACE uint16 path_len were checked against MeshCore a366955 Packet.cpp/Packet.h. +func ContentHash(rawHex string) string { + buf, err := hex.DecodeString(rawHex) + if err != nil || len(buf) < 2 { + if len(rawHex) >= 16 { + return rawHex[:16] + } + return rawHex + } + + headerByte := buf[0] + offset := 1 + if IsTransportRoute(int(headerByte & 0x03)) { + offset += 4 + } + if offset >= len(buf) { + if len(rawHex) >= 16 { + return rawHex[:16] + } + return rawHex + } + pathByte := buf[offset] + offset++ + hashSize := int((pathByte>>6)&0x3) + 1 + hashCount := int(pathByte & 0x3F) + pathBytes := hashSize * hashCount + + payloadStart := offset + pathBytes + if payloadStart > len(buf) { + if len(rawHex) >= 16 { + return rawHex[:16] + } + return rawHex + } + + payload := buf[payloadStart:] + + // Hash payload-type byte only (bits 2-5 of header), not the full header. + // Firmware: SHA256(payload_type + [path_len for TRACE] + payload) + // Using the full header caused different hashes for the same logical packet + // when route type or version bits differed. See issue #786. + payloadType := (headerByte >> 2) & 0x0F + toHash := []byte{payloadType} + if int(payloadType) == 0x09 { + // Firmware uses uint16_t path_len (2 bytes, little-endian) + toHash = append(toHash, pathByte, 0x00) + } + toHash = append(toHash, payload...) + + h := sha256.Sum256(toHash) + return hex.EncodeToString(h[:])[:16] +} diff --git a/internal/packetpath/hash_test.go b/internal/packetpath/hash_test.go new file mode 100644 index 000000000..386ba5943 --- /dev/null +++ b/internal/packetpath/hash_test.go @@ -0,0 +1,45 @@ +package packetpath + +import "testing" + +func TestContentHashAllRoutesKeepBaselineGolden(t *testing.T) { + // Packet.cpp and Packet.h at MeshCore a366955: only routes 0/3 carry + // transport codes; those codes and hop bytes do not enter packet identity. + for _, raw := range []string{ + "1401020304000102030405060708090a0b0c0d0e0f", + "15000102030405060708090a0b0c0d0e0f", + "16000102030405060708090a0b0c0d0e0f", + "1701020304000102030405060708090a0b0c0d0e0f", + "1502aabb0102030405060708090a0b0c0d0e0f", + "1541aabb0102030405060708090a0b0c0d0e0f", + "1581aabbcc0102030405060708090a0b0c0d0e0f", + } { + if got := ContentHash(raw); got != "fae0c9e6d357a814" { + t.Fatalf("hash %s=%s", raw, got) + } + } +} + +func TestContentHashMalformedFallbackPreserved(t *testing.T) { + for _, raw := range []string{"", "a", "zz", "00112233", "150faabb", "invalid-long-raw-frame"} { + want := raw + if len(want) > 16 { + want = want[:16] + } + if got := ContentHash(raw); got != want { + t.Fatalf("malformed fallback %q=%q; want %q", raw, got, want) + } + } +} + +func TestContentHashTraceAndAnonymousRequestGoldens(t *testing.T) { + for _, test := range []struct{ raw, want string }{ + {"250001020304", "240ed67f297f83f9"}, + {"2501aa01020304", "dd79cbe99f3fc8f5"}, + {"1d0001020304", "14d68bbc63e0902d"}, + } { + if got := ContentHash(test.raw); got != test.want { + t.Fatalf("hash %s=%s; want %s", test.raw, got, test.want) + } + } +} diff --git a/internal/pgutil/dump.go b/internal/pgutil/dump.go new file mode 100644 index 000000000..64215f18f --- /dev/null +++ b/internal/pgutil/dump.go @@ -0,0 +1,81 @@ +package pgutil + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "os/exec" + "strconv" + "strings" + "time" +) + +// CommandEnv supplies libpq credentials through the child environment, never +// through argv. It removes inherited PG settings so a dump cannot reach a +// different database than the application. Callers must never log this slice. +func CommandEnv(dsn string) ([]string, error) { + c, err := ParseConfig(dsn) + if err != nil { + return nil, err + } + u, _ := url.Parse(c.ConnString()) + env := make([]string, 0, len(os.Environ())+12) + for _, e := range os.Environ() { + if !strings.HasPrefix(strings.ToUpper(e), "PG") { + env = append(env, e) + } + } + env = append(env, "PGHOST="+c.Host, "PGPORT="+strconv.Itoa(int(c.Port)), "PGDATABASE="+c.Database, + "PGUSER="+c.User, "PGPASSWORD="+c.Password, "PGCONNECT_TIMEOUT="+strconv.Itoa(int(c.ConnectTimeout/time.Second))) + for _, pair := range [][2]string{{"sslmode", "PGSSLMODE"}, {"sslrootcert", "PGSSLROOTCERT"}, + {"sslcert", "PGSSLCERT"}, {"sslkey", "PGSSLKEY"}, {"sslpassword", "PGSSLPASSWORD"}, + {"target_session_attrs", "PGTARGETSESSIONATTRS"}, {"application_name", "PGAPPNAME"}} { + if value := u.Query().Get(pair[0]); value != "" { + env = append(env, pair[1]+"="+value) + } + } + return env, nil +} + +// Dump writes a consistent native custom-format archive with owner-only mode. +// It never overwrites an existing file, and removes its own partial output on +// any failure or cancellation. The PostgreSQL client executable must be on PATH. +func Dump(ctx context.Context, dsn, path string) (err error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + env, err := CommandEnv(dsn) + if err != nil { + return err + } + u, _ := url.Parse(dsn) + if u.Query().Get("search_path") != "" { + return errors.New("postgres: native backup requires a database URL without search_path") + } + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + if errors.Is(err, os.ErrExist) { + return errors.New("postgres: snapshot target already exists") + } + return fmt.Errorf("postgres: create snapshot: %w", err) + } + defer func() { + if closeErr := f.Close(); err == nil { + err = closeErr + } + if err != nil { + _ = os.Remove(path) + } + }() + cmd := exec.CommandContext(ctx, "pg_dump", "--format=custom", "--no-password", "--no-owner", "--no-privileges") + cmd.Env, cmd.Stdout = env, f + // Never return client stderr: libpq errors can include connection settings. + if runErr := cmd.Run(); runErr != nil { + if ctx.Err() != nil { + return ctx.Err() + } + return errors.New("postgres: pg_dump failed; check client installation, version and backup privileges") + } + return f.Sync() +} diff --git a/internal/pgutil/go.mod b/internal/pgutil/go.mod new file mode 100644 index 000000000..a26277f9d --- /dev/null +++ b/internal/pgutil/go.mod @@ -0,0 +1,14 @@ +module github.com/meshcore-analyzer/pgutil + +go 1.25.0 + +require github.com/jackc/pgx/v5 v5.7.6 + +require ( + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + golang.org/x/crypto v0.37.0 // indirect + golang.org/x/sync v0.13.0 // indirect + golang.org/x/text v0.24.0 // indirect +) diff --git a/internal/pgutil/go.sum b/internal/pgutil/go.sum new file mode 100644 index 000000000..7600ae8d6 --- /dev/null +++ b/internal/pgutil/go.sum @@ -0,0 +1,28 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/pgutil/pgtest/pgtest.go b/internal/pgutil/pgtest/pgtest.go new file mode 100644 index 000000000..1e17cff05 --- /dev/null +++ b/internal/pgutil/pgtest/pgtest.go @@ -0,0 +1,141 @@ +// Package pgtest provisions disposable PostgreSQL test namespaces. Tests fail +// rather than silently skipping when CORESCOPE_TEST_POSTGRES_URL is missing. +package pgtest + +import ( + "crypto/rand" + "database/sql" + "encoding/hex" + "net/url" + "os" + "strings" + "testing" + + "github.com/meshcore-analyzer/pgutil" +) + +func name() string { + var b [12]byte + if _, err := rand.Read(b[:]); err != nil { + panic(err) + } + return "corescope_test_" + hex.EncodeToString(b[:]) +} + +func admin(t testing.TB) (*sql.DB, *url.URL) { + t.Helper() + dsn := os.Getenv("CORESCOPE_TEST_POSTGRES_URL") + if dsn == "" { + t.Fatal("set CORESCOPE_TEST_POSTGRES_URL to a disposable PostgreSQL 18 administrator URL") + } + u, err := url.Parse(dsn) + if err != nil { + t.Fatal("invalid CORESCOPE_TEST_POSTGRES_URL") + } + db, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + return db, u +} + +// NewSchema returns a URL selecting a fresh empty schema. No fixture tables +// are created. Register pool cleanup after this call so pools close first. +func NewSchema(t testing.TB) string { + t.Helper() + db, u := admin(t) + schema := name() + if _, err := db.Exec(`CREATE SCHEMA ` + schema); err != nil { + db.Close() + t.Fatal(err) + } + t.Cleanup(func() { + defer db.Close() + if _, err := db.Exec(`DROP SCHEMA ` + schema + ` CASCADE`); err != nil { + t.Errorf("drop test schema: %v", err) + } + }) + q := u.Query() + q.Set("search_path", schema) + u.RawQuery = q.Encode() + return u.String() +} + +// NewDatabase returns a fresh empty logical database for isolation and restore +// tests. Only this randomly named database is removed by cleanup. +func NewDatabase(t testing.TB) string { + t.Helper() + db, u := admin(t) + database := name() + if _, err := db.Exec(`CREATE DATABASE ` + database + ` TEMPLATE template0`); err != nil { + db.Close() + t.Fatal(err) + } + t.Cleanup(func() { + defer db.Close() + if _, err := db.Exec(`DROP DATABASE ` + database + ` WITH (FORCE)`); err != nil { + t.Errorf("drop test database: %v", err) + } + }) + u.Path = "/" + database + q := u.Query() + q.Del("search_path") + u.RawQuery = q.Encode() + return u.String() +} + +// ReadOnly provisions a reader for the selected schema and future tables. +// It grants neither schema creation nor any write privilege. +func ReadOnly(t testing.TB, writerDSN string) string { + return role(t, writerDSN, false) +} + +// Writer provisions DML and sequence privileges, without schema ownership or DDL. +// A store can revoke writes on its readiness metadata after provisioning. +func Writer(t testing.TB, ownerDSN string) string { + return role(t, ownerDSN, true) +} + +func role(t testing.TB, writerDSN string, write bool) string { + t.Helper() + db, err := pgutil.Open(writerDSN, false) + if err != nil { + t.Fatal(err) + } + role, password := name(), name() + var schema string + if err := db.QueryRow(`SELECT current_schema()`).Scan(&schema); err != nil { + db.Close() + t.Fatal(err) + } + quotedSchema := `"` + schema + `"` + privileges := "SELECT" + if write { + privileges = "SELECT,INSERT,UPDATE,DELETE" + } + statements := []string{`CREATE ROLE ` + role + ` LOGIN PASSWORD '` + password + `'`, + `GRANT USAGE ON SCHEMA ` + quotedSchema + ` TO ` + role, + `GRANT ` + privileges + ` ON ALL TABLES IN SCHEMA ` + quotedSchema + ` TO ` + role, + `ALTER DEFAULT PRIVILEGES IN SCHEMA ` + quotedSchema + ` GRANT ` + privileges + ` ON TABLES TO ` + role} + sequencePrivileges := "SELECT" + if write { + sequencePrivileges = "USAGE,SELECT" + } + statements = append(statements, `GRANT `+sequencePrivileges+` ON ALL SEQUENCES IN SCHEMA `+quotedSchema+` TO `+role, + `ALTER DEFAULT PRIVILEGES IN SCHEMA `+quotedSchema+` GRANT `+sequencePrivileges+` ON SEQUENCES TO `+role) + // One implicit transaction preserves atomicity and avoids a durable commit + // per grant in suites with hundreds of isolated roles. + if _, err := db.Exec(strings.Join(statements, ";")); err != nil { + db.Close() + t.Fatal(err) + } + t.Cleanup(func() { + defer db.Close() + if _, err := db.Exec(`DROP OWNED BY ` + role + `;DROP ROLE ` + role); err != nil { + t.Errorf("drop test role: %v", err) + } + }) + u, _ := url.Parse(writerDSN) + u.User = url.UserPassword(role, password) + return u.String() +} diff --git a/internal/pgutil/pgutil.go b/internal/pgutil/pgutil.go new file mode 100644 index 000000000..38e733beb --- /dev/null +++ b/internal/pgutil/pgutil.go @@ -0,0 +1,150 @@ +// Package pgutil contains PostgreSQL connection and native backup helpers. +package pgutil + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/binary" + "errors" + "fmt" + "net/url" + "os" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/stdlib" +) + +// ParseConfig requires an explicit PostgreSQL URL. Errors never include the +// URL: driver parse errors can contain credentials, including URL parameters. +func ParseConfig(dsn string) (*pgx.ConnConfig, error) { + u, err := url.Parse(dsn) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + return nil, errors.New("postgres: expected a PostgreSQL database URL") + } + if u.Hostname() == "" || strings.Contains(u.Host, ",") || u.User == nil || u.User.Username() == "" || strings.Trim(u.Path, "/") == "" { + return nil, errors.New("postgres: database URL requires one host, username and database") + } + allowed := map[string]bool{"sslmode": true, "sslrootcert": true, "sslcert": true, "sslkey": true, "sslpassword": true, + "target_session_attrs": true, "connect_timeout": true, "application_name": true, "search_path": true} + q, err := url.ParseQuery(u.RawQuery) + if err != nil { + return nil, errors.New("postgres: invalid database URL query") + } + for key, values := range q { + if !allowed[key] || len(values) != 1 { + return nil, errors.New("postgres: unsupported or repeated database URL parameter") + } + } + // Normalize TLS environment defaults into the parsed URL so libpq and pgx + // receive the same connection policy. Service files/options and multi-host + // URLs are deliberately unsupported rather than silently changing a backup. + if os.Getenv("PGSERVICE") != "" || os.Getenv("PGOPTIONS") != "" { + return nil, errors.New("postgres: use explicit database URL settings instead of PGSERVICE or PGOPTIONS") + } + for _, pair := range [][2]string{{"sslmode", "PGSSLMODE"}, {"sslrootcert", "PGSSLROOTCERT"}, {"sslcert", "PGSSLCERT"}, {"sslkey", "PGSSLKEY"}, {"sslpassword", "PGSSLPASSWORD"}} { + if !q.Has(pair[0]) && os.Getenv(pair[1]) != "" { + q.Set(pair[0], os.Getenv(pair[1])) + } + } + if !q.Has("sslmode") { + q.Set("sslmode", "prefer") + } + u.RawQuery = q.Encode() + c, err := pgx.ParseConfig(u.String()) + if err != nil || c.Database == "" { + return nil, errors.New("postgres: invalid database URL or missing database name") + } + // Ten seconds is the connection upper bound; retain a shorter explicit limit. + if c.ConnectTimeout <= 0 || c.ConnectTimeout > 10*time.Second { + c.ConnectTimeout = 10 * time.Second + } + return c, nil +} + +// Open returns a bounded pgx database/sql pool. readOnly is defense in depth; +// deployments must also grant the runtime role only its required privileges. +func Open(dsn string, readOnly bool) (*sql.DB, error) { + c, err := ParseConfig(dsn) + if err != nil { + return nil, err + } + c.RuntimeParams["timezone"] = "UTC" + if readOnly { + c.RuntimeParams["default_transaction_read_only"] = "on" + } + db := stdlib.OpenDB(*c) + db.SetMaxOpenConns(8) + db.SetMaxIdleConns(2) + db.SetConnMaxIdleTime(5 * time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := db.PingContext(ctx); err != nil { + db.Close() + return nil, errors.New("postgres: connection failed; check database URL, credentials and server availability") + } + return db, nil +} + +// AssertReadOnly checks effective grants, including inherited grants. The +// session's read-only setting alone is insufficient: a client can change it. +func AssertReadOnly(db *sql.DB) error { + var writable bool + err := db.QueryRow(`SELECT + EXISTS (SELECT 1 FROM pg_roles WHERE rolname=current_user AND (rolsuper OR rolcreaterole OR rolcreatedb)) + OR has_database_privilege(current_database(), 'CREATE') + OR EXISTS (SELECT 1 FROM pg_namespace WHERE nspname !~ '^pg_' AND nspname <> 'information_schema' + AND has_schema_privilege(oid, 'CREATE')) + OR EXISTS (SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema' AND c.relkind IN ('r','p','v','m','f') + AND has_table_privilege(c.oid, 'INSERT,UPDATE,DELETE,TRUNCATE,TRIGGER'))`).Scan(&writable) + if err != nil { + return errors.New("postgres: cannot verify reader privileges") + } + if writable { + return errors.New("postgres: reader role has write or schema privileges") + } + return nil +} + +// SameDatabase compares the actual databases reached by two URLs, independent +// of aliases, proxies, usernames, search_path, and URL spelling. Advisory locks +// are database-local and need no DDL/data write privilege. Both transactions +// always roll back; the random lock leaves no persistent state. +func SameDatabase(ctx context.Context, left, right string) (bool, error) { + a, err := Open(left, true) + if err != nil { + return false, err + } + defer a.Close() + b, err := Open(right, true) + if err != nil { + return false, err + } + defer b.Close() + ta, err := a.BeginTx(ctx, nil) + if err != nil { + return false, errors.New("postgres: database identity check failed") + } + defer ta.Rollback() + tb, err := b.BeginTx(ctx, nil) + if err != nil { + return false, errors.New("postgres: database identity check failed") + } + defer tb.Rollback() + var key [8]byte + if _, err := rand.Read(key[:]); err != nil { + return false, err + } + id := int64(binary.BigEndian.Uint64(key[:])) + var locked bool + if err := ta.QueryRowContext(ctx, `SELECT pg_try_advisory_xact_lock($1)`, id).Scan(&locked); err != nil || !locked { + return false, errors.New("postgres: database identity lock unavailable") + } + if err := tb.QueryRowContext(ctx, `SELECT pg_try_advisory_xact_lock($1)`, id).Scan(&locked); err != nil { + return false, fmt.Errorf("postgres: database identity probe failed") + } + return !locked, nil +} diff --git a/internal/pgutil/pgutil_test.go b/internal/pgutil/pgutil_test.go new file mode 100644 index 000000000..7df8c60da --- /dev/null +++ b/internal/pgutil/pgutil_test.go @@ -0,0 +1,232 @@ +package pgutil_test + +import ( + "context" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +func TestParseConfigSecretsAndSupportedURL(t *testing.T) { + for _, dsn := range []string{ + "postgresql://hidden-user:hidden-password@host:bad/db", + "postgresql://hidden-user:hidden-password@host/db?password=hidden-password", + "postgresql://hidden-user:hidden-password@one,two/db", + "postgresql://hidden-user:hidden-password@host/db?hostaddr=elsewhere", + "postgresql://hidden-user:hidden-password@host/db?options=-csearch_path=elsewhere", + "postgresql://hidden-user:hidden-password@host/db?sslmode=disable&sslmode=require", + } { + _, err := pgutil.ParseConfig(dsn) + if err == nil { + t.Fatal("unsupported connection URL accepted") + } + if strings.Contains(err.Error(), "hidden-") { + t.Fatal("connection error leaked a credential") + } + } + c, err := pgutil.ParseConfig("postgresql://user:p%40ss@localhost:5433/accounts?sslmode=disable&search_path=test_schema") + if err != nil { + t.Fatal(err) + } + if c.Password != "p@ss" || c.Database != "accounts" || c.Port != 5433 || c.RuntimeParams["search_path"] != "test_schema" { + t.Fatal("URL parameters changed") + } +} + +func TestParseConfigRejectsMalformedQueryWithoutTLSDowngrade(t *testing.T) { + for _, key := range []string{"PGSSLMODE", "PGSSLROOTCERT", "PGSSLCERT", "PGSSLKEY", "PGSSLPASSWORD", "PGSERVICE", "PGOPTIONS"} { + t.Setenv(key, "") + } + for _, test := range []struct{ name, query string }{ + {"bad_value_escape", "sslmode=verify-full%ZZ"}, + {"unescaped_separator", "sslmode=verify-full;application_name=hidden-param"}, + {"bad_key_escape", "ssl%ZZmode=verify-full"}, + {"bad_other_parameter", "sslmode=verify-full&sslpassword=hidden-param%ZZ"}, + } { + t.Run(test.name, func(t *testing.T) { + dsn := "postgresql://hidden-user:hidden-password@localhost/accounts?" + test.query + if _, err := pgutil.ParseConfig(dsn); err == nil { + t.Fatal("malformed query accepted instead of preserving the requested connection policy") + } else if strings.Contains(err.Error(), "hidden-") || strings.Contains(err.Error(), test.query) { + t.Fatal("malformed query error leaked connection settings") + } + if _, err := pgutil.CommandEnv(dsn); err == nil { + t.Fatal("native client accepted the malformed query") + } + }) + } + c, err := pgutil.ParseConfig("postgresql://encoded%20user:p%40ss%25@localhost/accounts?ssl%6dode=verify%2Dfull&application_name=synthetic%3Btls%3Dstrict%26name%3Dread%2Bonly") + if err != nil { + t.Fatal("valid encoded URL refused", err) + } + if c.User != "encoded user" || c.Password != "p@ss%" || c.RuntimeParams["application_name"] != "synthetic;tls=strict&name=read+only" { + t.Fatal("valid encoded URL changed") + } + if c.TLSConfig == nil || c.TLSConfig.InsecureSkipVerify || len(c.Fallbacks) != 0 { + t.Fatal("valid strict-TLS URL was downgraded") + } +} + +func TestCommandEnvMatchesConnection(t *testing.T) { + dsn := "postgresql://someone:secret%40value@localhost:5433/accounts?sslmode=require" + env, err := pgutil.CommandEnv(dsn) + if err != nil { + t.Fatal(err) + } + values := map[string]string{} + for _, e := range env { + k, v, _ := strings.Cut(e, "=") + values[k] = v + } + for k, want := range map[string]string{"PGHOST": "localhost", "PGPORT": "5433", "PGDATABASE": "accounts", "PGUSER": "someone", "PGPASSWORD": "secret@value", "PGSSLMODE": "require"} { + if values[k] != want { + t.Errorf("%s did not preserve connection settings", k) + } + } +} + +func TestConnectTimeoutKeepsShorterLimitAndCapsLonger(t *testing.T) { + for _, test := range []struct { + seconds string + want time.Duration + }{{"1", time.Second}, {"30", 10 * time.Second}, {"0", 10 * time.Second}} { + c, err := pgutil.ParseConfig("postgresql://user:password@localhost/database?connect_timeout=" + test.seconds) + if err != nil { + t.Fatal(err) + } + if c.ConnectTimeout != test.want { + t.Fatalf("timeout %s became %s; want %s", test.seconds, c.ConnectTimeout, test.want) + } + } +} + +func TestSameDatabaseChecksActualTarget(t *testing.T) { + dsn := pgtest.NewSchema(t) + reader := pgtest.ReadOnly(t, dsn) + u, _ := url.Parse(reader) + q := u.Query() + q.Set("application_name", "identity-test") + q.Del("search_path") + u.RawQuery = q.Encode() + same, err := pgutil.SameDatabase(context.Background(), dsn, u.String()) + if err != nil || !same { + t.Fatalf("same database across roles/search paths=%v,%v", same, err) + } + other := pgtest.NewDatabase(t) + same, err = pgutil.SameDatabase(context.Background(), dsn, other) + if err != nil || same { + t.Fatalf("separate databases=%v,%v", same, err) + } +} + +func TestReadOnlyRoleDeniesWritesAndDDL(t *testing.T) { + dsn := pgtest.NewSchema(t) + owner, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + defer owner.Close() + if _, err := owner.Exec(`CREATE TABLE sample(id BIGINT PRIMARY KEY)`); err != nil { + t.Fatal(err) + } + readerDSN := pgtest.ReadOnly(t, dsn) + db, err := pgutil.Open(readerDSN, true) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := pgutil.AssertReadOnly(db); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`SET default_transaction_read_only=off`); err != nil { + t.Fatal(err) + } + for _, stmt := range []string{`INSERT INTO sample VALUES(1)`, `UPDATE sample SET id=2`, `DELETE FROM sample`, `CREATE TABLE forbidden(id int)`} { + if _, err := db.Exec(stmt); err == nil { + t.Fatalf("reader accepted write: %s", stmt) + } + } + u, _ := url.Parse(readerDSN) + var database string + if err := owner.QueryRow(`SELECT current_database()`).Scan(&database); err != nil { + t.Fatal(err) + } + if _, err := owner.Exec(`GRANT CREATE ON DATABASE "` + database + `" TO "` + u.User.Username() + `"`); err != nil { + t.Fatal(err) + } + if err := pgutil.AssertReadOnly(db); err == nil { + t.Fatal("database CREATE privilege was missed") + } +} + +func TestDumpRestoreAndFailureCleanup(t *testing.T) { + dsn := pgtest.NewDatabase(t) + db, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`CREATE TABLE sample(id BIGINT PRIMARY KEY,value TEXT); INSERT INTO sample VALUES(17,'unchanged')`); err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "snapshot.dump") + if err := pgutil.Dump(context.Background(), dsn, path); err != nil { + t.Fatal(err) + } + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(string(content), "PGDMP") { + t.Fatal("backup is not a native custom archive") + } + target := pgtest.NewDatabase(t) + env, err := pgutil.CommandEnv(target) + if err != nil { + t.Fatal(err) + } + cfg, err := pgutil.ParseConfig(target) + if err != nil { + t.Fatal(err) + } + cmd := exec.Command("pg_restore", "--no-password", "--exit-on-error", "--no-owner", "--no-privileges", "--dbname="+cfg.Database, path) + cmd.Env = env + if err := cmd.Run(); err != nil { + t.Fatal("native restore failed") + } + restored, err := pgutil.Open(target, false) + if err != nil { + t.Fatal(err) + } + defer restored.Close() + var value string + if err := restored.QueryRow(`SELECT value FROM sample WHERE id=17`).Scan(&value); err != nil || value != "unchanged" { + t.Fatalf("restored data=%q,%v", value, err) + } + if err := pgutil.Dump(context.Background(), dsn, path); err == nil { + t.Fatal("existing backup overwritten") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + partial := filepath.Join(t.TempDir(), "partial.dump") + if err := pgutil.Dump(ctx, dsn, partial); err == nil { + t.Fatal("canceled dump succeeded") + } + if _, err := os.Stat(partial); !os.IsNotExist(err) { + t.Fatal("partial backup remains") + } + t.Setenv("PATH", t.TempDir()) + if err := pgutil.Dump(context.Background(), dsn, partial); err == nil { + t.Fatal("dump succeeded without pg_dump") + } + if _, err := os.Stat(partial); !os.IsNotExist(err) { + t.Fatal("failed dump left partial backup") + } +} diff --git a/internal/users/audit.go b/internal/users/audit.go index ff3f47e6b..9a1091417 100644 --- a/internal/users/audit.go +++ b/internal/users/audit.go @@ -3,6 +3,7 @@ package users import ( "database/sql" "encoding/json" + "fmt" "strings" "time" ) @@ -44,7 +45,7 @@ func (s *Store) Audit(actor *int64, action string, target *int64, detail map[str if err != nil { return err } - _, err = s.db.Exec(`INSERT INTO audit_log (at, actor_user_id, action, target_user_id, detail) VALUES (?, ?, ?, ?, ?)`, + _, err = s.db.Exec(`INSERT INTO audit_log (at, actor_user_id, action, target_user_id, detail) VALUES ($1, $2, $3, $4, $5)`, unix(s.now()), nullInt(actor), action, nullInt(target), string(b)) return err } @@ -63,9 +64,9 @@ func (s *Store) AuditAllFor(userID int64) ([]AuditEntry, error) { return s.auditFor(userID, noLimit) } -func (s *Store) auditFor(userID int64, limit int) ([]AuditEntry, error) { +func (s *Store) auditFor(userID int64, limit any) ([]AuditEntry, error) { rows, err := s.db.Query(`SELECT id, at, actor_user_id, action, target_user_id, detail FROM audit_log - WHERE target_user_id = ? OR actor_user_id = ? ORDER BY at DESC, id DESC LIMIT ?`, userID, userID, limit) + WHERE target_user_id = $1 OR actor_user_id = $2 ORDER BY at DESC, id DESC LIMIT $3`, userID, userID, limit) if err != nil { return nil, err } @@ -80,29 +81,29 @@ func (s *Store) AuditList(f AuditFilter) ([]AuditEntry, error) { var ors []string for _, a := range f.Actions { if group, ok := strings.CutSuffix(a, ".*"); ok { - ors = append(ors, `action = ? OR action LIKE ? ESCAPE '\'`) + ors = append(ors, fmt.Sprintf(`action = $%d OR action LIKE $%d ESCAPE '\'`, len(args)+1, len(args)+2)) args = append(args, group, escapeLike(group)+".%") } else { - ors = append(ors, `action = ?`) + ors = append(ors, fmt.Sprintf(`action = $%d`, len(args)+1)) args = append(args, a) } } q += ` AND (` + strings.Join(ors, ` OR `) + `)` } if f.UserID != nil { - q += ` AND (actor_user_id = ? OR target_user_id = ?)` + q += fmt.Sprintf(` AND (actor_user_id = $%d OR target_user_id = $%d)`, len(args)+1, len(args)+2) args = append(args, *f.UserID, *f.UserID) } if f.From != nil { - q += ` AND at >= ?` + q += fmt.Sprintf(` AND at >= $%d`, len(args)+1) args = append(args, unix(*f.From)) } if f.To != nil { - q += ` AND at <= ?` + q += fmt.Sprintf(` AND at <= $%d`, len(args)+1) args = append(args, unix(*f.To)) } if f.BeforeID > 0 { - q += ` AND id < ?` + q += fmt.Sprintf(` AND id < $%d`, len(args)+1) args = append(args, f.BeforeID) } limit := f.Limit @@ -112,7 +113,7 @@ func (s *Store) AuditList(f AuditFilter) ([]AuditEntry, error) { if limit > AuditListMax { limit = AuditListMax } - q += ` ORDER BY id DESC LIMIT ?` + q += fmt.Sprintf(` ORDER BY id DESC LIMIT $%d`, len(args)+1) args = append(args, limit) rows, err := s.db.Query(q, args...) if err != nil { @@ -133,7 +134,7 @@ func (s *Store) PruneAudit(actions []string, maxAge time.Duration) (int64, error } args = append(args, unix(s.now())-int64(maxAge/time.Second)) ph := placeholders(len(actions)) - res, err := s.db.Exec(`DELETE FROM audit_log WHERE action IN (`+ph+`) AND at < ?`, args...) + res, err := s.db.Exec(`DELETE FROM audit_log WHERE action IN (`+ph+`) AND at < $`+fmt.Sprint(len(args)), args...) if err != nil { return 0, err } diff --git a/internal/users/audit_list_test.go b/internal/users/audit_list_test.go index a74f33eac..a438f9ae9 100644 --- a/internal/users/audit_list_test.go +++ b/internal/users/audit_list_test.go @@ -1,9 +1,7 @@ package users import ( - "database/sql" "fmt" - "path/filepath" "strings" "testing" "time" @@ -147,7 +145,7 @@ func TestUsersByID(t *testing.T) { func hasIndex(t *testing.T, st *Store, name string) bool { t.Helper() var n int - if err := st.db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = ?`, name).Scan(&n); err != nil { + if err := st.db.QueryRow(`SELECT COUNT(*) FROM pg_indexes WHERE schemaname=current_schema() AND indexname=$1`, name).Scan(&n); err != nil { t.Fatal(err) } return n == 1 @@ -160,32 +158,9 @@ func TestFreshDatabaseHasAuditTimeIndex(t *testing.T) { } } -// A users.db written by a v2 binary gains the audit_at index and keeps its rows. +// A native account schema at v2 binary gains the audit_at index and keeps its rows. func TestMigrateV2DatabaseToV3(t *testing.T) { - path := filepath.Join(t.TempDir(), "users.db") - db, err := sql.Open("sqlite", path) - if err != nil { - t.Fatal(err) - } - stmts := []string{ - `CREATE TABLE schema_version (version INTEGER NOT NULL)`, - `INSERT INTO schema_version (version) VALUES (2)`, - } - stmts = append(stmts, migrations[0]...) - stmts = append(stmts, migrations[1]...) - stmts = append(stmts, `INSERT INTO audit_log (at, action, detail) VALUES (1, 'user.register', '{}')`) - for _, s := range stmts { - if _, err := db.Exec(s); err != nil { - t.Fatalf("build v2 db: %v", err) - } - } - db.Close() - - st, err := Open(path) - if err != nil { - t.Fatalf("Open v2 db: %v", err) - } - defer st.Close() + st := migratedTestStore(t, 2, `INSERT INTO audit_log (at,action,detail) VALUES (1,'user.register','{}')`) if v, err := st.SchemaVersion(); err != nil || v != len(migrations) { t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, len(migrations)) } diff --git a/internal/users/export_test.go b/internal/users/export_test.go index 8da4e2b24..8dc3861af 100644 --- a/internal/users/export_test.go +++ b/internal/users/export_test.go @@ -66,7 +66,7 @@ func TestAllProposalsByUserHasNoCap(t *testing.T) { t.Fatal(err) } for i := 0; i <= ProposalListMax; i++ { - if _, err := tx.Exec(`INSERT INTO proposals (kind, subject, status, proposer_id, created_at) VALUES (?, ?, 'rejected', ?, ?)`, + if _, err := tx.Exec(`INSERT INTO proposals (kind, subject, status, proposer_id, created_at) VALUES ($1, $2, 'rejected', $3, $4)`, KindHashtagChannel, fmt.Sprintf("#p%d", i), u.ID, unix(clk.Now())); err != nil { tx.Rollback() t.Fatal(err) diff --git a/internal/users/go.mod b/internal/users/go.mod index ac4edb30b..537ea9d9d 100644 --- a/internal/users/go.mod +++ b/internal/users/go.mod @@ -1,20 +1,23 @@ module github.com/meshcore-analyzer/users -go 1.22 +go 1.25.0 require ( - golang.org/x/crypto v0.31.0 - modernc.org/sqlite v1.34.5 + github.com/jackc/pgx/v5 v5.7.6 + golang.org/x/crypto v0.37.0 ) require ( - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/ncruces/go-strftime v0.1.9 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/sys v0.28.0 // indirect - modernc.org/libc v1.55.3 // indirect - modernc.org/mathutil v1.6.0 // indirect - modernc.org/memory v1.8.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + golang.org/x/sync v0.13.0 // indirect + golang.org/x/text v0.24.0 // indirect ) + +require ( + github.com/meshcore-analyzer/pgutil v0.0.0 + golang.org/x/sys v0.32.0 // indirect +) + +replace github.com/meshcore-analyzer/pgutil => ../pgutil diff --git a/internal/users/go.sum b/internal/users/go.sum index aa3cb2b45..fa18bbda0 100644 --- a/internal/users/go.sum +++ b/internal/users/go.sum @@ -1,45 +1,30 @@ -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= -modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= -modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= -modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= -modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= -modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= -modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= -modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= -modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= -modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= -modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= -modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= -modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= -modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= -modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= -modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= -modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= -modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= -modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/users/helpers_test.go b/internal/users/helpers_test.go index e6efef154..698a73ad5 100644 --- a/internal/users/helpers_test.go +++ b/internal/users/helpers_test.go @@ -1,7 +1,10 @@ package users import ( - "path/filepath" + "database/sql" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" + "net/url" "testing" "time" ) @@ -11,10 +14,24 @@ type fakeClock struct{ t time.Time } func (c *fakeClock) Now() time.Time { return c.t } func (c *fakeClock) Advance(d time.Duration) { c.t = c.t.Add(d) } -// newTestStore opens a fresh users.db in a temp dir with a controllable clock. +// newTestStore bootstraps an isolated PostgreSQL schema with a controllable clock. func newTestStore(t *testing.T) (*Store, *fakeClock) { t.Helper() - st, err := Open(filepath.Join(t.TempDir(), "users.db")) + return newTestStoreAt(t, pgtest.NewSchema(t)) +} + +func newTestStoreAt(t *testing.T, dsn string) (*Store, *fakeClock) { + t.Helper() + db, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + if err := Apply(db); err != nil { + db.Close() + t.Fatal(err) + } + db.Close() + st, err := Open(testRuntimeURL(t, dsn)) if err != nil { t.Fatalf("Open: %v", err) } @@ -23,3 +40,67 @@ func newTestStore(t *testing.T) (*Store, *fakeClock) { t.Cleanup(func() { st.Close() }) return st, clk } + +func testRuntimeURL(t *testing.T, ownerURL string) string { + t.Helper() + dsn := pgtest.Writer(t, ownerURL) + u, _ := url.Parse(dsn) + db, err := pgutil.Open(ownerURL, false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`REVOKE INSERT,UPDATE,DELETE ON corescope_schema,schema_version FROM "` + u.User.Username() + `"`); err != nil { + t.Fatal(err) + } + return dsn +} + +// migratedTestStore exercises every intermediate native account layout while +// SQLite source normalization is covered by the offline importer's fixtures. +func migratedTestStore(t *testing.T, version int, seed string) *Store { + t.Helper() + dsn := pgtest.NewSchema(t) + db, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`CREATE TABLE schema_version(version INTEGER NOT NULL)`); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO schema_version VALUES ($1)`, version); err != nil { + t.Fatal(err) + } + for _, migration := range migrations[:version] { + for _, stmt := range migration { + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } + } + } + if seed != "" { + if _, err := db.Exec(seed); err != nil { + t.Fatal(err) + } + } + if err := Apply(db); err != nil { + t.Fatal(err) + } + st, err := Open(testRuntimeURL(t, dsn)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + return st +} + +func testOwner(t *testing.T, dsn string) *sql.DB { + t.Helper() + db, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + return db +} diff --git a/internal/users/legacy_schema.go b/internal/users/legacy_schema.go new file mode 100644 index 000000000..85ca25f4b --- /dev/null +++ b/internal/users/legacy_schema.go @@ -0,0 +1,130 @@ +package users + +// LegacyMigrations contains only immutable SQLite upgrade metadata for the offline +// importer. Runtime account connections never execute these statements. +var legacyMigrations = [][]string{ + { // v1: sub-project A + `CREATE TABLE users ( + id INTEGER PRIMARY KEY, + email TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + role TEXT NOT NULL DEFAULT 'user' CHECK (role IN ('user','admin')), + status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','active','disabled')), + created_at INTEGER NOT NULL, + activated_at INTEGER, + activated_by INTEGER, + last_login_at INTEGER, + email_bouncing INTEGER NOT NULL DEFAULT 0 + )`, + `CREATE TABLE sessions ( + id INTEGER PRIMARY KEY, + token_hash TEXT NOT NULL UNIQUE, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + csrf_token TEXT NOT NULL, + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + last_seen_at INTEGER NOT NULL, + user_agent TEXT NOT NULL DEFAULT '' + )`, + `CREATE INDEX sessions_user ON sessions(user_id)`, + `CREATE TABLE tokens ( + token_hash TEXT PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + purpose TEXT NOT NULL CHECK (purpose IN ('activate','reset','email_change')), + new_email TEXT, + expires_at INTEGER NOT NULL, + used_at INTEGER + )`, + `CREATE INDEX tokens_user ON tokens(user_id, purpose)`, + `CREATE TABLE audit_log ( + id INTEGER PRIMARY KEY, + at INTEGER NOT NULL, + actor_user_id INTEGER, + action TEXT NOT NULL, + target_user_id INTEGER, + detail TEXT NOT NULL DEFAULT '{}' + )`, + `CREATE INDEX audit_target ON audit_log(target_user_id)`, + `CREATE INDEX audit_actor ON audit_log(actor_user_id)`, + `CREATE TABLE mail_log ( + id INTEGER PRIMARY KEY, + user_id INTEGER REFERENCES users(id) ON DELETE SET NULL, + to_email TEXT NOT NULL, + purpose TEXT NOT NULL, + provider_message_id TEXT, + sent_at INTEGER NOT NULL, + last_event TEXT NOT NULL DEFAULT 'sent', + last_event_at INTEGER NOT NULL, + last_reason TEXT NOT NULL DEFAULT '' + )`, + `CREATE INDEX mail_user ON mail_log(user_id)`, + `CREATE UNIQUE INDEX mail_msgid ON mail_log(provider_message_id) WHERE provider_message_id IS NOT NULL`, + `CREATE TABLE mail_events ( + mail_id INTEGER NOT NULL REFERENCES mail_log(id) ON DELETE CASCADE, + event TEXT NOT NULL, + at INTEGER NOT NULL, + reason TEXT NOT NULL DEFAULT '', + UNIQUE (mail_id, event, at) + )`, + }, + { // v2: sub-project B, settings sync (one document per user) + `CREATE TABLE user_settings ( + user_id INTEGER PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + doc TEXT NOT NULL, + revision INTEGER NOT NULL, + generation TEXT NOT NULL, + updated_at INTEGER NOT NULL + )`, + }, + { // v3: sub-project C, global audit list ordered and filtered by time + `CREATE INDEX audit_at ON audit_log(at)`, + }, + { // v4: sub-project D, proposals (one generic table; this version ships the kind 'hashtag_channel') + `CREATE TABLE proposals ( + id INTEGER PRIMARY KEY, + kind TEXT NOT NULL, + subject TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending','approved','rejected','revoked')), + proposer_id INTEGER REFERENCES users(id) ON DELETE SET NULL, + reviewer_id INTEGER REFERENCES users(id) ON DELETE SET NULL, + note TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL, + decided_at INTEGER + )`, + `CREATE UNIQUE INDEX proposals_kind_subject ON proposals(kind, subject)`, + `CREATE INDEX proposals_status ON proposals(status)`, + }, + { // v5: sub-project E, node notifications + `CREATE TABLE notification_prefs ( + user_id INTEGER PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + enabled INTEGER NOT NULL DEFAULT 1, + events TEXT NOT NULL DEFAULT '', + unsub_token TEXT NOT NULL UNIQUE, + updated_at INTEGER NOT NULL + )`, + `CREATE TABLE notification_watches ( + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + pubkey TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (user_id, pubkey) + )`, + `CREATE TABLE notification_state ( + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + event TEXT NOT NULL, + subject TEXT NOT NULL, + state TEXT NOT NULL, + changed_at INTEGER NOT NULL, + PRIMARY KEY (user_id, event, subject) + )`, + }, +} + +// LegacyMigrations returns a copy of the five append-only SQLite migrations. +func LegacyMigrations() [][]string { + out := make([][]string, len(legacyMigrations)) + for i, m := range legacyMigrations { + out[i] = append([]string(nil), m...) + } + return out +} diff --git a/internal/users/mail.go b/internal/users/mail.go index 3bb2ca8b4..6b48a4c8d 100644 --- a/internal/users/mail.go +++ b/internal/users/mail.go @@ -56,12 +56,13 @@ func (s *Store) LogMail(userID *int64, to, purpose, messageID string) (int64, er if messageID != "" { mid = messageID } - res, err := s.db.Exec(`INSERT INTO mail_log (user_id, to_email, purpose, provider_message_id, sent_at, last_event, last_event_at) - VALUES (?, ?, ?, ?, ?, 'sent', ?)`, nullInt(userID), to, purpose, mid, now, now) + var id int64 + err := s.db.QueryRow(`INSERT INTO mail_log (user_id, to_email, purpose, provider_message_id, sent_at, last_event, last_event_at) + VALUES ($1, $2, $3, $4, $5, 'sent', $6) RETURNING id`, nullInt(userID), to, purpose, mid, now, now).Scan(&id) if err != nil { return 0, err } - return res.LastInsertId() + return id, nil } // RecordMailEvent appends a provider event to the mail with that provider @@ -75,7 +76,7 @@ func (s *Store) RecordMailEvent(messageID, event string, at time.Time, reason st defer tx.Rollback() var id, lastAt int64 var uid sql.NullInt64 - err = tx.QueryRow(`SELECT id, user_id, last_event_at FROM mail_log WHERE provider_message_id = ?`, messageID). + err = tx.QueryRow(`SELECT id, user_id, last_event_at FROM mail_log WHERE provider_message_id = $1 FOR UPDATE`, messageID). Scan(&id, &uid, &lastAt) if errors.Is(err, sql.ErrNoRows) { return nil, false, nil @@ -83,12 +84,12 @@ func (s *Store) RecordMailEvent(messageID, event string, at time.Time, reason st if err != nil { return nil, false, err } - if _, err := tx.Exec(`INSERT OR IGNORE INTO mail_events (mail_id, event, at, reason) VALUES (?, ?, ?, ?)`, + if _, err := tx.Exec(`INSERT INTO mail_events (mail_id, event, at, reason) VALUES ($1, $2, $3, $4) ON CONFLICT (mail_id,event,at) DO NOTHING`, id, event, unix(at), reason); err != nil { return nil, false, err } if unix(at) >= lastAt { - if _, err := tx.Exec(`UPDATE mail_log SET last_event = ?, last_event_at = ?, last_reason = ? WHERE id = ?`, + if _, err := tx.Exec(`UPDATE mail_log SET last_event = $1, last_event_at = $2, last_reason = $3 WHERE id = $4`, event, unix(at), reason, id); err != nil { return nil, false, err } @@ -104,7 +105,7 @@ func (s *Store) RecordMailEvent(messageID, event string, at time.Time, reason st } func (s *Store) eventsFor(mailID int64) ([]MailEvent, error) { - rows, err := s.db.Query(`SELECT event, at, reason FROM mail_events WHERE mail_id = ? ORDER BY at, rowid`, mailID) + rows, err := s.db.Query(`SELECT event, at, reason FROM mail_events WHERE mail_id = $1 ORDER BY at, id`, mailID) if err != nil { return nil, err } @@ -124,7 +125,7 @@ func (s *Store) eventsFor(mailID int64) ([]MailEvent, error) { // MailByID returns one record with its events. func (s *Store) MailByID(id int64) (*MailRecord, error) { - m, err := scanMail(s.db.QueryRow(`SELECT `+mailCols+` FROM mail_log WHERE id = ?`, id)) + m, err := scanMail(s.db.QueryRow(`SELECT `+mailCols+` FROM mail_log WHERE id = $1`, id)) if err != nil { return nil, err } @@ -148,8 +149,8 @@ func (s *Store) MailAllForUser(userID int64) ([]MailRecord, error) { return s.mailForUser(userID, noLimit) } -func (s *Store) mailForUser(userID int64, limit int) ([]MailRecord, error) { - rows, err := s.db.Query(`SELECT `+mailCols+` FROM mail_log WHERE user_id = ? ORDER BY sent_at DESC, id DESC LIMIT ?`, userID, limit) +func (s *Store) mailForUser(userID int64, limit any) ([]MailRecord, error) { + rows, err := s.db.Query(`SELECT `+mailCols+` FROM mail_log WHERE user_id = $1 ORDER BY sent_at DESC, id DESC LIMIT $2`, userID, limit) if err != nil { return nil, err } @@ -195,7 +196,7 @@ func (s *Store) LatestMailByUser() (map[int64]MailRecord, error) { // PruneMail deletes mail records (and their events) older than maxAge. func (s *Store) PruneMail(maxAge time.Duration) (int64, error) { - res, err := s.db.Exec(`DELETE FROM mail_log WHERE sent_at < ?`, unix(s.now())-int64(maxAge/time.Second)) + res, err := s.db.Exec(`DELETE FROM mail_log WHERE sent_at < $1`, unix(s.now())-int64(maxAge/time.Second)) if err != nil { return 0, err } diff --git a/internal/users/notify.go b/internal/users/notify.go index 4bad64573..9c4b6858f 100644 --- a/internal/users/notify.go +++ b/internal/users/notify.go @@ -3,6 +3,8 @@ package users import ( "database/sql" "errors" + "fmt" + "sort" "strings" "time" ) @@ -112,7 +114,17 @@ func canonicalEvents(list []string) []string { return out } -func placeholders(n int) string { return strings.TrimSuffix(strings.Repeat("?,", n), ",") } +func placeholders(n int, offset ...int) string { + start := 1 + if len(offset) > 0 { + start += offset[0] + } + out := make([]string, n) + for i := range out { + out[i] = fmt.Sprintf("$%d", start+i) + } + return strings.Join(out, ",") +} const prefsCols = `user_id, enabled, events, unsub_token, updated_at` @@ -134,7 +146,7 @@ func scanPrefs(row rowScanner) (*NotifyPrefs, error) { } func (s *Store) getPrefs(userID int64) (*NotifyPrefs, error) { - return scanPrefs(s.db.QueryRow(`SELECT `+prefsCols+` FROM notification_prefs WHERE user_id = ?`, userID)) + return scanPrefs(s.db.QueryRow(`SELECT `+prefsCols+` FROM notification_prefs WHERE user_id = $1`, userID)) } // NotifyPrefsFor returns the user's preferences, creating the default row @@ -151,8 +163,8 @@ func (s *Store) NotifyPrefsFor(userID int64) (NotifyPrefs, error) { return NotifyPrefs{}, err } d := DefaultNotifyPrefs(userID) - if _, err := s.db.Exec(`INSERT OR IGNORE INTO notification_prefs (user_id, enabled, events, unsub_token, updated_at) - VALUES (?, ?, ?, ?, ?)`, userID, d.Enabled, strings.Join(d.Events, ","), tok, unix(s.now())); err != nil { + if _, err := s.db.Exec(`INSERT INTO notification_prefs (user_id, enabled, events, unsub_token, updated_at) + VALUES ($1, $2, $3, $4, $5) ON CONFLICT(user_id) DO NOTHING`, userID, 1, strings.Join(d.Events, ","), tok, unix(s.now())); err != nil { return NotifyPrefs{}, err } p, err := s.getPrefs(userID) @@ -190,14 +202,14 @@ func (s *Store) SetNotifyPrefs(userID int64, enabled bool, events []string) (Not return NotifyPrefs{}, err } defer tx.Rollback() - if _, err := tx.Exec(`UPDATE notification_prefs SET enabled = ?, events = ?, updated_at = ? WHERE user_id = ?`, + if _, err := tx.Exec(`UPDATE notification_prefs SET enabled = $1, events = $2, updated_at = $3 WHERE user_id = $4`, en, strings.Join(ev, ","), unix(s.now()), userID); err != nil { return NotifyPrefs{}, err } - q := `DELETE FROM notification_state WHERE user_id = ?` + q := `DELETE FROM notification_state WHERE user_id = $1` args := []any{userID} if len(ev) > 0 { - q += ` AND event NOT IN (` + placeholders(len(ev)) + `)` + q += ` AND event NOT IN (` + placeholders(len(ev), 1) + `)` for _, e := range ev { args = append(args, e) } @@ -218,8 +230,13 @@ func (s *Store) DisableNotifyByToken(token string) (userID int64, wasEnabled boo if token == "" { return 0, false, ErrTokenInvalid } + tx, err := s.db.Begin() + if err != nil { + return 0, false, err + } + defer tx.Rollback() var en int - err = s.db.QueryRow(`SELECT user_id, enabled FROM notification_prefs WHERE unsub_token = ?`, token).Scan(&userID, &en) + err = tx.QueryRow(`SELECT user_id,enabled FROM notification_prefs WHERE unsub_token=$1 FOR UPDATE`, token).Scan(&userID, &en) if errors.Is(err, sql.ErrNoRows) { return 0, false, ErrTokenInvalid } @@ -227,11 +244,11 @@ func (s *Store) DisableNotifyByToken(token string) (userID int64, wasEnabled boo return 0, false, err } if en != 0 { - if _, err := s.db.Exec(`UPDATE notification_prefs SET enabled = 0, updated_at = ? WHERE user_id = ?`, unix(s.now()), userID); err != nil { + if _, err = tx.Exec(`UPDATE notification_prefs SET enabled=0,updated_at=$1 WHERE user_id=$2`, unix(s.now()), userID); err != nil { return 0, false, err } } - return userID, en != 0, nil + return userID, en != 0, tx.Commit() } // AllNotifyPrefs returns every preferences row, by user id. @@ -262,8 +279,11 @@ func (s *Store) AddWatches(userID int64, pubkeys []string, max int) (added, alre return 0, 0, 0, err } defer tx.Rollback() + if err := lockUser(tx, userID); err != nil { + return 0, 0, 0, err + } have := map[string]bool{} - rows, err := tx.Query(`SELECT pubkey FROM notification_watches WHERE user_id = ?`, userID) + rows, err := tx.Query(`SELECT pubkey FROM notification_watches WHERE user_id = $1`, userID) if err != nil { return 0, 0, 0, err } @@ -287,7 +307,7 @@ func (s *Store) AddWatches(userID int64, pubkeys []string, max int) (added, alre case max > 0 && len(have) >= max: overLimit++ default: - if _, err := tx.Exec(`INSERT INTO notification_watches (user_id, pubkey, created_at) VALUES (?, ?, ?)`, userID, pk, now); err != nil { + if _, err := tx.Exec(`INSERT INTO notification_watches (user_id, pubkey, created_at) VALUES ($1, $2, $3)`, userID, pk, now); err != nil { return 0, 0, 0, err } have[pk] = true @@ -321,10 +341,15 @@ func (s *Store) RemoveWatch(userID int64, pubkey string) error { return err } defer tx.Rollback() - if _, err := tx.Exec(`DELETE FROM notification_watches WHERE user_id = ? AND pubkey = ?`, userID, pubkey); err != nil { + if err := lockUser(tx, userID); errors.Is(err, ErrNotFound) { + return nil + } else if err != nil { + return err + } + if _, err := tx.Exec(`DELETE FROM notification_watches WHERE user_id = $1 AND pubkey = $2`, userID, pubkey); err != nil { return err } - if _, err := tx.Exec(`DELETE FROM notification_state WHERE user_id = ? AND subject = ? AND event IN (?, ?)`, + if _, err := tx.Exec(`DELETE FROM notification_state WHERE user_id = $1 AND subject = $2 AND event IN ($3, $4)`, userID, pubkey, NotifyNodeOffline, NotifyNodeBattery); err != nil { return err } @@ -352,7 +377,7 @@ func (s *Store) queryWatches(q string, args ...any) ([]NotifyWatch, error) { // WatchesFor returns the user's watches, oldest first. func (s *Store) WatchesFor(userID int64) ([]NotifyWatch, error) { - return s.queryWatches(`SELECT user_id, pubkey, created_at FROM notification_watches WHERE user_id = ? ORDER BY created_at, pubkey`, userID) + return s.queryWatches(`SELECT user_id, pubkey, created_at FROM notification_watches WHERE user_id = $1 ORDER BY created_at, pubkey`, userID) } // AllWatches returns every watch, by user and pubkey. @@ -390,14 +415,17 @@ func (s *Store) WriteNotifyStates(list []NotifyState) error { } defer tx.Rollback() stmt, err := tx.Prepare(`INSERT INTO notification_state (user_id, event, subject, state, changed_at) - SELECT ?, ?, ?, ?, ? WHERE EXISTS (SELECT 1 FROM users WHERE id = ?) + SELECT id, $2, $3, $4, $5 FROM users WHERE id=$1 FOR KEY SHARE ON CONFLICT (user_id, event, subject) DO UPDATE SET state = excluded.state, changed_at = excluded.changed_at`) if err != nil { return err } defer stmt.Close() - for _, st := range list { - if _, err := stmt.Exec(st.UserID, st.Event, st.Subject, st.State, unix(st.ChangedAt), st.UserID); err != nil { + // Match the janitor's ascending account lock order to avoid deadlocks. + ordered := append([]NotifyState(nil), list...) + sort.SliceStable(ordered, func(i, j int) bool { return ordered[i].UserID < ordered[j].UserID }) + for _, st := range ordered { + if _, err := stmt.Exec(st.UserID, st.Event, st.Subject, st.State, unix(st.ChangedAt)); err != nil { return err } } @@ -415,7 +443,7 @@ func (s *Store) DeleteNotifyStates(keys []NotifyKey) error { return err } defer tx.Rollback() - stmt, err := tx.Prepare(`DELETE FROM notification_state WHERE user_id = ? AND event = ? AND subject = ?`) + stmt, err := tx.Prepare(`DELETE FROM notification_state WHERE user_id = $1 AND event = $2 AND subject = $3`) if err != nil { return err } @@ -432,7 +460,7 @@ func (s *Store) DeleteNotifyStates(keys []NotifyKey) error { // total (deleted accounts included) and per existing user. func (s *Store) NotifyMailCounts(since time.Time) (total int, perUser map[int64]int, err error) { perUser = map[int64]int{} - rows, err := s.db.Query(`SELECT user_id, COUNT(*) FROM mail_log WHERE purpose = ? AND sent_at >= ? GROUP BY user_id`, + rows, err := s.db.Query(`SELECT user_id, COUNT(*) FROM mail_log WHERE purpose = $1 AND sent_at >= $2 GROUP BY user_id`, NotifyMailPurpose, unix(since)) if err != nil { return 0, nil, err diff --git a/internal/users/notify_test.go b/internal/users/notify_test.go index 668d68be7..0973457fd 100644 --- a/internal/users/notify_test.go +++ b/internal/users/notify_test.go @@ -1,9 +1,7 @@ package users import ( - "database/sql" "errors" - "path/filepath" "reflect" "strings" "testing" @@ -42,30 +40,9 @@ func statesByKey(t *testing.T, st *Store) map[NotifyKey]NotifyState { return out } -// A users.db written by a v4 binary gains the notification tables and keeps its rows. +// A native account schema at v4 binary gains the notification tables and keeps its rows. func TestMigrateV4DatabaseToV5(t *testing.T) { - path := filepath.Join(t.TempDir(), "users.db") - db, err := sql.Open("sqlite", path) - if err != nil { - t.Fatal(err) - } - stmts := []string{`CREATE TABLE schema_version (version INTEGER NOT NULL)`, `INSERT INTO schema_version (version) VALUES (4)`} - for _, m := range migrations[:4] { - stmts = append(stmts, m...) - } - stmts = append(stmts, `INSERT INTO users (email, display_name, password_hash, created_at) VALUES ('old@example.org', 'Old', 'x', 1)`) - for _, s := range stmts { - if _, err := db.Exec(s); err != nil { - t.Fatalf("build v4 db: %v", err) - } - } - db.Close() - - st, err := Open(path) - if err != nil { - t.Fatalf("Open v4 db: %v", err) - } - defer st.Close() + st := migratedTestStore(t, 4, `INSERT INTO users (email,display_name,password_hash,created_at) VALUES ('old@example.org','Old','x',1)`) if v, err := st.SchemaVersion(); err != nil || v != len(migrations) { t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, len(migrations)) } diff --git a/internal/users/onetime.go b/internal/users/onetime.go index 89d3556ba..aad937161 100644 --- a/internal/users/onetime.go +++ b/internal/users/onetime.go @@ -29,7 +29,10 @@ func (s *Store) IssueToken(userID int64, p Purpose, ttl time.Duration, newEmail return "", err } defer tx.Rollback() - if _, err := tx.Exec(`UPDATE tokens SET used_at = ? WHERE user_id = ? AND purpose = ? AND used_at IS NULL`, + if err := lockUser(tx, userID); err != nil { + return "", err + } + if _, err := tx.Exec(`UPDATE tokens SET used_at = $1 WHERE user_id = $2 AND purpose = $3 AND used_at IS NULL`, unix(now), userID, string(p)); err != nil { return "", err } @@ -37,7 +40,7 @@ func (s *Store) IssueToken(userID int64, p Purpose, ttl time.Duration, newEmail if newEmail != "" { ne = newEmail } - if _, err := tx.Exec(`INSERT INTO tokens (token_hash, user_id, purpose, new_email, expires_at) VALUES (?, ?, ?, ?, ?)`, + if _, err := tx.Exec(`INSERT INTO tokens (token_hash, user_id, purpose, new_email, expires_at) VALUES ($1, $2, $3, $4, $5)`, hash, userID, string(p), ne, unix(now.Add(ttl))); err != nil { return "", err } @@ -56,7 +59,7 @@ func (s *Store) checkToken(q tokenQuerier, hash string, p Purpose) (userID int64 var expires int64 var ne sql.NullString var used sql.NullInt64 - err = q.QueryRow(`SELECT user_id, purpose, new_email, expires_at, used_at FROM tokens WHERE token_hash = ?`, hash). + err = q.QueryRow(`SELECT user_id, purpose, new_email, expires_at, used_at FROM tokens WHERE token_hash = $1`, hash). Scan(&userID, &purpose, &ne, &expires, &used) if errors.Is(err, sql.ErrNoRows) { return 0, "", ErrTokenInvalid @@ -93,7 +96,14 @@ func (s *Store) ConsumeToken(raw string, p Purpose) (userID int64, newEmail stri if err != nil { return 0, "", err } - if err := expectOne(tx.Exec(`UPDATE tokens SET used_at = ? WHERE token_hash = ? AND used_at IS NULL`, unix(s.now()), hash)); err != nil { + if err := lockUser(tx, userID); err != nil { + return 0, "", ErrTokenInvalid + } + userID, newEmail, err = s.checkToken(tx, hash, p) + if err != nil { + return 0, "", err + } + if err := expectOne(tx.Exec(`UPDATE tokens SET used_at = $1 WHERE token_hash = $2 AND used_at IS NULL`, unix(s.now()), hash)); err != nil { if errors.Is(err, ErrNotFound) { return 0, "", ErrTokenInvalid } @@ -117,6 +127,9 @@ func (s *Store) ActivateWithToken(raw string, id int64, role Role, verifiedHash return err } defer tx.Rollback() + if err := lockUser(tx, id); err != nil { + return ErrAccountChanged + } uid, _, err := s.checkToken(tx, hash, PurposeActivate) if err != nil { return err @@ -125,14 +138,14 @@ func (s *Store) ActivateWithToken(raw string, id int64, role Role, verifiedHash return ErrTokenInvalid } now := unix(s.now()) - if err := expectOne(tx.Exec(`UPDATE tokens SET used_at = ? WHERE token_hash = ? AND used_at IS NULL`, now, hash)); err != nil { + if err := expectOne(tx.Exec(`UPDATE tokens SET used_at = $1 WHERE token_hash = $2 AND used_at IS NULL`, now, hash)); err != nil { if errors.Is(err, ErrNotFound) { return ErrTokenInvalid } return err } - err = expectOne(tx.Exec(`UPDATE users SET status = 'active', role = ?, activated_at = ?, activated_by = NULL - WHERE id = ? AND status = 'pending' AND password_hash = ?`, string(role), now, id, verifiedHash)) + err = expectOne(tx.Exec(`UPDATE users SET status = 'active', role = $1, activated_at = $2, activated_by = NULL + WHERE id = $3 AND status = 'pending' AND password_hash = $4`, string(role), now, id, verifiedHash)) if errors.Is(err, ErrNotFound) { return ErrAccountChanged } @@ -144,9 +157,20 @@ func (s *Store) ActivateWithToken(raw string, id int64, role Role, verifiedHash // InvalidateTokens burns all of a user's unused tokens for purpose p. func (s *Store) InvalidateTokens(userID int64, p Purpose) error { - _, err := s.db.Exec(`UPDATE tokens SET used_at = ? WHERE user_id = ? AND purpose = ? AND used_at IS NULL`, - unix(s.now()), userID, string(p)) - return err + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + if err := lockUser(tx, userID); errors.Is(err, ErrNotFound) { + return nil + } else if err != nil { + return err + } + if _, err = tx.Exec(`UPDATE tokens SET used_at=$1 WHERE user_id=$2 AND purpose=$3 AND used_at IS NULL`, unix(s.now()), userID, string(p)); err != nil { + return err + } + return tx.Commit() } // PendingEmailChange returns the new address of the user's unused, @@ -155,7 +179,7 @@ func (s *Store) InvalidateTokens(userID int64, p Purpose) error { func (s *Store) PendingEmailChange(userID int64) (string, error) { var ne sql.NullString err := s.db.QueryRow(`SELECT new_email FROM tokens - WHERE user_id = ? AND purpose = ? AND used_at IS NULL AND expires_at > ? + WHERE user_id = $1 AND purpose = $2 AND used_at IS NULL AND expires_at > $3 ORDER BY expires_at DESC LIMIT 1`, userID, string(PurposeEmailChange), unix(s.now())).Scan(&ne) if errors.Is(err, sql.ErrNoRows) { return "", nil @@ -165,7 +189,7 @@ func (s *Store) PendingEmailChange(userID int64) (string, error) { // PruneTokens deletes tokens that expired more than keep ago. func (s *Store) PruneTokens(keep time.Duration) (int64, error) { - res, err := s.db.Exec(`DELETE FROM tokens WHERE expires_at < ?`, unix(s.now())-int64(keep/time.Second)) + res, err := s.db.Exec(`DELETE FROM tokens WHERE expires_at < $1`, unix(s.now())-int64(keep/time.Second)) if err != nil { return 0, err } diff --git a/internal/users/postgres_test.go b/internal/users/postgres_test.go new file mode 100644 index 000000000..7cb73370e --- /dev/null +++ b/internal/users/postgres_test.go @@ -0,0 +1,389 @@ +package users + +import ( + "errors" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +func TestOpenRejectsCredentialsInErrors(t *testing.T) { + _, err := Open("postgresql://private-user:private-password@localhost:invalid/accounts") + if err == nil { + t.Fatal("invalid database URL accepted") + } + for _, secret := range []string{"private-user", "private-password"} { + if strings.Contains(err.Error(), secret) { + t.Errorf("connection error exposed credentials") + } + } +} + +func TestOpenRefusesSchemaOwner(t *testing.T) { + dsn := pgtest.NewSchema(t) + db, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := Apply(db); err != nil { + t.Fatal(err) + } + st, err := Open(dsn) + if st != nil { + st.Close() + } + if err == nil { + t.Fatal("runtime account connection accepted the migration owner") + } +} + +func TestPostgresSettingsCASAcrossPools(t *testing.T) { + st, _ := newTestStore(t) + other, err := Open(st.databaseURL) + if err != nil { + t.Fatal(err) + } + defer other.Close() + u := mustCreate(t, st, "cas@example.org", "CAS") + start := make(chan struct{}) + var done sync.WaitGroup + var won atomic.Int32 + for i := 0; i < 20; i++ { + done.Add(1) + go func(i int) { + defer done.Done() + <-start + s := st + if i%2 == 1 { + s = other + } + _, err := s.PutSettings(u.ID, SettingsVersion{}, `{"device":true}`) + if err == nil { + won.Add(1) + } else if !errors.Is(err, ErrSettingsConflict) { + t.Errorf("settings: %v", err) + } + }(i) + } + close(start) + done.Wait() + if got := won.Load(); got != 1 { + t.Fatalf("initial settings writers succeeded=%d; want 1", got) + } +} + +func TestPostgresProposalQuotaAcrossPools(t *testing.T) { + st, _ := newTestStore(t) + other, err := Open(st.databaseURL) + if err != nil { + t.Fatal(err) + } + defer other.Close() + u := mustCreate(t, st, "quota@example.org", "Quota") + start := make(chan struct{}) + var done sync.WaitGroup + var won atomic.Int32 + for i := 0; i < 20; i++ { + done.Add(1) + go func(i int) { + defer done.Done() + <-start + s := st + if i%2 == 1 { + s = other + } + _, err := s.Propose(KindHashtagChannel, strings.Repeat("x", i+1), u.ID, ProposalLimits{MaxPending: 1}) + if err == nil { + won.Add(1) + } else if !errors.Is(err, ErrProposalPendingLimit) { + t.Errorf("proposal: %v", err) + } + }(i) + } + close(start) + done.Wait() + if got := won.Load(); got != 1 { + t.Fatalf("pending proposals accepted=%d; want 1", got) + } +} + +func TestOpenRequiresBootstrap(t *testing.T) { + dsn := pgtest.NewSchema(t) + st, err := Open(dsn) + if st != nil { + st.Close() + } + if err == nil || !strings.Contains(err.Error(), "bootstrap") { + t.Fatal("runtime Open must report that the PostgreSQL schema needs bootstrap") + } +} + +func TestPostgresSingleUseAndLatestTokenAcrossPools(t *testing.T) { + st, _ := newTestStore(t) + other, err := Open(st.databaseURL) + if err != nil { + t.Fatal(err) + } + defer other.Close() + other.SetClock(st.now) + u := mustCreate(t, st, "token-race@example.org", "Token") + raw, err := st.IssueToken(u.ID, PurposeReset, time.Hour, "") + if err != nil { + t.Fatal(err) + } + var won atomic.Int32 + raceStores(st, other, func(s *Store, i int) { + _, _, err := s.ConsumeToken(raw, PurposeReset) + if err == nil { + won.Add(1) + } else if !errors.Is(err, ErrTokenInvalid) { + t.Errorf("consume: %v", err) + } + }) + if won.Load() != 1 { + t.Fatalf("token consumed %d times", won.Load()) + } + tokens := make([]string, 20) + raceStores(st, other, func(s *Store, i int) { + var err error + tokens[i], err = s.IssueToken(u.ID, PurposeReset, time.Hour, "") + if err != nil { + t.Errorf("issue: %v", err) + } + }) + var valid int + for _, raw := range tokens { + if _, err := st.TokenUser(raw, PurposeReset); err == nil { + valid++ + } else if !errors.Is(err, ErrTokenInvalid) { + t.Fatal(err) + } + } + if valid != 1 { + t.Fatalf("live reset tokens=%d; want 1", valid) + } +} + +func TestPostgresWatchAndApprovalQuotasAcrossPools(t *testing.T) { + st, _ := newTestStore(t) + other, err := Open(st.databaseURL) + if err != nil { + t.Fatal(err) + } + defer other.Close() + other.SetClock(st.now) + u := mustCreate(t, st, "watch-race@example.org", "Watch") + var watched atomic.Int32 + raceStores(st, other, func(s *Store, i int) { + err := s.AddWatch(u.ID, strings.Repeat("a", i+1), 1) + if err == nil { + watched.Add(1) + } else if !errors.Is(err, ErrWatchLimit) { + t.Errorf("watch: %v", err) + } + }) + if watched.Load() != 1 { + t.Fatalf("watches=%d; want 1", watched.Load()) + } + var proposals []*Proposal + for i := 0; i < 20; i++ { + p, err := st.Propose(KindHashtagChannel, strings.Repeat("x", i+1), u.ID, ProposalLimits{}) + if err != nil { + t.Fatal(err) + } + proposals = append(proposals, p) + } + var approved atomic.Int32 + raceStores(st, other, func(s *Store, i int) { + _, err := s.Decide(proposals[i].ID, ProposalApprove, u.ID, "", 1) + if err == nil { + approved.Add(1) + } else if !errors.Is(err, ErrProposalApprovedLimit) { + t.Errorf("approve: %v", err) + } + }) + if approved.Load() != 1 { + t.Fatalf("approved=%d; want 1", approved.Load()) + } +} + +func raceStores(first, second *Store, fn func(*Store, int)) { + start := make(chan struct{}) + var wg sync.WaitGroup + for i := 0; i < 20; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + s := first + if i%2 == 1 { + s = second + } + fn(s, i) + }(i) + } + close(start) + wg.Wait() +} + +func TestApplyForImportCannotAppearReady(t *testing.T) { + dsn := pgtest.NewSchema(t) + db := testOwner(t, dsn) + if err := ApplyForImport(db); err != nil { + t.Fatal(err) + } + if err := AssertReady(db); err == nil { + t.Fatal("unverified import is runtime-ready") + } + if err := Apply(db); err == nil { + t.Fatal("normal bootstrap bypassed import verification") + } + if err := ApplyForImport(db); err == nil { + t.Fatal("restarted bootstrap accepted partial import") + } + if _, err := db.Exec(`UPDATE corescope_schema SET ready=true WHERE kind='accounts'`); err != nil { + t.Fatal(err) + } + if err := AssertReady(db); err != nil { + t.Fatal(err) + } +} + +func TestAccountRuntimeCannotAlterSchemaOrReadiness(t *testing.T) { + st, _ := newTestStore(t) + for _, stmt := range []string{`CREATE TABLE forbidden(id int)`, `ALTER TABLE users ADD COLUMN forbidden int`, `UPDATE corescope_schema SET ready=false`, `DELETE FROM schema_version`} { + if _, err := st.db.Exec(stmt); err == nil { + t.Fatalf("runtime accepted privileged statement: %s", stmt) + } + } +} + +func TestNativeAccountTextUsesBinaryCollation(t *testing.T) { + st, _ := newTestStore(t) + var nonBinary int + if err := st.db.QueryRow(`SELECT count(*) FROM information_schema.columns WHERE table_schema=current_schema() AND data_type='text' AND collation_name IS DISTINCT FROM 'C'`).Scan(&nonBinary); err != nil { + t.Fatal(err) + } + if nonBinary != 0 { + t.Fatalf("%d account text columns inherit locale-dependent collation", nonBinary) + } + for _, email := range []string{"a@example.org", "B@example.org", "ä@example.org"} { + mustCreate(t, st, email, "Case") + } + rows, err := st.db.Query(`SELECT email FROM users ORDER BY email`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var got []string + for rows.Next() { + var email string + if err := rows.Scan(&email); err != nil { + t.Fatal(err) + } + got = append(got, email) + } + if strings.Join(got, ",") != "B@example.org,a@example.org,ä@example.org" { + t.Fatal("account byte ordering changed") + } +} + +func TestPendingPruneRechecksTokenAfterConcurrentIssue(t *testing.T) { + st, clk := newTestStore(t) + u := mustCreate(t, st, "pending-race@example.org", "Pending") + clk.Advance(8 * 24 * time.Hour) + other, err := pgutil.Open(st.databaseURL, false) + if err != nil { + t.Fatal(err) + } + defer other.Close() + tx, err := other.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + if err := lockUser(tx, u.ID); err != nil { + t.Fatal(err) + } + type result struct { + n int64 + err error + } + finished := make(chan result, 1) + go func() { n, err := st.PruneStalePending(7 * 24 * time.Hour); finished <- result{n, err} }() + deadline := time.Now().Add(5 * time.Second) + for { + var waiting int + if err := other.QueryRow(`SELECT COUNT(*) FROM pg_stat_activity WHERE usename=current_user AND wait_event_type='Lock'`).Scan(&waiting); err != nil { + t.Fatal(err) + } + if waiting > 0 { + break + } + if time.Now().After(deadline) { + t.Fatal("prune did not reach the account row lock") + } + time.Sleep(10 * time.Millisecond) + } + if _, err := tx.Exec(`INSERT INTO tokens(token_hash,user_id,purpose,expires_at) VALUES('prune-race',$1,'activate',$2)`, u.ID, unix(clk.Now().Add(time.Hour))); err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + got := <-finished + if got.err != nil || got.n != 0 { + t.Fatalf("pruned account after new activation token=%d,%v; want 0", got.n, got.err) + } + if _, err := st.GetByID(u.ID); err != nil { + t.Fatal("newly issued activation account was deleted") + } +} + +func TestNotifyStatesSkipConcurrentAccountDeletion(t *testing.T) { + st, clk := newTestStore(t) + u := mustCreate(t, st, "delete-race@example.org", "Deleted") + other, err := pgutil.Open(st.databaseURL, false) + if err != nil { + t.Fatal(err) + } + defer other.Close() + tx, err := other.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + if _, err := tx.Exec(`DELETE FROM users WHERE id=$1`, u.ID); err != nil { + t.Fatal(err) + } + finished := make(chan error, 1) + go func() { + finished <- st.WriteNotifyStates([]NotifyState{nState(u.ID, NotifyNodeOffline, "node", NotifyBad, clk.Now())}) + }() + deadline := time.Now().Add(5 * time.Second) + for { + var waiting int + if err := other.QueryRow(`SELECT COUNT(*) FROM pg_stat_activity WHERE usename=current_user AND wait_event_type='Lock'`).Scan(&waiting); err != nil { + t.Fatal(err) + } + if waiting > 0 { + break + } + if time.Now().After(deadline) { + t.Fatal("notification insert did not reach the account lock") + } + time.Sleep(10 * time.Millisecond) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + if err := <-finished; err != nil { + t.Fatalf("deleted account was not skipped: %v", err) + } +} diff --git a/internal/users/proposals.go b/internal/users/proposals.go index ba3d5bba2..ebe851522 100644 --- a/internal/users/proposals.go +++ b/internal/users/proposals.go @@ -3,6 +3,7 @@ package users import ( "database/sql" "errors" + "fmt" "time" ) @@ -116,8 +117,13 @@ func (s *Store) Propose(kind, subject string, userID int64, lim ProposalLimits) return nil, err } defer tx.Rollback() + // ponytail: serialize the small proposal queue across processes; replace + // with per-kind locks only if measured proposal throughput requires it. + if _, err := tx.Exec(`SELECT pg_advisory_xact_lock(hashtextextended(current_schema(),7289041403))`); err != nil { + return nil, err + } now := unix(s.now()) - cur, err := scanProposal(tx.QueryRow(`SELECT `+proposalCols+` FROM proposals WHERE kind = ? AND subject = ?`, kind, subject)) + cur, err := scanProposal(tx.QueryRow(`SELECT `+proposalCols+` FROM proposals WHERE kind = $1 AND subject = $2`, kind, subject)) switch { case errors.Is(err, ErrNotFound): cur = nil @@ -132,7 +138,7 @@ func (s *Store) Propose(kind, subject string, userID int64, lim ProposalLimits) } if lim.PerUserPerDay > 0 { var n int - if err := tx.QueryRow(`SELECT COUNT(*) FROM proposals WHERE proposer_id = ? AND created_at > ?`, + if err := tx.QueryRow(`SELECT COUNT(*) FROM proposals WHERE proposer_id = $1 AND created_at > $2`, userID, now-secs(24*time.Hour)).Scan(&n); err != nil { return nil, err } @@ -151,22 +157,19 @@ func (s *Store) Propose(kind, subject string, userID int64, lim ProposalLimits) } var id int64 if cur == nil { - res, err := tx.Exec(`INSERT INTO proposals (kind, subject, status, proposer_id, created_at) VALUES (?, ?, 'pending', ?, ?)`, - kind, subject, userID, now) + err := tx.QueryRow(`INSERT INTO proposals (kind, subject, status, proposer_id, created_at) VALUES ($1, $2, 'pending', $3, $4) RETURNING id`, + kind, subject, userID, now).Scan(&id) if err != nil { return nil, err } - if id, err = res.LastInsertId(); err != nil { - return nil, err - } } else { id = cur.ID - if _, err := tx.Exec(`UPDATE proposals SET status = 'pending', proposer_id = ?, reviewer_id = NULL, note = '', - created_at = ?, decided_at = NULL WHERE id = ?`, userID, now, id); err != nil { + if _, err := tx.Exec(`UPDATE proposals SET status = 'pending', proposer_id = $1, reviewer_id = NULL, note = '', + created_at = $2, decided_at = NULL WHERE id = $3`, userID, now, id); err != nil { return nil, err } } - p, err := scanProposal(tx.QueryRow(`SELECT `+proposalCols+` FROM proposals WHERE id = ?`, id)) + p, err := scanProposal(tx.QueryRow(`SELECT `+proposalCols+` FROM proposals WHERE id = $1`, id)) if err != nil { return nil, err } @@ -183,7 +186,12 @@ func (s *Store) Decide(id int64, action ProposalAction, reviewerID int64, note s return nil, err } defer tx.Rollback() - p, err := scanProposal(tx.QueryRow(`SELECT `+proposalCols+` FROM proposals WHERE id = ?`, id)) + // ponytail: serialize the small proposal queue across processes; replace + // with per-kind locks only if measured proposal throughput requires it. + if _, err := tx.Exec(`SELECT pg_advisory_xact_lock(hashtextextended(current_schema(),7289041403))`); err != nil { + return nil, err + } + p, err := scanProposal(tx.QueryRow(`SELECT `+proposalCols+` FROM proposals WHERE id = $1`, id)) if err != nil { return nil, err } @@ -200,7 +208,7 @@ func (s *Store) Decide(id int64, action ProposalAction, reviewerID int64, note s } if to == ProposalApproved && maxApproved > 0 { var n int - if err := tx.QueryRow(`SELECT COUNT(*) FROM proposals WHERE kind = ? AND status = 'approved'`, p.Kind).Scan(&n); err != nil { + if err := tx.QueryRow(`SELECT COUNT(*) FROM proposals WHERE kind = $1 AND status = 'approved'`, p.Kind).Scan(&n); err != nil { return nil, err } if n >= maxApproved { @@ -208,7 +216,7 @@ func (s *Store) Decide(id int64, action ProposalAction, reviewerID int64, note s } } now := s.now() - if _, err := tx.Exec(`UPDATE proposals SET status = ?, reviewer_id = ?, note = ?, decided_at = ? WHERE id = ?`, + if _, err := tx.Exec(`UPDATE proposals SET status = $1, reviewer_id = $2, note = $3, decided_at = $4 WHERE id = $5`, string(to), reviewerID, note, unix(now), id); err != nil { return nil, err } @@ -225,14 +233,14 @@ func (s *Store) ListProposals(f ProposalFilter) ([]Proposal, error) { q := `SELECT ` + proposalCols + ` FROM proposals WHERE 1=1` var args []any if f.Status != "" { - q += ` AND status = ?` + q += fmt.Sprintf(` AND status = $%d`, len(args)+1) args = append(args, string(f.Status)) } if f.Kind != "" { - q += ` AND kind = ?` + q += fmt.Sprintf(` AND kind = $%d`, len(args)+1) args = append(args, f.Kind) } - q += ` ORDER BY created_at DESC, id DESC LIMIT ?` + q += fmt.Sprintf(` ORDER BY created_at DESC, id DESC LIMIT $%d`, len(args)+1) args = append(args, ProposalListMax) return s.queryProposals(q, args...) } @@ -248,9 +256,9 @@ func (s *Store) AllProposalsByUser(userID int64) ([]Proposal, error) { return s.proposalsByUser(userID, noLimit) } -func (s *Store) proposalsByUser(userID int64, limit int) ([]Proposal, error) { - return s.queryProposals(`SELECT `+proposalCols+` FROM proposals WHERE proposer_id = ? - ORDER BY created_at DESC, id DESC LIMIT ?`, userID, limit) +func (s *Store) proposalsByUser(userID int64, limit any) ([]Proposal, error) { + return s.queryProposals(`SELECT `+proposalCols+` FROM proposals WHERE proposer_id = $1 + ORDER BY created_at DESC, id DESC LIMIT $2`, userID, limit) } func (s *Store) queryProposals(q string, args ...any) ([]Proposal, error) { @@ -274,10 +282,10 @@ func (s *Store) queryProposals(q string, args ...any) ([]Proposal, error) { // first, at most limit (0 = all). Never nil. cmd/ingestor runs the same // query as raw SQL (see ingestorApprovedQuery in proposals_test.go). func (s *Store) ApprovedSubjects(kind string, limit int) ([]string, error) { - q := `SELECT subject FROM proposals WHERE kind = ? AND status = 'approved' ORDER BY decided_at, id` + q := `SELECT subject FROM proposals WHERE kind = $1 AND status = 'approved' ORDER BY decided_at, id` args := []any{kind} if limit > 0 { - q += ` LIMIT ?` + q += ` LIMIT $2` args = append(args, limit) } rows, err := s.db.Query(q, args...) @@ -299,7 +307,7 @@ func (s *Store) ApprovedSubjects(kind string, limit int) ([]string, error) { // PruneProposals deletes rejected and revoked proposals decided more than // maxAge ago. Pending and approved rows are kept however old they are. func (s *Store) PruneProposals(maxAge time.Duration) (int64, error) { - res, err := s.db.Exec(`DELETE FROM proposals WHERE status IN ('rejected','revoked') AND decided_at < ?`, + res, err := s.db.Exec(`DELETE FROM proposals WHERE status IN ('rejected','revoked') AND decided_at < $1`, unix(s.now())-secs(maxAge)) if err != nil { return 0, err diff --git a/internal/users/proposals_test.go b/internal/users/proposals_test.go index 1154965d9..b635fe007 100644 --- a/internal/users/proposals_test.go +++ b/internal/users/proposals_test.go @@ -1,9 +1,7 @@ package users import ( - "database/sql" "errors" - "path/filepath" "strings" "testing" "time" @@ -13,7 +11,7 @@ import ( // literal (it does not import this package). Its test checks the literal is // still in this file; TestApprovedSubjects checks it still agrees with // ApprovedSubjects. -const ingestorApprovedQuery = `SELECT subject FROM proposals WHERE kind = 'hashtag_channel' AND status = 'approved' ORDER BY decided_at, id LIMIT ?` +const ingestorApprovedQuery = `SELECT subject FROM approved_channels WHERE kind = 'hashtag_channel' ORDER BY decided_at, id LIMIT $1` func mustPropose(t *testing.T, st *Store, subject string, uid int64) *Proposal { t.Helper() @@ -333,33 +331,9 @@ func TestProposalSurvivesProposerDeletion(t *testing.T) { } } -// A users.db written by a v3 binary gains the proposals table and keeps its rows. +// A native account schema at v3 binary gains the proposals table and keeps its rows. func TestMigrateV3DatabaseToV4(t *testing.T) { - path := filepath.Join(t.TempDir(), "users.db") - db, err := sql.Open("sqlite", path) - if err != nil { - t.Fatal(err) - } - stmts := []string{ - `CREATE TABLE schema_version (version INTEGER NOT NULL)`, - `INSERT INTO schema_version (version) VALUES (3)`, - } - for _, m := range migrations[:3] { - stmts = append(stmts, m...) - } - stmts = append(stmts, `INSERT INTO users (email, display_name, password_hash, created_at) VALUES ('old@example.org', 'Old', 'x', 1)`) - for _, s := range stmts { - if _, err := db.Exec(s); err != nil { - t.Fatalf("build v3 db: %v", err) - } - } - db.Close() - - st, err := Open(path) - if err != nil { - t.Fatalf("Open v3 db: %v", err) - } - defer st.Close() + st := migratedTestStore(t, 3, `INSERT INTO users (email,display_name,password_hash,created_at) VALUES ('old@example.org','Old','x',1)`) if v, err := st.SchemaVersion(); err != nil || v != len(migrations) { t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, len(migrations)) } diff --git a/internal/users/schema.go b/internal/users/schema.go index c7fd8d6a4..3ac76e6e4 100644 --- a/internal/users/schema.go +++ b/internal/users/schema.go @@ -6,81 +6,80 @@ import ( "fmt" ) -// migrations[i] upgrades the schema from version i to i+1. Forward-only: -// never edit a shipped entry, append a new one. var migrations = [][]string{ { // v1: sub-project A `CREATE TABLE users ( - id INTEGER PRIMARY KEY, - email TEXT NOT NULL UNIQUE, - display_name TEXT NOT NULL, - password_hash TEXT NOT NULL, - role TEXT NOT NULL DEFAULT 'user' CHECK (role IN ('user','admin')), - status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','active','disabled')), - created_at INTEGER NOT NULL, - activated_at INTEGER, - activated_by INTEGER, - last_login_at INTEGER, - email_bouncing INTEGER NOT NULL DEFAULT 0 + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + email TEXT COLLATE "C" NOT NULL UNIQUE, + display_name TEXT COLLATE "C" NOT NULL, + password_hash TEXT COLLATE "C" NOT NULL, + role TEXT COLLATE "C" NOT NULL DEFAULT 'user' CHECK (role IN ('user','admin')), + status TEXT COLLATE "C" NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','active','disabled')), + created_at BIGINT NOT NULL, + activated_at BIGINT, + activated_by BIGINT, + last_login_at BIGINT, + email_bouncing BIGINT NOT NULL DEFAULT 0 CHECK (email_bouncing IN (0,1)) )`, `CREATE TABLE sessions ( - id INTEGER PRIMARY KEY, - token_hash TEXT NOT NULL UNIQUE, - user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, - csrf_token TEXT NOT NULL, - created_at INTEGER NOT NULL, - expires_at INTEGER NOT NULL, - last_seen_at INTEGER NOT NULL, - user_agent TEXT NOT NULL DEFAULT '' + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + token_hash TEXT COLLATE "C" NOT NULL UNIQUE, + user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + csrf_token TEXT COLLATE "C" NOT NULL, + created_at BIGINT NOT NULL, + expires_at BIGINT NOT NULL, + last_seen_at BIGINT NOT NULL, + user_agent TEXT COLLATE "C" NOT NULL DEFAULT '' )`, `CREATE INDEX sessions_user ON sessions(user_id)`, `CREATE TABLE tokens ( - token_hash TEXT PRIMARY KEY, - user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, - purpose TEXT NOT NULL CHECK (purpose IN ('activate','reset','email_change')), - new_email TEXT, - expires_at INTEGER NOT NULL, - used_at INTEGER + token_hash TEXT COLLATE "C" PRIMARY KEY, + user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + purpose TEXT COLLATE "C" NOT NULL CHECK (purpose IN ('activate','reset','email_change')), + new_email TEXT COLLATE "C", + expires_at BIGINT NOT NULL, + used_at BIGINT )`, `CREATE INDEX tokens_user ON tokens(user_id, purpose)`, `CREATE TABLE audit_log ( - id INTEGER PRIMARY KEY, - at INTEGER NOT NULL, - actor_user_id INTEGER, - action TEXT NOT NULL, - target_user_id INTEGER, - detail TEXT NOT NULL DEFAULT '{}' + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + at BIGINT NOT NULL, + actor_user_id BIGINT, + action TEXT COLLATE "C" NOT NULL, + target_user_id BIGINT, + detail TEXT COLLATE "C" NOT NULL DEFAULT '{}' )`, `CREATE INDEX audit_target ON audit_log(target_user_id)`, `CREATE INDEX audit_actor ON audit_log(actor_user_id)`, `CREATE TABLE mail_log ( - id INTEGER PRIMARY KEY, - user_id INTEGER REFERENCES users(id) ON DELETE SET NULL, - to_email TEXT NOT NULL, - purpose TEXT NOT NULL, - provider_message_id TEXT, - sent_at INTEGER NOT NULL, - last_event TEXT NOT NULL DEFAULT 'sent', - last_event_at INTEGER NOT NULL, - last_reason TEXT NOT NULL DEFAULT '' + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + user_id BIGINT REFERENCES users(id) ON DELETE SET NULL, + to_email TEXT COLLATE "C" NOT NULL, + purpose TEXT COLLATE "C" NOT NULL, + provider_message_id TEXT COLLATE "C", + sent_at BIGINT NOT NULL, + last_event TEXT COLLATE "C" NOT NULL DEFAULT 'sent', + last_event_at BIGINT NOT NULL, + last_reason TEXT COLLATE "C" NOT NULL DEFAULT '' )`, `CREATE INDEX mail_user ON mail_log(user_id)`, `CREATE UNIQUE INDEX mail_msgid ON mail_log(provider_message_id) WHERE provider_message_id IS NOT NULL`, `CREATE TABLE mail_events ( - mail_id INTEGER NOT NULL REFERENCES mail_log(id) ON DELETE CASCADE, - event TEXT NOT NULL, - at INTEGER NOT NULL, - reason TEXT NOT NULL DEFAULT '', + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + mail_id BIGINT NOT NULL REFERENCES mail_log(id) ON DELETE CASCADE, + event TEXT COLLATE "C" NOT NULL, + at BIGINT NOT NULL, + reason TEXT COLLATE "C" NOT NULL DEFAULT '', UNIQUE (mail_id, event, at) )`, }, { // v2: sub-project B, settings sync (one document per user) `CREATE TABLE user_settings ( - user_id INTEGER PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, - doc TEXT NOT NULL, - revision INTEGER NOT NULL, - generation TEXT NOT NULL, - updated_at INTEGER NOT NULL + user_id BIGINT PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + doc TEXT COLLATE "C" NOT NULL, + revision BIGINT NOT NULL, + generation TEXT COLLATE "C" NOT NULL, + updated_at BIGINT NOT NULL )`, }, { // v3: sub-project C, global audit list ordered and filtered by time @@ -88,84 +87,163 @@ var migrations = [][]string{ }, { // v4: sub-project D, proposals (one generic table; this version ships the kind 'hashtag_channel') `CREATE TABLE proposals ( - id INTEGER PRIMARY KEY, - kind TEXT NOT NULL, - subject TEXT NOT NULL, - status TEXT NOT NULL CHECK (status IN ('pending','approved','rejected','revoked')), - proposer_id INTEGER REFERENCES users(id) ON DELETE SET NULL, - reviewer_id INTEGER REFERENCES users(id) ON DELETE SET NULL, - note TEXT NOT NULL DEFAULT '', - created_at INTEGER NOT NULL, - decided_at INTEGER + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + kind TEXT COLLATE "C" NOT NULL, + subject TEXT COLLATE "C" NOT NULL, + status TEXT COLLATE "C" NOT NULL CHECK (status IN ('pending','approved','rejected','revoked')), + proposer_id BIGINT REFERENCES users(id) ON DELETE SET NULL, + reviewer_id BIGINT REFERENCES users(id) ON DELETE SET NULL, + note TEXT COLLATE "C" NOT NULL DEFAULT '', + created_at BIGINT NOT NULL, + decided_at BIGINT )`, `CREATE UNIQUE INDEX proposals_kind_subject ON proposals(kind, subject)`, `CREATE INDEX proposals_status ON proposals(status)`, }, { // v5: sub-project E, node notifications `CREATE TABLE notification_prefs ( - user_id INTEGER PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, - enabled INTEGER NOT NULL DEFAULT 1, - events TEXT NOT NULL DEFAULT '', - unsub_token TEXT NOT NULL UNIQUE, - updated_at INTEGER NOT NULL + user_id BIGINT PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + enabled BIGINT NOT NULL DEFAULT 1 CHECK (enabled IN (0,1)), + events TEXT COLLATE "C" NOT NULL DEFAULT '', + unsub_token TEXT COLLATE "C" NOT NULL UNIQUE, + updated_at BIGINT NOT NULL )`, `CREATE TABLE notification_watches ( - user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, - pubkey TEXT NOT NULL, - created_at INTEGER NOT NULL, + user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + pubkey TEXT COLLATE "C" NOT NULL, + created_at BIGINT NOT NULL, PRIMARY KEY (user_id, pubkey) )`, `CREATE TABLE notification_state ( - user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, - event TEXT NOT NULL, - subject TEXT NOT NULL, - state TEXT NOT NULL, - changed_at INTEGER NOT NULL, + user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + event TEXT COLLATE "C" NOT NULL, + subject TEXT COLLATE "C" NOT NULL, + state TEXT COLLATE "C" NOT NULL, + changed_at BIGINT NOT NULL, PRIMARY KEY (user_id, event, subject) )`, }, } -func (s *Store) migrate() error { - if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER NOT NULL)`); err != nil { - return fmt.Errorf("users: schema_version: %w", err) - } - v, err := s.SchemaVersion() - if errors.Is(err, sql.ErrNoRows) { - if _, err := s.db.Exec(`INSERT INTO schema_version (version) VALUES (0)`); err != nil { - return fmt.Errorf("users: init schema_version: %w", err) - } - v = 0 - } else if err != nil { - return fmt.Errorf("users: read schema_version: %w", err) +// SchemaVersion is the account data layout version; PostgreSQL readiness is +// recorded separately so the legacy importer can preserve its source history. +const CurrentSchemaVersion = 5 +const PostgresSchemaVersion = 1 + +// Tables is the dependency-ordered account data manifest (metadata excluded). +func Tables() []string { + return []string{"users", "sessions", "tokens", "audit_log", "mail_log", "mail_events", "user_settings", "proposals", "notification_prefs", "notification_watches", "notification_state"} +} + +// Apply installs account schema using the migration owner's connection. Runtime +// Open never runs DDL. An advisory transaction lock serializes simultaneous +// bootstraps, and all DDL plus the readiness marker commit atomically. +func Apply(db *sql.DB) error { + return apply(db, true) +} + +// ApplyForImport creates the schema with readiness false in the same commit. +// Only the offline importer's verified completion may change it to true. +func ApplyForImport(db *sql.DB) error { return apply(db, false) } + +func apply(db *sql.DB, markReady bool) error { + tx, err := db.Begin() + if err != nil { + return err } - if v > len(migrations) { - return fmt.Errorf("users: database schema version %d is newer than this binary supports (%d)", v, len(migrations)) + defer tx.Rollback() + if err := applyTx(tx, markReady); err != nil { + return err } - for i := v; i < len(migrations); i++ { - tx, err := s.db.Begin() - if err != nil { + return tx.Commit() +} + +// ApplyForImportTx lets the offline importer atomically create schema and its +// source-bound progress marker, with readiness false throughout. +func ApplyForImportTx(tx *sql.Tx) error { return applyTx(tx, false) } + +func applyTx(tx *sql.Tx, markReady bool) error { + var err error + if _, err = tx.Exec(`SELECT pg_advisory_xact_lock(hashtextextended(current_schema(),7289041402))`); err != nil { + return err + } + if _, err = tx.Exec(`CREATE TABLE IF NOT EXISTS corescope_schema (version INTEGER NOT NULL, kind TEXT COLLATE "C" NOT NULL PRIMARY KEY CHECK (kind='accounts'), ready BOOLEAN NOT NULL)`); err != nil { + return err + } + var kind string + var version int + var ready bool + err = tx.QueryRow(`SELECT kind,version,ready FROM corescope_schema`).Scan(&kind, &version, &ready) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + if err == nil && !markReady { + return errors.New("users: import bootstrap requires a fresh account database") + } + if err == nil && (kind != "accounts" || version != PostgresSchemaVersion || !ready) { + return errors.New("users: incompatible or incomplete PostgreSQL account schema") + } + if _, err = tx.Exec(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER NOT NULL CHECK (version>=0))`); err != nil { + return err + } + var v, count int + if err = tx.QueryRow(`SELECT COUNT(*), COALESCE(MAX(version),0) FROM schema_version`).Scan(&count, &v); err != nil { + return err + } + if count > 1 { + return errors.New("users: invalid account schema version metadata") + } + if v > CurrentSchemaVersion { + return fmt.Errorf("users: database schema version %d is newer than supported %d", v, CurrentSchemaVersion) + } + if count == 0 { + if _, err = tx.Exec(`INSERT INTO schema_version VALUES (0)`); err != nil { return err } + } + for i := v; i < len(migrations); i++ { for _, stmt := range migrations[i] { - if _, err := tx.Exec(stmt); err != nil { - tx.Rollback() + if _, err = tx.Exec(stmt); err != nil { return fmt.Errorf("users: migration %d: %w", i+1, err) } } - if _, err := tx.Exec(`UPDATE schema_version SET version = ?`, i+1); err != nil { - tx.Rollback() - return fmt.Errorf("users: migration %d: %w", i+1, err) - } - if err := tx.Commit(); err != nil { - return fmt.Errorf("users: migration %d: %w", i+1, err) + if _, err = tx.Exec(`UPDATE schema_version SET version=$1`, i+1); err != nil { + return err } } + if _, err = tx.Exec(`CREATE OR REPLACE VIEW approved_channels AS SELECT kind,subject,decided_at,id FROM proposals WHERE status='approved'`); err != nil { + return err + } + if _, err = tx.Exec(`INSERT INTO corescope_schema(version,kind,ready) VALUES (1,'accounts',$1) ON CONFLICT(kind) DO NOTHING`, markReady); err != nil { + return err + } + return nil +} + +// AssertReady rejects missing, unsupported or partially imported account state. +func AssertReady(db *sql.DB) error { + var kind string + var version int + var ready bool + if err := db.QueryRow(`SELECT kind,version,ready FROM corescope_schema`).Scan(&kind, &version, &ready); err != nil { + return errors.New("users: account database needs bootstrap before runtime startup") + } + if kind != "accounts" || version != PostgresSchemaVersion || !ready { + return errors.New("users: account database is incompatible or import is incomplete") + } + var n, v int + if err := db.QueryRow(`SELECT COUNT(*),COALESCE(MAX(version),0) FROM schema_version`).Scan(&n, &v); err != nil { + return errors.New("users: account database needs bootstrap") + } + if v > CurrentSchemaVersion { + return fmt.Errorf("users: database schema version %d is newer than supported %d", v, CurrentSchemaVersion) + } + if n != 1 || v != CurrentSchemaVersion { + return errors.New("users: account database needs bootstrap") + } return nil } -// SchemaVersion returns the applied schema version (sql.ErrNoRows on a -// database that has never been migrated). func (s *Store) SchemaVersion() (int, error) { var v int err := s.db.QueryRow(`SELECT version FROM schema_version`).Scan(&v) diff --git a/internal/users/sessions.go b/internal/users/sessions.go index 92fe4f0a8..7caab7882 100644 --- a/internal/users/sessions.go +++ b/internal/users/sessions.go @@ -47,12 +47,9 @@ func (s *Store) CreateSession(userID int64, ttl time.Duration, userAgent string) userAgent = strings.ToValidUTF8(userAgent[:maxUserAgent], "") } now := s.now() - res, err := s.db.Exec(`INSERT INTO sessions (token_hash, user_id, csrf_token, created_at, expires_at, last_seen_at, user_agent) - VALUES (?, ?, ?, ?, ?, ?, ?)`, hash, userID, csrf, unix(now), unix(now.Add(ttl)), unix(now), userAgent) - if err != nil { - return "", nil, err - } - id, err := res.LastInsertId() + var id int64 + err = s.db.QueryRow(`INSERT INTO sessions (token_hash, user_id, csrf_token, created_at, expires_at, last_seen_at, user_agent) + VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id`, hash, userID, csrf, unix(now), unix(now.Add(ttl)), unix(now), userAgent).Scan(&id) if err != nil { return "", nil, err } @@ -63,7 +60,7 @@ func (s *Store) CreateSession(userID int64, ttl time.Duration, userAgent string) // LookupSession resolves a raw cookie token. Unknown and expired sessions // return ErrNotFound; expired ones are deleted on the way. func (s *Store) LookupSession(raw string) (*Session, error) { - sess, err := scanSession(s.db.QueryRow(`SELECT `+sessionCols+` FROM sessions WHERE token_hash = ?`, HashToken(raw))) + sess, err := scanSession(s.db.QueryRow(`SELECT `+sessionCols+` FROM sessions WHERE token_hash = $1`, HashToken(raw))) if errors.Is(err, sql.ErrNoRows) { return nil, ErrNotFound } @@ -71,7 +68,7 @@ func (s *Store) LookupSession(raw string) (*Session, error) { return nil, fmt.Errorf("users: lookup session: %w", err) } if !s.now().Before(sess.ExpiresAt) { - _, _ = s.db.Exec(`DELETE FROM sessions WHERE id = ?`, sess.ID) + _, _ = s.db.Exec(`DELETE FROM sessions WHERE id = $1`, sess.ID) return nil, ErrNotFound } return sess, nil @@ -80,29 +77,29 @@ func (s *Store) LookupSession(raw string) (*Session, error) { // ExtendSession marks the session seen now and moves its expiry to now+ttl. func (s *Store) ExtendSession(id int64, ttl time.Duration) error { now := s.now() - return expectOne(s.db.Exec(`UPDATE sessions SET last_seen_at = ?, expires_at = ? WHERE id = ?`, + return expectOne(s.db.Exec(`UPDATE sessions SET last_seen_at = $1, expires_at = $2 WHERE id = $3`, unix(now), unix(now.Add(ttl)), id)) } func (s *Store) DeleteSessionByToken(raw string) error { - _, err := s.db.Exec(`DELETE FROM sessions WHERE token_hash = ?`, HashToken(raw)) + _, err := s.db.Exec(`DELETE FROM sessions WHERE token_hash = $1`, HashToken(raw)) return err } // DeleteSession revokes one session, only if it belongs to userID. func (s *Store) DeleteSession(userID, sessionID int64) error { - return expectOne(s.db.Exec(`DELETE FROM sessions WHERE id = ? AND user_id = ?`, sessionID, userID)) + return expectOne(s.db.Exec(`DELETE FROM sessions WHERE id = $1 AND user_id = $2`, sessionID, userID)) } // DeleteUserSessions revokes all of a user's sessions except exceptID (0 = none kept). func (s *Store) DeleteUserSessions(userID, exceptID int64) error { - _, err := s.db.Exec(`DELETE FROM sessions WHERE user_id = ? AND id != ?`, userID, exceptID) + _, err := s.db.Exec(`DELETE FROM sessions WHERE user_id = $1 AND id != $2`, userID, exceptID) return err } // ListSessions returns a user's sessions, most recently seen first. func (s *Store) ListSessions(userID int64) ([]Session, error) { - rows, err := s.db.Query(`SELECT `+sessionCols+` FROM sessions WHERE user_id = ? ORDER BY last_seen_at DESC, id DESC`, userID) + rows, err := s.db.Query(`SELECT `+sessionCols+` FROM sessions WHERE user_id = $1 ORDER BY last_seen_at DESC, id DESC`, userID) if err != nil { return nil, err } @@ -119,7 +116,7 @@ func (s *Store) ListSessions(userID int64) ([]Session, error) { } func (s *Store) PruneExpiredSessions() (int64, error) { - res, err := s.db.Exec(`DELETE FROM sessions WHERE expires_at <= ?`, unix(s.now())) + res, err := s.db.Exec(`DELETE FROM sessions WHERE expires_at <= $1`, unix(s.now())) if err != nil { return 0, err } diff --git a/internal/users/settings.go b/internal/users/settings.go index 148298abb..a89569c7c 100644 --- a/internal/users/settings.go +++ b/internal/users/settings.go @@ -36,7 +36,7 @@ func newGeneration() (string, error) { func (s *Store) GetSettings(userID int64) (string, SettingsVersion, error) { var doc string var v SettingsVersion - err := s.db.QueryRow(`SELECT doc, revision, generation FROM user_settings WHERE user_id = ?`, userID).Scan(&doc, &v.Revision, &v.Generation) + err := s.db.QueryRow(`SELECT doc, revision, generation FROM user_settings WHERE user_id = $1`, userID).Scan(&doc, &v.Revision, &v.Generation) if errors.Is(err, sql.ErrNoRows) { return "", SettingsVersion{}, nil } @@ -59,7 +59,7 @@ type SettingsRecord struct { func (s *Store) SettingsRecordFor(userID int64) (*SettingsRecord, error) { var r SettingsRecord var updated int64 - err := s.db.QueryRow(`SELECT doc, revision, generation, updated_at FROM user_settings WHERE user_id = ?`, userID). + err := s.db.QueryRow(`SELECT doc, revision, generation, updated_at FROM user_settings WHERE user_id = $1`, userID). Scan(&r.Doc, &r.Version.Revision, &r.Version.Generation, &updated) if errors.Is(err, sql.ErrNoRows) { return nil, nil @@ -82,8 +82,11 @@ func (s *Store) PutSettings(userID int64, base SettingsVersion, doc string) (Set return SettingsVersion{}, err } defer tx.Rollback() + if err := lockUser(tx, userID); err != nil { + return SettingsVersion{}, err + } var cur SettingsVersion - err = tx.QueryRow(`SELECT revision, generation FROM user_settings WHERE user_id = ?`, userID).Scan(&cur.Revision, &cur.Generation) + err = tx.QueryRow(`SELECT revision, generation FROM user_settings WHERE user_id = $1`, userID).Scan(&cur.Revision, &cur.Generation) if err != nil && !errors.Is(err, sql.ErrNoRows) { return SettingsVersion{}, err } @@ -99,7 +102,7 @@ func (s *Store) PutSettings(userID int64, base SettingsVersion, doc string) (Set return cur, ErrSettingsConflict } next.Revision = cur.Revision + 1 - if _, err := tx.Exec(`INSERT INTO user_settings (user_id, doc, revision, generation, updated_at) VALUES (?, ?, ?, ?, ?) + if _, err := tx.Exec(`INSERT INTO user_settings (user_id, doc, revision, generation, updated_at) VALUES ($1, $2, $3, $4, $5) ON CONFLICT(user_id) DO UPDATE SET doc = excluded.doc, revision = excluded.revision, generation = excluded.generation, updated_at = excluded.updated_at`, userID, doc, next.Revision, next.Generation, unix(s.now())); err != nil { return SettingsVersion{}, err @@ -110,6 +113,18 @@ func (s *Store) PutSettings(userID int64, base SettingsVersion, doc string) (Set // DeleteSettings removes the user's synced settings; the next write starts // a new generation. No row is not an error. func (s *Store) DeleteSettings(userID int64) error { - _, err := s.db.Exec(`DELETE FROM user_settings WHERE user_id = ?`, userID) - return err + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + if err = lockUser(tx, userID); errors.Is(err, ErrNotFound) { + return nil + } else if err != nil { + return err + } + if _, err = tx.Exec(`DELETE FROM user_settings WHERE user_id=$1`, userID); err != nil { + return err + } + return tx.Commit() } diff --git a/internal/users/settings_test.go b/internal/users/settings_test.go index bf25a10ad..4e8f80474 100644 --- a/internal/users/settings_test.go +++ b/internal/users/settings_test.go @@ -1,9 +1,7 @@ package users import ( - "database/sql" "errors" - "path/filepath" "testing" "time" ) @@ -44,7 +42,7 @@ func TestSettingsPutRevisions(t *testing.T) { t.Fatalf("GetSettings = %q, %+v, %v", doc, got, err) } var at int64 - if err := st.db.QueryRow(`SELECT updated_at FROM user_settings WHERE user_id = ?`, u.ID).Scan(&at); err != nil || at != unix(clk.Now()) { + if err := st.db.QueryRow(`SELECT updated_at FROM user_settings WHERE user_id = $1`, u.ID).Scan(&at); err != nil || at != unix(clk.Now()) { t.Fatalf("updated_at = %d, %v; want %d", at, err, unix(clk.Now())) } } @@ -134,30 +132,9 @@ func TestSettingsSurviveDisable(t *testing.T) { } } -// A users.db written by a v1 binary gains user_settings and keeps its rows. +// A native account schema at v1 binary gains user_settings and keeps its rows. func TestMigrateV1DatabaseToV2(t *testing.T) { - path := filepath.Join(t.TempDir(), "users.db") - db, err := sql.Open("sqlite", path) - if err != nil { - t.Fatal(err) - } - stmts := append([]string{ - `CREATE TABLE schema_version (version INTEGER NOT NULL)`, - `INSERT INTO schema_version (version) VALUES (1)`, - }, migrations[0]...) - stmts = append(stmts, `INSERT INTO users (email, display_name, password_hash, created_at) VALUES ('old@example.org', 'Old', 'x', 1)`) - for _, s := range stmts { - if _, err := db.Exec(s); err != nil { - t.Fatalf("build v1 db: %v", err) - } - } - db.Close() - - st, err := Open(path) - if err != nil { - t.Fatalf("Open v1 db: %v", err) - } - defer st.Close() + st := migratedTestStore(t, 1, `INSERT INTO users (email,display_name,password_hash,created_at) VALUES ('old@example.org','Old','x',1)`) if v, err := st.SchemaVersion(); err != nil || v != len(migrations) { t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, len(migrations)) } diff --git a/internal/users/snapshot.go b/internal/users/snapshot.go index a4c2a0f4f..2112f7667 100644 --- a/internal/users/snapshot.go +++ b/internal/users/snapshot.go @@ -1,30 +1,21 @@ package users import ( + "context" "errors" - "fmt" - "io/fs" - "os" + "github.com/meshcore-analyzer/pgutil" ) -// Snapshot writes a consistent copy of users.db to path with VACUUM INTO -// (the technique of GET /api/backup) and makes it readable by the owner -// only: it holds password hashes and addresses. path must not exist; an -// empty file there is refused too, although VACUUM INTO would accept it. -// On failure no file is left at path. +// Snapshot writes a consistent PostgreSQL custom-format archive readable only +// by its owner. Restore with pg_restore; the destination must not exist. func (s *Store) Snapshot(path string) error { - if _, err := os.Lstat(path); err == nil { - return fmt.Errorf("users: snapshot target %s already exists", path) - } else if !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("users: snapshot target %s: %w", path, err) - } - if _, err := s.db.Exec(`VACUUM INTO ?`, path); err != nil { - os.Remove(path) - return fmt.Errorf("users: snapshot: %w", err) - } - if err := os.Chmod(path, 0o600); err != nil { - os.Remove(path) - return fmt.Errorf("users: snapshot mode: %w", err) + return s.SnapshotContext(context.Background(), path) +} + +// SnapshotContext cancels the dump when the caller disconnects or times out. +func (s *Store) SnapshotContext(ctx context.Context, path string) error { + if err := s.db.PingContext(ctx); err != nil { + return errors.New("users: account database is not available for snapshot") } - return nil + return pgutil.Dump(ctx, s.databaseURL, path) } diff --git a/internal/users/snapshot_test.go b/internal/users/snapshot_test.go index fdf282248..89c29c595 100644 --- a/internal/users/snapshot_test.go +++ b/internal/users/snapshot_test.go @@ -1,29 +1,70 @@ package users import ( + "context" + "errors" "os" + "os/exec" "path/filepath" "runtime" "strings" "testing" + "time" + + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" ) func TestSnapshotCopiesRows(t *testing.T) { - st, _ := newTestStore(t) + st, clk := newTestStoreAt(t, pgtest.NewDatabase(t)) u := mustCreate(t, st, "a@example.org", "Aaa") + if err := st.Activate(u.ID, RoleUser, nil); err != nil { + t.Fatal(err) + } + rawSession, session, err := st.CreateSession(u.ID, 24*time.Hour, "backup-test") + if err != nil { + t.Fatal(err) + } + usedToken, err := st.IssueToken(u.ID, PurposeReset, time.Hour, "") + if err != nil { + t.Fatal(err) + } + if _, _, err := st.ConsumeToken(usedToken, PurposeReset); err != nil { + t.Fatal(err) + } + liveToken, err := st.IssueToken(u.ID, PurposeEmailChange, time.Hour, "new@example.org") + if err != nil { + t.Fatal(err) + } uid := u.ID if err := st.Audit(&uid, "user.register", &uid, nil); err != nil { t.Fatal(err) } - path := filepath.Join(t.TempDir(), "snap.db") + path := filepath.Join(t.TempDir(), "snap.dump") if err := st.Snapshot(path); err != nil { t.Fatal(err) } - cp, err := Open(path) + target := pgtest.NewDatabase(t) + env, err := pgutil.CommandEnv(target) + if err != nil { + t.Fatal(err) + } + // --dbname is needed for restore mode; use a credential-free database name. + cfg, err := pgutil.ParseConfig(target) + if err != nil { + t.Fatal(err) + } + cmd := exec.Command("pg_restore", "--no-password", "--exit-on-error", "--no-owner", "--no-privileges", "--dbname="+cfg.Database, path) + cmd.Env = env + if err := cmd.Run(); err != nil { + t.Fatal("native account restore failed") + } + cp, err := Open(testRuntimeURL(t, target)) if err != nil { t.Fatal(err) } defer cp.Close() + cp.SetClock(clk.Now) got, err := cp.GetByEmail("a@example.org") if err != nil || got.ID != u.ID || got.DisplayName != "Aaa" { t.Fatalf("user in snapshot = %+v, %v", got, err) @@ -34,14 +75,26 @@ func TestSnapshotCopiesRows(t *testing.T) { if e, err := cp.AuditAllFor(u.ID); err != nil || len(e) != 1 { t.Fatalf("audit in snapshot = %d, %v", len(e), err) } + if got.PasswordHash != u.PasswordHash || got.Status != StatusActive { + t.Fatal("password hash or account status changed") + } + if restored, err := cp.LookupSession(rawSession); err != nil || restored.ID != session.ID || restored.CSRFToken != session.CSRFToken { + t.Fatal("restored session/CSRF changed") + } + if _, err := cp.TokenUser(usedToken, PurposeReset); !errors.Is(err, ErrTokenInvalid) { + t.Fatal("used token revived after restore") + } + if id, email, err := cp.ConsumeToken(liveToken, PurposeEmailChange); err != nil || id != u.ID || email != "new@example.org" { + t.Fatal("unconsumed token did not survive restore") + } } func TestSnapshotFileMode(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("Unix file modes") } - st, _ := newTestStore(t) - path := filepath.Join(t.TempDir(), "snap.db") + st, _ := newTestStoreAt(t, pgtest.NewDatabase(t)) + path := filepath.Join(t.TempDir(), "snap.dump") if err := st.Snapshot(path); err != nil { t.Fatal(err) } @@ -55,9 +108,9 @@ func TestSnapshotFileMode(t *testing.T) { } func TestSnapshotRefusesExistingTarget(t *testing.T) { - st, _ := newTestStore(t) - path := filepath.Join(t.TempDir(), "snap.db") - // An empty file: VACUUM INTO alone would accept and fill it. + st, _ := newTestStoreAt(t, pgtest.NewDatabase(t)) + path := filepath.Join(t.TempDir(), "snap.dump") + // Existing empty files must also be preserved. if err := os.WriteFile(path, nil, 0o600); err != nil { t.Fatal(err) } @@ -71,9 +124,9 @@ func TestSnapshotRefusesExistingTarget(t *testing.T) { } func TestSnapshotFailureLeavesNoFile(t *testing.T) { - st, _ := newTestStore(t) + st, _ := newTestStoreAt(t, pgtest.NewDatabase(t)) st.Close() - path := filepath.Join(t.TempDir(), "snap.db") + path := filepath.Join(t.TempDir(), "snap.dump") if err := st.Snapshot(path); err == nil { t.Fatal("snapshot of a closed store succeeded") } @@ -81,3 +134,16 @@ func TestSnapshotFailureLeavesNoFile(t *testing.T) { t.Fatalf("partial snapshot left behind: %v", err) } } + +func TestSnapshotCanceledLeavesNoFile(t *testing.T) { + st, _ := newTestStoreAt(t, pgtest.NewDatabase(t)) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + path := filepath.Join(t.TempDir(), "cancel.dump") + if err := st.SnapshotContext(ctx, path); err == nil { + t.Fatal("canceled snapshot succeeded") + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatal("canceled snapshot left a file") + } +} diff --git a/internal/users/stats.go b/internal/users/stats.go index d6b22731c..2883b5076 100644 --- a/internal/users/stats.go +++ b/internal/users/stats.go @@ -61,18 +61,18 @@ func (s *Store) Stats(now time.Time) (Stats, error) { t := unix(now) var st Stats if err := s.db.QueryRow(`SELECT COUNT(*), - COALESCE(SUM(status = 'active'), 0), - COALESCE(SUM(status = 'pending'), 0), - COALESCE(SUM(status = 'disabled'), 0), - COALESCE(SUM(role = 'admin' AND status = 'active'), 0), - COALESCE(SUM(status = 'pending' AND created_at < ?), 0), - COALESCE(SUM(email_bouncing != 0), 0) + COUNT(*) FILTER (WHERE status = 'active'), + COUNT(*) FILTER (WHERE status = 'pending'), + COUNT(*) FILTER (WHERE status = 'disabled'), + COUNT(*) FILTER (WHERE role = 'admin' AND status = 'active'), + COUNT(*) FILTER (WHERE status = 'pending' AND created_at < $1), + COUNT(*) FILTER (WHERE email_bouncing != 0) FROM users`, t-secs(StuckPendingAfter)). Scan(&st.Total, &st.Active, &st.Pending, &st.Disabled, &st.Admins, &st.StuckPending, &st.Bouncing); err != nil { return Stats{}, err } - if err := s.db.QueryRow(`SELECT COALESCE(SUM(at >= ?), 0), COUNT(*) FROM audit_log - WHERE action = 'user.register' AND at >= ? AND at <= ?`, t-secs(7*day), t-secs(30*day), t). + if err := s.db.QueryRow(`SELECT COUNT(*) FILTER (WHERE at >= $1), COUNT(*) FROM audit_log + WHERE action = 'user.register' AND at >= $2 AND at <= $3`, t-secs(7*day), t-secs(30*day), t). Scan(&st.New7d, &st.New30d); err != nil { return Stats{}, err } @@ -81,16 +81,16 @@ func (s *Store) Stats(now time.Time) (Stats, error) { return Stats{}, err } st.NewPerDay = perDay - if err := s.db.QueryRow(`SELECT COALESCE(SUM(last >= ?), 0), COUNT(*) FROM ( - SELECT MAX(COALESCE(u.last_login_at, 0), + if err := s.db.QueryRow(`SELECT COUNT(*) FILTER (WHERE last >= $1), COUNT(*) FROM ( + SELECT GREATEST(COALESCE(u.last_login_at, 0), COALESCE((SELECT MAX(last_seen_at) FROM sessions s WHERE s.user_id = u.id), 0)) AS last FROM users u - ) WHERE last >= ?`, t-secs(7*day), t-secs(30*day)). + ) activity WHERE last >= $2`, t-secs(7*day), t-secs(30*day)). Scan(&st.Active7d, &st.Active30d); err != nil { return Stats{}, err } - if err := s.db.QueryRow(`SELECT COALESCE(SUM(action = 'user.login'), 0), COALESCE(SUM(action = 'user.login.failed'), 0) - FROM audit_log WHERE action IN ('user.login', 'user.login.failed') AND at >= ?`, t-secs(day)). + if err := s.db.QueryRow(`SELECT COUNT(*) FILTER (WHERE action = 'user.login'), COUNT(*) FILTER (WHERE action = 'user.login.failed') + FROM audit_log WHERE action IN ('user.login', 'user.login.failed') AND at >= $1`, t-secs(day)). Scan(&st.Logins24h, &st.FailedLogins24h); err != nil { return Stats{}, err } @@ -107,8 +107,8 @@ func (s *Store) Stats(now time.Time) (Stats, error) { // statsPerDayDays days, today included. func (s *Store) registrationsPerDay(now time.Time) ([]DayCount, error) { first := now.UTC().Truncate(24*time.Hour).AddDate(0, 0, -(statsPerDayDays - 1)) - rows, err := s.db.Query(`SELECT (at - ?) / 86400, COUNT(*) FROM audit_log - WHERE action = 'user.register' AND at >= ? AND at <= ? GROUP BY 1`, unix(first), unix(first), unix(now)) + rows, err := s.db.Query(`SELECT (at - $1) / 86400, COUNT(*) FROM audit_log + WHERE action = 'user.register' AND at >= $2 AND at <= $3 GROUP BY 1`, unix(first), unix(first), unix(now)) if err != nil { return nil, err } @@ -134,7 +134,7 @@ func (s *Store) registrationsPerDay(now time.Time) ([]DayCount, error) { func (s *Store) mailCounts(since int64) (MailCounts, error) { var mc MailCounts - rows, err := s.db.Query(`SELECT last_event, COUNT(*) FROM mail_log WHERE sent_at >= ? GROUP BY last_event`, since) + rows, err := s.db.Query(`SELECT last_event, COUNT(*) FROM mail_log WHERE sent_at >= $1 GROUP BY last_event`, since) if err != nil { return mc, err } @@ -166,8 +166,8 @@ func (s *Store) mailCounts(since int64) (MailCounts, error) { func (s *Store) guessedAccounts(since int64) ([]GuessedAccount, error) { rows, err := s.db.Query(`SELECT a.target_user_id, COALESCE(u.display_name, ''), COUNT(*) AS n FROM audit_log a LEFT JOIN users u ON u.id = a.target_user_id - WHERE a.action = 'user.login.failed' AND a.at >= ? AND a.target_user_id IS NOT NULL - GROUP BY a.target_user_id HAVING n >= ? ORDER BY n DESC, a.target_user_id LIMIT ?`, + WHERE a.action = 'user.login.failed' AND a.at >= $1 AND a.target_user_id IS NOT NULL + GROUP BY a.target_user_id,u.display_name HAVING COUNT(*) >= $2 ORDER BY n DESC, a.target_user_id LIMIT $3`, since, GuessingThreshold, statsGuessingLimit) if err != nil { return nil, err diff --git a/internal/users/store.go b/internal/users/store.go index bb7b589dd..0ebd482dc 100644 --- a/internal/users/store.go +++ b/internal/users/store.go @@ -1,20 +1,17 @@ -// Package users is CoreScope's optional account store (user management, -// docs/specs/2026-10-06-user-management-design.md). It owns users.db, a -// SQLite file separate from the measurement database. cmd/server never -// writes measurement data (#1283); this package is the single, opt-in -// exception, and Open refuses to touch the measurement database by path. +// Package users is CoreScope's optional PostgreSQL account store. Account +// state lives in a separate database from telemetry; runtime connections +// never bootstrap schemas or write telemetry (#1283). package users import ( + "context" "database/sql" "errors" - "fmt" - "os" - "path/filepath" "strings" "time" - _ "modernc.org/sqlite" + "github.com/jackc/pgx/v5/pgconn" + "github.com/meshcore-analyzer/pgutil" ) var ( @@ -28,61 +25,82 @@ var ( ErrAccountChanged = errors.New("users: account changed since it was read") ) -// noLimit is SQLite's LIMIT value for "every row" (the account export). -const noLimit = -1 +// A SQL NULL limit means every row for full account exports. +var noLimit any = nil -// Store is the users.db handle. Safe for concurrent use. +// Store owns a separate account database. Safe for concurrent use. type Store struct { - db *sql.DB - now func() time.Time + db *sql.DB + databaseURL string + now func() time.Time } -// Open opens (creating if needed) the users database at path and applies -// pending migrations. forbidden lists paths Open must refuse; the server -// passes the measurement DB path so a misconfigured dbPath can never turn -// this package into a writer of measurement data. -func Open(path string, forbidden ...string) (*Store, error) { - if strings.ContainsAny(path, "?#") { - return nil, fmt.Errorf("users: database path %q must not contain '?' or '#'", path) - } - abs, err := filepath.Abs(path) - if err != nil { - return nil, fmt.Errorf("users: resolve %s: %w", path, err) +// Open connects using the account runtime role, never applying migrations. +// Every forbidden URL is checked against the effective PostgreSQL database, +// so alternate hostnames, credentials or search paths cannot bypass isolation. +func Open(databaseURL string, forbidden ...string) (*Store, error) { + if _, err := pgutil.ParseConfig(databaseURL); err != nil { + return nil, err } - for _, f := range forbidden { - if strings.TrimSpace(f) != "" && samePath(abs, f) { - return nil, fmt.Errorf("users: refusing to open %s: it is the measurement database", path) + for _, other := range forbidden { + if strings.TrimSpace(other) == "" { + continue + } + same, err := pgutil.SameDatabase(context.Background(), databaseURL, other) + if err != nil { + return nil, err + } + if same { + return nil, errors.New("users: refusing to open the measurement database") } } - if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil { - return nil, fmt.Errorf("users: create dir for %s: %w", path, err) - } - db, err := sql.Open("sqlite", abs+"?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)") + db, err := pgutil.Open(databaseURL, false) if err != nil { - return nil, fmt.Errorf("users: open %s: %w", path, err) + return nil, err } - // Account traffic is tiny; one connection removes SQLITE_BUSY between - // our own writers and keeps the per-connection pragmas in force. + // Account traffic is small. Keep baseline per-process concurrency while + // database locks also protect transactions across independent processes. db.SetMaxOpenConns(1) - s := &Store{db: db, now: time.Now} - if err := s.migrate(); err != nil { + db.SetMaxIdleConns(1) + if err := AssertReady(db); err != nil { db.Close() return nil, err } - return s, nil + if err := assertRuntimePrivileges(db); err != nil { + db.Close() + return nil, err + } + return &Store{db: db, databaseURL: databaseURL, now: time.Now}, nil } -func samePath(abs, other string) bool { - oa, err := filepath.Abs(other) +func assertRuntimePrivileges(db *sql.DB) error { + var elevated bool + err := db.QueryRow(`SELECT + EXISTS (SELECT 1 FROM pg_roles WHERE rolname=current_user AND (rolsuper OR rolcreaterole OR rolcreatedb OR rolreplication OR rolbypassrls)) + OR has_database_privilege(current_database(),'CREATE') + OR has_schema_privilege(current_schema(),'CREATE') + OR EXISTS (SELECT 1 FROM pg_class c JOIN pg_namespace n ON c.relnamespace=n.oid + WHERE n.nspname=current_schema() AND pg_has_role(c.relowner,'USAGE')) + OR has_table_privilege('corescope_schema','INSERT,UPDATE,DELETE,TRUNCATE,TRIGGER') + OR has_table_privilege('schema_version','INSERT,UPDATE,DELETE,TRUNCATE,TRIGGER')`).Scan(&elevated) if err != nil { - return false + return errors.New("users: cannot verify account runtime role privileges") } - a, errA := os.Stat(abs) - b, errB := os.Stat(oa) - if errA == nil && errB == nil { - return os.SameFile(a, b) + if elevated { + return errors.New("users: account runtime role must be separate from the migration owner and cannot modify schema metadata") + } + return nil +} + +// lockUser serializes account-level read/modify/write operations, including +// the initial insert where no settings, tokens or watch rows exist yet. +func lockUser(tx *sql.Tx, id int64) error { + var found int64 + err := tx.QueryRow(`SELECT id FROM users WHERE id=$1 FOR UPDATE`, id).Scan(&found) + if errors.Is(err, sql.ErrNoRows) { + return ErrNotFound } - return strings.EqualFold(filepath.Clean(abs), filepath.Clean(oa)) + return err } // Close closes the database. @@ -126,5 +144,6 @@ func expectOne(res sql.Result, err error) error { } func isUniqueViolation(err error) bool { - return err != nil && strings.Contains(err.Error(), "UNIQUE constraint failed") + var pgerr *pgconn.PgError + return errors.As(err, &pgerr) && pgerr.Code == "23505" } diff --git a/internal/users/store_test.go b/internal/users/store_test.go index 0db2c9950..4fa448296 100644 --- a/internal/users/store_test.go +++ b/internal/users/store_test.go @@ -1,72 +1,101 @@ package users import ( - "os" - "path/filepath" + "github.com/meshcore-analyzer/pgutil/pgtest" + "net/url" "strings" + "sync" "testing" ) func TestOpenCreatesSchemaAndIsIdempotent(t *testing.T) { - path := filepath.Join(t.TempDir(), "sub", "users.db") - st, err := Open(path) - if err != nil { - t.Fatalf("first Open: %v", err) - } - v, err := st.SchemaVersion() - if err != nil || v != len(migrations) { - t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, len(migrations)) - } - st.Close() - - st2, err := Open(path) - if err != nil { - t.Fatalf("second Open: %v", err) + dsn := pgtest.NewSchema(t) + db := testOwner(t, dsn) + if err := Apply(db); err != nil { + t.Fatal(err) } - defer st2.Close() - if v2, _ := st2.SchemaVersion(); v2 != len(migrations) { - t.Fatalf("version after reopen = %d", v2) + runtimeDSN := testRuntimeURL(t, dsn) + for i := 0; i < 2; i++ { + st, err := Open(runtimeDSN) + if err != nil { + t.Fatal(err) + } + if v, err := st.SchemaVersion(); err != nil || v != CurrentSchemaVersion { + t.Fatalf("schema=%d,%v", v, err) + } + st.Close() + if err := Apply(db); err != nil { + t.Fatal(err) + } } } func TestOpenRefusesForbiddenPath(t *testing.T) { - dir := t.TempDir() - measurement := filepath.Join(dir, "meshcore.db") - if err := os.WriteFile(measurement, []byte{}, 0o644); err != nil { - t.Fatal(err) - } - _, err := Open(measurement, measurement) - if err == nil || !strings.Contains(err.Error(), "measurement database") { - t.Fatalf("Open(measurement) err = %v; want refusal", err) - } - // A relative spelling of the same file is refused too. - wd, _ := os.Getwd() - defer os.Chdir(wd) - os.Chdir(dir) - if _, err := Open("meshcore.db", measurement); err == nil { - t.Fatal("relative path to the measurement DB was not refused") + dsn := pgtest.NewSchema(t) + alias, _ := url.Parse(dsn) + q := alias.Query() + q.Set("application_name", "different-client") + q.Set("search_path", "public") + alias.RawQuery = q.Encode() + if st, err := Open(dsn, alias.String()); err == nil || !strings.Contains(err.Error(), "measurement database") { + if st != nil { + st.Close() + } + t.Fatal("the same effective database was not refused") } } func TestOpenRejectsNewerSchema(t *testing.T) { - path := filepath.Join(t.TempDir(), "users.db") - st, err := Open(path) - if err != nil { + dsn := pgtest.NewSchema(t) + db := testOwner(t, dsn) + if err := Apply(db); err != nil { t.Fatal(err) } - if _, err := st.db.Exec(`UPDATE schema_version SET version = 999`); err != nil { + runtimeDSN := testRuntimeURL(t, dsn) + if _, err := db.Exec(`UPDATE schema_version SET version=999`); err != nil { t.Fatal(err) } - st.Close() - if _, err := Open(path); err == nil || !strings.Contains(err.Error(), "newer") { - t.Fatalf("Open with newer schema err = %v", err) + if st, err := Open(runtimeDSN); err == nil || !strings.Contains(err.Error(), "newer") { + if st != nil { + st.Close() + } + t.Fatalf("Open with newer schema error=%v", err) + } + if err := Apply(db); err == nil { + t.Fatal("bootstrap accepted a newer schema") } } func TestOpenRejectsDSNCharacters(t *testing.T) { - for _, p := range []string{"users?.db", "users#1.db"} { - if _, err := Open(filepath.Join(t.TempDir(), p)); err == nil { - t.Errorf("Open(%q) accepted", p) + for _, dsn := range []string{"users.db", "users?.db", "file:users.db", "postgresql://u:p@host/a?host=elsewhere", "postgresql://u:p@host1,host2/a"} { + if st, err := Open(dsn); err == nil { + st.Close() + t.Fatal("unsupported database address accepted") } } } + +func TestApplyConcurrentAndIncomplete(t *testing.T) { + dsn := pgtest.NewSchema(t) + db := testOwner(t, dsn) + var wg sync.WaitGroup + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if err := Apply(db); err != nil { + t.Errorf("concurrent bootstrap: %v", err) + } + }() + } + wg.Wait() + if _, err := db.Exec(`UPDATE corescope_schema SET ready=false`); err != nil { + t.Fatal(err) + } + if err := AssertReady(db); err == nil { + t.Fatal("incomplete import accepted") + } + if err := Apply(db); err == nil { + t.Fatal("bootstrap completed an unverified import") + } +} diff --git a/internal/users/users.go b/internal/users/users.go index 7ebe8cf99..5a90d9aba 100644 --- a/internal/users/users.go +++ b/internal/users/users.go @@ -3,6 +3,7 @@ package users import ( "database/sql" "errors" + "fmt" "strings" "time" ) @@ -69,56 +70,53 @@ func scanUser(row rowScanner) (*User, error) { // CreatePending inserts a new pending user. email must already be normalized. func (s *Store) CreatePending(email, displayName, passwordHash string) (*User, error) { - res, err := s.db.Exec(`INSERT INTO users (email, display_name, password_hash, role, status, created_at) - VALUES (?, ?, ?, 'user', 'pending', ?)`, email, displayName, passwordHash, unix(s.now())) + var id int64 + err := s.db.QueryRow(`INSERT INTO users (email, display_name, password_hash, role, status, created_at) + VALUES ($1, $2, $3, 'user', 'pending', $4) RETURNING id`, email, displayName, passwordHash, unix(s.now())).Scan(&id) if err != nil { if isUniqueViolation(err) { return nil, ErrEmailTaken } return nil, err } - id, err := res.LastInsertId() - if err != nil { - return nil, err - } return s.GetByID(id) } func (s *Store) GetByID(id int64) (*User, error) { - return scanUser(s.db.QueryRow(`SELECT `+userCols+` FROM users WHERE id = ?`, id)) + return scanUser(s.db.QueryRow(`SELECT `+userCols+` FROM users WHERE id = $1`, id)) } // GetByEmail looks up a normalized address. func (s *Store) GetByEmail(email string) (*User, error) { - return scanUser(s.db.QueryRow(`SELECT `+userCols+` FROM users WHERE email = ?`, email)) + return scanUser(s.db.QueryRow(`SELECT `+userCols+` FROM users WHERE email = $1`, email)) } // Activate moves a pending user to active with role. by is the admin who // activated manually, nil for link activation. ErrNotFound if not pending. func (s *Store) Activate(id int64, role Role, by *int64) error { - return expectOne(s.db.Exec(`UPDATE users SET status = 'active', role = ?, activated_at = ?, activated_by = ? - WHERE id = ? AND status = 'pending'`, string(role), unix(s.now()), nullInt(by), id)) + return expectOne(s.db.Exec(`UPDATE users SET status = 'active', role = $1, activated_at = $2, activated_by = $3 + WHERE id = $4 AND status = 'pending'`, string(role), unix(s.now()), nullInt(by), id)) } func (s *Store) SetStatus(id int64, st Status) error { - return expectOne(s.db.Exec(`UPDATE users SET status = ? WHERE id = ?`, string(st), id)) + return expectOne(s.db.Exec(`UPDATE users SET status = $1 WHERE id = $2`, string(st), id)) } func (s *Store) SetRole(id int64, r Role) error { - return expectOne(s.db.Exec(`UPDATE users SET role = ? WHERE id = ?`, string(r), id)) + return expectOne(s.db.Exec(`UPDATE users SET role = $1 WHERE id = $2`, string(r), id)) } func (s *Store) SetPassword(id int64, hash string) error { - return expectOne(s.db.Exec(`UPDATE users SET password_hash = ? WHERE id = ?`, hash, id)) + return expectOne(s.db.Exec(`UPDATE users SET password_hash = $1 WHERE id = $2`, hash, id)) } func (s *Store) SetDisplayName(id int64, name string) error { - return expectOne(s.db.Exec(`UPDATE users SET display_name = ? WHERE id = ?`, name, id)) + return expectOne(s.db.Exec(`UPDATE users SET display_name = $1 WHERE id = $2`, name, id)) } // SetEmail changes the address (normalized) and clears the bounce flag. func (s *Store) SetEmail(id int64, email string) error { - err := expectOne(s.db.Exec(`UPDATE users SET email = ?, email_bouncing = 0 WHERE id = ?`, email, id)) + err := expectOne(s.db.Exec(`UPDATE users SET email = $1, email_bouncing = 0 WHERE id = $2`, email, id)) if isUniqueViolation(err) { return ErrEmailTaken } @@ -130,11 +128,11 @@ func (s *Store) SetEmailBouncing(id int64, v bool) error { if v { b = 1 } - return expectOne(s.db.Exec(`UPDATE users SET email_bouncing = ? WHERE id = ?`, b, id)) + return expectOne(s.db.Exec(`UPDATE users SET email_bouncing = $1 WHERE id = $2`, b, id)) } func (s *Store) TouchLogin(id int64) error { - return expectOne(s.db.Exec(`UPDATE users SET last_login_at = ? WHERE id = ?`, unix(s.now()), id)) + return expectOne(s.db.Exec(`UPDATE users SET last_login_at = $1 WHERE id = $2`, unix(s.now()), id)) } // Delete removes a user; sessions and tokens cascade. Mail-log rows survive @@ -146,7 +144,7 @@ func (s *Store) Delete(id int64) error { } defer tx.Rollback() var exists int - if err := tx.QueryRow(`SELECT 1 FROM users WHERE id = ?`, id).Scan(&exists); err != nil { + if err := tx.QueryRow(`SELECT 1 FROM users WHERE id = $1 FOR UPDATE`, id).Scan(&exists); err != nil { if errors.Is(err, sql.ErrNoRows) { return ErrNotFound } @@ -155,7 +153,7 @@ func (s *Store) Delete(id int64) error { if err := hashMailLogTx(tx, id); err != nil { return err } - if _, err := tx.Exec(`DELETE FROM users WHERE id = ?`, id); err != nil { + if _, err := tx.Exec(`DELETE FROM users WHERE id = $1`, id); err != nil { return err } return tx.Commit() @@ -169,17 +167,16 @@ type ListFilter struct { Bouncing bool // only addresses whose mail bounces } -// List returns at most 1000 users, newest first. SQLite lower()/LIKE fold -// ASCII only, so a non-ASCII search is case-sensitive. +// List returns at most 1000 users, newest first, with case-insensitive search. func (s *Store) List(f ListFilter) ([]User, error) { q := `SELECT ` + userCols + ` FROM users WHERE 1=1` var args []any if f.Status != "" { - q += ` AND status = ?` + q += fmt.Sprintf(` AND status = $%d`, len(args)+1) args = append(args, string(f.Status)) } if f.Role != "" { - q += ` AND role = ?` + q += fmt.Sprintf(` AND role = $%d`, len(args)+1) args = append(args, string(f.Role)) } if f.Bouncing { @@ -187,7 +184,7 @@ func (s *Store) List(f ListFilter) ([]User, error) { } if t := strings.TrimSpace(f.Query); t != "" { like := "%" + escapeLike(strings.ToLower(t)) + "%" - q += ` AND (email LIKE ? ESCAPE '\' OR lower(display_name) LIKE ? ESCAPE '\')` + q += fmt.Sprintf(` AND (email LIKE $%d ESCAPE '\' OR lower(display_name) LIKE $%d ESCAPE '\')`, len(args)+1, len(args)+2) args = append(args, like, like) } q += ` ORDER BY created_at DESC, id DESC LIMIT 1000` @@ -254,9 +251,9 @@ func (s *Store) PruneStalePending(maxAge time.Duration) (int64, error) { return 0, err } defer tx.Rollback() - rows, err := tx.Query(`SELECT id FROM users WHERE status = 'pending' AND created_at < ? + rows, err := tx.Query(`SELECT id FROM users WHERE status = 'pending' AND created_at < $1 AND NOT EXISTS (SELECT 1 FROM tokens t WHERE t.user_id = users.id AND t.purpose = 'activate' - AND t.used_at IS NULL AND t.expires_at > ?)`, now-int64(maxAge/time.Second), now) + AND t.used_at IS NULL AND t.expires_at > $2) ORDER BY id FOR UPDATE`, now-int64(maxAge/time.Second), now) if err != nil { return 0, err } @@ -273,15 +270,28 @@ func (s *Store) PruneStalePending(maxAge time.Duration) (int64, error) { if err := rows.Err(); err != nil { return 0, err } + var removed int64 for _, id := range ids { + // The locking SELECT may have waited for a concurrent IssueToken. + // Recheck with a fresh READ COMMITTED snapshot after owning the user + // lock; its original NOT EXISTS snapshot cannot see that new token. + var eligible bool + if err := tx.QueryRow(`SELECT EXISTS(SELECT 1 FROM users WHERE id=$1 AND status='pending' AND created_at<$2 + AND NOT EXISTS(SELECT 1 FROM tokens WHERE user_id=$1 AND purpose='activate' AND used_at IS NULL AND expires_at>$3))`, id, now-int64(maxAge/time.Second), now).Scan(&eligible); err != nil { + return 0, err + } + if !eligible { + continue + } if err := hashMailLogTx(tx, id); err != nil { return 0, err } - if _, err := tx.Exec(`DELETE FROM users WHERE id = ?`, id); err != nil { + if _, err := tx.Exec(`DELETE FROM users WHERE id = $1`, id); err != nil { return 0, err } + removed++ } - return int64(len(ids)), tx.Commit() + return removed, tx.Commit() } // hashMailLogTx replaces the plaintext address of every mail_log row of the @@ -289,7 +299,7 @@ func (s *Store) PruneStalePending(maxAge time.Duration) (int64, error) { // change). Rows already hashed are skipped. The cursor is closed before any // UPDATE because the store uses a single connection. func hashMailLogTx(tx *sql.Tx, userID int64) error { - rows, err := tx.Query(`SELECT id, to_email FROM mail_log WHERE user_id = ? AND to_email NOT LIKE 'sha256:%'`, userID) + rows, err := tx.Query(`SELECT id, to_email FROM mail_log WHERE user_id = $1 AND to_email NOT LIKE 'sha256:%'`, userID) if err != nil { return err } @@ -311,7 +321,7 @@ func hashMailLogTx(tx *sql.Tx, userID int64) error { return err } for _, r := range list { - if _, err := tx.Exec(`UPDATE mail_log SET to_email = ? WHERE id = ?`, HashedEmail(r.email), r.id); err != nil { + if _, err := tx.Exec(`UPDATE mail_log SET to_email = $1 WHERE id = $2`, HashedEmail(r.email), r.id); err != nil { return err } } diff --git a/manage.sh b/manage.sh index 099c0fb71..01c36a4dd 100755 --- a/manage.sh +++ b/manage.sh @@ -91,6 +91,58 @@ dc_staging() { fi } +# Read the owner credential inside the database container, never from argv. +pg_exec() { + local environment="$1" database="$2" + shift 2 + case "$environment" in + prod) dc_prod exec -T postgres sh -c 'export PGHOST=127.0.0.1 PGPORT=5432 PGUSER=corescope_owner PGPASSWORD="$CORESCOPE_OWNER_PASSWORD" PGDATABASE="$1"; shift; exec "$@"' sh "$database" "$@" ;; + staging) dc_staging exec -T postgres sh -c 'export PGHOST=127.0.0.1 PGPORT=5432 PGUSER=corescope_owner PGPASSWORD="$CORESCOPE_OWNER_PASSWORD" PGDATABASE="$1"; shift; exec "$@"' sh "$database" "$@" ;; + *) err "Unknown PostgreSQL environment"; return 1 ;; + esac +} + +pg_empty() { + local count + count=$(pg_exec "$1" "$2" psql -X -A -t -v ON_ERROR_STOP=1 -c "SELECT COUNT(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname NOT IN ('pg_catalog','information_schema') AND n.nspname NOT LIKE 'pg_toast%' AND c.relkind IN ('r','p','v','m','S','f')") || return 1 + [ "$count" = 0 ] +} + +pg_dump_file() { + local environment="$1" database="$2" destination="$3" temporary="$3.partial" + if [ -e "$destination" ]; then err "Backup target already exists: $destination"; return 1; fi + ( + umask 077 + set -o noclobber + exec 3>"$temporary" || exit 1 + trap 'rm -f -- "$temporary"' EXIT HUP INT TERM + pg_exec "$environment" "$database" pg_dump --format=custom --no-password --no-owner --no-privileges >&3 || exit 1 + exec 3>&- + sync -f "$temporary" || exit 1 + ln -- "$temporary" "$destination" || exit 1 + rm -- "$temporary" || exit 1 + trap - EXIT HUP INT TERM + ) +} + +pg_restore_file() { + local environment="$1" database="$2" archive="$3" + if [ "$(head -c 5 "$archive")" != PGDMP ]; then err "SQLite files require docs/postgresql-upgrade.md; expected a PostgreSQL archive."; return 1; fi + pg_empty "$environment" "$database" || { err "Refusing to overwrite a nonempty PostgreSQL destination."; return 1; } + pg_exec "$environment" "$database" pg_restore --no-password --exit-on-error --single-transaction --no-owner --no-privileges --dbname="$database" < "$archive" || return 1 + pg_exec "$environment" "$database" psql -X -v ON_ERROR_STOP=1 < docker/postgres-grants.sql >/dev/null || return 1 + pg_exec "$environment" "$database" psql -X -v ON_ERROR_STOP=1 -c ANALYZE >/dev/null || return 1 +} + +require_database_credentials() { + local key value + if [ -z "$POSTGRES_ADMIN_PASSWORD" ]; then err "Set PostgreSQL credentials in .env before starting; see .env.example."; return 1; fi + for key in CORESCOPE_OWNER_PASSWORD CORESCOPE_READER_PASSWORD CORESCOPE_WRITER_PASSWORD CORESCOPE_ACCOUNTS_PASSWORD CORESCOPE_CHANNELS_PASSWORD; do + value=$(printenv "$key" || true) + if ! [[ "$value" =~ ^[a-fA-F0-9]{32,}$ ]]; then err "Set $key in .env to at least 32 hexadecimal characters; see docs/postgresql-upgrade.md."; return 1; fi + done +} + confirm() { read -p " $1 [y/N] " -n 1 -r echo @@ -287,7 +339,7 @@ write_env_managed_values() { local seen_data=0 local seen_disable_mosquitto=0 - : > "$tmp_file" + (umask 077; set -o noclobber; : > "$tmp_file") || return 1 while IFS= read -r line || [ -n "$line" ]; do case "$line" in PROD_HTTP_PORT=*) @@ -761,6 +813,7 @@ cmd_setup() { write_env_managed_values "$selected_http" "$selected_https" "$selected_mqtt" "$selected_data_dir" "$selected_disable_mosquitto" log "Saved negotiated ports to .env" + require_database_credentials show_env_port_summary "$selected_http" "$selected_https" "$selected_mqtt" "$selected_data_dir" "$selected_disable_mosquitto" echo " Resolved port mapping:" @@ -878,16 +931,25 @@ cmd_setup() { # ─── Staging Helpers ────────────────────────────────────────────────────── -# Copy production DB to staging data directory +# Copy telemetry into an empty staging database; accounts stay independent. prepare_staging_db() { mkdir -p "$STAGING_DATA" - if [ -f "$PROD_DATA/meshcore.db" ]; then - info "Copying production database to staging..." - cp "$PROD_DATA/meshcore.db" "$STAGING_DATA/meshcore.db" 2>/dev/null || true - log "Database snapshot copied to ${STAGING_DATA}/meshcore.db" - else - warn "No production database found at ${PROD_DATA}/meshcore.db — staging starts empty." - fi + dc_staging up -d postgres + local attempt + for attempt in $(seq 1 30); do + if pg_exec staging corescope_telemetry psql -X -A -t -c 'SELECT 1' >/dev/null 2>&1; then break; fi + [ "$attempt" -lt 30 ] || { err "Staging PostgreSQL did not become ready"; return 1; } + sleep 1 + done + if ! pg_empty staging corescope_telemetry; then info "Keeping existing staging PostgreSQL data; automatic replacement is refused."; return 0; fi + local copy_dir + copy_dir=$(mktemp -d) + chmod 700 "$copy_dir" + pg_dump_file prod corescope_telemetry "$copy_dir/telemetry.dump" || return 1 + pg_restore_file staging corescope_telemetry "$copy_dir/telemetry.dump" || return 1 + rm -f -- "$copy_dir/telemetry.dump" + rmdir -- "$copy_dir" + log "Telemetry restored into the empty staging database. Staging accounts remain independent." } # Copy config.prod.json → config.staging.json with siteName change @@ -1029,6 +1091,7 @@ ensure_config() { } cmd_start() { + require_database_credentials local WITH_STAGING=false if [ "$1" = "--with-staging" ]; then WITH_STAGING=true @@ -1046,14 +1109,16 @@ cmd_start() { # Always check prod config ensure_config "$PROD_DATA" + # Compose completes PostgreSQL/bootstrap dependencies before staging copies data. + info "Starting production container (corescope-prod) on ports ${PROD_HTTP_PORT:-80}/${PROD_HTTPS_PORT:-443}..." + dc_prod up -d prod + if $WITH_STAGING; then # Prepare staging data and config prepare_staging_db prepare_staging_config - info "Starting production container (corescope-prod) on ports ${PROD_HTTP_PORT:-80}/${PROD_HTTPS_PORT:-443}..." info "Starting staging container (${STAGING_CONTAINER}) on port ${STAGING_GO_HTTP_PORT:-82}..." - dc_prod up -d prod dc_staging up -d staging-go if is_true "${DISABLE_MOSQUITTO:-false}"; then log "Production started on ports ${PROD_HTTP_PORT:-80}/${PROD_HTTPS_PORT:-443} (MQTT disabled)" @@ -1063,8 +1128,6 @@ cmd_start() { log "Staging started on port ${STAGING_GO_HTTP_PORT:-82} (MQTT: ${STAGING_GO_MQTT_PORT:-1885})" fi else - info "Starting production container (corescope-prod) on ports ${PROD_HTTP_PORT:-80}/${PROD_HTTPS_PORT:-443}..." - dc_prod up -d prod log "Production started. Staging NOT running (use --with-staging to start both)." fi } @@ -1200,18 +1263,11 @@ cmd_status() { echo "" # Disk usage - if [ -d "$PROD_DATA" ] && [ -f "$PROD_DATA/meshcore.db" ]; then - local db_size - db_size=$(du -h "$PROD_DATA/meshcore.db" 2>/dev/null | cut -f1) - info "Production DB: ${db_size}" - fi - if [ -d "$STAGING_DATA" ] && [ -f "$STAGING_DATA/meshcore.db" ]; then - local staging_db_size - staging_db_size=$(du -h "$STAGING_DATA/meshcore.db" 2>/dev/null | cut -f1) - info "Staging DB: ${staging_db_size}" - fi + local database size + for database in corescope_telemetry corescope_accounts; do + if size=$(pg_exec prod "$database" psql -X -A -t -c 'SELECT pg_size_pretty(pg_database_size(current_database()))' 2>/dev/null); then info "Production $database: $size"; else warn "PostgreSQL size unavailable for $database"; fi + done - echo "" } # ─── Logs ───────────────────────────────────────────────────────────────── @@ -1269,18 +1325,8 @@ cmd_promote() { exit 0 fi - # Backup production DB - info "Backing up production database..." local BACKUP_DIR="./backups/pre-promotion-$(date +%Y%m%d-%H%M%S)" - mkdir -p "$BACKUP_DIR" - if [ -f "$PROD_DATA/meshcore.db" ]; then - cp "$PROD_DATA/meshcore.db" "$BACKUP_DIR/" - elif container_running "corescope-prod"; then - docker cp corescope-prod:/app/data/meshcore.db "$BACKUP_DIR/" - else - warn "Could not backup production database." - fi - log "Backup saved to ${BACKUP_DIR}/" + cmd_backup "$BACKUP_DIR" # Restart prod with latest image info "Restarting production with latest image..." @@ -1298,7 +1344,7 @@ cmd_promote() { if [ "$i" -eq 30 ]; then err "Production failed health check after 30s" warn "Check logs: ./manage.sh logs prod" - warn "Rollback: cp ${BACKUP_DIR}/meshcore.db ${PROD_DATA}/ && ./manage.sh restart prod" + warn "Native archives are in $BACKUP_DIR. Restore only into a fresh target; see docs/postgresql-upgrade.md." exit 1 fi sleep 1 @@ -1366,145 +1412,48 @@ cmd_update() { # ─── Backup ─────────────────────────────────────────────────────────────── cmd_backup() { - TIMESTAMP=$(date +%Y%m%d-%H%M%S) - BACKUP_DIR="${1:-./backups/corescope-${TIMESTAMP}}" - mkdir -p "$BACKUP_DIR" - - info "Backing up to ${BACKUP_DIR}/" - - # Database - # Always use bind mount path (from .env or default) - DB_PATH="$PROD_DATA/meshcore.db" - if [ -f "$DB_PATH" ]; then - cp "$DB_PATH" "$BACKUP_DIR/meshcore.db" - log "Database ($(du -h "$BACKUP_DIR/meshcore.db" | cut -f1))" - elif container_running "corescope-prod"; then - docker cp corescope-prod:/app/data/meshcore.db "$BACKUP_DIR/meshcore.db" 2>/dev/null && \ - log "Database (via docker cp)" || warn "Could not backup database" - else - warn "Database not found (container not running?)" - fi - - # Config (now lives in data dir) - if [ -f "$PROD_DATA/config.json" ]; then - cp "$PROD_DATA/config.json" "$BACKUP_DIR/config.json" - log "config.json" - elif [ -f config.json ]; then - cp config.json "$BACKUP_DIR/config.json" - log "config.json (legacy repo root)" - fi - - # Caddyfile - if [ -f caddy-config/Caddyfile ]; then - cp caddy-config/Caddyfile "$BACKUP_DIR/Caddyfile" - log "Caddyfile" - fi - - # Theme - # Always use bind mount path (from .env or default) - THEME_PATH="$PROD_DATA/theme.json" - if [ -f "$THEME_PATH" ]; then - cp "$THEME_PATH" "$BACKUP_DIR/theme.json" - log "theme.json" - elif [ -f theme.json ]; then - cp theme.json "$BACKUP_DIR/theme.json" - log "theme.json" - fi - - # Summary - TOTAL=$(du -sh "$BACKUP_DIR" | cut -f1) - FILES=$(ls "$BACKUP_DIR" | wc -l) - echo "" - log "Backup complete: ${FILES} files, ${TOTAL} total → ${BACKUP_DIR}/" + local timestamp backup_dir source name + timestamp=$(date +%Y%m%d-%H%M%S) + backup_dir="$1" + [ -n "$backup_dir" ] || backup_dir="./backups/corescope-$timestamp" + umask 077 + mkdir -p "$backup_dir" + info "Writing native PostgreSQL archives to $backup_dir/" + pg_dump_file prod corescope_telemetry "$backup_dir/telemetry.dump" || return 1 + pg_dump_file prod corescope_accounts "$backup_dir/accounts.dump" || return 1 + for name in config.json theme.json; do + source="$PROD_DATA/$name" + [ -f "$source" ] || source="$name" + if [ -f "$source" ]; then cp -- "$source" "$backup_dir/$name"; chmod 600 "$backup_dir/$name"; fi + done + if [ -f caddy-config/Caddyfile ]; then cp -- caddy-config/Caddyfile "$backup_dir/Caddyfile"; chmod 600 "$backup_dir/Caddyfile"; fi + log "Both database archives and available configuration saved. Keep the bundle private and encrypted off-host." } # ─── Restore ────────────────────────────────────────────────────────────── cmd_restore() { - if [ -z "$1" ]; then - err "Usage: ./manage.sh restore " - if [ -d "./backups" ]; then - echo "" - echo " Available backups:" - ls -dt ./backups/meshcore-* ./backups/corescope-* 2>/dev/null | head -10 | while read d; do - if [ -d "$d" ]; then - echo " $d/ ($(ls "$d" | wc -l) files)" - elif [ -f "$d" ]; then - echo " $d ($(du -h "$d" | cut -f1))" - fi - done - fi - exit 1 - fi - - # Accept either a directory (full backup) or a single .db file - if [ -d "$1" ]; then - DB_FILE="$1/meshcore.db" - CONFIG_FILE="$1/config.json" - CADDY_FILE="$1/Caddyfile" - THEME_FILE="$1/theme.json" - elif [ -f "$1" ]; then - DB_FILE="$1" - CONFIG_FILE="" - CADDY_FILE="" - THEME_FILE="" - else - err "Not found: $1" - exit 1 - fi - - if [ ! -f "$DB_FILE" ]; then - err "No meshcore.db found in $1" - exit 1 - fi - - echo "" - info "Will restore from: $1" - [ -f "$DB_FILE" ] && echo " • Database" - [ -n "$CONFIG_FILE" ] && [ -f "$CONFIG_FILE" ] && echo " • config.json" - [ -n "$CADDY_FILE" ] && [ -f "$CADDY_FILE" ] && echo " • Caddyfile" - [ -n "$THEME_FILE" ] && [ -f "$THEME_FILE" ] && echo " • theme.json" - echo "" - - if ! confirm "Continue? (current state will be backed up first)"; then - echo " Aborted." - exit 0 - fi - - # Backup current state first - info "Backing up current state..." - cmd_backup "./backups/corescope-pre-restore-$(date +%Y%m%d-%H%M%S)" - - dc_prod stop prod 2>/dev/null || true - - # Restore database - mkdir -p "$PROD_DATA" - DEST_DB="$PROD_DATA/meshcore.db" - cp "$DB_FILE" "$DEST_DB" - log "Database restored" - - # Restore config if present - if [ -n "$CONFIG_FILE" ] && [ -f "$CONFIG_FILE" ]; then - cp "$CONFIG_FILE" "$PROD_DATA/config.json" - log "config.json restored to ${PROD_DATA}/" - fi - - # Restore Caddyfile if present - if [ -n "$CADDY_FILE" ] && [ -f "$CADDY_FILE" ]; then - mkdir -p caddy-config - cp "$CADDY_FILE" caddy-config/Caddyfile - log "Caddyfile restored" - fi - - # Restore theme if present - if [ -n "$THEME_FILE" ] && [ -f "$THEME_FILE" ]; then - DEST_THEME="$PROD_DATA/theme.json" - cp "$THEME_FILE" "$DEST_THEME" - log "theme.json restored" - fi - - dc_prod up -d prod - log "Restored and restarted." + local bundle="$1" file source destination stamp + if [ ! -d "$bundle" ]; then err "Usage: ./manage.sh restore . SQLite files require docs/postgresql-upgrade.md."; return 1; fi + for file in telemetry.dump accounts.dump; do + if [ ! -f "$bundle/$file" ] || [ "$(head -c 5 "$bundle/$file")" != PGDMP ]; then err "Backup must contain native telemetry.dump and accounts.dump; SQLite files are not accepted."; return 1; fi + done + if container_running corescope-prod; then err "Stop CoreScope before restoring; leave target PostgreSQL running."; return 1; fi + if ! pg_empty prod corescope_telemetry || ! pg_empty prod corescope_accounts; then err "Restore requires empty PostgreSQL databases. Preserve the current cluster and configure a fresh destination."; return 1; fi + if ! confirm "Restore both native archives into these empty databases?"; then return 0; fi + pg_restore_file prod corescope_telemetry "$bundle/telemetry.dump" || return 1 + pg_restore_file prod corescope_accounts "$bundle/accounts.dump" || return 1 + mkdir -p "$PROD_DATA" caddy-config + stamp="$(date +%Y%m%d-%H%M%S)-$$" + for file in config.json theme.json Caddyfile; do + source="$bundle/$file" + [ -f "$source" ] || continue + destination="$PROD_DATA/$file" + [ "$file" != Caddyfile ] || destination=caddy-config/Caddyfile + if [ -f "$destination" ]; then cp -- "$destination" "$destination.pre-restore-$stamp"; fi + cp -- "$source" "$destination" + done + log "Native archives restored. Services remain stopped: review restored accounts, sessions and tokens before ./manage.sh start." } # ─── MQTT Test ──────────────────────────────────────────────────────────── @@ -1563,7 +1512,7 @@ cmd_help() { echo "" printf '%b\n' " ${BOLD}Run${NC}" echo " start Start production container" - echo " start --with-staging Start production + staging-go (copies prod DB + config)" + echo " start --with-staging Start production + staging-go (clones into empty staging telemetry + config)" echo " stop [prod|staging|all] Stop specific or all containers (default: all)" echo " restart [prod|staging|all] Restart specific or all containers" echo " status Show health, stats, and service status" @@ -1572,8 +1521,8 @@ cmd_help() { printf '%b\n' " ${BOLD}Maintain${NC}" echo " update [version] Update to version (no arg=latest tag, 'latest'=master tip, or e.g. v3.1.0)" echo " promote Promote staging → production (backup + restart)" - echo " backup [dir] Full backup: database + config + theme" - echo " restore Restore from backup dir or .db file" + echo " backup [dir] Native PostgreSQL archives + config + theme" + echo " restore Restore native archives into empty PostgreSQL databases" echo " mqtt-test Check if MQTT data is flowing" echo "" echo "Prod uses docker-compose.yml; staging uses ${STAGING_COMPOSE_FILE}." diff --git a/public/admin-audit.js b/public/admin-audit.js index c141b1a5b..1618a9129 100644 --- a/public/admin-audit.js +++ b/public/admin-audit.js @@ -22,7 +22,7 @@ ['user.email.*', 'Address changes'], ['user.mail.refresh', 'Mail status refreshed'], ['user.export', 'Data exports'], - ['user.backup', 'users.db downloads'], + ['user.backup', 'Account backup downloads'], ['proposal.*', 'Channel proposals'], ['notify.*', 'Notification settings'] ]; diff --git a/public/geofilter-docs.html b/public/geofilter-docs.html index f445cebc3..3f492363c 100644 --- a/public/geofilter-docs.html +++ b/public/geofilter-docs.html @@ -120,22 +120,14 @@

Legacy bounding box

Prefer the polygon format — it supports irregular shapes and the bufferKm margin.

Cleaning up historical nodes

-

The ingestor prevents new out-of-bounds nodes from being ingested, but does not retroactively remove nodes stored before the filter was configured. Use the prune script for that:

-
# Dry run — shows what would be deleted without making any changes
-python3 scripts/prune-nodes-outside-geo-filter.py --dry-run
-
-# Default paths: /app/data/meshcore.db and /app/config.json
-python3 scripts/prune-nodes-outside-geo-filter.py
-
-# Custom paths
-python3 scripts/prune-nodes-outside-geo-filter.py /path/to/meshcore.db \
-  --config /path/to/config.json
-
-# In Docker — run inside the container
-docker exec -it meshcore-analyzer \
-  python3 /app/scripts/prune-nodes-outside-geo-filter.py --dry-run
-

The script reads geo_filter.polygon and geo_filter.bufferKm from config, lists nodes that fall outside, then asks for yes confirmation before deleting. Nodes without coordinates are always kept.

-

This is a one-time migration tool — run it once after first configuring geo_filter to clean up pre-filter data.

+

The ingestor prevents new out-of-bounds nodes from being ingested. To review older nodes, open the Customizer's Admin Geo Filter settings and save the desired polygon and buffer first.

+
    +
  1. In Prune historical nodes, choose Preview prune.
  2. +
  3. Review the listed nodes outside the active filter. Nodes without coordinates are kept.
  4. +
  5. Choose Delete nodes only after reviewing the preview.
  6. +
+

The server submits the request to the shared state-directory queue; the ingestor performs the database changes. Keep both processes on the same configured state directory and allow the ingestor to finish before checking the result.

+

The old Python prune script is offline legacy SQLite tooling. It must not be used for PostgreSQL maintenance.

diff --git a/public/perf.js b/public/perf.js index 1ec00dd5b..e077e146f 100644 --- a/public/perf.js +++ b/public/perf.js @@ -117,11 +117,11 @@ function renderVersionCard(health) { try { // #1258: /api/health was awaited AFTER Promise.all, adding a full RTT // (~50-200ms) on every 5s refresh. Issue it in parallel with the rest. - const [server, client, ioStats, sqliteStats, writeSources, health] = await Promise.all([ + const [server, client, ioStats, postgresStats, writeSources, health] = await Promise.all([ fetch('/api/perf').then(r => r.json()), Promise.resolve(window.apiPerf ? window.apiPerf() : null), fetch('/api/perf/io').then(r => r.json()).catch(() => null), - fetch('/api/perf/sqlite').then(r => r.json()).catch(() => null), + fetch('/api/perf/postgres').then(r => r.json()).catch(() => null), fetch('/api/perf/write-sources').then(r => r.json()).catch(() => null), fetch('/api/health').then(r => r.json()).catch(() => null) ]); @@ -213,7 +213,7 @@ function renderVersionCard(health) { const keys = Object.keys(src).sort((a, b) => (src[b] || 0) - (src[a] || 0)); html += '

Write Sources

'; if (keys.length === 0) { - html += '

No ingestor stats yet (waiting for /tmp/corescope-ingestor-stats.json)

'; + html += '

No ingestor stats yet (waiting for the ingestor statistics snapshot)

'; } else { // Anomaly detection (#1120 acceptance): flag any component whose // per-second write rate exceeds 10× its 5-minute rolling baseline. @@ -246,17 +246,16 @@ function renderVersionCard(health) { } } - // SQLite perf (separate from existing SQLite block — focused on WAL + cache hit) (#1120) - if (sqliteStats) { - const walMB = sqliteStats.walSizeMB || 0; - const walFlag = walMB > 100 ? ' ' : ''; - const hitRate = (sqliteStats.cacheHitRate || 0) * 100; + // Database-wide PostgreSQL counters and this server's connection pool. + if (postgresStats && postgresStats.engine === 'postgresql') { + const hitRate = postgresStats.cacheHitRate == null ? null : Number(postgresStats.cacheHitRate) * 100; const hitFlag = hitRate > 0 && hitRate < 90 ? ' ' : ''; - html += `

SQLite (WAL + Cache Hit)

-
${walMB.toFixed(1)}MB${walFlag}
WAL Size
-
${(sqliteStats.pageCount || 0).toLocaleString()}
Page Count
-
${sqliteStats.pageSize || 0}
Page Size
-
${hitRate.toFixed(1)}%${hitFlag}
Cache Hit Rate
+ const hitText = hitRate != null && Number.isFinite(hitRate) ? hitRate.toFixed(1) + '%' : '—'; + html += `

PostgreSQL connections

+
${Number(postgresStats.inUseConnections || 0)} / ${Number(postgresStats.openConnections || 0)}
Connections in use
+
${Number(postgresStats.connectionWaitCount || 0).toLocaleString()}
Connection waits
+
${Number(postgresStats.connectionWaitMs || 0).toFixed(1)}ms
Total connection wait
+
${hitText}${hitFlag}
Database Block Cache Hit Rate
`; } @@ -299,23 +298,24 @@ function renderVersionCard(health) {
`; } - // SQLite stats - if (server.sqlite && !server.sqlite.error) { - const sq = server.sqlite; - const walColor = sq.walSizeMB > 50 ? 'var(--status-red)' : sq.walSizeMB > 10 ? 'var(--status-yellow)' : 'var(--status-green)'; - const freelistColor = sq.freelistMB > 10 ? 'var(--status-yellow)' : 'var(--status-green)'; - html += `

SQLite

+ // PostgreSQL size includes this database's tables and indexes, not cluster WAL. + if (server.postgres && server.postgres.rows && server.postgres.dbSizeMB != null) { + const sq = server.postgres; + html += `

PostgreSQL data

${sq.dbSizeMB}MB
DB Size
-
${sq.walSizeMB}MB
WAL Size
-
${sq.freelistMB}MB
Freelist
${(sq.rows.transmissions || 0).toLocaleString()}
Transmissions
${(sq.rows.observations || 0).toLocaleString()}
Observations
${sq.rows.nodes || 0}
Nodes
${sq.rows.observers || 0}
Observers
`; - if (sq.walPages) { - html += `
${sq.walPages.busy}
WAL Busy Pages
`; - } html += `
`; + if (sq.sampledAt) { + const sampled = new Date(sq.sampledAt); + const sampleTime = Number.isFinite(sampled.getTime()) ? sampled.toLocaleString() : 'unknown time'; + const interval = Number(sq.sampleIntervalSeconds); + html += `
${sq.stale ? 'Last verified PostgreSQL sample' : 'PostgreSQL sample'}: ${sampleTime}${Number.isFinite(interval) && interval > 0 ? ' · refreshes every ' + interval + 's' : ''}${sq.stale ? ' · refresh unavailable' : ''}
`; + } + } else if (server.postgres) { + html += '

PostgreSQL data

PostgreSQL storage unavailable
'; } // Server endpoints table — sort by total time (count * avg) DESC. diff --git a/scripts/bash-path.js b/scripts/bash-path.js new file mode 100644 index 000000000..cb1551afb --- /dev/null +++ b/scripts/bash-path.js @@ -0,0 +1,15 @@ +// Resolve the shell used by tests that exercise the real management/CI scripts. +const fs = require('node:fs'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); + +module.exports = function bashPath() { + if (process.env.BASH_PATH) return process.env.BASH_PATH; + if (process.platform === 'win32') { + // --exec-path is stable whether PATH finds Git in cmd/, bin/ or mingw64/bin/. + const git = spawnSync('git', ['--exec-path'], { encoding: 'utf8' }); + const candidate = path.resolve((git.stdout || '').trim(), '../../../bin/bash.exe'); + if (git.status === 0 && fs.existsSync(candidate)) return candidate; + } + return 'bash'; +}; diff --git a/scripts/install-postgres-ci.sh b/scripts/install-postgres-ci.sh new file mode 100755 index 000000000..3af4b0ff1 --- /dev/null +++ b/scripts/install-postgres-ci.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Exact PostgreSQL client/server packages for disposable Ubuntu CI runners. +set -euo pipefail +mode=${1:-client} +[[ ${GITHUB_ACTIONS:-} == true ]] || { echo 'This installer is for disposable GitHub Actions runners.' >&2; exit 1; } +[[ $mode == client || $mode == server ]] || { echo 'Usage: install-postgres-ci.sh client|server' >&2; exit 1; } +sudo install -d /usr/share/postgresql-common/pgdg +curl --fail --show-error --silent https://www.postgresql.org/media/keys/ACCC4CF8.asc | + sudo tee /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc >/dev/null +. /etc/os-release +printf 'deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt-archive.postgresql.org/pub/repos/apt %s-pgdg-archive main\n' "$VERSION_CODENAME" | + sudo tee /etc/apt/sources.list.d/corescope-pgdg.list >/dev/null +sudo apt-get update -qq +client_version=$(apt-cache madison postgresql-client-18 | awk '$3 ~ /^18[.]6-/ && !found {print $3; found=1} END {if(!found) exit 1}') +packages=("postgresql-client-18=$client_version" sqlite3) +if [[ $mode == server ]]; then + # The benchmark owns its own cluster. Package installation must not launch + # an extra unmeasured PostgreSQL service on the runner. + sudo install -d /etc/postgresql-common + printf 'create_main_cluster = false\n' | sudo tee /etc/postgresql-common/createcluster.conf >/dev/null + server_version=$(apt-cache madison postgresql-18 | awk '$3 ~ /^18[.]6-/ && !found {print $3; found=1} END {if(!found) exit 1}') + packages+=("postgresql-18=$server_version") +fi +sudo apt-get install -y --no-install-recommends "${packages[@]}" +echo /usr/lib/postgresql/18/bin >> "$GITHUB_PATH" +/usr/lib/postgresql/18/bin/pg_dump --version | grep -E '18[.]6([ (]|$)' +if [[ $mode == server ]]; then /usr/lib/postgresql/18/bin/postgres --version | grep -E '18[.]6([ (]|$)'; fi diff --git a/scripts/migrate-dedup.js b/scripts/migrate-dedup.js index 8e2fd3a4f..cdbdd831c 100644 --- a/scripts/migrate-dedup.js +++ b/scripts/migrate-dedup.js @@ -1,20 +1,26 @@ #!/usr/bin/env node /** - * Milestone 1: Packet Dedup Schema Migration + * Offline legacy SQLite tooling. Not part of PostgreSQL upgrades. * * Creates `transmissions` and `observations` tables from the existing `packets` table. * Idempotent — drops and recreates new tables on each run. * Does NOT touch the original `packets` table. * - * Usage: node scripts/migrate-dedup.js + * Usage: node scripts/migrate-dedup.js --legacy-sqlite-offline */ +if (process.argv[2] !== '--legacy-sqlite-offline') { + console.error('Offline legacy SQLite tooling only. Use docs/postgresql-upgrade.md for current upgrades. To operate on a stopped, disposable recovery copy, pass --legacy-sqlite-offline explicitly.'); + process.exit(1); +} +process.argv.splice(2, 1); + const Database = require('better-sqlite3'); const path = require('path'); const dbPath = process.argv[2]; if (!dbPath) { - console.error('Usage: node scripts/migrate-dedup.js '); + console.error('Usage: node scripts/migrate-dedup.js --legacy-sqlite-offline '); process.exit(1); } diff --git a/scripts/migrate-fixture-hashes.go b/scripts/migrate-fixture-hashes.go index a0acded93..7c356ecf6 100644 --- a/scripts/migrate-fixture-hashes.go +++ b/scripts/migrate-fixture-hashes.go @@ -1,136 +1,109 @@ -// migrate-fixture-hashes recomputes content hashes in a fixture DB using the -// current ComputeContentHash formula. Run once; idempotent. +// migrate-fixture-hashes updates only an explicit disposable legacy SQLite fixture. +// Run from cmd/migrate before offline PostgreSQL import. It shares the runtime +// packet identity algorithm and refuses collisions instead of dropping records. package main import ( - "crypto/sha256" "database/sql" - "encoding/hex" "fmt" "log" + "net/url" "os" + "path/filepath" + "runtime" _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/packetpath" ) -func computeContentHash(rawHex string) string { - buf, err := hex.DecodeString(rawHex) - if err != nil || len(buf) < 2 { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex - } - - headerByte := buf[0] - offset := 1 - routeType := int(headerByte & 0x03) - if routeType == 2 || routeType == 3 { // transport - offset += 4 +func main() { + if len(os.Args) != 2 { + log.Fatal("usage: migrate-fixture-hashes ") } - if offset >= len(buf) { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex + if err := migrateFixtureHashes(os.Args[1]); err != nil { + log.Fatal(err) } - pathByte := buf[offset] - offset++ - hashSize := int((pathByte>>6)&0x3) + 1 - hashCount := int(pathByte & 0x3F) - pathBytes := hashSize * hashCount +} - payloadStart := offset + pathBytes - if payloadStart > len(buf) { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex +func migrateFixtureHashes(path string) error { + info, err := os.Stat(path) + if err != nil { + return err } - - payload := buf[payloadStart:] - payloadType := (headerByte >> 2) & 0x0F - toHash := []byte{payloadType} - - // TRACE = payload type 7 - if int(payloadType) == 7 { - toHash = append(toHash, pathByte, 0x00) + if !info.Mode().IsRegular() { + return fmt.Errorf("fixture must be an existing regular file") } - toHash = append(toHash, payload...) - - h := sha256.Sum256(toHash) - return hex.EncodeToString(h[:])[:16] -} - -func main() { - if len(os.Args) < 2 { - fmt.Fprintf(os.Stderr, "usage: %s \n", os.Args[0]) - os.Exit(1) + absolute, err := filepath.Abs(path) + if err != nil { + return err } - dbPath := os.Args[1] - - db, err := sql.Open("sqlite3", dbPath) + absolute = filepath.ToSlash(absolute) + if runtime.GOOS == "windows" { + absolute = "/" + absolute + } + u := url.URL{Scheme: "file", Path: absolute} + q := url.Values{"mode": {"rw"}, "_foreign_keys": {"on"}, "_synchronous": {"FULL"}} + u.RawQuery = q.Encode() + db, err := sql.Open("sqlite3", u.String()) if err != nil { - log.Fatal(err) + return err } defer db.Close() - - rows, err := db.Query("SELECT id, raw_hex, hash FROM transmissions") + db.SetMaxOpenConns(1) + if _, err := db.Exec(`PRAGMA trusted_schema=OFF`); err != nil { + return err + } + rows, err := db.Query(`SELECT id,raw_hex,hash FROM transmissions ORDER BY id LIMIT 10001`) if err != nil { - log.Fatal(err) + return err } - type update struct { - id int - newHash string + id int64 + hash string } var updates []update - + seen := make(map[string]int64) + count := 0 for rows.Next() { - var id int - var rawHex, oldHash string - if err := rows.Scan(&id, &rawHex, &oldHash); err != nil { - log.Printf("scan: %v", err) - continue + var id int64 + var raw, old string + if err := rows.Scan(&id, &raw, &old); err != nil { + rows.Close() + return err + } + count++ + if count > 10000 { + rows.Close() + return fmt.Errorf("fixture exceeds 10000 transmissions; this is not a production repair tool") } - newHash := computeContentHash(rawHex) - if newHash != oldHash { - updates = append(updates, update{id, newHash}) + hash := packetpath.ContentHash(raw) + if previous, exists := seen[hash]; exists { + rows.Close() + return fmt.Errorf("content-hash collision between fixture IDs %d and %d requires explicit review; no rows changed", previous, id) + } + seen[hash] = id + if hash != old { + updates = append(updates, update{id, hash}) } } - rows.Close() - - if len(updates) == 0 { - fmt.Println("All hashes already match current formula.") - return + if err := rows.Err(); err != nil { + rows.Close() + return err } - + rows.Close() tx, err := db.Begin() if err != nil { - log.Fatal(err) - } - stmt, err := tx.Prepare("UPDATE transmissions SET hash = ? WHERE id = ?") - if err != nil { - log.Fatal(err) + return err } - merged := 0 + defer tx.Rollback() for _, u := range updates { - if _, err := stmt.Exec(u.newHash, u.id); err != nil { - // UNIQUE constraint = duplicate (same content, different old hash). - // Move observations to the surviving tx, then delete the dup. - log.Printf("update id %d: %v — merging duplicate", u.id, err) - // Find surviving tx id - var survID int - if err2 := tx.QueryRow("SELECT id FROM transmissions WHERE hash = ?", u.newHash).Scan(&survID); err2 == nil { - tx.Exec("UPDATE observations SET transmission_id = ? WHERE transmission_id = ?", survID, u.id) - tx.Exec("DELETE FROM transmissions WHERE id = ?", u.id) - merged++ - } + if _, err := tx.Exec(`UPDATE transmissions SET hash=? WHERE id=?`, u.hash, u.id); err != nil { + return fmt.Errorf("fixture hash update failed; no changes committed: %w", err) } } - stmt.Close() if err := tx.Commit(); err != nil { - log.Fatal(err) + return err } - fmt.Printf("Migrated %d hashes, merged %d duplicates.\n", len(updates)-merged, merged) + fmt.Printf("Updated %d fixture hashes; preserved %d transmissions and every observation.\n", len(updates), count) + return nil } diff --git a/scripts/non-unit-tests.json b/scripts/non-unit-tests.json index a5fbb97a8..0a1c87d1b 100644 --- a/scripts/non-unit-tests.json +++ b/scripts/non-unit-tests.json @@ -106,6 +106,7 @@ "tests/e2e/test-packets-compact-columns-e2e.js", "tests/e2e/test-packets-scope-column.js", "tests/e2e/test-path-inspector-coverage-e2e.js", + "tests/e2e/test-postgres-backup-e2e.js", "tests/e2e/test-rx-coverage-mobile-nav-e2e.js", "tests/e2e/test-rx-coverage-noise-e2e.js", "tests/e2e/test-rx-coverage-viewport-e2e.js", diff --git a/scripts/postgres-benchmark/.gitignore b/scripts/postgres-benchmark/.gitignore new file mode 100644 index 000000000..c18dd8d83 --- /dev/null +++ b/scripts/postgres-benchmark/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/scripts/postgres-benchmark/README.md b/scripts/postgres-benchmark/README.md new file mode 100644 index 000000000..4a0c29c2c --- /dev/null +++ b/scripts/postgres-benchmark/README.md @@ -0,0 +1,181 @@ +# PostgreSQL comparison harness + +This harness measures the actual CoreScope application methods and server. It +does not replace them with standalone SQL insert loops. Production files are +archived at exact commits and remain unchanged; Go build overlays add only test +drivers. Fixture construction happens outside measured ingestion. + +## Status + +The generator, protocol vectors, bounded fixture/handler test, and harness unit +checks have been exercised locally. The Linux controller, full S/B corpora, +paired runtime profiles, and resource enforcement still need an isolated Linux +pilot. **No empirical database comparison is included in this change.** + +## Prerequisites + +- One disposable Linux host: at least four CPUs and 8 GiB RAM for `primary`. +- Go **1.27.2**, a C compiler, Git, Python 3.12+, and native PostgreSQL **18.6**. +- PostgreSQL tools default to `/usr/lib/postgresql/18/bin`; override with + `--postgres-bin`. The controller uses `postgres`, `initdb`, `psql`, and + `pg_isready`, and never contacts an existing PostgreSQL service. +- Run as a non-root user. PostgreSQL starts on a newly selected loopback port + with a private data directory and synthetic, local roles. +- Primary runs require a delegated cgroup v2 parent with CPU and memory + controllers enabled. The runner creates its own child and writes its limits + before any application/database child executes. All descendants inherit the + same **3 CPU / 6 GiB** application budget. The coordinator and event generator + stay outside that child. +- The coordinator must be inside a sibling of the measured child under the + delegated parent, or otherwise have permission to migrate its children at + their common cgroup ancestor. Chowning an unrelated destination alone is not + sufficient. `memory.reclaim` is required to remove inactive prior-backend + charges between serial runs; the script never clears the host page cache. +- Capacity guards are S:4 GiB, B:12 GiB, L:128 GiB free before generation. A + second guard uses eight times the *measured* canonical file size before each + restore/import. These are conservative guards, not dataset-size claims. + +## Commands + +First run the portable controls and compile/protocol checks: + +```sh +python3 -m unittest discover -s scripts/postgres-benchmark -p 'test_*.py' +``` + +Pilot on the same Linux host intended for the comparison: + +```sh +python3 scripts/postgres-benchmark/run.py --repo . \ + --candidate-sha --corpus S --pairs 1 \ + --profile smoke --ingest-rate 10 --http-rate 5 \ + --output /tmp/corescope-bench-pilot +``` + +Smoke results are explicitly ineligible for a primary comparison. Primary: + +```sh +python3 scripts/postgres-benchmark/run.py --repo . \ + --candidate-sha --corpus B --pairs 5 \ + --profile primary --cgroup "$CORESCOPE_BENCH_CGROUP" \ + --output /tmp/corescope-bench-primary +``` + +The baseline is fixed to `9dbc287579a237ffa744dd0c91fa7227d09763ac`. Floating +revisions and existing output directories are refused. Primary defaults to +60 seconds of warmup and 180 measured seconds per backend, with 100 offered +ingest events/second and 20 HTTP requests/second. Five alternating A/B pairs +therefore spend 40 minutes in load windows, plus preparation, imports, +validation, startup and drain time. The longer design profile is explicit: +`--warmup 120 --seconds 600`. Record and compare identical locked settings; +changing rates after a pilot produces a separately identified experiment. + +## Data and correctness + +The immutable baseline creates the complete canonical SQLite schema and seeds +all 18 telemetry tables. S has 30,000 transmissions / 90,000 observations; B has +128,000 / 2,048,000; L has 1,000,000 / 16,000,000. S cycles through fanouts 1–5; +B/L use 50%×4, 20%×22, 20%×23 and 10%×50. Inputs include missing observers, +nullable and fractional signal values, integral scores, resolved/unresolved paths, and +one-, two-, and three-byte hop hashes. S/B have seven retained days and one +expired day. The current day includes a real hot-startup window. + +Scores in the timed corpus and replay are integral values supported by both +revisions. The unchanged SQLite server silently drops fractional-score rows; +including those would compare different loaded datasets after the PostgreSQL +loader correction. Separate native PostgreSQL regressions exercise fractional, +large integral and null scores through every loader and the existing HTTP +response. The benchmark never patches the baseline application to fix that bug. + +Adverts have deterministic Ed25519 identities and unique synthetic advert +labels; channel frames use the repository's existing AES/HMAC test helper. +Both are checked through the real decoder. Protocol references are MeshCore +`a366955cb2f67b8e6842d4f00d2b6a554dddd88a`: `src/Packet.cpp`, +`src/Mesh.cpp::createAdvert`, `src/helpers/AdvertDataHelpers.h`, and +`docs/packet_format.md`. No real packet bodies, identities, or locations are +fetched. The generated coordinates, names and messages are fictional. + +Each pair receives one recorded receipt-time offset. All persisted receipt and +bookkeeping timestamps shift together, preserving fractional precision. Signed +wire payloads and emission timestamps stay unchanged, so their identities +remain stable. Both backends use exactly that same pair copy and pre-generated +event file. The event file contains 50% new transmissions, 30% distinct later +observations, 15% exact duplicates and 5% late observations. Replayed existing +transmissions are selected from retained history so retention does not +introduce nondeterministic delete/recreate races. + +The shipping importer loads PostgreSQL. Before timing, every table's ordered, +typed logical digest and row count must match the pair's SQLite copy; the source +must remain unchanged. After replay and retention, counts, write errors, +relationships, and full logical parity are checked again. Only documented +operational receipt/creation times are excluded from the final digest; packet +timestamps, raw frames, IDs, scores and nulls remain part of it. A separate +200-envelope `handleMessage` control follows parity and verifies real durable +effects, rather than trusting the void callback return. + +## Measurements + +- Real server launch to first HTTP, ready header/health, and full background + startup completion, with `hotStartupHours` 0 and 1. Loaded transmission and + observation counts must agree across each pair under identical memory limits. + For S and B, both counts must also equal the complete retained corpus computed + independently from SQLite; two equally incomplete loads cannot pass. L records + bounded memory coverage for optional capacity qualification. +- Startup uses new application processes after complete data validation, so the + OS cache is warm. Mixed requests have a separate application warmup. The + manifest records these actual per-phase policies; there is no cache-state + switch or unsupported OS-cold label. +- Real `InsertTransmission` latency and durable effects; a bounded 1024-event + queue reports every dropped schedule and drain time. +- The fixed weighted API mix and production caches: nodes 20%, channels/messages + 20%, observers 10%, metrics 10%, reach/RX 10%, packets/details 15%, analytics + 10%, stats 5%. Nonempty data and requested page sizes are checked. Individual + HTTP cache hits are **unobserved**, not inferred from a fake cache-bust flag. +- Separate real channel-message SQL misses (using the existing cache-clear seam) + and observer-metric SQL queries. These are not confused with warm HTTP hits. +- Real WebSocket receipts correlated to committed new-transmission hashes. + Negative receipt-minus-return lags are retained and counted: the baseline can + expose a transmission before all its auto-committed writes finish. Multiple + hash receipts may be legitimate observation updates, and are not mislabeled + as duplicate packet bugs. +- The real 250-transmission retention path during replay, followed by an empty + retention pass. A packet-detail request for expired history becomes an + explicitly separate expected-404 class after retention starts. +- Whole application/database cgroup CPU, memory and I/O; per-process RSS/PSS, + database/WAL storage, and remaining filesystem headroom. RSS is not used as a + substitute for shared-memory-aware PSS/cgroup memory. +- Actual compiled retention and advert-preservation query plans in a separate + diagnostic phase. PostgreSQL uses `EXPLAIN (ANALYZE, BUFFERS, SETTINGS, FORMAT + JSON)`; SQLite uses `EXPLAIN QUERY PLAN`. Plan collection is outside headlines. + +The runner never weakens WAL durability, adds a cache service, or changes the +poll interval. SQLite WAL/FULL/FK and PostgreSQL fsync/synchronous-commit/full-page +writes are checked on real connections. Actual toolchain versions, binary +hashes, schemas, run order, and production-tree hashes before and after the run +are retained. The unit of variation is a paired run, not millions of requests +treated as independent experiments. Per-class p99 requires at least 1,000 +successful samples; unavailable values remain null, never zero. + +## Artifacts and limits + +Upload **only `/public/`**. It is an explicit sanitized whitelist and is +also produced on controller failure. It contains manifests, typed validation +digests, per-run JSONL/CSV samples, plans, summaries and the report. Do not upload +`.runtime`, control files, binaries, databases, raw event payload streams or +unfiltered build logs. Completed temporary per-run databases are removed only +after verification; failure preserves them. The original canonical corpus stays +available locally for investigation. + +`summary.csv` contains absolute per-run values. `paired-summary.json` contains +paired ratios, sample/pair counts and observed spread. Raw data remains the +authority. Missing prerequisites, wrong source hashes, unsafe destinations, +unverified cgroup limits, empty results, errors, loss or parity mismatches fail +the run. A timeout, error or partial artifact must never be relabeled zero work. + +The controller does not claim verified OS-cold results, a 65-minute maintenance +soak, size-target 5.5/13-GiB qualification, or an optional million-mobile-RX +extension. L is capacity qualification only when actually run. Preparation, +receipt-time freshening and protocol generation stay outside timed ingestion. +Migration, startup, diagnostics, and steady-state profiles remain separate in +the resource samples. Results measure the conversion as shipped, including its +transaction/batching/index changes; they cannot isolate the database engine alone. diff --git a/scripts/postgres-benchmark/overlays/clock_linux_test.go b/scripts/postgres-benchmark/overlays/clock_linux_test.go new file mode 100644 index 000000000..0a75fd57c --- /dev/null +++ b/scripts/postgres-benchmark/overlays/clock_linux_test.go @@ -0,0 +1,18 @@ +//go:build linux + +package main + +import ( + "syscall" + "unsafe" +) + +// CLOCK_MONOTONIC is shared across processes on this one Linux host. +func benchMono() int64 { + var ts syscall.Timespec + _, _, e := syscall.RawSyscall(syscall.SYS_CLOCK_GETTIME, 1, uintptr(unsafe.Pointer(&ts)), 0) + if e != 0 { + panic(e) + } + return ts.Sec*1e9 + ts.Nsec +} diff --git a/scripts/postgres-benchmark/overlays/clock_other_test.go b/scripts/postgres-benchmark/overlays/clock_other_test.go new file mode 100644 index 000000000..8ebc3054f --- /dev/null +++ b/scripts/postgres-benchmark/overlays/clock_other_test.go @@ -0,0 +1,6 @@ +//go:build !linux + +package main + +// Portable protocol-only tests compile here; measured runs are Linux-only. +func benchMono() int64 { panic("measurement requires Linux CLOCK_MONOTONIC") } diff --git a/scripts/postgres-benchmark/overlays/common_test.go b/scripts/postgres-benchmark/overlays/common_test.go new file mode 100644 index 000000000..de2d9a8e9 --- /dev/null +++ b/scripts/postgres-benchmark/overlays/common_test.go @@ -0,0 +1,122 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +type benchShape struct { + Transmissions int `json:"transmissions"` + Observations int `json:"observations"` + Nodes int `json:"nodes"` + Observers int `json:"observers"` + Days int `json:"days"` +} +type benchConfig struct { + EventsFile string `json:"events_file"` + CorpusInfo string `json:"corpus_info"` + Mode string `json:"mode"` + Corpus string `json:"corpus"` + Shape benchShape `json:"shape"` + Seed int64 `json:"seed"` + WireEpoch int64 `json:"wire_epoch"` + Epoch int64 `json:"epoch"` + SQLite string `json:"sqlite"` + Output string `json:"output"` + StateDir string `json:"state_dir"` + BaseURL string `json:"base_url"` + StartFile string `json:"start_file"` + Warmup int `json:"warmup"` + Seconds int `json:"seconds"` + IngestRate int `json:"ingest_rate"` + HTTPRate int `json:"http_rate"` +} + +func benchRead(t *testing.T) benchConfig { + t.Helper() + path := os.Getenv("CORESCOPE_BENCH_CONFIG") + if path == "" { + t.Skip("opt-in benchmark overlay") + } + b, e := os.ReadFile(path) + if e != nil { + t.Fatal(e) + } + var c benchConfig + if e = json.Unmarshal(b, &c); e != nil { + t.Fatal(e) + } + if !strings.Contains(filepath.Clean(c.Output), "corescope-bench-") { + t.Fatal("output is not an allowlisted disposable benchmark directory") + } + if c.Output == "" || c.Shape.Transmissions <= 0 || c.Shape.Nodes <= 0 || c.Shape.Observers < 1 { + t.Fatal("incomplete benchmark configuration") + } + return c +} +func benchJSON(t *testing.T, path string, value any) { + t.Helper() + b, e := json.MarshalIndent(value, "", " ") + if e != nil { + t.Fatal(e) + } + if e = os.WriteFile(path, append(b, '\n'), 0600); e != nil { + t.Fatal(e) + } +} +func benchOutput(t *testing.T, c benchConfig, name string) *os.File { + t.Helper() + f, e := os.OpenFile(filepath.Join(c.Output, name), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + if e != nil { + t.Fatal(e) + } + t.Cleanup(func() { f.Close() }) + return f +} +func benchStart(c benchConfig) (int64, error) { + for i := 0; i < 2400; i++ { + if b, e := os.ReadFile(c.StartFile); e == nil { + var start struct { + Mono int64 `json:"mono_ns"` + } + if e = json.Unmarshal(b, &start); e != nil { + return 0, e + } + return start.Mono, nil + } + time.Sleep(50 * time.Millisecond) + } + return 0, fmt.Errorf("benchmark start barrier timed out") +} +func benchSleep(until int64) { + if delay := until - benchMono(); delay > 0 { + time.Sleep(time.Duration(delay)) + } +} +func benchFanout(c benchConfig, i int) int { + if c.Corpus == "S" { + return i%5 + 1 + } + switch { + case i%10 < 5: + return 4 + case i%10 < 7: + return 22 + case i%10 < 9: + return 23 + default: + return 50 + } +} +func benchStamp(c benchConfig, i int) int64 { + day := (i / 10) % c.Shape.Days + if day == 0 { + return c.Epoch - 2700 + int64(i%1800) + } + return c.Epoch - int64(day)*86400 - 43200 + int64(i%3600) +} diff --git a/scripts/postgres-benchmark/overlays/ingestor_test.go b/scripts/postgres-benchmark/overlays/ingestor_test.go new file mode 100644 index 000000000..da9fa0334 --- /dev/null +++ b/scripts/postgres-benchmark/overlays/ingestor_test.go @@ -0,0 +1,529 @@ +package main + +import ( + "bytes" + "crypto/ed25519" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" +) + +// Wire format is pinned to MeshCore a366955: Packet.cpp and Mesh.cpp::createAdvert; +// group encryption reuses this revision's firmware-derived test helper. +type benchFixture struct { + c benchConfig + public []string + private []ed25519.PrivateKey + channels map[string]string +} + +func benchFixtureFor(c benchConfig) *benchFixture { + f := &benchFixture{c: c, channels: map[string]string{}} + for i := 0; i < c.Shape.Nodes; i++ { + seed := sha256.Sum256([]byte(fmt.Sprintf("corescope-bench/%d/node/%d", c.Seed, i))) + key := ed25519.NewKeyFromSeed(seed[:]) + f.private = append(f.private, key) + f.public = append(f.public, hex.EncodeToString(key.Public().(ed25519.PublicKey))) + } + for i := 0; i < 20; i++ { + name := fmt.Sprintf("#bench-%02d", i) + f.channels[name] = deriveHashtagChannelKey(name) + } + return f +} +func (f *benchFixture) packet(index, variant int, stamp int64) (*PacketData, error) { + node := index % len(f.public) + payloadType := 5 + var payload []byte + if index%10 == 0 { + payloadType = 4 + public := f.private[node].Public().(ed25519.PublicKey) + app := []byte{0x92} + location := make([]byte, 8) + binary.LittleEndian.PutUint32(location, uint32(20_000_000+node*100)) + binary.LittleEndian.PutUint32(location[4:], uint32(30_000_000+node*100)) + app = append(app, location...) + app = append(app, []byte(fmt.Sprintf("Bench%04d-%07d", node, index))...) + signed := append([]byte{}, public...) + clock := make([]byte, 4) + binary.LittleEndian.PutUint32(clock, uint32(stamp)) + signed = append(signed, clock...) + signed = append(signed, app...) + payload = append(payload, public...) + payload = append(payload, clock...) + payload = append(payload, ed25519.Sign(f.private[node], signed)...) + payload = append(payload, app...) + } else { + channelIndex := 0 + if index%10 >= 5 { + channelIndex = 1 + index%19 + } + channel := fmt.Sprintf("#bench-%02d", channelIndex) + key := f.channels[channel] + inner := make([]byte, 5) + binary.LittleEndian.PutUint32(inner, uint32(stamp)) + inner = append(inner, []byte(fmt.Sprintf("Bench%04d: synthetic sequence %d seed %d %s", node, index, f.c.Seed, strings.Repeat("x", index%40)))...) + cipher, mac := buildChannelEncrypted(key, inner) + keyBytes, _ := hex.DecodeString(key) + sum := sha256.Sum256(keyBytes) + macBytes, _ := hex.DecodeString(mac) + cipherBytes, _ := hex.DecodeString(cipher) + payload = append([]byte{sum[0]}, macBytes...) + payload = append(payload, cipherBytes...) + } + hashSize := 1 + index%3 + hopCount := 1 + index%3 + route := 1 + if index%5 == 0 && variant == 0 { + hopCount = 0 + } + if payloadType == 4 && variant == 1 { + route = 2 + hopCount = 0 + } + var path []byte + for hop := 0; hop < hopCount; hop++ { + key, _ := hex.DecodeString(f.public[(index+variant*7+hop+1)%len(f.public)]) + path = append(path, key[:hashSize]...) + } + if variant >= 1000 { + hashSize = 3 + hopCount = 2 + route = 1 + path = []byte{0xfa, byte(variant >> 8), byte(variant), 0xfb, byte(index >> 8), byte(index)} + } + wire := append([]byte{byte(payloadType<<2 | route), byte((hashSize-1)<<6 | hopCount)}, path...) + wire = append(wire, payload...) + raw := hex.EncodeToString(wire) + decoded, err := DecodePacket(raw, f.channels, true) + if err != nil { + return nil, err + } + score, snr, rssi := float64(index%5), float64(index%20)-10.5, -100.5+float64(index%25) + msg := &MQTTPacketMessage{Raw: raw, SNR: &snr, RSSI: &rssi, Score: &score, Timestamp: time.Unix(stamp, 0).UTC().Format(time.RFC3339)} + p := BuildPacketData(msg, decoded, f.public[0], "AAA", nil) + p.Timestamp = msg.Timestamp + if p.Hash == "" || p.PayloadType != payloadType { + return nil, fmt.Errorf("generated frame failed native decoder") + } + return p, nil +} + +func TestCoreScopeBenchmarkProtocol(t *testing.T) { + c := benchConfig{Seed: 20261008, Epoch: 1791451200, Shape: benchShape{Nodes: 80, Observers: 8, Transmissions: 100, Days: 8}} + f := benchFixtureFor(c) + a, _ := f.packet(0, 0, c.Epoch) + repeated, _ := f.packet(18000, 0, c.Epoch) + if a.Hash == repeated.Hash { + t.Fatal("repeated advertiser and emission time collapsed distinct corpus transmissions") + } + seen := map[string]bool{} + for i := 0; i < 100; i++ { + a, e := f.packet(i, 0, c.Epoch-int64(i)) + if e != nil { + t.Fatal(e) + } + b, e := f.packet(i, 1000, c.Epoch-int64(i)) + if e != nil { + t.Fatal(e) + } + if a.Hash != b.Hash { + t.Fatal("path change altered content identity") + } + if seen[a.Hash] { + t.Fatal("generator accidentally deduplicated distinct transmissions") + } + if a.Score == nil || *a.Score != math.Trunc(*a.Score) { + t.Fatal("timed score corpus must stay integral so the unmodified baseline loads the same rows") + } + seen[a.Hash] = true + if i%10 != 0 && !strings.HasPrefix(a.ChannelHash, "#bench-") { + t.Fatal("generated channel message did not decrypt") + } + } + for _, name := range []string{"S", "B", "L"} { + c.Corpus = name + total := 0 + for i := 0; i < 100; i++ { + total += benchFanout(c, i) + } + want := 1600 + if name == "S" { + want = 300 + } + if total != want { + t.Fatalf("fanout %s=%d", name, total) + } + } +} + +func TestCoreScopeBenchmarkEventStream(t *testing.T) { + dir := t.TempDir() + c := benchConfig{Corpus: "S", Seed: 20261008, Epoch: 1791454800, WireEpoch: 1791451200, Shape: benchShape{Nodes: 200, Observers: 16, Transmissions: 100, Observations: 300, Days: 2}, Output: dir, EventsFile: filepath.Join(dir, "one.jsonl"), Seconds: 4, IngestRate: 10} + benchMakeEvents(t, c) + first, e := os.ReadFile(c.EventsFile) + if e != nil { + t.Fatal(e) + } + c.EventsFile = filepath.Join(dir, "two.jsonl") + benchMakeEvents(t, c) + second, e := os.ReadFile(c.EventsFile) + if e != nil { + t.Fatal(e) + } + if !bytes.Equal(first, second) { + t.Fatal("same seed/epochs produced different event streams") + } + dec := json.NewDecoder(bytes.NewReader(first)) + counts := map[string]int{} + var additional *PacketData + for i := 0; i < 40; i++ { + var input benchInput + if e := dec.Decode(&input); e != nil { + t.Fatal(e) + } + if input.Sequence != i { + t.Fatal("sequence drift") + } + counts[input.Class]++ + if i == 10 { + additional = input.Packet + } + if i == 16 && (input.Packet.Hash != additional.Hash || input.Packet.PathJSON != additional.PathJSON) { + t.Fatal("duplicate event does not hit the same observation key") + } + } + for class, want := range map[string]int{"new": 20, "additional": 12, "duplicate": 6, "late": 2} { + if counts[class] != want { + t.Fatalf("event class %s=%d want %d", class, counts[class], want) + } + } +} + +func TestCoreScopeBenchmark(t *testing.T) { + c := benchRead(t) + if runtime.GOOS != "linux" { + t.Fatal("measurement requires Linux") + } + if c.Mode == "prepare" { + if err := benchPrepare(c); err != nil { + t.Fatal(err) + } + return + } + if c.Mode == "events" { + benchMakeEvents(t, c) + return + } + s, err := benchOpen(c) + if err != nil { + t.Fatal(err) + } + defer s.Close() + s.WaitForAsyncMigrations() + settings, err := benchSettings(s) + if err != nil { + t.Fatal(err) + } + benchJSON(t, filepath.Join(c.Output, "database-settings.json"), settings) + if err = s.RefreshPrefixIndex(); err != nil { + t.Fatal(err) + } + switch c.Mode { + case "replay": + benchReplay(t, c, s) + case "retention": + benchRetention(t, c, s) + case "handler": + benchHandler(t, c, s, benchFixtureFor(c)) + case "plans": + if e := os.MkdirAll(filepath.Join(c.Output, "plans"), 0700); e != nil { + t.Fatal(e) + } + for _, query := range []struct { + name, sql string + args []any + }{ + {"retention-empty", pruneAgedTransmissionIDs, []any{time.Now().UTC().AddDate(0, 0, -7).Format(time.RFC3339), pruneBatchTransmissions}}, + {"advert-preservation", legacyAdvertObservationSQL, []any{1, 1, "[]"}}, + } { + plan, e := benchExplain(s, query.sql, query.args...) + if e != nil { + t.Fatal(e) + } + benchJSON(t, filepath.Join(c.Output, "plans", query.name+".json"), map[string]any{"sql": query.sql, "args": query.args, "plan": plan, "context": "separate diagnostic after replay and retention; excluded from headline timings"}) + } + default: + t.Fatal("unsupported benchmark mode") + } +} + +type benchEvent struct { + Sequence int `json:"sequence"` + Class string `json:"class"` + Hash string `json:"hash"` + Scheduled int64 `json:"scheduled_ns"` + Started int64 `json:"started_ns"` + Completed int64 `json:"completed_ns"` + Measured bool `json:"measured"` + Dropped bool `json:"dropped"` + New bool `json:"new_transmission"` + Error string `json:"error,omitempty"` +} + +type benchInput struct { + Sequence int `json:"sequence"` + Class string `json:"class"` + Packet *PacketData `json:"packet"` +} + +func benchMakeEvents(t *testing.T, c benchConfig) { + f := benchFixtureFor(c) + file, e := os.OpenFile(c.EventsFile, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if e != nil { + t.Fatal(e) + } + defer file.Close() + enc := json.NewEncoder(file) + for sequence := 0; sequence < (c.Warmup+c.Seconds)*c.IngestRate; sequence++ { + class, index, variant := "new", c.Shape.Transmissions+sequence, 0 + part, block := sequence%20, sequence/20 + stamp := c.Epoch + int64(sequence/c.IngestRate) + if part >= 10 { + stride := c.Shape.Days * 10 + blocks := c.Shape.Transmissions / stride + if blocks < 1 { + blocks = 1 + } + index = (block%blocks)*stride + block%10 + wire := c + if c.WireEpoch != 0 { + wire.Epoch = c.WireEpoch + } + stamp = benchStamp(wire, index) + class = "additional" + variant = 1000 + part + if part >= 16 && part < 19 { + class = "duplicate" + variant = 1010 + } + if part == 19 { + class = "late" + variant = 1019 + } + } + packet, e := f.packet(index, variant, stamp) + if e != nil { + t.Fatal(e) + } + packet.ObserverID = f.public[0] + packet.Timestamp = time.Unix(c.Epoch+int64(sequence/c.IngestRate), 0).UTC().Format(time.RFC3339) + if class == "late" { + packet.Timestamp = time.Unix(benchStamp(c, index)-60, 0).UTC().Format(time.RFC3339) + } + if e := enc.Encode(benchInput{sequence, class, packet}); e != nil { + t.Fatal(e) + } + } +} + +func benchReplay(t *testing.T, c benchConfig, s *Store) { + var initialTx, initialObs int64 + if err := s.db.QueryRow(`SELECT count(*) FROM transmissions`).Scan(&initialTx); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow(`SELECT count(*) FROM observations`).Scan(&initialObs); err != nil { + t.Fatal(err) + } + file := benchOutput(t, c, "ingest.jsonl") + encoder := json.NewEncoder(file) + var outputMu sync.Mutex + emit := func(e benchEvent) { + outputMu.Lock() + defer outputMu.Unlock() + if err := encoder.Encode(e); err != nil { + t.Error(err) + } + } + if err := os.WriteFile(filepath.Join(c.Output, "replay-ready"), []byte("ready"), 0600); err != nil { + t.Fatal(err) + } + start, err := benchStart(c) + if err != nil { + t.Fatal(err) + } + benchSleep(start) + queue := make(chan benchInput, 1024) + done := make(chan struct{}) + var successful, newTx, newObs, dropped int64 + seenReplay := make(map[string]bool) + var firstError error + go func() { + defer close(done) + for input := range queue { + sequence, class, p := input.Sequence, input.Class, input.Packet + var e error + event := benchEvent{Sequence: sequence, Class: class, Scheduled: start + int64(sequence)*int64(time.Second)/int64(c.IngestRate), Measured: sequence >= c.Warmup*c.IngestRate} + if p != nil { + event.Hash = p.Hash + event.Started = benchMono() + beforeErrors := s.Stats.WriteErrors.Load() + event.New, e = s.InsertTransmission(p) + if e == nil && s.Stats.WriteErrors.Load() != beforeErrors { + e = fmt.Errorf("write-error counter advanced despite handler return") + } + event.Completed = benchMono() + if e == nil && event.New != (class == "new") { + e = fmt.Errorf("event class %s had unexpected transmission creation", class) + } + } + if e != nil { + event.Error = e.Error() + if firstError == nil { + firstError = e + } + } else { + successful++ + if event.New { + newTx++ + } + key := p.Hash + "/" + p.ObserverID + "/" + p.PathJSON + if !seenReplay[key] { + newObs++ + seenReplay[key] = true + } + } + emit(event) + } + }() + retained := make(chan benchPruneResult, 1) + go func() { + benchSleep(start + int64(c.Warmup+c.Seconds*3/4)*int64(time.Second)) + _ = os.WriteFile(filepath.Join(c.Output, "retention-started"), []byte("started"), 0600) + retained <- benchPrune(s) + }() + events := (c.Warmup + c.Seconds) * c.IngestRate + inputFile, err := os.Open(c.EventsFile) + if err != nil { + close(queue) + <-done + t.Fatal(err) + } + defer inputFile.Close() + inputDecoder := json.NewDecoder(inputFile) + var producerError error + for sequence := 0; sequence < events; sequence++ { + var input benchInput + if err := inputDecoder.Decode(&input); err != nil { + producerError = err + break + } + if input.Sequence != sequence || input.Packet == nil { + producerError = fmt.Errorf("invalid pre-generated event sequence") + break + } + scheduled := start + int64(sequence)*int64(time.Second)/int64(c.IngestRate) + benchSleep(scheduled) + select { + case queue <- input: + default: + dropped++ + emit(benchEvent{Sequence: sequence, Class: "schedule", Scheduled: scheduled, Dropped: true, Measured: sequence >= c.Warmup*c.IngestRate}) + } + } + close(queue) + select { + case <-done: + case <-time.After(120 * time.Second): + t.Fatal("bounded ingestion queue did not drain in 120 seconds") + } + retention := <-retained + benchJSON(t, filepath.Join(c.Output, "retention.json"), retention) + if retention.Error != "" { + t.Fatal(retention.Error) + } + var actualTx, actualObs int64 + if err := s.db.QueryRow(`SELECT count(*) FROM transmissions`).Scan(&actualTx); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow(`SELECT count(*) FROM observations`).Scan(&actualObs); err != nil { + t.Fatal(err) + } + verified := dropped == 0 && successful == int64(events) && producerError == nil && firstError == nil && s.Stats.WriteErrors.Load() == 0 && actualTx == initialTx+newTx-retention.Transmissions && actualObs == initialObs+newObs-retention.Observations + benchJSON(t, filepath.Join(c.Output, "ingest-validation.json"), map[string]any{"verified": verified, "scheduled": events, "completed": successful, "dropped_schedules": dropped, "expected_transmissions": initialTx + newTx - retention.Transmissions, "actual_transmissions": actualTx, "expected_observations": initialObs + newObs - retention.Observations, "actual_observations": actualObs, "write_errors": s.Stats.WriteErrors.Load(), "writer_wait_hold": s.WriterStatsSnapshot(), "finished_ns": benchMono()}) + if !verified { + t.Fatalf("durable ingest effects did not match: dropped=%d completed=%d/%d tx=%d/%d obs=%d/%d first error=%v", dropped, successful, events, actualTx, initialTx+newTx-retention.Transmissions, actualObs, initialObs+newObs-retention.Observations, firstError) + } +} + +type benchPruneResult struct { + Elapsed int64 `json:"elapsed_ns"` + Transmissions int64 `json:"transmissions_deleted"` + Observations int64 `json:"observations_deleted"` + Error string `json:"error,omitempty"` +} + +func benchPrune(s *Store) benchPruneResult { + result := benchPruneResult{} + cutoff := time.Now().UTC().AddDate(0, 0, -7).Format(time.RFC3339) + q := fmt.Sprintf("SELECT count(*) FROM observations o JOIN transmissions t ON t.id=o.transmission_id WHERE t.first_seen < '%s'", cutoff) + if err := s.db.QueryRow(q).Scan(&result.Observations); err != nil { + result.Error = err.Error() + return result + } + started := benchMono() + n, err := s.PruneOldPackets(7) + result.Elapsed = benchMono() - started + result.Transmissions = n + if err != nil { + result.Error = err.Error() + } + return result +} +func benchRetention(t *testing.T, c benchConfig, s *Store) { + r := benchPrune(s) + benchJSON(t, filepath.Join(c.Output, "retention-empty.json"), r) + if r.Error != "" || r.Transmissions != 0 || r.Observations != 0 { + t.Fatal("empty retention pass found unexpected work or failed") + } +} + +func benchHandler(t *testing.T, c benchConfig, s *Store, f *benchFixture) { + var before, after int64 + if err := s.db.QueryRow(`SELECT count(*) FROM transmissions`).Scan(&before); err != nil { + t.Fatal(err) + } + samples := make([]int64, 0, 200) + for i := 0; i < 200; i++ { + p, err := f.packet(c.Shape.Transmissions+10000000+i, 0, c.Epoch+int64(i)) + if err != nil { + t.Fatal(err) + } + payload, err := json.Marshal(map[string]any{"raw": p.RawHex, "origin": "Synthetic bench", "SNR": 6.5, "RSSI": -95.5}) + if err != nil { + t.Fatal(err) + } + msg := &mockMessage{topic: "meshcore/AAA/" + f.public[0] + "/packets", payload: payload} + started := benchMono() + handleMessage(s, "bench", MQTTSource{Name: "bench"}, msg, f.channels, nil, &Config{}) + samples = append(samples, benchMono()-started) + } + if err := s.db.QueryRow(`SELECT count(*) FROM transmissions`).Scan(&after); err != nil { + t.Fatal(err) + } + valid := after-before == 200 && s.Stats.WriteErrors.Load() == 0 + benchJSON(t, filepath.Join(c.Output, "handler.json"), map[string]any{"verified": valid, "samples_ns": samples, "new_transmissions": after - before, "classification": "separate 200-envelope mechanism control; not mixed-load throughput"}) + if !valid { + t.Fatal("real MQTT handler dropped valid synthetic envelopes") + } +} diff --git a/scripts/postgres-benchmark/overlays/postgres_adapter_test.go b/scripts/postgres-benchmark/overlays/postgres_adapter_test.go new file mode 100644 index 000000000..6d8870393 --- /dev/null +++ b/scripts/postgres-benchmark/overlays/postgres_adapter_test.go @@ -0,0 +1,48 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/url" + "os" + "strings" +) + +func benchExplain(s *Store, query string, args ...any) (any, error) { + var raw []byte + if e := s.db.QueryRow("EXPLAIN (ANALYZE,BUFFERS,SETTINGS,FORMAT JSON) "+query, args...).Scan(&raw); e != nil { + return nil, e + } + if !json.Valid(raw) { + return nil, fmt.Errorf("invalid PostgreSQL plan JSON") + } + return json.RawMessage(raw), nil +} + +func benchOpen(c benchConfig) (*Store, error) { + dsn := os.Getenv("CORESCOPE_WRITER_DATABASE_URL") + u, e := url.Parse(dsn) + if e != nil || u.Hostname() != "127.0.0.1" || !strings.HasPrefix(strings.TrimPrefix(u.Path, "/"), "bench_") { + return nil, fmt.Errorf("refusing non-disposable PostgreSQL benchmark target") + } + return OpenStoreWithState(dsn, c.StateDir, 300) +} +func benchPrepare(c benchConfig) error { + return fmt.Errorf("canonical corpus must be generated by the unmodified SQLite baseline") +} +func benchSettings(s *Store) (map[string]string, error) { + out := map[string]string{} + for _, name := range []string{"server_version", "fsync", "synchronous_commit", "full_page_writes", "shared_buffers", "work_mem", "maintenance_work_mem", "effective_cache_size", "max_connections", "jit", "max_parallel_workers_per_gather", "checkpoint_timeout", "max_wal_size", "autovacuum"} { + var v string + if e := s.db.QueryRow("SHOW " + name).Scan(&v); e != nil { + return nil, e + } + out[name] = v + } + for _, key := range []string{"fsync", "synchronous_commit", "full_page_writes"} { + if out[key] != "on" { + return nil, fmt.Errorf("PostgreSQL %s is not durable", key) + } + } + return out, nil +} diff --git a/scripts/postgres-benchmark/overlays/server_test.go b/scripts/postgres-benchmark/overlays/server_test.go new file mode 100644 index 000000000..c415cd5b3 --- /dev/null +++ b/scripts/postgres-benchmark/overlays/server_test.go @@ -0,0 +1,355 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gorilla/websocket" +) + +type benchCorpusInfo struct { + Node string `json:"node"` + Dense string `json:"dense_observer"` + Sparse string `json:"sparse_observer"` + NewPacket string `json:"new_packet"` + OldPacket string `json:"old_packet"` +} + +func benchInfo(c benchConfig) (benchCorpusInfo, error) { + var info benchCorpusInfo + b, e := os.ReadFile(c.CorpusInfo) + if e == nil { + e = json.Unmarshal(b, &info) + } + return info, e +} + +func TestCoreScopeBenchmark(t *testing.T) { + c := benchRead(t) + if runtime.GOOS != "linux" { + t.Fatal("measurement requires Linux") + } + switch c.Mode { + case "queries": + benchQueries(t, c) + case "client": + benchClient(t, c) + default: + t.Fatal("unsupported server benchmark mode") + } +} + +type benchRequest struct { + Class string `json:"class"` + Scheduled int64 `json:"scheduled_ns"` + Dispatched int64 `json:"dispatched_ns"` + Completed int64 `json:"completed_ns"` + Status int `json:"status"` + ExpectedStatus int `json:"expected_status"` + Bytes int `json:"bytes"` + SHA256 string `json:"response_sha256,omitempty"` + Error string `json:"error,omitempty"` + Measured bool `json:"measured"` + Cache string `json:"cache"` +} + +func benchQueries(t *testing.T, c benchConfig) { + target := os.Getenv("CORESCOPE_READER_DATABASE_URL") + if target == "" { + target = c.SQLite + } + db, e := OpenDB(target) + if e != nil { + t.Fatal(e) + } + defer db.Close() + info, e := benchInfo(c) + if e != nil { + t.Fatal(e) + } + out := json.NewEncoder(benchOutput(t, c, "sql-queries.jsonl")) + for i := 0; i < 200; i++ { + sample := benchRequest{Measured: true, Status: 200, Cache: "explicit SQL miss"} + var value any + if i%2 == 0 { + db.msgCacheMu.Lock() + db.msgCache = nil + db.msgCacheMu.Unlock() + sample.Class = "channel_messages_sql" + sample.Dispatched = benchMono() + rows, total, err := db.GetChannelMessages("#bench-00", 50, (i%4)*500) + sample.Completed = benchMono() + e = err + value = rows + if e == nil && (len(rows) == 0 || total < 50) { + e = fmt.Errorf("channel SQL query returned no meaningful data") + } + } else { + sample.Class = "observer_metrics_sql" + sample.Dispatched = benchMono() + rows, _, err := db.GetObserverMetrics(info.Dense, time.Unix(c.Epoch-86400, 0).UTC().Format(time.RFC3339), time.Unix(c.Epoch, 0).UTC().Format(time.RFC3339), "5m", 300) + sample.Completed = benchMono() + e = err + value = rows + if e == nil && len(rows) == 0 { + e = fmt.Errorf("metrics SQL query returned no meaningful data") + } + } + if e != nil { + sample.Error = e.Error() + } + b, _ := json.Marshal(value) + sum := sha256.Sum256(b) + sample.SHA256 = hex.EncodeToString(sum[:]) + sample.Bytes = len(b) + if e := out.Encode(sample); e != nil { + t.Fatal(e) + } + if sample.Error != "" { + t.Fatal(sample.Error) + } + } +} + +type benchEndpoint struct { + Class string `json:"class"` + Path string `json:"path"` +} + +func benchEndpoints(c benchConfig, info benchCorpusInfo) []benchEndpoint { + since24 := url.QueryEscape(time.Unix(c.Epoch-86400, 0).UTC().Format(time.RFC3339)) + since7 := url.QueryEscape(time.Unix(c.Epoch-7*86400, 0).UTC().Format(time.RFC3339)) + until := url.QueryEscape(time.Unix(c.Epoch, 0).UTC().Format(time.RFC3339)) + groups := []struct { + weight int + items []benchEndpoint + }{ + {20, []benchEndpoint{{"nodes_50", "/api/nodes?limit=50"}, {"nodes_2000", "/api/nodes?limit=2000"}, {"nodes_region", "/api/nodes?limit=50®ion=AAA&search=Synthetic"}}}, + {20, []benchEndpoint{{"channels", "/api/channels"}, {"messages_0", "/api/channels/%23bench-00/messages?limit=50&offset=0"}, {"messages_1000", "/api/channels/%23bench-00/messages?limit=50&offset=1000"}}}, + {10, []benchEndpoint{{"observers", "/api/observers"}}}, + {10, []benchEndpoint{{"metrics_dense_24h", "/api/observers/" + info.Dense + "/metrics?since=" + since24 + "&until=" + until + "&resolution=5m"}, {"metrics_sparse_7d", "/api/observers/" + info.Sparse + "/metrics?since=" + since7 + "&until=" + until + "&resolution=1h"}}}, + {10, []benchEndpoint{{"reach", "/api/nodes/" + info.Node + "/reach"}, {"rx_coverage", "/api/nodes/" + info.Node + "/rx-coverage?bbox=19,29,22,32&z=14"}}}, + {15, []benchEndpoint{{"packets_memory", "/api/packets?limit=50"}, {"packet_detail_memory", "/api/packets/" + info.NewPacket}, {"packet_detail_sql", "/api/packets/" + info.OldPacket}}}, + {10, []benchEndpoint{{"analytics_rf", "/api/analytics/rf"}, {"analytics_topology", "/api/analytics/topology"}, {"analytics_channels", "/api/analytics/channels"}, {"analytics_rf_filtered", "/api/analytics/rf?region=AAA"}}}, + {5, []benchEndpoint{{"stats", "/api/stats"}}}, + } + var result []benchEndpoint + for _, group := range groups { + for i := 0; i < group.weight; i++ { + result = append(result, group.items[i%len(group.items)]) + } + } + return result +} + +// Ignore timestamp/version metadata when deciding whether a response carries +// useful data. A successful empty response must not masquerade as fast SQL. +func benchNonempty(value any) bool { + switch v := value.(type) { + case []any: + return len(v) > 0 + case map[string]any: + if _, bad := v["error"]; bad { + return false + } + if hash, ok := v["hash"].(string); ok && hash != "" { + return true + } + for key, child := range v { + switch key { + case "total", "totalNodes", "nodeCount", "packetCount", "totalPackets", "totalTransmissions", "totalMessages", "count", "mobile_receptions", "totalLoaded": + if n, ok := child.(float64); ok && n > 0 { + return true + } + } + switch child.(type) { + case []any, map[string]any: + if benchNonempty(child) { + return true + } + } + } + } + return false +} + +func benchArrayCount(v any) int { + if rows, ok := v.([]any); ok { + return len(rows) + } + if obj, ok := v.(map[string]any); ok { + for _, key := range []string{"nodes", "packets", "messages", "metrics", "samples", "observers", "features", "data"} { + if child, found := obj[key]; found { + if n := benchArrayCount(child); n > 0 { + return n + } + } + } + } + return 0 +} + +func benchClient(t *testing.T, c benchConfig) { + info, e := benchInfo(c) + if e != nil { + t.Fatal(e) + } + endpoints := benchEndpoints(c, info) + benchJSON(t, filepath.Join(c.Output, "http-workload.json"), endpoints) + requests := json.NewEncoder(benchOutput(t, c, "requests.jsonl")) + messages := json.NewEncoder(benchOutput(t, c, "websocket.jsonl")) + conn, _, e := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(c.BaseURL, "http")+"/ws", nil) + if e != nil { + t.Fatal(e) + } + defer conn.Close() + conn.SetReadLimit(8 << 20) + var closing atomic.Bool + var wsErr atomic.Pointer[string] + wsDone := make(chan struct{}) + go func() { + defer close(wsDone) + for { + _, raw, err := conn.ReadMessage() + if err != nil { + if !closing.Load() { + value := err.Error() + wsErr.Store(&value) + } + return + } + var frame struct { + Type string `json:"type"` + Data struct { + Hash string `json:"hash"` + ID int64 `json:"id"` + } `json:"data"` + } + if err = json.Unmarshal(raw, &frame); err != nil { + value := err.Error() + wsErr.Store(&value) + return + } + if frame.Type == "packet" && frame.Data.Hash != "" { + if err := messages.Encode(map[string]any{"hash": frame.Data.Hash, "id": frame.Data.ID, "received_ns": benchMono(), "bytes": len(raw)}); err != nil { + value := err.Error() + wsErr.Store(&value) + return + } + } + } + }() + if e := os.WriteFile(filepath.Join(c.Output, "client-ready"), []byte("ready"), 0600); e != nil { + t.Fatal(e) + } + start, e := benchStart(c) + if e != nil { + t.Fatal(e) + } + client := &http.Client{Timeout: 15 * time.Second} + defer client.CloseIdleConnections() + limit := make(chan struct{}, 64) + var wg sync.WaitGroup + var mu sync.Mutex + var measuredErrors int64 + emit := func(sample benchRequest) { + mu.Lock() + defer mu.Unlock() + if sample.Measured && sample.Error != "" { + measuredErrors++ + } + if e := requests.Encode(sample); e != nil { + t.Error(e) + } + } + for i := 0; i < (c.Warmup+c.Seconds)*c.HTTPRate; i++ { + endpoint := endpoints[i%len(endpoints)] + sample := benchRequest{Class: endpoint.Class, Scheduled: start + int64(i)*int64(time.Second)/int64(c.HTTPRate), Measured: i >= c.Warmup*c.HTTPRate, Cache: "production cache; individual hit/miss unobserved"} + benchSleep(sample.Scheduled) + select { + case limit <- struct{}{}: + default: + sample.Error = "load generator concurrency cap reached" + sample.Completed = benchMono() + emit(sample) + continue + } + wg.Add(1) + go func(endpoint benchEndpoint, sample benchRequest) { + defer wg.Done() + defer func() { <-limit }() + sample.Dispatched = benchMono() + response, err := client.Get(c.BaseURL + endpoint.Path) + if err != nil { + sample.Error = err.Error() + } else { + sample.Status = response.StatusCode + raw, readErr := io.ReadAll(io.LimitReader(response.Body, (8<<20)+1)) + response.Body.Close() + sample.Bytes = len(raw) + sum := sha256.Sum256(raw) + sample.SHA256 = hex.EncodeToString(sum[:]) + var value any + switch { + case readErr != nil: + sample.Error = readErr.Error() + case len(raw) > 8<<20: + sample.Error = "response exceeded bounded client limit" + case sample.Status == 404 && endpoint.Class == "packet_detail_sql": + if _, err := os.Stat(filepath.Join(c.Output, "retention-started")); err != nil { + sample.Error = "history packet missing before retention" + } else { + sample.Class = "packet_detail_pruned" + sample.ExpectedStatus = 404 + } + case sample.Status != 200: + sample.Error = fmt.Sprintf("HTTP %d", sample.Status) + case json.Unmarshal(raw, &value) != nil: + sample.Error = "invalid JSON response" + case (endpoint.Class == "nodes_50" || endpoint.Class == "messages_0" || endpoint.Class == "messages_1000" || endpoint.Class == "packets_memory") && benchArrayCount(value) != 50: + sample.Error = "bounded page did not return the expected 50 real records" + case endpoint.Class == "nodes_2000" && benchArrayCount(value) != 2000: + sample.Error = "node directory did not return the expected 2000 records" + case !benchNonempty(value): + sample.Error = "empty or invalid benchmark result" + } + } + sample.Completed = benchMono() + emit(sample) + }(endpoint, sample) + } + wg.Wait() + // The writer may still be draining its bounded queue. The completion marker + // is authoritative; allow the unchanged poller to catch up afterward. + deadline := time.Now().Add(150 * time.Second) + for time.Now().Before(deadline) { + if _, e := os.Stat(filepath.Join(c.Output, "ingest-validation.json")); e == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + time.Sleep(10 * time.Second) + closing.Store(true) + conn.Close() + <-wsDone + benchJSON(t, filepath.Join(c.Output, "client-validation.json"), map[string]any{"verified": measuredErrors == 0 && wsErr.Load() == nil, "measured_errors": measuredErrors, "websocket_error": wsErr.Load(), "finished_ns": benchMono()}) + if measuredErrors > 0 { + t.Fatalf("%d measured HTTP requests failed validation", measuredErrors) + } + if e := wsErr.Load(); e != nil { + t.Fatal(*e) + } +} diff --git a/scripts/postgres-benchmark/overlays/sqlite_adapter_test.go b/scripts/postgres-benchmark/overlays/sqlite_adapter_test.go new file mode 100644 index 000000000..ab9a3b60a --- /dev/null +++ b/scripts/postgres-benchmark/overlays/sqlite_adapter_test.go @@ -0,0 +1,306 @@ +package main + +import ( + "database/sql" + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/meshcore-analyzer/packetpath" +) + +func TestCoreScopeBenchmarkSmallCorpus(t *testing.T) { + dir := t.TempDir() + c := benchConfig{Corpus: "S", Seed: 20261008, Epoch: 1791451200, SQLite: filepath.Join(dir, "corpus.sqlite"), Output: dir, Shape: benchShape{Transmissions: 100, Observations: 300, Nodes: 200, Observers: 16, Days: 2}} + if e := benchPrepare(c); e != nil { + t.Fatal(e) + } + db, e := sql.Open("sqlite3", c.SQLite+"?mode=ro") + if e != nil { + t.Fatal(e) + } + defer db.Close() + for table, want := range map[string]int{"transmissions": 100, "observations": 300, "nodes": 200, "observers": 16} { + var n int + if e := db.QueryRow("SELECT count(*) FROM " + table).Scan(&n); e != nil { + t.Fatal(e) + } + if n != want { + t.Fatalf("%s=%d want %d", table, n, want) + } + } + var invalid int + if e := db.QueryRow(`SELECT count(*) FROM observations o LEFT JOIN transmissions t ON t.id=o.transmission_id WHERE t.id IS NULL`).Scan(&invalid); e != nil { + t.Fatal(e) + } + if invalid != 0 { + t.Fatal("generator created orphans") + } + if e := db.QueryRow(`SELECT count(*) FROM observations WHERE score IS NOT NULL AND typeof(score) != 'integer'`).Scan(&invalid); e != nil { + t.Fatal(e) + } + if invalid != 0 { + t.Fatalf("%d fractional scores would make the unchanged baseline silently discard corpus rows", invalid) + } + // The handler returns void, so prove its durable effects rather than + // treating delivery to the callback as a successful ingestion. + db.Close() + store, e := OpenStore(c.SQLite) + if e != nil { + t.Fatal(e) + } + defer store.Close() + store.WaitForAsyncMigrations() + f := benchFixtureFor(c) + packet, e := f.packet(1_000_001, 0, c.Epoch) + if e != nil { + t.Fatal(e) + } + body, _ := json.Marshal(map[string]any{"raw": packet.RawHex, "origin": "Synthetic bench", "SNR": 6.5, "RSSI": -95.5}) + handleMessage(store, "bench", MQTTSource{Name: "bench"}, &mockMessage{topic: "meshcore/AAA/" + f.public[0] + "/packets", payload: body}, f.channels, nil, &Config{}) + var inserted int + if e := store.db.QueryRow(`SELECT count(*) FROM transmissions WHERE hash=?`, packet.Hash).Scan(&inserted); e != nil { + t.Fatal(e) + } + if inserted != 1 || store.Stats.WriteErrors.Load() != 0 { + t.Fatal("handler callback did not produce a durable synthetic transmission") + } +} + +func benchOpen(c benchConfig) (*Store, error) { return OpenStore(c.SQLite) } +func benchSettings(s *Store) (map[string]string, error) { + out := map[string]string{} + for _, name := range []string{"journal_mode", "synchronous", "foreign_keys", "busy_timeout", "cache_size", "page_size", "mmap_size", "cache_spill", "journal_size_limit", "wal_autocheckpoint"} { + var v string + if e := s.db.QueryRow("PRAGMA " + name).Scan(&v); e != nil { + return nil, e + } + out[name] = v + } + var version string + if e := s.db.QueryRow(`SELECT sqlite_version()`).Scan(&version); e != nil { + return nil, e + } + out["sqlite_version"] = version + if out["journal_mode"] != "wal" || out["synchronous"] != "2" || out["foreign_keys"] != "1" { + return nil, fmt.Errorf("SQLite durability differs from WAL/FULL/FK contract") + } + return out, nil +} + +func benchExplain(s *Store, query string, args ...any) (any, error) { + rows, e := s.db.Query("EXPLAIN QUERY PLAN "+query, args...) + if e != nil { + return nil, e + } + defer rows.Close() + type step struct { + ID int `json:"id"` + Parent int `json:"parent"` + Detail string `json:"detail"` + } + var result []step + for rows.Next() { + var item step + var unused int + if e := rows.Scan(&item.ID, &item.Parent, &unused, &item.Detail); e != nil { + return nil, e + } + result = append(result, item) + } + return result, rows.Err() +} + +func benchPrepare(c benchConfig) error { + if _, e := os.Stat(c.SQLite); !os.IsNotExist(e) { + return fmt.Errorf("corpus path must be new") + } + s, e := OpenStore(c.SQLite) + if e != nil { + return e + } + defer s.Close() + s.WaitForAsyncMigrations() + f := benchFixtureFor(c) + tx, e := s.db.Begin() + if e != nil { + return e + } + defer tx.Rollback() + stamp := time.Unix(c.Epoch, 0).UTC().Format(time.RFC3339) + exec := func(q string, args ...any) error { _, e := tx.Exec(q, args...); return e } + for i, pk := range f.public { + role := []string{"repeater", "companion", "room", "sensor"}[i%4] + if e := exec(`INSERT INTO nodes(public_key,name,role,lat,lon,first_seen,last_seen,advert_count) VALUES(?,?,?,?,?,?,?,0)`, pk, fmt.Sprintf("Synthetic %04d", i), role, 20+float64(i%2000)/10000, 30+float64(i%2000)/10000, stamp, stamp); e != nil { + return e + } + } + for i := 0; i < c.Shape.Observers; i++ { + if e := exec(`INSERT INTO observers(rowid,id,name,iata,last_seen,first_seen,packet_count,noise_floor,can_relay,can_relay_seen) VALUES(?,?,?,?,?,?,0,-110.5,1,1)`, i+1, f.public[i], fmt.Sprintf("Synthetic observer %d", i), []string{"AAA", "BBB", "CCC", "DDD"}[i%4], stamp, stamp); e != nil { + return e + } + } + transmission, e := tx.Prepare(`INSERT INTO transmissions(id,raw_hex,hash,first_seen,route_type,payload_type,payload_version,decoded_json,channel_hash,from_pubkey,last_seen,created_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)`) + if e != nil { + return e + } + defer transmission.Close() + observation, e := tx.Prepare(`INSERT INTO observations(transmission_id,observer_idx,direction,snr,rssi,score,path_json,timestamp,raw_hex,resolved_path) VALUES(?,?,?,?,?,?,?,?,?,?)`) + if e != nil { + return e + } + defer observation.Close() + evidence, e := tx.Prepare(`INSERT INTO advert_route_evidence(tx_id,bit) VALUES(?,?)`) + if e != nil { + return e + } + defer evidence.Close() + var observationCount int64 + countsByDay := make([]int, c.Shape.Days) + var newest, oldest string + for i := 0; i < c.Shape.Transmissions; i++ { + at := benchStamp(c, i) + p, e := f.packet(i, 0, at) + if e != nil { + return e + } + count := benchFanout(c, i) + if _, e = transmission.Exec(i+1, p.RawHex, p.Hash, p.Timestamp, p.RouteType, p.PayloadType, p.PayloadVersion, p.DecodedJSON, p.ChannelHash, nilIfEmpty(p.FromPubkey), at+int64(count-1), stamp); e != nil { + return e + } + bits := packetpath.AdvertRouteEvidence(p.RawHex) + for o := 0; o < count; o++ { + seen, e := f.packet(i, o, at) + if e != nil { + return e + } + var observer any = (o % c.Shape.Observers) + 1 + if o%17 == 16 { + observer = nil + } + var snr any = float64((i+o)%20) - 10.5 + if o%11 == 10 { + snr = nil + } + var resolved any + if i%3 != 0 && seen.PathJSON != "[]" { + b, _ := json.Marshal([]string{f.public[(i+o*7+1)%len(f.public)]}) + resolved = string(b) + } + if _, e = observation.Exec(i+1, observer, "rx", snr, -100.5+float64(o%25), p.Score, seen.PathJSON, at+int64(o), seen.RawHex, resolved); e != nil { + return e + } + observationCount++ + bits |= packetpath.AdvertRouteEvidence(seen.RawHex) + } + if p.PayloadType == 4 { + for _, bit := range []uint8{1, 2} { + if bits&bit != 0 { + if _, e = evidence.Exec(i+1, bit); e != nil { + return e + } + } + } + } + countsByDay[(i/10)%c.Shape.Days]++ + if i == 0 { + newest = p.Hash + } + if i == 70 { + oldest = p.Hash + } + } + if observationCount != int64(c.Shape.Observations) { + return fmt.Errorf("fanout generated %d observations, expected %d", observationCount, c.Shape.Observations) + } + if e := exec(`UPDATE nodes SET advert_count=(SELECT count(*) FROM transmissions t WHERE t.payload_type=4 AND t.from_pubkey=nodes.public_key)`); e != nil { + return e + } + for i := 0; i < c.Shape.Nodes; i++ { + if e := exec(`INSERT INTO neighbor_edges(node_a,node_b,count,last_seen) VALUES(?,?,?,?)`, f.public[i], f.public[(i+1)%len(f.public)], 1+i%100, stamp); e != nil { + return e + } + } + metric, e := tx.Prepare(`INSERT INTO observer_metrics(observer_id,timestamp,noise_floor,tx_air_secs,rx_air_secs,recv_errors,battery_mv,packets_sent,packets_recv) VALUES(?,?,?,?,?,?,?,?,?)`) + if e != nil { + return e + } + defer metric.Close() + for observer := 0; observer < c.Shape.Observers; observer++ { + for sample := 0; sample < c.Shape.Days*288; sample++ { + if observer == c.Shape.Observers-1 && sample%48 != 0 { + continue + } + at := time.Unix(c.Epoch-int64(c.Shape.Days*86400)+int64(sample*300), 0).UTC().Format(time.RFC3339) + if _, e := metric.Exec(f.public[observer], at, -110.5+float64(observer%5), sample*2, sample*9, sample%3, 3700+observer%200, sample*3, sample*20); e != nil { + return e + } + } + } + // Every enabled mobile feature has deterministic data; this is the base + // corpus extension, not a claim to have run the optional million-RX corpus. + reception, e := tx.Prepare(`INSERT INTO client_receptions(rx_pubkey,heard_key,heard_keylen,rssi,snr,lat,lon,pos_acc_m,rx_at,ingested_at,src) VALUES(?,?,?,?,?,?,?,?,?,?,?)`) + if e != nil { + return e + } + defer reception.Close() + for i := 0; i < c.Shape.Transmissions; i++ { + key := f.public[i%len(f.public)] + size := []int{32, 8, 3, 2}[i%4] + at := time.Unix(benchStamp(c, i), int64(i)).UTC().Format("2006-01-02T15:04:05.000000000Z") + if _, e := reception.Exec(f.public[(i+100)%len(f.public)], key[:size*2], size, -95, 6.5, 20+float64(i%2000)/10000, 30+float64(i%2000)/10000, 5, at, stamp, "synthetic"); e != nil { + return e + } + } + for i := 0; i < 100; i++ { + pk := f.public[i%len(f.public)] + at := time.Unix(c.Epoch-int64(i*300), 0).UTC().Format(time.RFC3339) + if e := exec(`INSERT INTO client_observers(pubkey,name,last_seen) VALUES(?,?,?)`, pk, fmt.Sprintf("Synthetic mobile %d", i), stamp); e != nil { + return e + } + if e := exec(`INSERT INTO node_declared_regions(target,rx_pubkey,observed_at,ingested_at,regions_csv,truncated,lat,lon) VALUES(?,?,?,?,?,0,20,30)`, pk, pk, at, stamp, "#bench"); e != nil { + return e + } + if e := exec(`INSERT INTO client_rf_samples(rx_pubkey,sampled_at,ingested_at,lat,lon,stationary,uptime_secs,noise_floor,last_snr) VALUES(?,?,?,20,30,0,1000,-110,6.5)`, pk, at, stamp); e != nil { + return e + } + if e := exec(`INSERT INTO dropped_packets(hash,raw_hex,reason,observer_id,dropped_at) VALUES(?,?,?,?,?)`, fmt.Sprintf("synthetic-drop-%d", i), "", "synthetic-rejection", pk, stamp); e != nil { + return e + } + } + if e := exec(`INSERT INTO client_rx_observations(rx_pubkey,rx_at,ingested_at,pkt_hash,route_type,payload_type,hash_size,hop_count,lat,lon) SELECT ?,first_seen,?,hash,route_type,payload_type,1,0,20,30 FROM transmissions WHERE id<=100`, f.public[0], stamp); e != nil { + return e + } + if e := exec(`INSERT INTO inactive_nodes(public_key,name,role,first_seen,last_seen,advert_count) VALUES('ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff','Synthetic inactive','repeater',?,?,0)`, stamp, stamp); e != nil { + return e + } + if e := exec(`INSERT INTO advert_evidence_backfill(id,tx_cursor,obs_cursor) VALUES(1,?,?)`, c.Shape.Transmissions, c.Shape.Observations); e != nil { + return e + } + if e := exec(`UPDATE _async_migrations SET started_at=?,ended_at=?`, stamp, stamp); e != nil { + return e + } + if e := exec(`INSERT OR REPLACE INTO _async_migrations(name,status,started_at,ended_at) VALUES('advert_route_evidence_v1','done',?,?)`, stamp, stamp); e != nil { + return e + } + if e := exec(`UPDATE scope_match_totals SET since_unix=?,updated_unix=?`, c.Epoch, c.Epoch); e != nil { + return e + } + if e := tx.Commit(); e != nil { + return e + } + if _, e := s.db.Exec(`ANALYZE`); e != nil { + return e + } + summary := map[string]any{"transmissions": c.Shape.Transmissions, "observations": observationCount, "transmissions_by_day": countsByDay, "nodes": c.Shape.Nodes, "observers": c.Shape.Observers, "channels": 20, "rx_rows": c.Shape.Transmissions, "dense_observer": f.public[0], "sparse_observer": f.public[c.Shape.Observers-1], "node": f.public[0], "new_packet": newest, "old_packet": oldest, "epoch": c.Epoch, "protocol_source": "meshcore-dev/MeshCore@a366955cb2f67b8e6842d4f00d2b6a554dddd88a"} + b, e := json.MarshalIndent(summary, "", " ") + if e != nil { + return e + } + return os.WriteFile(filepath.Join(c.Output, "corpus.json"), b, 0600) +} + +var _ *sql.DB // database/sql remains the same application boundary on both sides. diff --git a/scripts/postgres-benchmark/run.py b/scripts/postgres-benchmark/run.py new file mode 100644 index 000000000..59dc8ca40 --- /dev/null +++ b/scripts/postgres-benchmark/run.py @@ -0,0 +1,1009 @@ +#!/usr/bin/env python3 +"""Isolated, serial, source-pinned CoreScope application comparison. + +No production service is contacted. A successful invocation means the requested +measurements and validation completed, not that PostgreSQL is faster. +""" +import argparse +import csv +import decimal +import datetime +import errno +import hashlib +import json +import math +import os +from pathlib import Path +import platform +import re +import shutil +import signal +import socket +import sqlite3 +import subprocess +import sys +import tarfile +import threading +import time +import urllib.error +import urllib.parse +import urllib.request + +BASELINE = "9dbc287579a237ffa744dd0c91fa7227d09763ac" +FIRMWARE = "a366955cb2f67b8e6842d4f00d2b6a554dddd88a" +HERE = Path(__file__).resolve().parent +PROCESSES = [] +TABLES = ["nodes", "inactive_nodes", "observers", "transmissions", "observations", "observer_metrics", "neighbor_edges", "dropped_packets", "client_receptions", "client_observers", "client_rx_observations", "client_rf_samples", "node_declared_regions", "scope_match_totals", "_migrations", "_async_migrations", "advert_route_evidence", "advert_evidence_backfill"] +# These are operational receipt/creation times, not packet timestamps. Full +# pre-run parity includes them; only post-replay logical parity excludes them. +VOLATILE = {"transmissions": {"created_at"}, "observers": {"last_seen", "last_packet_at", "first_seen"}, "_async_migrations": {"started_at", "ended_at"}, "scope_match_totals": {"updated_unix", "since_unix"}} +TEXT_TIMES = {"first_seen", "last_seen", "created_at", "last_packet_at", "clock_last_naive_at", "configured_scope_at", "timestamp", "dropped_at", "rx_at", "ingested_at", "sampled_at", "observed_at", "started_at", "ended_at"} + + +def shift_timestamp(value, delta): + if value is None or value == "": + return value + if not re.fullmatch(r"\d{4}-\d\d-\d\d[T ]\d\d:\d\d:\d\d(?:\.\d+)?(?:Z|[+-]\d\d:\d\d)?", value): + raise ValueError("unsupported synthetic timestamp") + head = datetime.datetime.strptime(value[:19].replace("T", " "), "%Y-%m-%d %H:%M:%S") + datetime.timedelta(seconds=delta) + return head.strftime("%Y-%m-%d" + value[10] + "%H:%M:%S") + value[19:] + + +def freshen(path, delta): + """One recorded receipt-time offset; signed wire bytes remain unchanged.""" + db = sqlite3.connect(path) + try: + db.create_function("bench_shift", 1, lambda value: shift_timestamp(value, delta)) + for table in TABLES: + columns = list(db.execute(f'PRAGMA table_info("{table}")')) + updates = [] + for _, name, kind, *_ in columns: + if name not in TEXT_TIMES and not (table == "scope_match_totals" and name in ("since_unix", "updated_unix")): + continue + if kind.upper() in ("INTEGER", "INT", "BIGINT"): + updates.append(f'"{name}"="{name}"+{int(delta)}') + elif kind.upper() in ("TEXT", "DATETIME"): + updates.append(f'"{name}"=bench_shift("{name}")') + if updates: + db.execute(f'UPDATE "{table}" SET ' + ",".join(updates)) + db.commit() + db.execute("PRAGMA wal_checkpoint(TRUNCATE)") + finally: + db.close() + + +def corpus(name): + return {"S": dict(transmissions=30_000, observations=90_000, nodes=2000, observers=32, days=8), "B": dict(transmissions=128_000, observations=2_048_000, nodes=2000, observers=128, days=8), "L": dict(transmissions=1_000_000, observations=16_000_000, nodes=5000, observers=512, days=30)}[name].copy() + + +def fanout(name, index): + return index % 5 + 1 if name == "S" else (4 if index % 10 < 5 else 22 if index % 10 < 7 else 23 if index % 10 < 9 else 50) + + +def pair_order(pairs): + return [(i, backend) for i in range(pairs) for backend in (("sqlite", "postgres") if i % 2 == 0 else ("postgres", "sqlite"))] + + +def percentiles(values): + values = sorted(values) + def q(p): + return values[max(0, math.ceil(p * len(values)) - 1)] if values else None + return dict(samples=len(values), p50=q(.50), p95=q(.95), p99=q(.99) if len(values) >= 1000 else None, maximum=q(1)) + + +def visibility(ingest, messages): + sent = {e["hash"]: e["completed_ns"] for e in ingest if e.get("new_transmission") and not e.get("error") and not e.get("dropped")} + received, duplicate_receipts = {}, 0 + for message in messages: + key = message.get("hash") + if key not in sent: + continue + if key in received: + duplicate_receipts += 1 + received[key] = min(received.get(key, message["received_ns"]), message["received_ns"]) + missing = sorted(set(sent) - set(received)) + lags = [received[key] - committed for key, committed in sent.items() if key in received] + return dict(verified=not missing, sentinels=len(sent), received=len(received), missing=len(missing), missing_hashes=missing[:100], duplicate_hash_receipts=duplicate_receipts, visible_before_ingest_return=sum(x < 0 for x in lags), lag_ns=percentiles(lags), samples_ns=lags) + + +def json_lines(path): + with open(path) as stream: + for line in stream: + yield json.loads(line) + + +def canonical_row(row): + out = [] + for value in row: + if value is None: + out.append(["null"]) + elif isinstance(value, (int, float, decimal.Decimal)) and not isinstance(value, bool): + number = decimal.Decimal(str(value)) + if not number.is_finite(): + raise ValueError("non-finite number in corpus") + out.append(["number", str(number.normalize()) if number else "0"]) + elif isinstance(value, str): + out.append(["text", value]) + else: + raise ValueError("unsupported corpus cell type") + return (json.dumps(out, ensure_ascii=False, separators=(",", ":")) + "\n").encode() + + +def safe_output(path): + path = Path(path).absolute() + if not path.name.startswith("corescope-bench-") or path.exists() or path.is_symlink(): + raise ValueError("output must be a new directory named corescope-bench-") + if not path.parent.is_dir(): + raise ValueError("output parent must already exist") + return path.resolve() + + +def check_revisions(baseline, candidate): + if baseline != BASELINE or not re.fullmatch(r"[0-9a-f]{40}", candidate): + raise ValueError("use the fixed upstream baseline and a complete 40-character candidate commit SHA") + + +def validate_profile(profile, pairs, cgroup): + if profile == "primary" and (pairs != 5 or not cgroup): + raise ValueError("primary requires five paired runs and a delegated CPU/memory cgroup") + if not 1 <= pairs <= 10: + raise ValueError("pairs must be between one and ten") + + +def write_json(path, value): + Path(path).parent.mkdir(parents=True, exist_ok=True) + temporary = Path(str(path) + ".tmp") + temporary.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + temporary.replace(path) + + +def safe_diagnostics(text): + """Keep diagnostic lines only; never publish arbitrary child log tails.""" + out = [] + for line in text.splitlines(): + line = line.strip() + if not re.match(r"(?:--- FAIL:|FAIL\b|panic:|fatal error:|[^\s]*\.go:\d+(?::\d+)?:|ERROR:|FATAL:|error:|benchmark:|migrate:)", line): + codes = re.findall(r"SQLSTATE [0-9A-Z]{5}", line) + if codes: + out.extend(codes) + continue + if line.startswith("panic:") and not line.startswith("panic: runtime error:"): + line = "panic: " + elif re.search(r"\b(?:host|hostaddr|user|dbname|password|sslmode)\s*=", line, re.I): + line = "database connection error: " + else: + line = re.sub(r"[a-zA-Z][a-zA-Z0-9+.-]*://\S+", "", line) + line = re.sub(r"\b[A-Za-z]:[\\/]\S+|(?", line) + line = re.sub(r"\{.*\}|\[.*\]", "", line) + line = re.sub(r'"[^"\n]*"|\x27[^\x27\n]*\x27', "", line) + line = re.sub(r"\b(?:raw|payload|packet|message|body|authorization|cookie|password|passwd|secret|token|dsn|database_url|credentials?)\s*[:=].*", "", line, flags=re.I) + line = re.sub(r"\b[0-9a-fA-F]{16,}\b|(?", line) + out.append(line[:240]) + return out[-10:] + + +def record_failure(log, exit_code): + log = Path(log) + with open(log, "rb") as stream: + stream.seek(max(0, log.stat().st_size - 8192)) + tail = stream.read(8192).decode("utf-8", errors="replace") + result = dict(log=log.name, exit_code=exit_code, diagnostics=safe_diagnostics(tail), policy="At most 10 sanitized diagnostic lines from the final 8 KiB; raw logs and event bodies excluded") + write_json(log.with_name("failure-" + log.stem + ".json"), result) + return result + + +def public_bundle(output, repository=None): + """Explicit whitelist: never export databases, event bodies or controls.""" + output = Path(output).resolve() + public = output / "public" + public.mkdir(exist_ok=True) + patterns = ["manifest.json", "summary*.csv", "paired-summary.json", "report.md", "logs/failure-*.json", "corpus/*.json", "pair-*/corpus-offset.json", "pair-*/events.json", "pair-*/*/failure-*.json", "pair-*/*/*.jsonl", "pair-*/*/resources.csv", "pair-*/*/processes.csv", "pair-*/*/resource-final.json", "pair-*/*/*validation*.json", "pair-*/*/startup*.json", "pair-*/*/migration.json", "pair-*/*/retention*.json", "pair-*/*/handler.json", "pair-*/*/database-settings.json", "pair-*/*/reader-settings.json", "pair-*/*/http-workload.json", "pair-*/*/visibility.json", "pair-*/*/plans/*"] + for pattern in patterns: + for source in sorted(output.glob(pattern)): + if not source.is_file() or source.is_symlink(): + continue + destination = public / source.relative_to(output) + destination.parent.mkdir(parents=True, exist_ok=True) + text = source.read_text() + text = text.replace(str(output), "") + if repository: + text = text.replace(str(Path(repository).resolve()), "") + destination.write_text(text) + + +def sha256(path): + h = hashlib.sha256() + with open(path, "rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + h.update(block) + return h.hexdigest() + + +def remove_working_directory(path, runtime_root): + path, runtime_root = Path(path).resolve(), Path(runtime_root).resolve() + if runtime_root.name != ".runtime" or not path.is_relative_to(runtime_root) or not re.fullmatch(r"config-\d+-(sqlite|postgres)", path.name): + raise ValueError("refusing cleanup outside an owned benchmark run") + shutil.rmtree(path) + + +def command(args, *, cwd=None, env=None, log=None, input=None, timeout=900, budget=None): + args = [str(x) for x in args] + proc = subprocess.Popen(budget.arguments(args) if budget else args, cwd=cwd, env=env, stdin=subprocess.PIPE, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, start_new_session=True) + PROCESSES.append(proc) + try: + result, _ = proc.communicate(input, timeout=timeout) + except BaseException: + stop(proc) + raise + if log: + Path(log).write_text(result) + if proc.returncode: + if not log: + diagnostic="; ".join(safe_diagnostics(result[-8192:])) or "no recognized safe diagnostic" + raise RuntimeError(f"{Path(str(args[0])).name} failed (exit {proc.returncode}): {diagnostic}") + record_failure(log, proc.returncode) + raise RuntimeError(f"{Path(str(args[0])).name} failed (exit {proc.returncode}); see failure-{Path(log).stem}.json") + return result + + +def production_hash(root): + h = hashlib.sha256() + for folder in ("cmd", "internal", "public"): + for path in sorted((Path(root) / folder).rglob("*")): + if path.is_file() and not path.name.endswith("_test.go"): + h.update(path.relative_to(root).as_posix().encode() + b"\0") + h.update(bytes.fromhex(sha256(path))) + return h.hexdigest() + + +def archive(repo, revision, destination): + actual = command(["git", "rev-parse", revision + "^{commit}"], cwd=repo).strip() + if actual != revision: + raise ValueError("revision did not resolve to the exact requested commit") + destination.mkdir() + proc = subprocess.Popen(["git", "archive", "--format=tar", revision], cwd=repo, stdout=subprocess.PIPE) + try: + with tarfile.open(fileobj=proc.stdout, mode="r|") as archive_file: + archive_file.extractall(destination, filter="data") + finally: + proc.stdout.close() + if proc.wait() != 0: + raise RuntimeError("git archive failed") + + +class Budget: + def __init__(self, parent, required): + self.path = None + self.required = required + self.info = {"enforced": False, "cpu_limit": None, "memory_limit_bytes": None} + if not parent: + return + parent = Path(parent).resolve() + if not str(parent).startswith("/sys/fs/cgroup/") or parent == Path("/sys/fs/cgroup"): + raise ValueError("cgroup must be a delegated subtree, not the global cgroup root") + self.path = parent / ("corescope-bench-" + str(os.getpid())) + self.path.mkdir() + try: + (self.path / "cpu.max").write_text("300000 100000") + (self.path / "memory.max").write_text(str(6 * 1024 ** 3)) + if (self.path / "memory.swap.max").exists(): + (self.path / "memory.swap.max").write_text("0") + cpu, memory = 3.0, 6 * 1024 ** 3 + for ancestor in (self.path, *self.path.parents): + if ancestor == Path("/sys/fs"): + break + if (ancestor / "cpu.max").exists(): + q, period = (ancestor / "cpu.max").read_text().split() + if q != "max": + cpu = min(cpu, int(q) / int(period)) + if (ancestor / "memory.max").exists(): + value = (ancestor / "memory.max").read_text().strip() + if value != "max": + memory = min(memory, int(value)) + if cpu < 3 or memory < 6 * 1024 ** 3: + raise ValueError("ancestor cgroup limits are below the shared 3-CPU/6-GiB budget") + cpuset = self.path / "cpuset.cpus.effective" + allowed = set(os.sched_getaffinity(0)) + if cpuset.exists(): + effective = set() + for part in cpuset.read_text().strip().split(","): + bounds = [int(x) for x in part.split("-")] + effective.update(range(bounds[0], bounds[-1] + 1)) + allowed &= effective + if len(allowed) < 3: + raise ValueError("effective CPU affinity provides fewer than three application CPUs") + self.info = dict(enforced=True, cpu_limit=cpu, memory_limit_bytes=memory, allowed_cpus=sorted(allowed)) + except Exception: + self.path.rmdir() + self.path = None + raise + + def arguments(self, args): + if not self.path: + return args + # No Python preexec_fn: the resource sampler is threaded. Arguments are + # positional shell parameters, never interpolated into shell source. + return ["/bin/sh", "-c", 'printf "%s" "$$" > "$1/cgroup.procs" || exit 125; shift; exec "$@"', "corescope-bench", str(self.path), *args] + + def verify(self, pid): + if self.path: + for _ in range(100): + if str(pid) in (self.path / "cgroup.procs").read_text().split(): + return + time.sleep(.01) + raise RuntimeError("application/database process failed to enter the benchmark cgroup") + + def reset(self): + if not self.path: + return + if (self.path / "cgroup.procs").read_text().strip(): + raise RuntimeError("previous backend is still active; serial comparison refused") + reclaim = self.path / "memory.reclaim" + if not reclaim.exists(): + raise RuntimeError("primary cgroup needs memory.reclaim to remove inactive prior-backend charges") + amount = int((self.path / "memory.current").read_text()) + if amount: + try: + reclaim.write_text(str(amount)) + except OSError as error: + if error.errno != errno.EAGAIN: + raise + remaining = int((self.path / "memory.current").read_text()) + if remaining > 64 * 1024 ** 2: + raise RuntimeError("more than 64 MiB of inactive prior-backend cgroup memory remains") + self.info["inactive_charge_reset_bytes"] = remaining + try: + (self.path / "memory.peak").write_text("0") + self.info["memory_peak_reset"] = True + except OSError: + self.info["memory_peak_reset"] = False + + def close(self): + if self.path: + if (self.path / "cgroup.procs").read_text().strip(): + raise RuntimeError("owned benchmark processes remain; refusing cgroup removal") + self.path.rmdir() + + +def spawn(args, *, cwd, env, log, budget=None): + stream = open(log, "w") + args = [str(x) for x in args] + proc = subprocess.Popen(budget.arguments(args) if budget else args, cwd=cwd, env=env, stdout=stream, stderr=subprocess.STDOUT, start_new_session=True) + PROCESSES.append(proc) + proc.bench_log = stream + if budget: + budget.verify(proc.pid) + return proc + + +def finish(proc, timeout=900): + try: + result = proc.wait(timeout=timeout) + if result: + record_failure(proc.bench_log.name, result) + raise RuntimeError(f"benchmark child exited {result}; see failure-{Path(proc.bench_log.name).stem}.json") + finally: + if proc.poll() is None: + os.killpg(proc.pid, signal.SIGTERM) + try: + proc.wait(timeout=15) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + proc.bench_log.close() + + +def stop(proc): + if proc and proc.poll() is None: + os.killpg(proc.pid, signal.SIGTERM) + try: + proc.wait(timeout=30) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + if proc and getattr(proc, "bench_log", None): + proc.bench_log.close() + + +def free_port(): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +class Postgres: + def __init__(self, runtime, bin_dir, budget, logs): + self.root, self.bin, self.budget, self.logs = runtime, Path(bin_dir), budget, logs + self.data, self.port = runtime / "pgdata", free_port() + self.proc = None + command([self.bin / "initdb", "-D", self.data, "--no-locale", "--encoding=UTF8", "--auth-local=trust", "--auth-host=trust", "-U", "bench_admin"], log=logs / "initdb.log", budget=budget) + self.env = dict({k:v for k,v in os.environ.items() if not k.startswith("PG")}, PGHOST="127.0.0.1", PGPORT=str(self.port), PGUSER="bench_admin", PGDATABASE="postgres") + + def start(self): + self.proc = spawn([self.bin / "postgres", "-D", self.data, "-h", "127.0.0.1", "-p", self.port, "-c", "fsync=on", "-c", "synchronous_commit=on", "-c", "full_page_writes=on", "-c", "shared_buffers=256MB", "-c", "max_connections=32"], cwd=self.root, env=self.env, log=self.logs / "postgres.log", budget=self.budget) + for _ in range(200): + if self.proc.poll() is not None: + record_failure(self.proc.bench_log.name, self.proc.returncode) + raise RuntimeError("private PostgreSQL failed to start") + if subprocess.run([str(self.bin / "pg_isready"), "-q"], env=self.env).returncode == 0: + self.sql("SELECT 1") + return + time.sleep(.1) + raise TimeoutError("private PostgreSQL readiness timeout") + + def stop(self): + stop(self.proc) + self.proc = None + + def sql(self, query, database="postgres"): + return command([self.bin / "psql", "-X", "-qAt", "-v", "ON_ERROR_STOP=1"], env=dict(self.env, PGDATABASE=database), input=query, log=None) + + def create(self, name): + if not re.fullmatch(r"bench_[a-z0-9_]+", name): + raise ValueError("non-benchmark database name") + self.sql(f"CREATE ROLE {name}_owner LOGIN; CREATE ROLE {name}_writer LOGIN; CREATE ROLE {name}_reader LOGIN; CREATE DATABASE {name} OWNER {name}_owner TEMPLATE template0 LC_COLLATE 'C' LC_CTYPE 'C';") + return {role: f"postgres://{name}_{role}@127.0.0.1:{self.port}/{name}?sslmode=disable" for role in ("owner", "writer", "reader")} + + def grant(self, name): + self.sql(f"GRANT USAGE ON SCHEMA public TO {name}_writer,{name}_reader; GRANT SELECT ON ALL TABLES IN SCHEMA public TO {name}_writer,{name}_reader; GRANT INSERT,UPDATE,DELETE ON ALL TABLES IN SCHEMA public TO {name}_writer; GRANT USAGE,SELECT ON ALL SEQUENCES IN SCHEMA public TO {name}_writer; DO $$DECLARE t record;BEGIN FOR t IN SELECT tablename FROM pg_tables WHERE schemaname='public' AND tablename LIKE 'corescope_%' LOOP EXECUTE format('REVOKE INSERT,UPDATE,DELETE,TRUNCATE,TRIGGER ON %I FROM {name}_writer',t.tablename);END LOOP;END$$; ANALYZE;", name) + + def drop(self, name): + if not re.fullmatch(r"bench_[a-z0-9_]+", name): + raise ValueError("non-benchmark database name") + self.sql(f"DROP DATABASE {name} WITH (FORCE); DROP ROLE {name}_reader,{name}_writer,{name}_owner;") + + +def table_manifest(source): + with sqlite3.connect(f"file:{source}?mode=ro", uri=True) as db: + manifest = [] + for table in TABLES: + columns = [row[1] for row in db.execute(f'PRAGMA table_info("{table}")')] + if not columns: + raise ValueError("corpus lacks canonical table " + table) + if table == "observers": + columns.insert(0, "rowid") + manifest.append(dict(name=table, columns=columns)) + return manifest + + +def validate(source, manifest, pg=None, database=None, logical=False): + result = {} + db = sqlite3.connect(f"file:{source}?mode=ro", uri=True) if pg is None else None + try: + for table in manifest: + name = table["name"] + columns = [c for c in table["columns"] if not logical or c not in VOLATILE.get(name, set())] + select = ",".join('"' + c + '"' for c in columns) + # Ordering by the complete row is deterministic even with optional + # nullable keys. Diagnostic sorting is outside headline timing. + if pg: + # SQLite sorts NULL before values; PostgreSQL's default is last. + order = ",".join('"' + c + '" NULLS FIRST' for c in columns) + query = f'SELECT json_build_array({select})::text FROM "{name}" ORDER BY {order};' + proc = subprocess.Popen([str(pg.bin / "psql"), "-X", "-qAt", "-v", "ON_ERROR_STOP=1"], env=dict(pg.env, PGDATABASE=database), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True) + PROCESSES.append(proc) + proc.stdin.write(query) + proc.stdin.close() + rows = (json.loads(line, parse_float=decimal.Decimal) for line in proc.stdout) + else: + rows = db.execute(f'SELECT {select} FROM "{name}" ORDER BY {select}') + digest, count = hashlib.sha256(), 0 + for row in rows: + digest.update(canonical_row(row)) + count += 1 + if pg: + diagnostic = proc.stderr.read() + if proc.wait() != 0: + raise RuntimeError("PostgreSQL corpus validation query failed: " + "; ".join(safe_diagnostics(diagnostic))) + result[name] = dict(rows=count, sha256=digest.hexdigest()) + finally: + if db: + db.close() + return result + + +class Resources: + def __init__(self, path, budget, data_paths): + self.path, self.budget, self.data_paths = path, budget, data_paths + self.phase="prepare" + self.done = threading.Event() + self.error = None + self.thread = threading.Thread(target=self.sample, daemon=True) + + def sample(self): + try: + self.sample_rows() + except BaseException as error: + self.error = error + + def sample_rows(self): + with open(self.path, "w", newline="") as stream, open(self.path.parent/"processes.csv","w",newline="") as processes: + process_writer=csv.DictWriter(processes,fieldnames=["mono_ns","phase","pid","command","rss_bytes","pss_bytes"]) + process_writer.writeheader() + writer = csv.DictWriter(stream, fieldnames=["mono_ns", "phase", "filesystem_free_bytes", "cgroup_memory_bytes", "cgroup_memory_peak_bytes", "cgroup_cpu_usec", "rss_bytes", "pss_bytes", "storage_bytes", "wal_bytes", "io_stat", "memory_events"]) + writer.writeheader() + while not self.done.is_set(): + row = dict(mono_ns=time.monotonic_ns(), phase=self.phase, filesystem_free_bytes=shutil.disk_usage(self.path.parent).free, cgroup_memory_bytes="not measured", cgroup_memory_peak_bytes="not measured", cgroup_cpu_usec="not measured", rss_bytes="not measured", pss_bytes="not measured", storage_bytes=0, wal_bytes=0, io_stat="not measured", memory_events="not measured") + if self.budget.path: + c = self.budget.path + row["cgroup_memory_bytes"] = (c / "memory.current").read_text().strip() + if self.budget.info.get("memory_peak_reset"): + row["cgroup_memory_peak_bytes"]=(c/"memory.peak").read_text().strip() + cpu = dict(line.split() for line in (c / "cpu.stat").read_text().splitlines()) + row["cgroup_cpu_usec"] = cpu.get("usage_usec", "not measured") + row["io_stat"] = (c / "io.stat").read_text().strip() + row["memory_events"] = (c / "memory.events").read_text().strip() + rss = pss = 0 + for pid in (c / "cgroup.procs").read_text().split(): + try: + process_rss=process_pss=0 + for line in Path("/proc", pid, "smaps_rollup").read_text().splitlines(): + if line.startswith("Rss:"): + process_rss += int(line.split()[1]) * 1024 + if line.startswith("Pss:"): + process_pss += int(line.split()[1]) * 1024 + rss+=process_rss;pss+=process_pss + process_writer.writerow(dict(mono_ns=row["mono_ns"],phase=self.phase,pid=pid,command=Path("/proc",pid,"comm").read_text().strip(),rss_bytes=process_rss,pss_bytes=process_pss)) + except (FileNotFoundError, PermissionError, ProcessLookupError): + pass + row.update(rss_bytes=rss, pss_bytes=pss) + for root in self.data_paths: + for file in root.rglob("*") if root.is_dir() else [root, Path(str(root) + "-wal")]: + try: + if file.is_file(): + size = file.stat().st_size + row["storage_bytes"] += size + if file.name.endswith("-wal") or "pg_wal" in file.parts: + row["wal_bytes"] += size + except FileNotFoundError: + pass + writer.writerow(row) + stream.flush() + processes.flush() + self.done.wait(1) + + def __enter__(self): + self.thread.start() + return self + + def __exit__(self, *args): + self.done.set() + self.thread.join(timeout=10) + if self.thread.is_alive(): + raise RuntimeError("resource sampler failed to stop") + if self.error: + raise RuntimeError("resource sampler failed") from self.error + if self.budget.path: + c=self.budget.path + write_json(self.path.parent/"resource-final.json",dict(memory_peak_bytes=int((c/"memory.peak").read_text()) if self.budget.info.get("memory_peak_reset") else None,cpu_stat=(c/"cpu.stat").read_text(),io_stat=(c/"io.stat").read_text(),memory_events=(c/"memory.events").read_text(),budget=self.budget.info)) + + +def wait_file(path, processes, seconds=120): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + if path.exists(): + return + if any(p.poll() is not None for p in processes): + for proc in processes: + if proc.poll() is not None and getattr(proc, "bench_log", None): + record_failure(proc.bench_log.name, proc.returncode) + raise RuntimeError("benchmark worker exited before the start barrier") + time.sleep(.05) + raise TimeoutError("benchmark start barrier timeout") + + +def retained_counts(source, since_epoch): + db = sqlite3.connect(f"file:{source}?mode=ro", uri=True) + try: + tx = db.execute("SELECT count(*) FROM transmissions WHERE last_seen >= ?", (since_epoch,)).fetchone()[0] + obs = db.execute("SELECT count(*) FROM observations o JOIN transmissions t ON t.id=o.transmission_id WHERE t.last_seen >= ?", (since_epoch,)).fetchone()[0] + return dict(transmissions=tx, observations=obs) + finally: + db.close() + + +def check_loaded_rows(health, expected): + if health.get("loadedTx") != expected["transmissions"] or health.get("loadedObs") != expected["observations"]: + raise RuntimeError("startup did not load the complete retained corpus: " + json.dumps(dict(actual=health, expected=expected))) + + +def startup(server, base_url, output, expected, launched_ns, hot_hours, retained_expected=None): + start, first, ready = launched_ns, None, None + deadline = time.monotonic() + 600 + last = None + while time.monotonic() < deadline: + if server.poll() is not None: + record_failure(server.bench_log.name, server.returncode) + raise RuntimeError("server exited during startup") + try: + with urllib.request.urlopen(base_url + "/api/healthz", timeout=5) as response: + body = json.load(response) + first = first or time.monotonic_ns() + last = body + if response.status == 200 and response.headers.get("X-CoreScope-Load-Status") == "ready": + ready = time.monotonic_ns() + break + except urllib.error.HTTPError as error: + first = first or time.monotonic_ns() + last = {"ready":False,"status":error.code} + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError): + pass + time.sleep(.1) + if ready is None: + raise TimeoutError("server did not reach full health/readiness") + full = ready + perf = None + while time.monotonic() < deadline: + with urllib.request.urlopen(base_url + "/api/perf", timeout=30) as response: + perf = json.load(response) + store = perf.get("packetStore", {}) + if store.get("backgroundLoadFailed"): + raise RuntimeError("background startup load failed") + if not hot_hours or store.get("backgroundLoadComplete") is True: + full = time.monotonic_ns() + break + time.sleep(.1) + else: + raise TimeoutError("background startup did not complete") + initial_health=last + with urllib.request.urlopen(base_url+"/api/healthz",timeout=30) as response: + last=json.load(response) + if not last or last.get("loadedTx",0) <= 0 or last.get("loadedObs",0) <= 0: + raise RuntimeError("ready server did not load a meaningful corpus") + if retained_expected is not None: + check_loaded_rows(last, retained_expected) + with urllib.request.urlopen(base_url + "/api/stats", timeout=30) as response: + stats = json.load(response) + result = dict(first_http_ns=first-start, ready_ns=ready-start, full_ready_ns=full-start, hot_startup_hours=hot_hours, initial_health=initial_health, health=last, stats=stats, perf=perf, durable_expected=expected, retained_expected=retained_expected, loaded_count_status="complete retained corpus verified" if retained_expected is not None else "bounded capacity load; actual counts recorded under common memory/retention limits") + write_json(output, result) + return result + + +def summarize(output): + rows=[] + def add(pair,backend,scenario,name,values,errors=0,drops=0,throughput=None): + stats=percentiles(values) + rows.append(dict(pair=pair,backend=backend,scenario=scenario,event_class=name,samples=stats["samples"],p50_ns=stats["p50"],p95_ns=stats["p95"],p99_ns=stats["p99"],maximum_ns=stats["maximum"],errors=errors,drops=drops,completed_per_second_including_drain=throughput)) + for directory in sorted(output.glob("pair-*/*")): + if not directory.is_dir():continue + pair,backend=directory.parent.name,directory.name + for file,scenario in (("requests.jsonl","mixed-http"),("sql-queries.jsonl","sql-miss"),("ingest.jsonl","durable-ingest")): + path=directory/file + if not path.exists():continue + groups={} + for item in json_lines(path): + if not item.get("measured"):continue + group=groups.setdefault(item["class"],dict(values=[],errors=0,drops=0,first=None,last=None)) + if item.get("dropped"): + group["drops"]+=1;continue + if item.get("error") or item.get("status",200)!=(item.get("expected_status") or 200): + group["errors"]+=1;continue + begun=item.get("dispatched_ns",item.get("started_ns")) + group["values"].append(item["completed_ns"]-begun) + scheduled=item.get("scheduled_ns") or begun + group["first"]=min(group["first"] or scheduled,scheduled) + group["last"]=max(group["last"] or item["completed_ns"],item["completed_ns"]) + for name,group in sorted(groups.items()): + elapsed=(group["last"] or 0)-(group["first"] or 0) + rate=len(group["values"])/(elapsed/1e9) if elapsed>0 else None + add(pair,backend,scenario,name,group["values"],group["errors"],group["drops"],rate) + for filename in ("startup.json","startup-hot1.json"): + if (directory/filename).exists(): + obj=json.loads((directory/filename).read_text()) + for phase in ("first_http_ns","ready_ns","full_ready_ns"): + add(pair,backend,filename.removesuffix(".json"),phase,[obj[phase]]) + for filename,key in (("migration.json","migration_tool_ns"),("retention.json","elapsed_ns"),("retention-empty.json","elapsed_ns")): + if (directory/filename).exists(): + obj=json.loads((directory/filename).read_text());add(pair,backend,filename.removesuffix(".json"),key,[obj[key]]) + if (directory/"handler.json").exists(): + obj=json.loads((directory/"handler.json").read_text());add(pair,backend,"handler-control","200-valid-envelopes",obj["samples_ns"]) + if (directory/"visibility.json").exists(): + obj=json.loads((directory/"visibility.json").read_text());add(pair,backend,"websocket","new-transmission-sentinels",obj["samples_ns"],obj["missing"]) + if rows: + with open(output/"summary.csv","w",newline="") as stream: + writer=csv.DictWriter(stream,fieldnames=list(rows[0]));writer.writeheader();writer.writerows(rows) + resource_rows=[] + for path in sorted(output.glob("pair-*/*/resources.csv")): + groups={} + with open(path,newline="") as stream: + for row in csv.DictReader(stream):groups.setdefault(row["phase"],[]).append(row) + for phase,samples in groups.items(): + def numbers(key):return [int(r[key]) for r in samples if str(r.get(key,"")).isdigit()] + resource_rows.append(dict(pair=path.parts[-3],backend=path.parts[-2],phase=phase,samples=len(samples),observed_cgroup_memory_peak_bytes=max(numbers("cgroup_memory_bytes"),default=None),observed_pss_peak_bytes=max(numbers("pss_bytes"),default=None),observed_storage_peak_bytes=max(numbers("storage_bytes"),default=None),observed_wal_peak_bytes=max(numbers("wal_bytes"),default=None),minimum_filesystem_free_bytes=min(numbers("filesystem_free_bytes"),default=None))) + if resource_rows: + with open(output/"summary-resources.csv","w",newline="") as stream: + writer=csv.DictWriter(stream,fieldnames=list(resource_rows[0]));writer.writeheader();writer.writerows(resource_rows) + paired=[] + keys=sorted({(r["scenario"],r["event_class"]) for r in rows}) + for scenario,name in keys: + for metric in ("p50_ns","p95_ns","p99_ns","completed_per_second_including_drain"): + ratios=[];absolute=[] + for pair in sorted({r["pair"] for r in rows}): + values={r["backend"]:r[metric] for r in rows if (r["pair"],r["scenario"],r["event_class"])==(pair,scenario,name)} + if set(values)=={"sqlite","postgres"} and values["sqlite"] not in (None,0) and values["postgres"] is not None: + ratios.append(values["postgres"]/values["sqlite"]);absolute.append(dict(pair=pair,**values)) + if ratios: + paired.append(dict(scenario=scenario,event_class=name,metric=metric,pairs=len(ratios),candidate_over_baseline_median=percentiles(ratios)["p50"],minimum=min(ratios),maximum=max(ratios),absolute_runs=absolute)) + write_json(output/"paired-summary.json",paired) + return rows + + +def build(root, backend, binaries, env, logs): + before = production_hash(root) + replacements = {} + for package in ("ingestor", "server"): + templates = ["common_test.go", "clock_linux_test.go", "clock_other_test.go", package + "_test.go"] + if package == "ingestor": + templates.append(backend + "_adapter_test.go") + for template in templates: + replacements[str(root / "cmd" / package / ("zz_benchmark_" + template))] = str(HERE / "overlays" / template) + overlay = binaries / (backend + "-overlay.json") + write_json(overlay, {"Replace": replacements}) + for package in ("ingestor", "server"): + command(["go", "test", "-overlay", overlay, "-c", "-o", binaries / (backend + "-" + package + ".test"), "."], cwd=root / "cmd" / package, env=env, log=logs / (backend + "-" + package + "-build.log")) + command(["go", "build", "-trimpath", "-o", binaries / (backend + "-server"), "."], cwd=root / "cmd/server", env=env, log=logs / (backend + "-server-binary.log")) + if backend == "postgres": + command(["go", "build", "-trimpath", "-o", binaries / "migrate", "."], cwd=root / "cmd/migrate", env=env, log=logs / "migrate-build.log") + if production_hash(root) != before: + raise RuntimeError("build changed production sources") + return before + + +def worker(binary, config, mode, env, log, budget=None): + cfg = dict(config, mode=mode) + path = Path(config["control_dir"]) / (mode + "-config.json") + write_json(path, cfg) + return spawn([binary, "-test.run=^TestCoreScopeBenchmark$", "-test.count=1", "-test.timeout=30m", "-test.v"], cwd=path.parent, env=dict(env, CORESCOPE_BENCH_CONFIG=str(path)), log=log, budget=budget) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, default=Path.cwd()) + parser.add_argument("--baseline-sha", default=BASELINE) + parser.add_argument("--candidate-sha", required=True) + parser.add_argument("--corpus", choices=("S", "B", "L"), default="B") + parser.add_argument("--pairs", type=int, default=5) + parser.add_argument("--profile", choices=("primary", "smoke"), default="primary") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--cgroup", default=os.environ.get("CORESCOPE_BENCH_CGROUP")) + parser.add_argument("--postgres-bin", type=Path, default=Path("/usr/lib/postgresql/18/bin")) + parser.add_argument("--postgres-version", default="18.6") + parser.add_argument("--go-version", default="1.27.2") + parser.add_argument("--seed", type=int, default=20261008) + parser.add_argument("--warmup", type=int) + parser.add_argument("--seconds", type=int) + parser.add_argument("--ingest-rate", type=int, default=100) + parser.add_argument("--http-rate", type=int, default=20) + args = parser.parse_args(argv) + check_revisions(args.baseline_sha, args.candidate_sha) + validate_profile(args.profile, args.pairs, args.cgroup) + if sys.version_info < (3,12): + raise ValueError("Python 3.12 or newer is required for safe archive extraction") + if sys.platform != "linux": + raise ValueError("measured runs require one isolated Linux host; use portable unit/overlay checks here") + if os.geteuid() == 0: + raise ValueError("run as an unprivileged account with delegated cgroup permissions; initdb refuses root") + if args.profile == "primary" and (len(os.sched_getaffinity(0)) < 4 or os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE") < 8 * 1024 ** 3): + raise ValueError("primary requires at least four host CPUs and 8 GiB RAM") + for executable in ("git", "go", "cc"): + if not shutil.which(executable): + raise ValueError("missing prerequisite: " + executable) + go_version = command(["go", "version"]).strip() + if not re.search(r"\bgo" + re.escape(args.go_version) + r"\b", go_version): + raise ValueError("Go version differs from the common toolchain pin") + for executable in ("postgres", "initdb", "psql", "pg_isready"): + if not (args.postgres_bin / executable).is_file(): + raise ValueError("missing PostgreSQL prerequisite: " + executable) + version = command([args.postgres_bin / "postgres", "--version"]).strip() + if not re.search(r"\b" + re.escape(args.postgres_version) + r"\b", version): + raise ValueError("PostgreSQL patch version differs from requested benchmark pin") + output = safe_output(args.output) + minimum = {"S": 4, "B": 12, "L": 128}[args.corpus] * 1024 ** 3 + if shutil.disk_usage(output.parent).free < minimum: + raise ValueError("insufficient free space for the declared corpus safety guard (S:4/B:12/L:128 GiB); these are capacity guards, not measured dataset sizes") + output.mkdir(mode=0o700) + runtime, logs, binaries = output / ".runtime", output / "logs", output / ".runtime/binaries" + for path in (runtime, logs, binaries): + path.mkdir(exist_ok=True) + budget = Budget(args.cgroup, args.profile == "primary") + env = {k:v for k,v in os.environ.items() if not k.startswith(("CORESCOPE_", "PG")) and k not in ("DB_PATH","MQTT_BROKER","MQTT_TOPIC","GOFLAGS")} + env.update(CGO_ENABLED="1",GOMAXPROCS="3",GOMEMLIMIT="512MiB",GOTOOLCHAIN="local",GOFLAGS="") + pg, server = None, None + manifest = dict(status="incomplete", baseline_sha=BASELINE, candidate_sha=args.candidate_sha, seed=args.seed, corpus=args.corpus, shape=corpus(args.corpus), firmware_reference=FIRMWARE, harness_sha256=hashlib.sha256(b"".join(bytes.fromhex(sha256(p)) for p in sorted(HERE.rglob("*")) if p.is_file() and "__pycache__" not in p.parts)).hexdigest(), host=dict(system=platform.system(), kernel=platform.release(), architecture=platform.machine(), cpus=os.cpu_count(), ram_bytes=os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE")), postgres_version=version, go_version=command(["go", "version"]).strip(), compiler=command(["cc", "--version"]).splitlines()[0], resource_budget=budget.info, cache_policy=dict(startup="new application process after full validation; OS cache warm", mixed="warmup then application-warm measurement", sql="channel application cache explicitly cleared; OS cache warm"), os_cold="not measured", comparison_eligible=False, extended_120s_600s_profile="not run unless explicitly requested by duration flags", order=pair_order(args.pairs)) + write_json(output / "manifest.json", manifest) + try: + roots = {} + for backend, revision in (("sqlite", BASELINE), ("postgres", args.candidate_sha)): + root = runtime / backend + archive(args.repo.resolve(), revision, root) + roots[backend] = root + manifest[backend + "_production_hash_before"] = build(root, backend, binaries, env, logs) + epoch = int(time.time()) + canonical = runtime / "corpus.sqlite" + prepare_dir = output / "corpus" + prepare_dir.mkdir() + base_config = dict(control_dir=str(runtime/"prepare-controls"), corpus_info=str(prepare_dir / "corpus.json"), corpus=args.corpus, shape=corpus(args.corpus), seed=args.seed, epoch=epoch, sqlite=str(canonical), output=str(prepare_dir), warmup=0, seconds=0, ingest_rate=args.ingest_rate, http_rate=args.http_rate) + finish(worker(binaries / "sqlite-ingestor.test", base_config, "prepare", env, logs / "prepare.log"), timeout=1800) + tables = table_manifest(canonical) + canonical_validation = validate(canonical, tables) + if canonical_validation["transmissions"]["rows"] != corpus(args.corpus)["transmissions"] or canonical_validation["observations"]["rows"] != corpus(args.corpus)["observations"]: + raise RuntimeError("canonical corpus counts differ from the declared shape") + write_json(prepare_dir / "validation.json", canonical_validation) + manifest.update(epoch=epoch, canonical_file_sha256=sha256(canonical), table_manifest=tables) + manifest["binary_sha256"] = {p.name: sha256(p) for p in binaries.iterdir() if p.is_file() and not p.name.endswith(".json")} + pg = Postgres(runtime, args.postgres_bin, budget, logs) + warmup = args.warmup if args.warmup is not None else (60 if args.profile == "primary" else 2) + seconds = args.seconds if args.seconds is not None else (180 if args.profile == "primary" else 10) + if warmup < 0 or seconds < 1 or args.ingest_rate < 1 or args.http_rate < 1 or (warmup+seconds)*max(args.ingest_rate,args.http_rate)>1_000_000: + raise ValueError("invalid workload duration or offered rate") + manifest["workload"] = dict(warmup_seconds=warmup, measured_seconds=seconds, ingest_rate=args.ingest_rate, http_rate=args.http_rate, queue_limit=1024, http_concurrency=64, server_memory_mib=1536, ingestor_memory_mib=512, packet_store_mib=1024, retention_hours=168, sql_reader_pool=4, sql_writer_pool=1) + write_json(output / "manifest.json", manifest) + after_by_pair = {} + paired_sources = {} + for pair, backend in pair_order(args.pairs): + budget.reset() + run_dir = output / f"pair-{pair:02d}" / backend + run_dir.mkdir(parents=True) + if shutil.disk_usage(output).free < max(2 * 1024 ** 3, canonical.stat().st_size * 8): + raise RuntimeError("insufficient measured-corpus headroom before the next restore/import") + config_dir = runtime / f"config-{pair:02d}-{backend}" + config_dir.mkdir() + state_dir = config_dir / "state" + state_dir.mkdir() + sqlite_path = config_dir / "telemetry.sqlite" + if pair not in paired_sources: + pair_source=runtime/f"pair-{pair:02d}.sqlite" + shutil.copy2(canonical,pair_source) + delta=int(time.time())-epoch + freshen(pair_source,delta) + paired_validation=validate(pair_source,tables) + paired_sources[pair]=(pair_source,delta,paired_validation) + write_json(output/f"pair-{pair:02d}"/"corpus-offset.json",dict(receipt_epoch_offset_seconds=delta,wire_epoch_unchanged=epoch,sha256=sha256(pair_source),validation=paired_validation)) + pair_source,delta,paired_validation=paired_sources[pair] + command(["cp","--reflink=never",pair_source,sqlite_path],budget=budget,log=run_dir/"restore-copy.log") + config = dict(base_config, control_dir=str(config_dir/"controls"), events_file=str(runtime/f"pair-{pair:02d}-events.jsonl"), epoch=epoch+delta, wire_epoch=epoch, output=str(run_dir), sqlite=str(sqlite_path), state_dir=str(state_dir), warmup=warmup, seconds=seconds, start_file=str(run_dir / "start.json")) + if not Path(config["events_file"]).exists(): + finish(worker(binaries/"sqlite-ingestor.test",config,"events",env,run_dir/"event-generation.log"),timeout=600) + write_json(output/f"pair-{pair:02d}"/"events.json",dict(sha256=sha256(config["events_file"]),rows=(warmup+seconds)*args.ingest_rate,source="one identical baseline-generated stream for both engines")) + event_hash = json.loads((output/f"pair-{pair:02d}"/"events.json").read_text())["sha256"] + if sha256(config["events_file"]) != event_hash: + raise RuntimeError("paired event stream changed before replay") + run_env = dict(env,CORESCOPE_INGESTOR_STATS=str(state_dir/"ingestor-stats.json")) + database = f"bench_p{pair:02d}" + with Resources(run_dir / "resources.csv", budget, [pg.data] if backend == "postgres" else [sqlite_path]) as resources: + if backend == "postgres": + pg.start() + urls = pg.create(database) + run_env.update(CORESCOPE_DATABASE_URL=urls["writer"], CORESCOPE_WRITER_DATABASE_URL=urls["writer"], CORESCOPE_READER_DATABASE_URL=urls["reader"]) + before_source = sha256(sqlite_path) + resources.phase="migration" + migration_start = time.monotonic_ns() + command([binaries / "migrate", "-offline", "-from-sqlite", sqlite_path, "-state-dir", config_dir / "migration"], env=dict(env, CORESCOPE_DATABASE_URL=urls["owner"]), log=run_dir / "migration.log", timeout=1800, budget=budget) + tool_finished=time.monotonic_ns() + pg.grant(database) + grants_finished=time.monotonic_ns() + migrated = validate(sqlite_path, tables, pg, database) + valid = migrated == paired_validation and sha256(sqlite_path) == before_source + write_json(run_dir / "migration.json", dict(elapsed_ns=time.monotonic_ns()-migration_start, migration_tool_ns=tool_finished-migration_start, role_grants_ns=grants_finished-tool_finished, harness_validation_ns=time.monotonic_ns()-grants_finished, verified=valid, source_unchanged=sha256(sqlite_path) == before_source, tables=migrated)) + if not valid: + raise RuntimeError("migration parity/source immutability failed") + else: + with sqlite3.connect(sqlite_path) as db: + db.execute("ANALYZE") + pre = validate(sqlite_path, tables, pg if backend == "postgres" else None, database) + if pre != paired_validation: + raise RuntimeError("initial backend state differs from the canonical corpus") + write_json(run_dir / "validation-before.json", pre) + retained_expected = retained_counts(sqlite_path, int(time.time())-168*3600) if args.corpus in ("S", "B") else None + app_config = dict(hashChannels=[f"#bench-{i:02d}" for i in range(20)], dbPath=str(sqlite_path), stateDir=str(state_dir), port=free_port(), packetStore=dict(retentionHours=168, maxMemoryMB=1024, hotStartupHours=0), runtime=dict(maxMemoryMB=1536), clientRxCoverage=dict(enabled=True), userManagement=dict(enabled=False)) + if backend == "postgres": + app_config.pop("dbPath") + write_json(config_dir / "config.json", app_config) + server_env = dict(run_env, GOMEMLIMIT="1536MiB", CORESCOPE_DATABASE_URL=run_env.get("CORESCOPE_READER_DATABASE_URL", "")) + resources.phase="startup_full" + launched_ns=time.monotonic_ns() + server = spawn([binaries / (backend + "-server"), "-config-dir", config_dir, "-public", roots[backend] / "public", "-port", app_config["port"], "-poll-ms", 1000], cwd=roots[backend], env=server_env, log=run_dir / "server.log", budget=budget) + config["base_url"] = f"http://127.0.0.1:{app_config['port']}" + startup(server,config["base_url"],run_dir/"startup.json",pre,launched_ns,0,retained_expected) + stop(server) + app_config["packetStore"]["hotStartupHours"]=1 + write_json(config_dir/"config.json",app_config) + resources.phase="startup_hot" + launched_ns=time.monotonic_ns() + server=spawn([binaries/(backend+"-server"),"-config-dir",config_dir,"-public",roots[backend]/"public","-port",app_config["port"],"-poll-ms",1000],cwd=roots[backend],env=server_env,log=run_dir/"server-hot.log",budget=budget) + startup(server,config["base_url"],run_dir/"startup-hot1.json",pre,launched_ns,1,retained_expected) + resources.phase="sql_queries" + finish(worker(binaries / (backend + "-server.test"), config, "queries", server_env, run_dir / "queries.log", budget), timeout=180) + resources.phase="mixed_with_retention" + client = worker(binaries / (backend + "-server.test"), config, "client", server_env, run_dir / "client.log") + replay = worker(binaries / (backend + "-ingestor.test"), config, "replay", run_env, run_dir / "replay.log", budget) + try: + wait_file(run_dir / "client-ready", [client, replay, server]) + wait_file(run_dir / "replay-ready", [client, replay, server]) + write_json(run_dir / "start.json", dict(mono_ns=time.monotonic_ns() + 2_000_000_000)) + finish(replay, timeout=warmup + seconds + 180) + finish(client, timeout=warmup + seconds + 180) + finally: + stop(replay) + stop(client) + observed=visibility(json_lines(run_dir/"ingest.jsonl"),json_lines(run_dir/"websocket.jsonl")) + write_json(run_dir/"visibility.json",observed) + if not observed["verified"]: + raise RuntimeError("durable new-transmission sentinels were not observed on the real WebSocket") + resources.phase="retention_empty" + finish(worker(binaries / (backend + "-ingestor.test"), config, "retention", run_env, run_dir / "retention.log", budget), timeout=600) + stop(server) + server = None + resources.phase="plans" + finish(worker(binaries/(backend+"-ingestor.test"),config,"plans",run_env,run_dir/"plans.log",budget),timeout=180) + resources.phase="post_validation" + post = validate(sqlite_path, tables, pg if backend == "postgres" else None, database, logical=True) + write_json(run_dir / "validation-after.json", post) + # Separate real-handler control follows parity, so its deliberate + # receipt-time timestamps do not weaken the deterministic replay digest. + resources.phase="handler_control" + finish(worker(binaries / (backend + "-ingestor.test"), config, "handler", run_env, run_dir / "handler.log", budget), timeout=180) + after_by_pair.setdefault(pair, {})[backend] = post + if len(after_by_pair[pair])==2: + coverage={} + for kind in ("sqlite","postgres"): + coverage[kind]=[json.loads((output/f"pair-{pair:02d}"/kind/file).read_text())["health"] for file in ("startup.json","startup-hot1.json")] + for position in (0,1): + for field in ("loadedTx","loadedObs"): + if coverage["sqlite"][position][field]!=coverage["postgres"][position][field]: + raise RuntimeError("startup memory coverage differs under equal retention/memory settings") + if len(after_by_pair[pair]) == 2 and after_by_pair[pair]["sqlite"] != after_by_pair[pair]["postgres"]: + raise RuntimeError("post-replay/retention logical parity failed; retain both validation artifacts") + if backend == "postgres": + pg.drop(database) + pg.stop() + remove_working_directory(config_dir,runtime) + if sha256(config["events_file"]) != event_hash: + raise RuntimeError("paired event stream changed during replay") + if len(after_by_pair[pair])==2: + pair_source.unlink() + Path(config["events_file"]).unlink() + if sha256(canonical) != manifest["canonical_file_sha256"]: + raise RuntimeError("canonical corpus was modified") + for backend, root in roots.items(): + after = production_hash(root) + manifest[backend + "_production_hash_after"] = after + if after != manifest[backend + "_production_hash_before"]: + raise RuntimeError("production source changed during measurements") + summary = summarize(output) + manifest.update(status="completed", comparison_eligible=args.profile=="primary", measured_rows=len(summary), completed_pairs=args.pairs) + write_json(output / "manifest.json", manifest) + (output / "report.md").write_text("# CoreScope paired database measurements\n\nResults are scoped to this exact manifest, corpus, host, cache state and conversion.\nSee summary.csv and the raw per-run artifacts. p99 is omitted below 1,000 samples.\nOS-cold behavior and the optional 65-minute maintenance soak were not measured.\nNo engine-wide speed claim follows from these results.\n") + except BaseException as error: + for child in reversed(PROCESSES): + stop(child) + try: + summarize(output) + except (OSError,ValueError,KeyError) as summary_error: + manifest["partial_summary_error"]=str(summary_error) + manifest.update(status="failed", comparison_eligible=False, error=str(error)) + write_json(output / "manifest.json", manifest) + raise + finally: + for child in reversed(PROCESSES): + stop(child) + PROCESSES.clear() + stop(server) + if pg: + pg.stop() + try: + budget.close() + except Exception as cleanup_error: + manifest.update(status="failed",comparison_eligible=False,cleanup_error=str(cleanup_error)) + write_json(output/"manifest.json",manifest) + raise + finally: + public_bundle(output,args.repo) + + +if __name__ == "__main__": + try: + main() + except (ValueError, RuntimeError, OSError, subprocess.SubprocessError) as error: + print("benchmark:", error, file=sys.stderr) + sys.exit(1) diff --git a/scripts/postgres-benchmark/test_harness.py b/scripts/postgres-benchmark/test_harness.py new file mode 100644 index 000000000..6fb66888c --- /dev/null +++ b/scripts/postgres-benchmark/test_harness.py @@ -0,0 +1,168 @@ +import importlib.util +import pathlib +import tempfile +import types +import unittest + +HERE = pathlib.Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location("bench", HERE / "run.py") +bench = importlib.util.module_from_spec(spec) +spec.loader.exec_module(bench) + + +class HarnessTests(unittest.TestCase): + def test_corpus_counts_are_exact(self): + for name, transmissions, observations in [("S", 30_000, 90_000), ("B", 128_000, 2_048_000), ("L", 1_000_000, 16_000_000)]: + shape = bench.corpus(name) + self.assertEqual(shape["transmissions"], transmissions) + self.assertEqual(sum(bench.fanout(name, i) for i in range(transmissions)), observations) + + def test_pair_order_is_serial_and_alternates(self): + self.assertEqual(bench.pair_order(3), [(0, "sqlite"), (0, "postgres"), (1, "postgres"), (1, "sqlite"), (2, "sqlite"), (2, "postgres")]) + + def test_sparse_p99_is_not_reported_as_zero(self): + self.assertIsNone(bench.percentiles([1, 2, 3])["p99"]) + self.assertIsNone(bench.percentiles([])["p50"]) + self.assertEqual(bench.percentiles(range(1, 1001))["p99"], 990) + + def test_logical_digest_distinguishes_null_and_empty_and_normalizes_numbers(self): + self.assertNotEqual(bench.canonical_row([None]), bench.canonical_row([""])) + self.assertEqual(bench.canonical_row([1]), bench.canonical_row([1.0])) + self.assertNotEqual(bench.canonical_row(["1"]), bench.canonical_row([1])) + + def test_destinations_fail_closed(self): + with self.assertRaises(ValueError): + bench.safe_output(pathlib.Path("/tmp")) + with tempfile.TemporaryDirectory() as directory: + target = pathlib.Path(directory) / "corescope-bench-fixture" + self.assertEqual(bench.safe_output(target), target.resolve()) + target.mkdir() + with self.assertRaises(ValueError): + bench.safe_output(target) + + def test_floating_or_wrong_baseline_is_rejected(self): + with self.assertRaises(ValueError): + bench.check_revisions("master", "1" * 40) + with self.assertRaises(ValueError): + bench.check_revisions("2" * 40, "1" * 40) + bench.check_revisions(bench.BASELINE, "1" * 40) + + def test_primary_requires_five_pairs_and_resource_controls(self): + with self.assertRaises(ValueError): + bench.validate_profile("primary", 1, None) + with self.assertRaises(ValueError): + bench.validate_profile("primary", 5, None) + bench.validate_profile("smoke", 1, None) + + def test_epoch_shift_preserves_nanoseconds_and_nulls(self): + self.assertEqual(bench.shift_timestamp("2026-10-08T12:00:00.123456789Z", 3600), "2026-10-08T13:00:00.123456789Z") + self.assertEqual(bench.shift_timestamp("2026-10-08 12:00:00", -1), "2026-10-08 11:59:59") + self.assertIsNone(bench.shift_timestamp(None, 1)) + with self.assertRaises(ValueError): + bench.shift_timestamp("unexpected", 1) + + def test_public_bundle_excludes_databases_payloads_and_controls(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + (root / ".runtime").mkdir() + (root / ".runtime" / "events.jsonl").write_text("private packet payload") + (root / "manifest.json").write_text('{"path":"' + str(root).replace('\\', '\\\\') + '"}') + run = root / "pair-00" / "sqlite" + run.mkdir(parents=True) + (run / "replay-config.json").write_text("private control") + (run / "requests.jsonl").write_text('{"class":"nodes"}\n') + bench.public_bundle(root) + files = {p.relative_to(root / "public").as_posix() for p in (root / "public").rglob("*") if p.is_file()} + self.assertEqual(files, {"manifest.json", "pair-00/sqlite/requests.jsonl"}) + + def test_visibility_is_bound_to_durable_sent_hashes(self): + sent = [{"hash":"a", "completed_ns":100, "new_transmission":True}, {"hash":"b", "completed_ns":200, "new_transmission":True}] + result = bench.visibility(sent,[{"hash":"a","received_ns":90},{"hash":"a","received_ns":110},{"hash":"unrelated","received_ns":300}]) + self.assertFalse(result["verified"]) + self.assertEqual(result["missing_hashes"],["b"]) + self.assertEqual(result["duplicate_hash_receipts"],1) + self.assertEqual(result["visible_before_ingest_return"],1) + self.assertIsNone(result["lag_ns"]["p99"]) + + def test_cgroup_launch_preserves_arguments_without_shell_interpolation(self): + budget = bench.Budget.__new__(bench.Budget) + budget.path = pathlib.Path("/sys/fs/cgroup/bench-test") + arguments = ["/tmp/application", "$(must-not-execute)", "a b", "a'b"] + launched = budget.arguments(arguments) + self.assertEqual(launched[-len(arguments):], arguments) + self.assertEqual(launched[-len(arguments)-1], str(budget.path)) + self.assertNotIn("must-not-execute", launched[2]) + budget.path = None + self.assertEqual(budget.arguments(arguments), arguments) + + def test_failure_diagnostic_keeps_errors_but_excludes_sensitive_log_data(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + log = root / "replay.log" + log.write_text('ordinary packet body: do not publish me\n' + ' zz_benchmark_ingestor_test.go:459: ERROR: missing relation "private_table" (SQLSTATE 42P01)\n' + ' zz_benchmark_ingestor_test.go:460: failed postgres://writer:topsecret@private.example/db\n' + ' zz_benchmark_ingestor_test.go:461: failed at /home/private/state/config.json and C:\\Private\\state.json\n' + ' zz_benchmark_ingestor_test.go:462: payload={"text":"private body"} raw=00112233445566778899aabbccddeeff\n' + ' zz_benchmark_ingestor_test.go:463: host=private.example password=topsecret\n' + '--- FAIL: TestCoreScopeBenchmark (1.20s)\n') + diagnostic = bench.record_failure(log, 1) + artifact = root / "failure-replay.json" + text = artifact.read_text() + self.assertEqual(diagnostic["log"], "replay.log") + self.assertEqual(diagnostic["exit_code"], 1) + self.assertIn("SQLSTATE 42P01", text) + self.assertIn("TestCoreScopeBenchmark", text) + for private in ("do not publish me", "topsecret", "private.example", "/home/private", "Private", "private body", "00112233", "private_table"): + self.assertNotIn(private, text) + self.assertLess(len(text), 5000) + + def test_public_bundle_includes_only_sanitized_failure_artifact(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + logs = root / "logs" + logs.mkdir() + log = logs / "postgres-build.log" + log.write_text("private raw log\n./main.go:27: undefined: MissingSymbol\n") + bench.record_failure(log, 1) + bench.public_bundle(root) + self.assertTrue((root / "public/logs/failure-postgres-build.json").is_file()) + self.assertFalse((root / "public/logs/postgres-build.log").exists()) + + def test_failure_diagnostic_bounds_long_logs(self): + with tempfile.TemporaryDirectory() as directory: + log = pathlib.Path(directory) / "worker.log" + log.write_text("ignored prefix\n" * 10000 + "panic: " + "x" * 20000 + "\n--- FAIL: TestCoreScopeBenchmark (1s)\n") + result = bench.record_failure(log, 2) + self.assertLessEqual(sum(len(x) for x in result["diagnostics"]), 2400) + self.assertIn("--- FAIL: TestCoreScopeBenchmark (1s)", result["diagnostics"]) + + def test_finish_records_failed_child_before_raising(self): + with tempfile.TemporaryDirectory() as directory: + log = pathlib.Path(directory) / "replay.log" + log.write_text(" zz_benchmark_ingestor_test.go:460: durable effects did not match\n") + child = types.SimpleNamespace(wait=lambda timeout: 1, poll=lambda: 1, bench_log=open(log, "a")) + with self.assertRaisesRegex(RuntimeError, "failure-replay.json"): + bench.finish(child) + self.assertTrue(child.bench_log.closed) + self.assertIn("durable effects did not match", (log.parent / "failure-replay.json").read_text()) + + def test_retained_load_requires_every_eligible_row(self): + with tempfile.TemporaryDirectory() as directory: + source = pathlib.Path(directory) / "corpus.sqlite" + with bench.sqlite3.connect(source) as db: + db.executescript("CREATE TABLE transmissions(id INTEGER,last_seen INTEGER);" + "CREATE TABLE observations(transmission_id INTEGER);" + "INSERT INTO transmissions VALUES(1,90),(2,100),(3,110);" + "INSERT INTO observations VALUES(1),(2),(2),(3);") + db.close() + expected = bench.retained_counts(source, 100) + self.assertEqual(expected, {"transmissions": 2, "observations": 3}) + bench.check_loaded_rows({"loadedTx": 2, "loadedObs": 3}, expected) + for incomplete in ({"loadedTx": 1, "loadedObs": 3}, {"loadedTx": 2, "loadedObs": 2}): + with self.assertRaisesRegex(RuntimeError, "retained corpus"): + bench.check_loaded_rows(incomplete, expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/prepare-postgres-fixture.go b/scripts/prepare-postgres-fixture.go new file mode 100644 index 000000000..85a2c4511 --- /dev/null +++ b/scripts/prepare-postgres-fixture.go @@ -0,0 +1,142 @@ +// Run from cmd/migrate: +// go run ../../scripts/prepare-postgres-fixture.go -source ../../test-fixtures/e2e-fixture.db -destination /private/fixture.sqlite +// This explicitly repairs the historical test fixture on a new disposable copy. +// Production duplicates require a reviewed repair/upgrade with the last SQLite +// release while retaining the raw recovery snapshot; the importer never repairs +// or discards incompatible source records silently. +package main + +import ( + "crypto/sha256" + "database/sql" + "errors" + "flag" + "fmt" + "io" + "net/url" + "os" + "path/filepath" + "runtime" + + _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbschema/legacy" +) + +func main() { + source := flag.String("source", "", "closed historical SQLite fixture") + destination := flag.String("destination", "", "new disposable destination copy (must not exist)") + flag.Parse() + if err := prepareFixture(*source, *destination); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func prepareFixture(source, destination string) error { + if source == "" || destination == "" { + return errors.New("-source and -destination are required") + } + if info, err := os.Stat(source + "-wal"); err == nil && info.Size() > 0 { + return errors.New("fixture has WAL data; close/checkpoint its writer before preparing a disposable test copy") + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + before, err := fileHash(source) + if err != nil { + return err + } + in, err := os.Open(source) + if err != nil { + return err + } + defer in.Close() + if err := os.MkdirAll(filepath.Dir(destination), 0o700); err != nil { + return err + } + out, err := os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return err + } + _, copyErr := io.Copy(out, in) + syncErr := out.Sync() + closeErr := out.Close() + if err := errors.Join(copyErr, syncErr, closeErr); err != nil { + return err + } + abs, err := filepath.Abs(destination) + if err != nil { + return err + } + abs = filepath.ToSlash(abs) + if runtime.GOOS == "windows" { + abs = "/" + abs + } + u := url.URL{Scheme: "file", Path: abs} + q := url.Values{"mode": {"rw"}} + u.RawQuery = q.Encode() + db, err := sql.Open("sqlite3", u.String()) + if err != nil { + return err + } + defer db.Close() + db.SetMaxOpenConns(1) + if _, err := db.Exec(`PRAGMA trusted_schema=OFF`); err != nil { + return err + } + beforeRows, err := counts(db) + if err != nil { + return err + } + // This step is deliberate and visible in the before/after counts. It is + // kept outside the production import command and runs only on destination. + if err := legacy.Apply(db, nil); err != nil { + return err + } + if err := legacy.Normalize(db, nil); err != nil { + return err + } + afterRows, err := counts(db) + if err != nil { + return err + } + if err := db.Close(); err != nil { + return err + } + after, err := fileHash(source) + if err != nil { + return err + } + if after != before { + return errors.New("original fixture changed during preparation; discard the disposable destination") + } + if info, err := os.Stat(source + "-wal"); err == nil && info.Size() > 0 { + return errors.New("fixture writer became active during preparation; discard the disposable destination") + } + fmt.Printf("transmissions: %d -> %d; observations: %d -> %d; original fixture unchanged\n", beforeRows[0], afterRows[0], beforeRows[1], afterRows[1]) + return nil +} + +func counts(db *sql.DB) ([2]int64, error) { + var result [2]int64 + for i, table := range []string{"transmissions", "observations"} { + if err := db.QueryRow(`SELECT count(*) FROM ` + table).Scan(&result[i]); err != nil { + return result, err + } + } + return result, nil +} + +func fileHash(path string) ([32]byte, error) { + var result [32]byte + file, err := os.Open(path) + if err != nil { + return result, err + } + defer file.Close() + h := sha256.New() + if _, err := io.Copy(h, file); err != nil { + return result, err + } + copy(result[:], h.Sum(nil)) + return result, nil +} diff --git a/scripts/prune-nodes-outside-geo-filter.py b/scripts/prune-nodes-outside-geo-filter.py index 6f34eb08e..fd8e1bb0e 100644 --- a/scripts/prune-nodes-outside-geo-filter.py +++ b/scripts/prune-nodes-outside-geo-filter.py @@ -1,10 +1,13 @@ #!/usr/bin/env python3 """ +Offline legacy SQLite tooling; use the Admin Geo Filter workflow for PostgreSQL. +Operate only on a stopped, disposable recovery copy. + Delete nodes from the database that fall outside the configured geo_filter polygon + bufferKm. Nodes with no GPS coordinates are always kept. Usage: - python3 prune-nodes-outside-geo-filter.py [db_path] [--config config.json] [--dry-run] + python3 prune-nodes-outside-geo-filter.py --legacy-sqlite-offline [db_path] [--config config.json] [--dry-run] db_path Path to meshcore.db (default: /app/data/meshcore.db) --config PATH Path to config.json (default: /app/config.json) @@ -165,4 +168,7 @@ def main(): if __name__ == '__main__': + if '--legacy-sqlite-offline' not in sys.argv: + sys.exit('Offline legacy SQLite tooling only. Use the Admin Geo Filter workflow for PostgreSQL. Pass --legacy-sqlite-offline only for a stopped, disposable recovery copy.') + sys.argv.remove('--legacy-sqlite-offline') main() diff --git a/scripts/test-postgres-compose.sh b/scripts/test-postgres-compose.sh new file mode 100755 index 000000000..5a099af0b --- /dev/null +++ b/scripts/test-postgres-compose.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash +# CI-only packaged startup/ingestion smoke. No image publication or host ports. +set +x +set -euo pipefail +[[ ${GITHUB_ACTIONS:-} == true && $(uname -s) == Linux ]] || { echo 'This smoke requires a disposable Linux GitHub Actions runner.' >&2; exit 1; } +[[ ${GITHUB_SHA:-} =~ ^[0-9a-f]{40}$ && ${1:-} == "corescope-pg-smoke:$GITHUB_SHA" ]] || { echo 'An exact locally built candidate image is required.' >&2; exit 1; } +repo=$(cd "$(dirname "$0")/.." && pwd -P) +temp_root=$(realpath -e "${RUNNER_TEMP:?RUNNER_TEMP is required}") +[[ -x "$repo/docker/postgres-init.sh" ]] || { echo 'PostgreSQL initialization must be executable, not sourced by the vendor entrypoint.' >&2; exit 1; } +image=$1 +image_id=$(timeout 15s docker image inspect --format '{{.Id}}' "$image") +umask 077 +work=$(mktemp -d "$temp_root/corescope-pg-smoke-XXXXXXXX") +project=$(basename "$work" | tr '[:upper:]' '[:lower:]') +owned_project=false +phase=configuration +dc=(docker compose --ansi never --project-name "$project" --env-file "$work/.env" + -f "$repo/docker-compose.example.yml" -f "$work/override.yml") + +project_resources() { + timeout 10s docker container ls -aq --filter "label=com.docker.compose.project=$project" || return 1 + timeout 10s docker network ls -q --filter "label=com.docker.compose.project=$project" || return 1 + timeout 10s docker volume ls -q --filter "label=com.docker.compose.project=$project" +} + +finish() { + local result=$? cleanup_ok=true + trap - EXIT INT TERM + set +e + if [ "$result" -ne 0 ] && $owned_project; then + timeout 20s "${dc[@]}" logs --no-color --tail 100 > "$work/container.raw.log" 2>&1 + fi + if $owned_project; then + timeout 60s "${dc[@]}" down --volumes --remove-orphans --timeout 20 >> "$work/actions.raw.log" 2>&1 || cleanup_ok=false + remaining=$(project_resources) || cleanup_ok=false + [ -z "$remaining" ] || cleanup_ok=false + fi + if ! $cleanup_ok; then result=1; phase=cleanup; fi + # Persist only bounded, redacted diagnostics; never print Compose's full config. + python3 - "$work" <<'PY' || result=1 +import os,re,sys +from pathlib import Path +root=Path(sys.argv[1]) +text="" +for name in ("actions.raw.log","container.raw.log"): + path=root/name + if path.exists(): + with path.open("rb") as source: + text += source.read(262144).decode("utf-8",errors="replace")+"\n" +for key,value in os.environ.items(): + if key.endswith("_PASSWORD") and value: + text=text.replace(value,"[redacted]") +text=re.sub(r"postgres(?:ql)?://[^\s'\"<>]+","[redacted database URL]",text) +(root/"failure.log").write_text(text,encoding="utf-8") +PY + # Remove only this mktemp directory's private data, after the project is gone. + if [[ $(realpath -e "$work") == "$temp_root"/corescope-pg-smoke-* && ! -L "$work" ]]; then + if $cleanup_ok; then + sudo rm -rf -- "$work/state" "$work/postgres" || result=1 + rm -f -- "$work/.env" "$work/compose.json" "$work/override.yml" "$work/packet.json" "$work/stats.json" "$work/states.json" "$work/health.json" "$work/actions.raw.log" "$work/container.raw.log" || result=1 + fi + if [ "$result" -eq 0 ]; then + rm -f -- "$work/failure.log" + rmdir -- "$work" || result=1 + else + echo "Compose smoke failed during $phase; sanitized diagnostics: $work/failure.log" >&2 + tail -n 120 "$work/failure.log" >&2 + fi + else + echo 'Refusing cleanup outside the owned smoke directory.' >&2 + result=1 + fi + exit "$result" +} +trap finish EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +for key in POSTGRES_ADMIN_PASSWORD CORESCOPE_OWNER_PASSWORD CORESCOPE_READER_PASSWORD CORESCOPE_WRITER_PASSWORD CORESCOPE_ACCOUNTS_PASSWORD CORESCOPE_CHANNELS_PASSWORD; do + password=$(openssl rand -hex 32) + export "$key=$password" + printf '%s=%s\n' "$key" "${!key}" >> "$work/.env" +done +unset password +export CORESCOPE_IMAGE="$image" DATA_DIR="$work/state" POSTGRES_DATA_DIR="$work/postgres" +export DISABLE_MOSQUITTO=false DISABLE_CADDY=true +mkdir "$work/state" +python3 - "$repo" "$work" <<'PY' +import json,re,sys +from pathlib import Path +repo,work=map(Path,sys.argv[1:]) +config={"siteName":"CI packaged PostgreSQL smoke","port":3000,"stateDir":"/app/data", + "mqttSources":[{"name":"ci-smoke","broker":"mqtt://localhost:1883","topics":["meshcore/SJC/ci-smoke/packets"]}], + "userManagement":{"enabled":False},"autoRegionKeys":{"enabled":False}} +(work/"state/config.json").write_text(json.dumps(config),encoding="utf-8") +# Reuse the existing ingestor regression's packet, not invented wire bytes. +test=(repo/"cmd/ingestor/main_test.go").read_text(encoding="utf-8").split("func TestHandleMessageRawPacket(t *testing.T) {",1)[1].split("\n}",1)[0] +raw=re.search(r'rawHex := "([0-9A-Fa-f]+)"',test).group(1) +bytes.fromhex(raw) +(work/"packet.json").write_text(json.dumps({"raw":raw,"SNR":5.5,"RSSI":-100.0,"origin":"ci-smoke"})+"\n",encoding="utf-8") +PY +cat > "$work/override.yml" <<'YAML' +services: + postgres: + restart: "no" + volumes: + - postgres-smoke:/var/lib/postgresql + bootstrap: + pull_policy: never + corescope: + pull_policy: never + restart: "no" + ports: !reset [] +networks: + default: + internal: true +volumes: + postgres-smoke: +YAML + +timeout 30s "${dc[@]}" config --format json > "$work/compose.json" 2>> "$work/actions.raw.log" +python3 - "$repo" "$work" <<'PY' +import json,sys +from pathlib import Path +repo,work=map(Path,sys.argv[1:]) +config=json.loads((work/"compose.json").read_text()) +assert config["name"]==work.name.lower() +assert set(config["services"])=={"postgres","bootstrap","corescope"} +assert set(config["networks"])=={"default"} +assert config["networks"]["default"]["internal"] is True +allowed={(work/"state").resolve(),(repo/"docker/postgres-init.sh").resolve()} +for service in config["services"].values(): + assert not service.get("ports"), "smoke must publish no ports" + assert not service.get("network_mode") and set(service["networks"])=={"default"} + for mount in service.get("volumes",[]): + if mount["type"]=="bind": + assert Path(mount["source"]).resolve() in allowed, "unexpected host mount" +for volume in config["volumes"].values(): + assert not volume.get("external") and volume["name"].startswith(config["name"]+"_") +PY +existing=$(project_resources) +[[ -z "$existing" ]] || { echo 'Smoke project already exists; refusing to touch it.' >&2; exit 1; } +owned_project=true +phase=startup +timeout 210s "${dc[@]}" up -d --wait --wait-timeout 180 >> "$work/actions.raw.log" 2>&1 +bootstrap=$(timeout 10s "${dc[@]}" ps -a -q bootstrap) +application=$(timeout 10s "${dc[@]}" ps -q corescope) +phase=bootstrap-order +[[ $(timeout 10s docker inspect --format '{{.Image}}' "$application") == "$image_id" ]] +timeout 10s docker inspect --format '{{json .State}}' "$bootstrap" "$application" > "$work/states.json" +python3 - "$work/states.json" <<'PY' +import datetime,json,sys +bootstrap,app=map(json.loads,open(sys.argv[1],encoding="utf-8")) +assert bootstrap["Status"]=="exited" and bootstrap["ExitCode"]==0 +assert app["Running"] +stamp=lambda value: datetime.datetime.fromisoformat(value.replace("Z","+00:00")) +assert stamp(app["StartedAt"]) >= stamp(bootstrap["FinishedAt"]), "application preceded bootstrap completion" +PY +timeout 10s "${dc[@]}" exec -T corescope pg_dump --version | grep -Eq '^pg_dump .* 18[.]' + +phase=schema-readiness +owner_sql() { + timeout 10s "${dc[@]}" exec -T postgres sh -eu -c ' + export PGHOST=127.0.0.1 PGUSER=corescope_owner PGPASSWORD="$CORESCOPE_OWNER_PASSWORD" PGDATABASE="$1" + exec psql -X -A -t -v ON_ERROR_STOP=1 + ' sh "$1" 2>> "$work/actions.raw.log" +} +[[ $(owner_sql corescope_telemetry <<'SQL' +SELECT version=1 AND ready AND current_setting('server_version_num')='180006' FROM corescope_schema WHERE kind='telemetry'; +SQL +) == t ]] +[[ $(owner_sql corescope_accounts <<'SQL' +SELECT version=1 AND ready AND (SELECT version=5 FROM schema_version) FROM corescope_schema WHERE kind='accounts'; +SQL +) == t ]] + +phase=mqtt-ingestion +timeout 10s "${dc[@]}" exec -T corescope mosquitto_pub -h 127.0.0.1 -q 1 -r -t meshcore/SJC/ci-smoke/packets -s < "$work/packet.json" >> "$work/actions.raw.log" 2>&1 +for attempt in $(seq 1 30); do + if timeout 5s "${dc[@]}" exec -T corescope wget -qO- http://127.0.0.1:3000/api/healthz > "$work/health.json" 2>> "$work/actions.raw.log" && + timeout 5s "${dc[@]}" exec -T corescope wget -qO- http://127.0.0.1:3000/api/stats > "$work/stats.json" 2>> "$work/actions.raw.log" && + python3 - "$work" "$GITHUB_SHA" <<'PY' +import json,sys +from pathlib import Path +root=Path(sys.argv[1]) +health=json.loads((root/"health.json").read_text()) +stats=json.loads((root/"stats.json").read_text()) +sys.exit(0 if health.get("ready") is True and stats.get("totalTransmissions")==1 and stats.get("totalObservations")==1 and stats.get("commit")==sys.argv[2][:7] else 1) +PY + then break; fi + [ "$attempt" -lt 30 ] || { echo 'Published fixture did not become visible through the packaged API.' >&2; exit 1; } + sleep 1 +done +phase=runtime-roles +[[ $(owner_sql corescope_telemetry <<'SQL' +SELECT (SELECT count(*)=1 FROM transmissions) + AND (SELECT count(*)=1 FROM observations) + AND EXISTS(SELECT 1 FROM pg_stat_activity WHERE datname=current_database() AND usename='corescope_reader') + AND EXISTS(SELECT 1 FROM pg_stat_activity WHERE datname=current_database() AND usename='corescope_writer') + AND NOT has_table_privilege('corescope_reader','transmissions','INSERT,UPDATE,DELETE') + AND NOT has_schema_privilege('corescope_reader','public','CREATE') + AND NOT has_schema_privilege('corescope_writer','public','CREATE'); +SQL +) == t ]] +echo 'Packaged Compose bootstrap, restricted PostgreSQL runtime, MQTT ingestion, API visibility and pg_dump availability passed.' diff --git a/test-all.sh b/test-all.sh index d9225a139..c650b924a 100755 --- a/test-all.sh +++ b/test-all.sh @@ -200,4 +200,6 @@ node tests/unit/test-warmup-banner.js node tests/unit/test-ws-stale-watchdog-1074.js node tests/unit/test-xss-escape-sinks.js +node tests/unit/test-postgres-ops.js + echo "All standalone frontend suites passed" diff --git a/tests/e2e/test-postgres-backup-e2e.js b/tests/e2e/test-postgres-backup-e2e.js new file mode 100644 index 000000000..fbb3a6972 --- /dev/null +++ b/tests/e2e/test-postgres-backup-e2e.js @@ -0,0 +1,124 @@ +/** + * Native PostgreSQL backup and restored-browser-session proof. + * Run test-user-management-e2e.js first to create its synthetic accounts. + * + * BASE_URL=http://localhost:13582 E2E_BACKUP_DIR= + * node tests/e2e/test-postgres-backup-e2e.js capture + * Restore telemetry.dump and accounts.dump into fresh PostgreSQL databases, + * grant the restricted runtime roles, and restart the fixture server against + * those databases. Then use the SAME directory and local browser origin: + * node tests/e2e/test-postgres-backup-e2e.js verify + * + * The directory contains account snapshots and session cookies. Keep it private + * and outside the checkout; never publish it as a CI artifact. + */ +'use strict'; +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { isDeepStrictEqual } = require('node:util'); +const { chromium } = require('playwright'); + +const BASE = process.env.BASE_URL || 'http://localhost:13582'; +const DIR = process.env.E2E_BACKUP_DIR || ''; +const MODE = process.argv[2]; +const HASH = 'fae0c9e6d357a814'; +const ACCOUNTS = ['admin@e2e.test', 'sync@e2e.test']; +let passed = 0; + +function pass(name) { passed++; console.log(' ✓ ' + name); } +function save(name, value) { + fs.writeFileSync(path.join(DIR, name), JSON.stringify(value), { mode: 0o600, flag: 'wx' }); +} +async function json(page, route) { + const response = await page.request.get(BASE + route); + assert.equal(response.status(), 200, route + ' status'); + return response.json(); +} +async function telemetry(page) { + const stats = await json(page, '/api/stats'); + const counts = {}; + for (const key of ['totalTransmissions', 'totalObservations', 'totalNodes', 'totalObservers']) { + assert(Number.isInteger(stats[key]) && stats[key] > 0, key + ' must contain fixture data'); + counts[key] = stats[key]; + } + const detail = await json(page, '/api/packets/' + HASH); + assert.equal(detail.packet.hash, HASH); + assert.equal(detail.observations.length, 3, 'seeded observation count'); + return { counts, hash: detail.packet.hash, observations: detail.observations.map(o => ({ + id: o.id, raw_hex: o.raw_hex, path_json: o.path_json, + })).sort((a, b) => a.id - b.id) }; +} + +(async () => { + assert(['capture', 'verify'].includes(MODE), 'use capture or verify'); + assert(['localhost', '127.0.0.1', '[::1]'].includes(new URL(BASE).hostname), 'local fixture server required'); + assert(path.isAbsolute(DIR), 'E2E_BACKUP_DIR must be a private absolute directory'); + const repo = path.resolve(__dirname, '../..'); + const relative = path.relative(repo, DIR); + assert(relative.startsWith('..' + path.sep) || path.isAbsolute(relative), 'backup directory must be outside the checkout'); + fs.mkdirSync(DIR, { recursive: true, mode: 0o700 }); + const expected = MODE === 'verify' ? JSON.parse(fs.readFileSync(path.join(DIR, 'expected.json'), 'utf8')) : { accounts: [] }; + const browser = await chromium.launch({ headless: true, + executablePath: process.env.CHROMIUM_PATH || undefined, + args: ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage'] }); + try { + const pages = []; + for (let i = 0; i < ACCOUNTS.length; i++) { + const statePath = path.join(DIR, 'browser-' + i + '.json'); + const context = await browser.newContext(MODE === 'verify' ? { storageState: statePath } : {}); + const page = await context.newPage(); + page.setDefaultTimeout(10000); + if (MODE === 'capture') { + await page.goto(BASE + '/#/account/login', { waitUntil: 'domcontentloaded' }); + await page.fill('#loginEmail', ACCOUNTS[i]); + await page.fill('#loginPassword', 'correct horse battery'); + await page.click('#loginForm button[type="submit"]'); + } else { + // No login request: the pre-backup cookie must resolve the restored session. + await page.goto(BASE + '/#/account', { waitUntil: 'domcontentloaded' }); + } + await page.waitForSelector('#profileForm'); + await page.evaluate(() => window.CSAuth.ready()); + if (MODE === 'capture') await page.evaluate(() => window.CSSettingsSync.syncNow()); + const identity = await json(page, '/api/auth/me'); + assert.equal(identity.email, ACCOUNTS[i]); + const settings = (await json(page, '/api/account/settings')).doc; + if (MODE === 'capture') { + expected.accounts.push({ identity, settings }); + save('browser-' + i + '.json', await context.storageState()); + } else { + // Keep session tokens and account contents out of failure output. + assert(isDeepStrictEqual(identity, expected.accounts[i].identity), 'restored identity and CSRF token'); + assert(isDeepStrictEqual(settings, expected.accounts[i].settings), 'restored account settings'); + } + pages.push(page); + pass(MODE + ': ' + (i === 0 ? 'admin' : 'user') + ' session and settings'); + } + assert.equal(expected.accounts[1].settings.keys['meshcore-time-window'], '180', 'synced fixture setting'); + const actualTelemetry = await telemetry(pages[0]); + if (MODE === 'capture') expected.telemetry = actualTelemetry; + else assert(isDeepStrictEqual(actualTelemetry, expected.telemetry), 'restored telemetry API identity and observation bytes'); + pass(MODE + ': telemetry counts and distinct observation bytes'); + + const anonymous = await (await browser.newContext()).newPage(); + // Telemetry falls back to the API-key gate; account backups use withAdmin. + for (const [route, filename, anonymousStatus] of [['/api/backup', 'telemetry.dump', 403], ['/api/admin/users-backup', 'accounts.dump', 401]]) { + assert.equal((await anonymous.request.get(BASE + route)).status(), anonymousStatus, route + ' anonymous denial'); + assert.equal((await pages[1].request.get(BASE + route)).status(), 403, route + ' non-admin denial'); + const response = await pages[0].request.get(BASE + route); + assert.equal(response.status(), 200, route + ' admin download'); + assert.match(response.headers()['content-disposition'], /^attachment; filename="corescope-[^"]+\.dump"$/); + assert.equal(response.headers()['content-type'], 'application/octet-stream'); + assert.equal(response.headers()['cache-control'], 'no-store'); + const body = await response.body(); + assert.equal(body.subarray(0, 5).toString(), 'PGDMP', route + ' native archive magic'); + if (MODE === 'capture') fs.writeFileSync(path.join(DIR, filename), body, { mode: 0o600, flag: 'wx' }); + pass(MODE + ': protected native ' + filename + ' download'); + } + if (MODE === 'capture') save('expected.json', expected); + } finally { + await browser.close(); + } + console.log('\n' + passed + '/' + passed + ' tests passed'); +})().catch(err => { console.error('PostgreSQL backup E2E failed: ' + err.message); process.exitCode = 1; }); diff --git a/tests/e2e/test-user-management-e2e.js b/tests/e2e/test-user-management-e2e.js index 0d6a1aea7..a9b6985ec 100644 --- a/tests/e2e/test-user-management-e2e.js +++ b/tests/e2e/test-user-management-e2e.js @@ -3,15 +3,18 @@ * BASE_URL server with userManagement on + fake mailer (-tags e2etest build) * BASE_URL_OFF the regular fixture server (feature off) * - * Local run (never point the servers at the tracked fixture; migrate a copy): - * cp test-fixtures/e2e-fixture.db "$TMP/on.db"; cp test-fixtures/e2e-fixture.db "$TMP/off.db" - * corescope-migrate -db "$TMP/on.db"; corescope-migrate -db "$TMP/off.db" + * Local run: prepare/freshen a disposable copy of the tracked SQLite fixture, + * apply the CI synthetic seeds and migrate-fixture-hashes.go, then import it + * with corescope-migrate -offline -from-sqlite . Use owner URLs only for + * migration; grant a read-only telemetry role and a separate account writer. * (cd cmd/server && go build -o ../../corescope-server . && go build -tags e2etest -o ../../corescope-server-e2e .) * # config.json for the on-server (in $CFGDIR): port 13582, userManagement {enabled: true, - * # dbPath "users.db", adminEmails ["admin@e2e.test"], publicBaseUrl "http://localhost:13582", + * # adminEmails ["admin@e2e.test"], publicBaseUrl "http://localhost:13582", * # mail {provider "fake", fromEmail "noreply@e2e.test"}, channelProposals {enabled: true}, notifications {enabled: true}} - * corescope-server -port 13581 -db "$TMP/off.db" -public public & - * (cd "$CFGDIR" && corescope-server-e2e -config-dir . -port 13582 -db "$TMP/on.db" -public /public) & + * # Set CORESCOPE_READER_DATABASE_URL for both processes and + * # CORESCOPE_USERS_DATABASE_URL for the on-server only (fresh account DB). + * corescope-server -port 13581 -state-dir "$TMP/off-state" -public public & + * (cd "$CFGDIR" && corescope-server-e2e -config-dir . -port 13582 -state-dir "$TMP/on-state" -public /public) & * BASE_URL=http://localhost:13582 BASE_URL_OFF=http://localhost:13581 node tests/e2e/test-user-management-e2e.js * Set CHROMIUM_PATH to a Chrome/Chromium binary if Playwright's own is not installed. */ diff --git a/tests/unit/test-issue-1648-m2-emoji-scan.js b/tests/unit/test-issue-1648-m2-emoji-scan.js index beb98ebaa..a4997b2ef 100644 --- a/tests/unit/test-issue-1648-m2-emoji-scan.js +++ b/tests/unit/test-issue-1648-m2-emoji-scan.js @@ -48,7 +48,7 @@ const MIN_USE_REFS = { 'map.js': 10, 'node-analytics.js': 4, 'traces.js': 1, - 'perf.js': 9, + 'perf.js': 8, // PostgreSQL replaces the SQLite-only WAL-size warning card. 'audio-lab.js': 5, }; diff --git a/tests/unit/test-issue-1956-release-routing.js b/tests/unit/test-issue-1956-release-routing.js index 89ee1738c..2b0419450 100644 --- a/tests/unit/test-issue-1956-release-routing.js +++ b/tests/unit/test-issue-1956-release-routing.js @@ -11,13 +11,7 @@ const { spawnSync } = require('node:child_process'); const read = name => fs.readFileSync(path.join(REPO_ROOT, '.github/workflows', name), 'utf8').replace(/\r/g, ''); const fast = read('release-fast-path.yml'); const deploy = read('deploy.yml'); -let bash = process.env.BASH_PATH || 'bash'; -if (!process.env.BASH_PATH && process.platform === 'win32') { - // Prefer Git Bash over Windows' WSL launcher; CI uses the native Linux bash. - const git = spawnSync('git', ['--exec-path'], { encoding: 'utf8' }); - const gitBash = path.resolve((git.stdout || '').trim(), '../../../bin/bash.exe'); - if (git.status === 0 && fs.existsSync(gitBash)) bash = gitBash; -} +const bash = require('../../scripts/bash-path')(); // Extract known YAML blocks, retaining the actual expressions and shell code. // Full YAML syntax is separately checked by actionlint; no YAML dependency here. @@ -56,11 +50,11 @@ function runSteps(source, context, edge, mutateFails = false) { const output = path.join(dir, 'output'); const log = path.join(dir, 'commands'); fs.writeFileSync(log, ''); - fs.mkdirSync(path.join(dir, 'cmd/decrypt'), { recursive: true }); + for (const name of ['decrypt', 'migrate']) fs.mkdirSync(path.join(dir, 'cmd', name), { recursive: true }); // go() only logs, so nothing real is produced; the release job's own // static/runnable verification step still needs these to exist. - for (const arch of ['amd64', 'arm64']) { - fs.writeFileSync(path.join(dir, `corescope-decrypt-linux-${arch}`), + for (const arch of ['amd64', 'arm64']) for (const name of ['decrypt', 'migrate']) { + fs.writeFileSync(path.join(dir, `corescope-${name}-linux-${arch}`), '#!/bin/sh\necho "corescope-decrypt stub"\n', { mode: 0o755 }); } const stubs = ` @@ -130,7 +124,7 @@ function runSteps(source, context, edge, mutateFails = false) { const result = spawnSync(bash, ['--noprofile', '--norc', '-e', '-o', 'pipefail'], { input: stubs + '\n' + expand(script, context), cwd: dir, encoding: 'utf8', timeout: 15000, env: { - ...process.env, GITHUB_REF: context.github.ref, GITHUB_SHA: context.github.sha, + ...process.env, CC: '', GITHUB_REF: context.github.ref, GITHUB_REF_NAME: context.github.ref_name, GITHUB_SHA: context.github.sha, GITHUB_OUTPUT: bashPath(output), COMMAND_LOG: bashPath(log), TMPDIR: bashPath(dir), EDGE_CONFIG: edge === null ? 'missing' : JSON.stringify({ config: { Labels: { 'org.opencontainers.image.revision': edge } } }), // :edge is a two-platform index plus the two buildx attestation @@ -259,7 +253,14 @@ for (const [ref, event] of [['refs/heads/master', 'push'], ['refs/heads/master', // The cross-toolchain gate: since the SQLite driver became cgo, every // event must build both architectures, without publishing, beside the tests. const checkSteps = steps(block(deploy, 'image-check', 2)).filter(step => step.includes('uses: docker/build-push-action')); - assert.equal(checkSteps.length, 1, 'image-check runs exactly one build'); + assert.equal(checkSteps.length, 2, 'image-check has one two-arch build and one cached amd64 load'); + for (const step of checkSteps) assert.equal(value(step, 'push', 10), 'false', 'neither image-check build may publish'); + assert.equal(value(checkSteps[1], 'platforms', 10), 'linux/amd64', 'Compose smoke must load the native runner architecture only'); + assert.equal(value(checkSteps[1], 'load', 10), 'true', 'Compose smoke requires a locally loaded image'); + assert.equal(value(checkSteps[1], 'context', 10), value(checkSteps[0], 'context', 10), 'smoke must build the same source'); + assert.equal(value(checkSteps[1], 'build-args', 10), value(checkSteps[0], 'build-args', 10), 'smoke must reuse identical version/commit/time metadata'); + assert.equal(value(checkSteps[1], 'cache-from', 10), value(checkSteps[0], 'cache-from', 10), 'smoke must reuse the validated build cache'); + assert.equal(value(checkSteps[1], 'if', 8), '', 'packaged smoke must run on every image-check event'); assert.equal(value(checkSteps[0], 'push', 10), 'false', 'the image check must never publish'); assert.equal(value(checkSteps[0], 'platforms', 10), 'linux/amd64,linux/arm64', 'the image check must cover both shipped architectures'); assert.equal(value(checkSteps[0], 'if', 8), '', 'the image check runs on every event'); @@ -286,21 +287,25 @@ assert.equal(value(dispatchInput, 'default', 8), 'false', 'manual and fallback d const release = block(deploy, 'release-artifacts', 2); const builds = runSteps(release, context(), null).commands.filter(command => command[0] === 'go'); -// CGO_ENABLED=1 since the SQLite driver became github.com/mattn/go-sqlite3, and -// CC must be zig targeting musl — that is what makes the artifact static and -// cross-buildable. A silent revert to the Go-only toolchain fails here. +// Exercise the actual shell: native PostgreSQL tools use pure Go; the offline +// importer alone retains SQLite/cgo and a static musl cross-compiler. assert.deepEqual(builds.map(command => command.slice(1, 5)), [ + ['linux', 'amd64', '0', ''], ['linux', 'amd64', '1', 'zig cc -target x86_64-linux-musl'], + ['linux', 'arm64', '0', ''], ['linux', 'arm64', '1', 'zig cc -target aarch64-linux-musl'], ]); for (const command of builds) { - assert.ok(command.includes("-ldflags=-s -w -extldflags '-static -Wl,-s' -X main.version=v9.8.7"), 'binary version must come from tag, and the artifact must stay static'); - assert.ok(command.includes('-tags'), 'netgo/osusergo/sqlite_omit_load_extension must survive'); + const importer = command.includes('netgo,osusergo,sqlite_omit_load_extension'); + assert.ok(command.includes(importer + ? "-ldflags=-s -w -extldflags '-static -Wl,-s'" + : '-ldflags=-s -w -X main.version=v9.8.7'), 'static linking/version flags must match the tool'); + assert.ok(command.includes('-tags'), 'runtime resolver tags must survive'); } const upload = steps(release).filter(step => step.includes('uses: softprops/action-gh-release@v2')); assert.equal(upload.length, 1, 'publish both architectures together, before the release becomes immutable'); assert.equal(value(upload[0], 'fail_on_unmatched_files', 10), 'true', 'missing assets must prevent publication'); -assert.deepEqual(value(upload[0], 'files', 10).trim().split('\n').map(line => line.trim()), ['corescope-decrypt-linux-amd64', 'corescope-decrypt-linux-arm64']); +assert.deepEqual(value(upload[0], 'files', 10).trim().split('\n').map(line => line.trim()), ['corescope-decrypt-linux-amd64', 'corescope-decrypt-linux-arm64', 'corescope-migrate-linux-amd64', 'corescope-migrate-linux-arm64']); assert.equal(value(upload[0], 'draft', 10), '', 'standard release action must finalize after both uploads'); assert.equal(value(upload[0], 'prerelease', 10), '', 'standard release action must upload before publishing'); const checkout = steps(release).find(step => step.includes('uses: actions/checkout@')); @@ -326,3 +331,18 @@ console.log('PASS failed retag/Go gates, branch/PR routes, and complete tagged r assert.deepEqual(commands.filter(command => command[0] === 'crane' && command[1] === 'tag').map(command => command.at(-1)), ['v9.8', 'v9', 'latest']); console.log('PASS dispatched republish: tags rebuilt, no second release dispatch'); } + +// Benchmark dispatches cannot publish or deploy, even on trusted master with +// staging enabled. Ordinary dispatch routing above remains unchanged. +for (const ref of ['refs/heads/master', 'refs/heads/codex/postgres', 'refs/tags/v9.8.7']) { + const ctx = context(ref, 'workflow_dispatch', { postgres_benchmark: true, candidate_sha: 'a'.repeat(40) }); + ctx.vars.ENABLE_STAGING_DEPLOY = 'true'; + const jobs = route(ctx); + for (const name of ['build-and-publish', 'release-artifacts', 'deploy', 'publish']) { + assert.equal(jobs[name].result, 'skipped', 'benchmark dispatch must skip ' + name); + } + assert.equal(evaluate(value(block(deploy, 'postgres-benchmark', 2), 'if', 4), ctx), true); +} +const benchmarkCall = block(deploy, 'postgres-benchmark', 2); +assert.ok(!benchmarkCall.includes('secrets:') && !benchmarkCall.includes('environment:'), 'benchmark must receive no deployment secrets/environment'); +console.log('PASS benchmark-only dispatch cannot publish or deploy'); diff --git a/tests/unit/test-perf-disk-io-1120.js b/tests/unit/test-perf-disk-io-1120.js index 35509fda4..61fc1b7cc 100644 --- a/tests/unit/test-perf-disk-io-1120.js +++ b/tests/unit/test-perf-disk-io-1120.js @@ -1,4 +1,4 @@ -/* Tests for perf.js Disk I/O + Write Sources + SQLite sections (#1120) */ +/* Tests for perf.js Disk I/O + Write Sources + PostgreSQL sections (#1120) */ 'use strict'; const vm = require('vm'); const fs = require('fs'); @@ -45,12 +45,12 @@ function loadPerf() { return sb; } -function stubFetch(sb, perfData, healthData, ioData, sqliteData, sourcesData) { +function stubFetch(sb, perfData, healthData, ioData, postgresData, sourcesData) { sb.ctx.fetch = (url) => { if (url === '/api/perf') return Promise.resolve({ json: () => Promise.resolve(perfData) }); if (url === '/api/health') return Promise.resolve({ json: () => Promise.resolve(healthData) }); if (url === '/api/perf/io') return Promise.resolve({ json: () => Promise.resolve(ioData) }); - if (url === '/api/perf/sqlite') return Promise.resolve({ json: () => Promise.resolve(sqliteData) }); + if (url === '/api/perf/postgres') return Promise.resolve({ json: () => Promise.resolve(postgresData) }); if (url === '/api/perf/write-sources') return Promise.resolve({ json: () => Promise.resolve(sourcesData) }); return Promise.resolve({ json: () => Promise.resolve({}) }); }; @@ -58,7 +58,7 @@ function stubFetch(sb, perfData, healthData, ioData, sqliteData, sourcesData) { const basePerf = { totalRequests: 100, avgMs: 5, uptime: 3600, - slowQueries: [], endpoints: {}, cache: null, packetStore: null, sqlite: null + slowQueries: [], endpoints: {}, cache: null, packetStore: null, postgres: null }; const goRuntime = { goroutines: 17, numGC: 31, pauseTotalMs: 2.1, lastPauseMs: 0.03, @@ -70,9 +70,9 @@ const ioData = { readBytesPerSec: 1024, writeBytesPerSec: 2048, syscallsRead: 10, syscallsWrite: 20 }; -const sqliteData = { - walSizeMB: 12.3, walSize: 12900000, pageCount: 4096, pageSize: 4096, - cacheSize: 2000, cacheHitRate: 0.987 +const postgresData = { + engine: 'postgresql', databaseBytes: 12900000, openConnections: 4, inUseConnections: 2, + idleConnections: 2, connectionWaitCount: 3, connectionWaitMs: 1.25, cacheHitRate: 0.987 }; const sourcesData = { sources: { tx_inserted: 25, obs_inserted: 1787, backfill_path_json: 0, node_upserts: 329, observer_upserts: 1823, walCommits: 100 }, @@ -84,7 +84,7 @@ console.log('\n🧪 perf.js — Disk I/O + Write Sources (#1120)\n'); (async () => { await test('Renders Disk I/O section', async () => { const sb = loadPerf(); - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sqliteData, sourcesData); + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, postgresData, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); const html = sb.getHtml(); @@ -94,7 +94,7 @@ await test('Renders Disk I/O section', async () => { await test('Renders Write Sources section with non-zero rates', async () => { const sb = loadPerf(); - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sqliteData, sourcesData); + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, postgresData, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); const html = sb.getHtml(); @@ -103,13 +103,15 @@ await test('Renders Write Sources section with non-zero rates', async () => { assert.ok(html.includes('obs_inserted'), 'should list obs_inserted source'); }); -await test('Renders SQLite section with WAL + cache hit rate', async () => { +await test('Renders PostgreSQL connection and cache statistics', async () => { const sb = loadPerf(); - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sqliteData, sourcesData); + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, postgresData, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); const html = sb.getHtml(); - assert.ok(/WAL/i.test(html), 'should show WAL info'); + assert.ok(html.includes('PostgreSQL'), 'should name the active database engine'); + assert.ok(html.includes('Connections in use'), 'should show PostgreSQL pool utilization'); + assert.ok(!html.includes('WAL Size') && !html.includes('Page Count'), 'must not invent SQLite metrics'); assert.ok(/Cache Hit/i.test(html) || /cacheHitRate/i.test(html), 'should show cache hit rate'); }); @@ -118,7 +120,7 @@ await test('Renders SQLite section with WAL + cache hit rate', async () => { await test('Renders cancelledWriteBytesPerSec for server process', async () => { const sb = loadPerf(); const io = { ...ioData, cancelledWriteBytesPerSec: 4096 }; - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, io, sqliteData, sourcesData); + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, io, postgresData, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); const html = sb.getHtml(); @@ -139,7 +141,7 @@ await test('Renders ingestor row alongside server row in Disk I/O', async () => syscallsWrite: 0, }, }; - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, io, sqliteData, sourcesData); + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, io, postgresData, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); const html = sb.getHtml(); @@ -147,34 +149,25 @@ await test('Renders ingestor row alongside server row in Disk I/O', async () => assert.ok(/1\.0\s*MB/.test(html), 'should render ingestor write 1 MB/s'); }); -await test('WAL >100 MB fires ⚠️ flag', async () => { +await test('Unavailable PostgreSQL cache statistics render as unknown', async () => { const sb = loadPerf(); - const sql = { ...sqliteData, walSizeMB: 150, walSize: 150 * 1048576 }; - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sql, sourcesData); + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, { ...postgresData, cacheHitRate: null }, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); - const html = sb.getHtml(); - // The warning appears in the WAL Size card; assert proximity by extracting - // the WAL Size card's text content. - const walSection = html.match(/150\.0MB\s*]*>]*#ph-warning"/); - assert.ok(walSection, 'expected ⚠️ next to 150MB WAL value, html=' + html.slice(html.indexOf('WAL Size') - 200, html.indexOf('WAL Size') + 200)); + assert.ok(sb.getHtml().includes('—
Database Block Cache Hit Rate'), 'missing metrics must not become zero'); }); -await test('WAL <100 MB does NOT fire ⚠️ flag', async () => { +await test('PostgreSQL numeric string metrics render without an exception', async () => { const sb = loadPerf(); - const sql = { ...sqliteData, walSizeMB: 12.3 }; - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sql, sourcesData); + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, { ...postgresData, cacheHitRate: '0.987', connectionWaitMs: '1.25' }, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); - const html = sb.getHtml(); - const walIdx = html.indexOf('WAL Size'); - const slice = html.slice(Math.max(0, walIdx - 200), walIdx); - assert.ok(!/12\.3MB\s*]*>]*#ph-warning"/.test(slice), 'expected NO warning icon next to 12.3MB WAL value'); + assert.ok(sb.getHtml().includes('98.7%') && sb.getHtml().includes('1.3ms'), 'string metrics must format numerically'); }); await test('Cache hit <90% fires ⚠️ flag', async () => { const sb = loadPerf(); - const sql = { ...sqliteData, cacheHitRate: 0.85 }; + const sql = { ...postgresData, cacheHitRate: 0.85 }; stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sql, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); @@ -184,7 +177,7 @@ await test('Cache hit <90% fires ⚠️ flag', async () => { await test('Cache hit ≥90% does NOT fire ⚠️ flag', async () => { const sb = loadPerf(); - const sql = { ...sqliteData, cacheHitRate: 0.987 }; + const sql = { ...postgresData, cacheHitRate: 0.987 }; stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sql, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); @@ -194,31 +187,28 @@ await test('Cache hit ≥90% does NOT fire ⚠️ flag', async () => { // === #1167 must-fix #7: threshold boundary cases === -await test('WAL exactly 100 MB does NOT fire ⚠️ (boundary, strict >)', async () => { +await test('Unavailable database diagnostics do not invent a PostgreSQL panel', async () => { const sb = loadPerf(); - const sql = { ...sqliteData, walSizeMB: 100 }; - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sql, sourcesData); + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, null, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); - const html = sb.getHtml(); - const walIdx = html.indexOf('WAL Size'); - const slice = html.slice(Math.max(0, walIdx - 200), walIdx); - assert.ok(!/100\.0MB\s*]*>]*#ph-warning"/.test(slice), 'expected NO warning icon at exactly 100 MB WAL (boundary), slice=' + slice); + assert.ok(!sb.getHtml().includes('PostgreSQL connections'), 'unavailable statistics should stay unavailable'); }); -await test('WAL infinitesimally over 100 MB DOES fire ⚠️', async () => { +await test('PostgreSQL data panel reports counts without SQLite storage fields', async () => { const sb = loadPerf(); - const sql = { ...sqliteData, walSizeMB: 100.01 }; - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sql, sourcesData); + const postgres = { engine: 'postgresql', dbSizeMB: 42, rows: { transmissions: 30000, observations: 90000, nodes: 2000, observers: 32 } }; + stubFetch(sb, { ...basePerf, goRuntime, postgres }, goHealth, ioData, postgresData, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); const html = sb.getHtml(); - assert.ok(/100\.0MB\s*]*>]*#ph-warning"/.test(html), 'expected warning icon next to 100.0MB WAL value (just over threshold)'); + assert.ok(html.includes('PostgreSQL data') && html.includes('42MB') && html.includes((30000).toLocaleString()), 'native storage figures missing'); + assert.ok(!html.includes('Freelist') && !html.includes('WAL Size'), 'SQLite-only metrics must not survive the conversion'); }); await test('Cache hit exactly 90% does NOT fire ⚠️ (boundary, strict <)', async () => { const sb = loadPerf(); - const sql = { ...sqliteData, cacheHitRate: 0.90 }; + const sql = { ...postgresData, cacheHitRate: 0.90 }; stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sql, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); @@ -228,7 +218,7 @@ await test('Cache hit exactly 90% does NOT fire ⚠️ (boundary, strict <)', as await test('Cache hit infinitesimally below 90% DOES fire ⚠️', async () => { const sb = loadPerf(); - const sql = { ...sqliteData, cacheHitRate: 0.8999 }; + const sql = { ...postgresData, cacheHitRate: 0.8999 }; stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sql, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); @@ -236,6 +226,24 @@ await test('Cache hit infinitesimally below 90% DOES fire ⚠️', async () => { assert.ok(/90\.0%\s*]*>]*#ph-warning"/.test(html), 'expected warning icon next to 90.0% cache hit value (just under threshold)'); }); +await test('Stale PostgreSQL sample is labelled and retains verified counts', async () => { + const sb = loadPerf(); + const postgres = { engine: 'postgresql', dbSizeMB: 42, rows: { transmissions: 30000, observations: 90000, nodes: 2000, observers: 32 }, sampledAt: '2026-01-01T12:00:00Z', sampleIntervalSeconds: 30, stale: true, error: 'unavailable' }; + stubFetch(sb, { ...basePerf, goRuntime, postgres }, goHealth, ioData, { ...postgresData, sampledAt: postgres.sampledAt, sampleIntervalSeconds: 30, stale: true, cacheHitRate: null }, sourcesData); + await sb.pages.perf.init({ set innerHTML(v) {} }); + await new Promise(r => setTimeout(r, 100)); + assert.ok(sb.getHtml().includes('42MB') && sb.getHtml().includes('Last verified PostgreSQL sample'), 'stale data must remain labelled as a prior verified sample'); +}); + +await test('Initial PostgreSQL sampling failure does not invent zero storage', async () => { + const sb = loadPerf(); + const postgres = { engine: 'postgresql', dbSizeMB: null, rows: null, stale: true, error: 'unavailable' }; + stubFetch(sb, { ...basePerf, goRuntime, postgres }, goHealth, ioData, { ...postgresData, databaseBytes: null, cacheHitRate: null, stale: true }, sourcesData); + await sb.pages.perf.init({ set innerHTML(v) {} }); + await new Promise(r => setTimeout(r, 100)); + assert.ok(sb.getHtml().includes('PostgreSQL storage unavailable'), 'failed first sample must be explicit'); + assert.ok(!sb.getHtml().includes('0MB
DB Size'), 'unknown storage must not become zero'); +}); await test('Backfill anomaly: rate >10× its stable rolling baseline shows a warning', async () => { // The current detector compares a source with its own rolling baseline. // Prime a 60-second baseline at 1/s, then jump to 1000/s for the next tick. @@ -246,7 +254,7 @@ await test('Backfill anomaly: rate >10× its stable rolling baseline shows a war { sources: { tx_inserted: 405, backfill_path_json: 1060 }, sampleAt: '2026-01-01T00:01:01Z' }, ]; for (const sample of samples) { - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sqliteData, sample); + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, postgresData, sample); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 50)); } @@ -266,10 +274,10 @@ await test('Backfill anomaly: insufficient history suppresses the warning', asyn const t1 = '2026-01-01T00:00:01Z'; const phase1 = { sources: { tx_inserted: 5, backfill_path_json: 0 }, sampleAt: t0 }; const phase2 = { sources: { tx_inserted: 6, backfill_path_json: 1000 }, sampleAt: t1 }; - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sqliteData, phase1); + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, postgresData, phase1); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 50)); - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sqliteData, phase2); + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, postgresData, phase2); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 50)); const html = sb.getHtml(); diff --git a/tests/unit/test-postgres-ops.js b/tests/unit/test-postgres-ops.js new file mode 100644 index 000000000..87adf7e84 --- /dev/null +++ b/tests/unit/test-postgres-ops.js @@ -0,0 +1,126 @@ +// Exercise the actual management functions with a fake container boundary. +// No running Docker service or real database is touched. +const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { spawnSync } = require('child_process'); + +const root = path.resolve(__dirname, '../..'); +const source = fs.readFileSync(path.join(root, 'manage.sh'), 'utf8').replace(/\r/g, ''); +const bash = require('../../scripts/bash-path')(); +const functions = ['pg_exec', 'pg_empty', 'pg_dump_file', 'pg_restore_file', 'cmd_backup', 'cmd_restore', 'prepare_staging_db', 'prepare_staging_config', 'write_env_managed_values', 'is_true', 'cmd_start'] + .map(name => source.match(new RegExp(`^${name}\\(\\)\\s*\\{[\\s\\S]*?^}`, 'm'))?.[0] || '').join('\n'); +const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'corescope-pg-ops-')); +const quote = value => "'" + value.replace(/'/g, "'\\''") + "'"; +const posix = value => value.replace(/\\/g, '/'); +function run(command, occupied = '0', failRestore = '0', prodReady = '1') { + const setup = ` +set -e +PROD_DATA=${quote(posix(path.join(temp, 'state')))} +STAGING_DATA=${quote(posix(path.join(temp, 'staging')))} +OCCUPIED=${quote(occupied)} +FAIL_RESTORE=${quote(failRestore)} +PROD_READY=${quote(prodReady)} +CALLS=${quote(posix(path.join(temp, 'calls')))} +CORESCOPE_OWNER_PASSWORD='must-not-appear-in-argv' +log(){ :; }; info(){ :; }; warn(){ :; }; err(){ printf '%s\\n' "$*" >&2; } +confirm(){ return 0; }; container_running(){ return 1; } +docker(){ return 0; }; require_database_credentials(){ :; } +preflight_validate_prod_ports(){ return 0; }; migrate_config(){ :; }; ensure_config(){ :; } +database_reply(){ + case "$*" in + *pg_dump*) [ "$PROD_READY" = 1 ] || { err 'Production PostgreSQL is not running'; return 1; }; printf 'PGDMP-native-test-archive';; + *pg_restore*) cat >/dev/null; [ "$FAIL_RESTORE" != 1 ];; + *SELECT*COUNT*) printf '%s\\n' "$OCCUPIED";; + *) :;; + esac +} +dc_prod(){ + printf 'prod %s\\n' "$*" >> "$CALLS" + if [ "$*" = 'up -d prod' ]; then + [ "$PROD_READY" != fail ] || return 1 + PROD_READY=1 + fi + database_reply "$@" +} +dc_staging(){ printf 'staging %s\\n' "$*" >> "$CALLS"; database_reply "$@"; } +${functions} +${command} +`; + // Keep the extracted shell source off Windows' bounded command line. + return spawnSync(bash, ['--noprofile', '--norc'], { input: setup, cwd: temp, encoding: 'utf8', timeout: 20000 }); +} +try { + fs.mkdirSync(path.join(temp, 'state')); + fs.mkdirSync(path.join(temp, 'docker')); + fs.writeFileSync(path.join(temp, 'docker/postgres-grants.sql'), 'SELECT 1;'); + fs.writeFileSync(path.join(temp, 'state/meshcore.db'), 'SQLite format 3\0legacy recovery'); + fs.writeFileSync(path.join(temp, 'state/config.json'), '{"siteName":"test"}'); + const backup = path.join(temp, 'backup'); + const made = run(`cmd_backup ${quote(posix(backup))}`); + assert.strictEqual(made.status, 0, made.stderr || made.error?.message); + for (const name of ['telemetry.dump', 'accounts.dump']) { + assert(fs.existsSync(path.join(backup, name)), `native archive missing: ${name}`); + assert(fs.readFileSync(path.join(backup, name), 'utf8').startsWith('PGDMP')); + } + assert(fs.existsSync(path.join(backup, 'config.json')), 'configuration omitted'); + assert(!fs.existsSync(path.join(backup, 'meshcore.db')), 'backup copied a legacy live SQLite file'); + const blocked = run(`cmd_restore ${quote(posix(backup))}`, '1'); + assert.notStrictEqual(blocked.status, 0, 'restore overwrote a nonempty destination'); + assert(!fs.readFileSync(path.join(temp, 'calls'), 'utf8').includes('pg_restore'), 'restore ran before emptiness check'); + const failedRestore = run(`cmd_restore ${quote(posix(backup))}`, '0', '1'); + assert.notStrictEqual(failedRestore.status, 0, 'failed native restore was reported as success'); + const restored = run(`cmd_restore ${quote(posix(backup))}`); + assert.strictEqual(restored.status, 0, restored.stderr || restored.error?.message); + const calls = fs.readFileSync(path.join(temp, 'calls'), 'utf8'); + assert(calls.includes('pg_restore'), 'native restore was not invoked'); + assert(!calls.includes('must-not-appear-in-argv'), 'database password appeared in command arguments'); + const legacy = run(`cmd_restore ${quote(posix(path.join(temp, 'state/meshcore.db')))}`); + assert.notStrictEqual(legacy.status, 0, 'SQLite restore accepted without offline import'); + fs.writeFileSync(path.join(temp, 'calls'), ''); + const staged = run('prepare_staging_db'); + assert.strictEqual(staged.status, 0, staged.stderr || staged.error?.message); + assert(!fs.readFileSync(path.join(temp, 'calls'), 'utf8').includes('corescope_accounts'), 'staging copied production accounts'); + fs.writeFileSync(path.join(temp, 'calls'), ''); + const productionOnly = run("cmd_start ''", '0', '0', '0'); + assert.strictEqual(productionOnly.status, 0, productionOnly.stderr || productionOnly.error?.message); + assert.deepStrictEqual(fs.readFileSync(path.join(temp, 'calls'), 'utf8').trim().split('\n'), ['prod up -d prod'], 'ordinary cold start changed'); + fs.writeFileSync(path.join(temp, 'calls'), ''); + const coldStaging = run('cmd_start --with-staging', '0', '0', '0'); + assert.strictEqual(coldStaging.status, 0, coldStaging.stderr || coldStaging.error?.message); + const startCalls = fs.readFileSync(path.join(temp, 'calls'), 'utf8').trim().split('\n'); + assert.strictEqual(startCalls.filter(call => call === 'prod up -d prod').length, 1, 'production must start once through normal Compose dependencies'); + assert(startCalls.indexOf('prod up -d prod') < startCalls.findIndex(call => call.includes('pg_dump')), 'staging dump preceded production bootstrap'); + const restoreIndex = startCalls.findIndex(call => call.includes('pg_restore')); + assert(restoreIndex >= 0 && restoreIndex < startCalls.indexOf('staging up -d staging-go'), 'staging application started before restore'); + assert(!startCalls.some(call => call.includes('corescope_accounts')), 'cold staging start copied production accounts'); + fs.writeFileSync(path.join(temp, 'calls'), ''); + const retainedStaging = run('cmd_start --with-staging', '1', '0', '0'); + assert.strictEqual(retainedStaging.status, 0, retainedStaging.stderr || retainedStaging.error?.message); + assert(!/pg_dump|pg_restore/.test(fs.readFileSync(path.join(temp, 'calls'), 'utf8')), 'cold start replaced existing staging data'); + fs.writeFileSync(path.join(temp, 'calls'), ''); + const failedStart = run('cmd_start --with-staging', '0', '0', 'fail'); + assert.notStrictEqual(failedStart.status, 0, 'failed production startup reported success'); + assert.deepStrictEqual(fs.readFileSync(path.join(temp, 'calls'), 'utf8').trim().split('\n'), ['prod up -d prod'], 'staging started after production bootstrap failed'); + for (const [executable, script] of [ + [process.execPath, 'scripts/migrate-dedup.js'], + [process.platform === 'win32' ? 'python' : 'python3', 'scripts/prune-nodes-outside-geo-filter.py'] + ]) { + const retired = spawnSync(executable, [path.join(root, script)], { cwd: temp, encoding: 'utf8', timeout: 10000 }); + assert.notStrictEqual(retired.status, 0, 'legacy tool executed without explicit offline opt-in'); + assert(/offline legacy SQLite/i.test(retired.stderr), retired.stderr); + } + fs.writeFileSync(path.join(temp, '.env'), 'CORESCOPE_OWNER_PASSWORD=private-test-value\nPROD_HTTP_PORT=80\n', { mode: 0o600 }); + const configured = run("write_env_managed_values 8080 8443 1883 ./state false"); + assert.strictEqual(configured.status, 0, configured.stderr); + const config = fs.readFileSync(path.join(temp, '.env'), 'utf8'); + assert(config.includes('CORESCOPE_OWNER_PASSWORD=private-test-value'), 'setup lost database credentials'); + assert(config.includes('PROD_HTTP_PORT=8080'), 'setup did not update the requested port'); + if (process.platform !== 'win32') assert.strictEqual(fs.statSync(path.join(temp, '.env')).mode & 0o777, 0o600, 'setup exposed private credentials'); + console.log('PostgreSQL management backup/restore safeguards passed'); +} finally { + const base = path.resolve(os.tmpdir()) + path.sep; + assert(path.resolve(temp).startsWith(base) && path.basename(temp).startsWith('corescope-pg-ops-')); + fs.rmSync(temp, { recursive: true }); +} From 6377102fe5b549fa8f7132237531557e07be5b0c Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 8 Oct 2026 23:59:08 -0400 Subject: [PATCH 02/67] fix: grant the ingestor access to its readiness view --- docker/postgres-grants.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/postgres-grants.sql b/docker/postgres-grants.sql index 177a20a9a..e446efa3e 100644 --- a/docker/postgres-grants.sql +++ b/docker/postgres-grants.sql @@ -9,6 +9,7 @@ GRANT USAGE ON SCHEMA public TO corescope_reader, corescope_writer; GRANT SELECT ON ALL TABLES IN SCHEMA public TO corescope_reader; GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO corescope_reader; GRANT SELECT ON corescope_schema TO corescope_writer; +GRANT SELECT ON packets_v TO corescope_writer; SELECT format('GRANT SELECT,INSERT,UPDATE,DELETE ON TABLE %I.%I TO corescope_writer',schemaname,tablename) FROM pg_tables WHERE schemaname='public' AND tablename NOT LIKE 'corescope\_%' ESCAPE '\' \gexec GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO corescope_writer; From cc89006c88a6e90c43c0ed0ce77f79e0f950575d Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 00:16:35 -0400 Subject: [PATCH 03/67] fix: support opted-in PR benchmarks and restore database URLs safely --- .github/workflows/deploy.yml | 32 ++++- .github/workflows/postgres-benchmark.yml | 14 ++- scripts/postgres-benchmark/README.md | 33 ++++++ tests/unit/test-issue-1956-release-routing.js | 111 +++++++++++++++++- 4 files changed, 184 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8263e421b..3f22852d8 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -56,12 +56,17 @@ env: jobs: postgres-benchmark: name: Isolated PostgreSQL comparison - if: github.event_name == 'workflow_dispatch' && inputs.postgres_benchmark + needs: [changes] + if: | + (github.event_name == 'workflow_dispatch' && inputs.postgres_benchmark) + || (github.event_name == 'pull_request' + && needs.changes.outputs.code == 'true' + && contains(github.event.pull_request.body, '[x] Run PostgreSQL comparison')) permissions: contents: read uses: ./.github/workflows/postgres-benchmark.yml with: - candidate_sha: ${{ inputs.candidate_sha }} + candidate_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || inputs.candidate_sha }} # ─────────────────────────────────────────────────────────────── # 0. Change scope — decides whether the expensive jobs below run. @@ -824,8 +829,27 @@ jobs: PGUSER=corescope_owner PGPASSWORD="$(cat "$RUNNER_TEMP/corescope-e2e-owner-password")" \ psql -X -v ON_ERROR_STOP=1 -d "corescope_restore_$kind" -c 'ANALYZE' done - export CORESCOPE_READER_DATABASE_URL="${CORESCOPE_READER_DATABASE_URL/\/corescope_telemetry/\/corescope_restore_telemetry}" - export CORESCOPE_USERS_DATABASE_URL="${CORESCOPE_USERS_DATABASE_URL/\/corescope_accounts/\/corescope_restore_accounts}" + # Switch restored database URLs. + restore_urls=$(python3 - <<'PY' + import os + from urllib.parse import urlsplit, urlunsplit + for key, database in ( + ("CORESCOPE_READER_DATABASE_URL", "corescope_restore_telemetry"), + ("CORESCOPE_USERS_DATABASE_URL", "corescope_restore_accounts"), + ): + raw = os.environ[key] + try: + parsed = urlsplit(raw) + except ValueError: + raise SystemExit("Invalid restore database URL") from None + if parsed.scheme not in ("postgres", "postgresql") or not parsed.netloc or "\n" in raw or "\r" in raw: + raise SystemExit("Invalid restore database URL") + print(key + "=" + urlunsplit(parsed._replace(path="/" + database))) + PY + ) + while IFS= read -r assignment; do export "$assignment"; done <<< "$restore_urls" + unset restore_urls assignment + # Restart restored servers. env -u PGPASSWORD -u PGUSER -u CORESCOPE_TEST_POSTGRES_URL ./corescope-server -port 13581 -state-dir "$RUNNER_TEMP/corescope-state" -public public-instrumented & echo $! > .server.pid env -u PGPASSWORD -u PGUSER -u CORESCOPE_TEST_POSTGRES_URL ./corescope-server-e2e -config-dir /tmp/cs-um -port 13582 -state-dir /tmp/cs-um -public public-instrumented & diff --git a/.github/workflows/postgres-benchmark.yml b/.github/workflows/postgres-benchmark.yml index 12bc47b2b..bd3dea97c 100644 --- a/.github/workflows/postgres-benchmark.yml +++ b/.github/workflows/postgres-benchmark.yml @@ -43,15 +43,27 @@ jobs: - name: Checkout the exact candidate uses: actions/checkout@v5 with: + repository: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name || github.repository }} ref: ${{ inputs.candidate_sha }} fetch-depth: 0 persist-credentials: false - name: Verify candidate and workflow snapshot + env: + EXPECTED_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + WORKFLOW_SHA: ${{ github.workflow_sha }} run: | set -euo pipefail test "$(git rev-parse HEAD)" = "$CANDIDATE_SHA" - test "$GITHUB_SHA" = "$CANDIDATE_SHA" || { echo 'Dispatch this workflow from the candidate commit/branch so workflow and code match' >&2; exit 1; } + test "$EXPECTED_SHA" = "$CANDIDATE_SHA" || { echo 'Candidate must match the PR head or dispatched workflow commit' >&2; exit 1; } + [[ "$WORKFLOW_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo 'A full workflow commit SHA is required' >&2; exit 1; } + # PR workflows run from a merge ref, while the experiment measures the + # exact head. Refuse a merge that changes the executed workflow/harness. + git fetch --no-tags --depth=1 "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git" "$WORKFLOW_SHA" + git diff --quiet "$WORKFLOW_SHA" "$CANDIDATE_SHA" -- \ + .github/workflows/deploy.yml .github/workflows/postgres-benchmark.yml \ + scripts/postgres-benchmark scripts/install-postgres-ci.sh \ + || { echo 'Workflow or benchmark harness differs from candidate; update the candidate before rerunning' >&2; exit 1; } git cat-file -e 9dbc287579a237ffa744dd0c91fa7227d09763ac^{commit} - name: Test benchmark controls diff --git a/scripts/postgres-benchmark/README.md b/scripts/postgres-benchmark/README.md index 4a0c29c2c..3995442a9 100644 --- a/scripts/postgres-benchmark/README.md +++ b/scripts/postgres-benchmark/README.md @@ -35,6 +35,39 @@ pilot. **No empirical database comparison is included in this change.** second guard uses eight times the *measured* canonical file size before each restore/import. These are conservative guards, not dataset-size claims. +## GitHub Actions opt-in + +The existing CI workflow can run the comparison for a pull request. Add this +checked item to the PR body **before pushing the reviewed candidate**: + +```markdown +- [x] Run PostgreSQL comparison +``` + +The checkbox is evaluated on the normal PR events: opened, reopened and head +updates. Editing the body alone does not launch or cancel CI. Without the checked +item, the benchmark is skipped. A documentation-only PR also skips it through +the change-scope gate. + +The comparison checks out the PR's exact head SHA from its head repository, +including forks. It does not benchmark GitHub's synthetic merge commit. The +executed workflow and benchmark harness must match that head; mismatches fail +before running the experiment. The job has read-only repository access, persists +no checkout credentials, inherits no deployment secrets and cannot publish or +deploy. Other PR checks continue normally. + +Wait for the comparison to finish, then **uncheck the item before pushing a +documentation-only results/report update**. Change scope covers the full PR diff, +so a PR that still contains implementation changes would otherwise request +another primary run. Existing PR concurrency can also cancel an unfinished run +when a new commit arrives. + +Manual dispatch remains available: use `deploy.yml` with +`postgres_benchmark=true` and the full `candidate_sha`, from the matching +candidate branch/commit. That dispatch runs the comparison alone and skips normal +publishing/deployment jobs. Both entry points run the small pilot first and then +five primary pairs; upload only the sanitized `public/` outputs. + ## Commands First run the portable controls and compile/protocol checks: diff --git a/tests/unit/test-issue-1956-release-routing.js b/tests/unit/test-issue-1956-release-routing.js index 2b0419450..312de970a 100644 --- a/tests/unit/test-issue-1956-release-routing.js +++ b/tests/unit/test-issue-1956-release-routing.js @@ -39,7 +39,8 @@ const steps = source => source.split(/(?=^ - name:)/m).slice(1); function evaluate(expression, context) { if (!expression) return true; return vm.runInNewContext(expression.replace(/^\$\{\{|\}\}$/g, '').trim(), { - ...context, startsWith: (text, prefix) => text.startsWith(prefix), cancelled: () => false + ...context, startsWith: (text, prefix) => text.startsWith(prefix), cancelled: () => false, + contains: (text, part) => String(text ?? '').toLowerCase().includes(String(part).toLowerCase()) }); } const expand = (script, context) => script.replace(/\$\{\{(.*?)\}\}/g, (_, expression) => String(evaluate(expression, context))); @@ -346,3 +347,111 @@ for (const ref of ['refs/heads/master', 'refs/heads/codex/postgres', 'refs/tags/ const benchmarkCall = block(deploy, 'postgres-benchmark', 2); assert.ok(!benchmarkCall.includes('secrets:') && !benchmarkCall.includes('environment:'), 'benchmark must receive no deployment secrets/environment'); console.log('PASS benchmark-only dispatch cannot publish or deploy'); + +// PR benchmarks are deliberate opt-ins and consume the head repository/SHA, +// while ordinary CI continues to test GitHub's merge ref. +const benchmarkWorkflow = read('postgres-benchmark.yml'); +const prBenchmark = context('refs/pull/1/merge', 'pull_request'); +prBenchmark.github.workflow_sha = 'c'.repeat(40); +prBenchmark.github.event = { pull_request: { + body: '- [x] Run PostgreSQL comparison', + head: { sha: 'b'.repeat(40), repo: { full_name: 'contributor/project' } } +} }; +for (const [body, code, expected] of [ + ['- [x] Run PostgreSQL comparison', 'true', true], + ['- [X] Run PostgreSQL comparison', 'true', true], + ['- [ ] Run PostgreSQL comparison', 'true', false], + ['', 'true', false], + [null, 'true', false], + ['- [x] Run PostgreSQL comparison', 'false', false] +]) { + prBenchmark.github.event.pull_request.body = body; + prBenchmark.needs.changes = { result: 'success', outputs: { code } }; + assert.equal(evaluate(value(benchmarkCall, 'if', 4), prBenchmark), expected, 'PR checkbox/code-scope routing'); +} +assert.equal(value(benchmarkCall, 'needs', 4), '[changes]', 'PR benchmarks must wait for code scope'); +const dispatchBenchmark = context('refs/heads/candidate', 'workflow_dispatch', { postgres_benchmark: true, candidate_sha: 'a'.repeat(40) }); +dispatchBenchmark.needs.changes = { result: 'success', outputs: { code: 'false' } }; +assert.equal(evaluate(value(benchmarkCall, 'if', 4), dispatchBenchmark), true, 'benchmark-only dispatch must survive its normal code=false output'); +assert.equal(evaluate(value(block(benchmarkCall, 'with', 4), 'candidate_sha', 6), prBenchmark), prBenchmark.github.event.pull_request.head.sha, 'benchmark candidate must be the PR head, not merge SHA'); +assert.equal(value(block(benchmarkCall, 'permissions', 4), 'contents', 6), 'read'); +const benchmarkSteps = steps(block(benchmarkWorkflow, 'benchmark', 2)); +const candidateCheckout = benchmarkSteps.find(step => value(step, '- name', 6) === 'Checkout the exact candidate'); +assert.equal(value(candidateCheckout, 'persist-credentials', 10), 'false'); +assert.equal(evaluate(value(candidateCheckout, 'repository', 10), prBenchmark), 'contributor/project'); +assert.equal(evaluate(value(candidateCheckout, 'repository', 10), context()), 'example/corescope'); +assert.equal(value(candidateCheckout, 'ref', 10), '${{ inputs.candidate_sha }}'); +assert.equal(value(block(benchmarkWorkflow, 'permissions', 0), 'contents', 2), 'read'); + +// Execute the real snapshot guard with only Git's external boundary stubbed. +// A PR's merge SHA differs from the benchmark head; mismatched code/workflows +// must still fail before package installation or benchmark execution. +const snapshotStep = benchmarkSteps.find(step => value(step, '- name', 6) === 'Verify candidate and workflow snapshot'); +function snapshotGuard({ candidate = 'b'.repeat(40), checkout = candidate, expected = candidate, merge = 'a'.repeat(40), matches = true } = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'corescope-benchmark-guard-')); + const log = path.join(dir, 'git-calls'); + fs.writeFileSync(log, ''); + try { + const stubs = ` + git() { + printf '%s\\n' "$*" >> "$CHECK_LOG" + case "$1" in + rev-parse) printf '%s\\n' "$CHECKOUT_SHA" ;; + fetch|cat-file) return 0 ;; + diff) [ "$WORKFLOW_MATCHES" = true ] ;; + *) return 99 ;; + esac + } + `; + const result = spawnSync(bash, ['--noprofile', '--norc'], { + input: stubs + value(snapshotStep, 'run', 8), cwd: dir, encoding: 'utf8', + env: { ...process.env, CANDIDATE_SHA: candidate, CHECKOUT_SHA: checkout, + EXPECTED_SHA: expected, GITHUB_SHA: merge, WORKFLOW_SHA: 'c'.repeat(40), + GITHUB_REPOSITORY: 'example/corescope', GITHUB_SERVER_URL: 'https://github.com', + WORKFLOW_MATCHES: String(matches), CHECK_LOG: bashPath(log) } + }); + return { status: result.status, error: result.stderr, calls: fs.readFileSync(log, 'utf8') }; + } finally { fs.rmSync(dir, { recursive: true, force: true }); } +} +const acceptedHead = snapshotGuard(); +assert.equal(acceptedHead.status, 0, 'PR head must be accepted despite a distinct merge SHA: ' + acceptedHead.error); +assert.ok(acceptedHead.calls.includes('fetch --no-tags --depth=1 https://github.com/example/corescope.git ' + 'c'.repeat(40)), 'verify the actual caller workflow commit'); +for (const path of ['.github/workflows/deploy.yml', '.github/workflows/postgres-benchmark.yml', 'scripts/postgres-benchmark', 'scripts/install-postgres-ci.sh']) { + assert.ok(acceptedHead.calls.includes(path), 'snapshot comparison omitted ' + path); +} +assert.notEqual(snapshotGuard({ matches: false }).status, 0, 'different workflow/harness content accepted'); +assert.notEqual(snapshotGuard({ checkout: 'd'.repeat(40) }).status, 0, 'wrong checkout accepted'); +assert.notEqual(snapshotGuard({ expected: 'd'.repeat(40) }).status, 0, 'candidate differs from event head'); +assert.equal(snapshotGuard({ candidate: 'a'.repeat(40), merge: 'a'.repeat(40) }).status, 0, 'ordinary exact-SHA dispatch failed'); +const snapshotEnv = block(snapshotStep, 'env', 8); +assert.equal(evaluate(value(snapshotEnv, 'EXPECTED_SHA', 10), prBenchmark), 'b'.repeat(40)); +assert.equal(evaluate(value(snapshotEnv, 'EXPECTED_SHA', 10), context()), 'a'.repeat(40)); +assert.equal(evaluate(value(snapshotEnv, 'WORKFLOW_SHA', 10), prBenchmark), 'c'.repeat(40)); +console.log('PASS opted-in PR benchmark uses exact head and matching workflow/harness, without persisted credentials'); + +// Execute the real restore-URL transformation. A role and its database can have +// the same name; replacing the first matching substring silently changes login. +const restoreStep = steps(block(deploy, 'e2e-shard', 2)).find(step => value(step, '- name', 6) === 'Restore native HTTP backups and verify browser sessions'); +const restoreScript = value(restoreStep, 'run', 8); +const switchURLs = restoreScript.split('# Switch restored database URLs.\n')[1]?.split('# Restart restored servers.')[0]; +assert.ok(switchURLs, 'missing restore URL transformation'); +const sourceURLs = { + CORESCOPE_USERS_DATABASE_URL: 'postgresql://corescope_accounts:p%40ss%2F%3A%3F%23@db.example.invalid:5433/corescope_accounts?sslmode=verify-full&application_name=corescope_accounts', + CORESCOPE_READER_DATABASE_URL: 'postgresql://corescope_reader:p%2540%3A%2F@[::1]:5432/corescope_telemetry?sslmode=require&application_name=reader%3Bsmoke' +}; +const rewritten = spawnSync(bash, ['--noprofile', '--norc', '-e', '-o', 'pipefail'], { + input: (process.platform === 'win32' ? 'python3(){ python "$@"; }\n' : '') + switchURLs + + '\nprintf "%s\\n" "$CORESCOPE_USERS_DATABASE_URL" "$CORESCOPE_READER_DATABASE_URL"\n', + encoding: 'utf8', env: { ...process.env, ...sourceURLs } +}); +assert.equal(rewritten.status, 0, rewritten.stderr); +const resultURLs = rewritten.stdout.trim().split('\n'); +for (const [index, key] of Object.keys(sourceURLs).entries()) { + const original = new URL(sourceURLs[key]); + const restored = new URL(resultURLs[index]); + for (const field of ['protocol', 'username', 'password', 'host', 'search', 'hash']) { + assert.equal(restored[field], original[field], 'restore URL changed ' + field); + } + assert.equal(restored.pathname, key === 'CORESCOPE_USERS_DATABASE_URL' ? '/corescope_restore_accounts' : '/corescope_restore_telemetry'); +} +console.log('PASS native backup restore changes only database URL paths'); From 7f655ae71ee7d3b4f29df2629c2152153a0d4aa4 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 00:42:12 -0400 Subject: [PATCH 04/67] fix: run blacklist QA against native PostgreSQL --- qa/README.md | 41 ++- qa/plans/v3.6.0-rc.md | 2 +- qa/scripts/blacklist-test.sh | 217 ++++++------ qa/scripts/postgres-connect.py | 93 ++++++ qa/scripts/test-blacklist-sql.sh | 497 +++++++++++++--------------- qa/scripts/test-postgres-connect.py | 136 ++++++++ 6 files changed, 609 insertions(+), 377 deletions(-) create mode 100644 qa/scripts/postgres-connect.py create mode 100644 qa/scripts/test-postgres-connect.py diff --git a/qa/README.md b/qa/README.md index 938b8ac1f..3a9310f78 100644 --- a/qa/README.md +++ b/qa/README.md @@ -10,7 +10,9 @@ qa/ ├── plans/ │ └── .md ← per-release test plans (one file per RC) └── scripts/ - └── api-contract-diff.sh ← CoreScope-tuned API contract diff + ├── api-contract-diff.sh ← CoreScope-tuned API contract diff + ├── blacklist-test.sh ← hide/retain/recover deployment check + └── test-blacklist-sql.sh ← disposable native PostgreSQL security gate ``` ## How to run @@ -43,3 +45,40 @@ Authoring rules from the qa-suite skill: - Distinguish HTTP errors from jq parse failures - Don't silence stderr — script bugs must surface - Exit code = number of failures + +## PostgreSQL blacklist retention probe + +The active blacklist check reads PostgreSQL using a restricted telemetry reader. +Set `CORESCOPE_READER_DATABASE_URL` privately on the QA runner, or configure native +libpq variables/service files (`PGDATABASE`, `PGUSER`, `PGPASSFILE`, `PGSERVICE`, +and connection/TLS settings). A URL belongs in `CORESCOPE_READER_DATABASE_URL`; +`PGDATABASE` itself does not expand URLs. Do not put credentials in command-line +arguments. `TARGET_DB_PATH` is obsolete and rejected. + +With no explicit local connection, the script probes the target container and +then the target host over its existing SSH transport. Configure a private reader +connection there. PostgreSQL 18 `psql` is required; URL parsing also needs Python +3 (`CORESCOPE_QA_PYTHON` can select the executable). Native libpq settings do not +need Python on the target. The shipped container has `psql`; use its private +libpq settings or run the URL-based reader probe locally/on a Python-equipped host. + +Every probe checks actual read-only grants and the completed telemetry readiness +marker, then binds the pubkey with psql's native extended-query protocol. SQL and +hex-encoded values travel on stdin. Connection credentials remain in the private +environment. Failures report SQLSTATE classes without printing connection strings. + +The local security gate contacts no HTTP endpoint or SSH host: + +```sh +# Export CORESCOPE_TEST_POSTGRES_URL privately for a disposable PostgreSQL 18 admin. +bash qa/scripts/test-blacklist-sql.sh +``` + +The gate requires Python 3, `psql`, Go and a C compiler. It creates temporary +schemas and separate owner/reader LOGIN roles, bootstraps with the shipping +migration tool, and verifies both legitimate and hostile pubkeys. It also prepares +and imports a disposable copy of the committed historical SQLite fixture before +comparing native PostgreSQL counts. SQLite is used only by these explicit offline +tools. Missing database/client prerequisites fail the gate; no SQL checks skip. +`CORESCOPE_QA_MIGRATOR` and `CORESCOPE_QA_PREPARER` may select previously built +matching helper binaries. Temporary objects and private files are removed on exit. diff --git a/qa/plans/v3.6.0-rc.md b/qa/plans/v3.6.0-rc.md index 77bcb8a72..35eefcf6d 100644 --- a/qa/plans/v3.6.0-rc.md +++ b/qa/plans/v3.6.0-rc.md @@ -95,7 +95,7 @@ Run `scripts/api-contract-diff.sh BASELINE_URL TARGET_URL` once. Report the scri | 10.1 | Add node pubkey to nodeBlacklist config; restart | Hidden from listings/map/neighbor graph | #742 | auto: blacklist-test.sh | | 10.2 | Packets still in DB | Yes (filter not delete) | #742 | auto: blacklist-test.sh | -`blacklist-test.sh` covers both 10.1 and 10.2 in one run. Required env: `TEST_NODE_PUBKEY` (hex, of a real visible node on TARGET), `TARGET_SSH_HOST`, `TARGET_CONFIG_PATH`, `TARGET_CONTAINER`. Optional: `TARGET_DB_PATH` or `ADMIN_API_TOKEN` for §10.2 probe; `TARGET_SSH_KEY` (default `/root/.ssh/id_ed25519`). Mandatory teardown removes the pubkey and verifies the node returns to listings. +`blacklist-test.sh` covers both 10.1 and 10.2 in one run. Required env: `TEST_NODE_PUBKEY` (hex, of a real visible node on TARGET), `TARGET_SSH_HOST`, `TARGET_CONFIG_PATH`, `TARGET_CONTAINER`. Configure §10.2 with a private `CORESCOPE_READER_DATABASE_URL` or native libpq reader settings locally/on the target; `ADMIN_API_TOKEN` remains optional. `TARGET_DB_PATH` is obsolete. `TARGET_SSH_KEY` defaults to `/root/.ssh/id_ed25519`. See `qa/README.md` for PostgreSQL client/role requirements. Mandatory teardown removes the pubkey and verifies the node returns to listings. ### 11. Deploy/ops | # | Step | Pass criteria | Source | Mode | diff --git a/qa/scripts/blacklist-test.sh b/qa/scripts/blacklist-test.sh index 205ff8f7e..6904f2bdf 100755 --- a/qa/scripts/blacklist-test.sh +++ b/qa/scripts/blacklist-test.sh @@ -15,8 +15,10 @@ # TARGET_CONFIG_PATH — absolute path to config.json on the target # TARGET_CONTAINER — docker container name on the target # Optional env: -# TARGET_DB_PATH — sqlite db path on the target (for §10.2 sqlite probe) -# ADMIN_API_TOKEN — if /api/admin/transmissions exists, use it instead of ssh+sqlite +# CORESCOPE_READER_DATABASE_URL — private local PostgreSQL reader URL for §10.2 +# PGSERVICE / PGDATABASE — alternatively configure private native libpq settings +# CORESCOPE_QA_PYTHON — Python 3 executable for URL parsing (default python3) +# ADMIN_API_TOKEN — if /api/admin/transmissions exists, use it instead of the PostgreSQL probe # (read from env, not argv — never appears in ps) # CURL_TIMEOUT — per-request curl timeout, seconds (default 60) # RESTART_WAIT_S — max wait for /api/stats after restart (default 120) @@ -27,9 +29,9 @@ # hide-failed → blacklisted pubkey still surfaced via API (§10.1 fail) # retain-failed → no transmissions.from_pubkey rows (ADVERTs) for the # blacklisted pubkey in the DB (§10.2 fail), or the -# §10.2 probe could not run at all — no sqlite3 on the target -# able to bind a parameter. The message names what is needed; -# there is no fallback to interpolated SQL. +# §10.2 probe cannot verify a ready, restricted PostgreSQL +# reader. Native parameter binding is mandatory; there is no +# interpolated SQL or SQLite runtime fallback. # teardown-failed→ post-test removal did not restore listing # # Exit code = number of failures (0 = pass). @@ -158,125 +160,130 @@ node_visible() { # ----------------------------------------------------------------------------- # §10.2 DB probe — bind the pubkey, do not interpolate it (issue #1977) # ----------------------------------------------------------------------------- -# Batch flags, all in service of "the count is parseable and errors are visible": -# -bail stop at the first SQL error instead of running on -# -init /dev/null ignore the operator's ~/.sqliterc — a stray .mode there -# would make the count unparseable -# -noheader -list stdout is exactly the number, nothing else -SQLITE_ARGS=(-batch -bail -init /dev/null -noheader -list) -# Round-trip probe token. The value is arbitrary; it only has to come back intact. -SQLITE_PROBE_TOKEN="corescope-probe-ok" -SQLITE_RUNNER="" # "container" | "host", set by resolve_sqlite_runner -RETAIN_COUNT="" # set by read_retain_count +# Native psql uses the extended query protocol (\bind), not text substitution. +# Queries and hex-only parameter values travel on stdin; connection credentials +# stay in private libpq environment/service settings, never process arguments. +POSTGRES_RUNNER="" # local | container | host +POSTGRES_PROBE_TOKEN="corescope-probe-ok" +RETAIN_COUNT="" +POSTGRES_CONNECT_PY="$(cat "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/postgres-connect.py")" -# Hex-encode a value for embedding in SQL as a blob literal. -# -# Why hex rather than quoting: the output alphabet is [0-9a-f], so no byte the -# caller passes can terminate a string literal or add a dot-command argument. -# That holds for arbitrary input, which is the point — the SQL layer stops -# depending on main()'s hex gate in order to be safe. -# -# `od -v` is load-bearing: without it od collapses runs of identical lines to -# '*' and long repetitive values encode wrongly. +# A psql meta-command argument whose only variable bytes are [0-9a-f]. od -v +# prevents repeated long values collapsing to '*'. Empty input binds as ''. sql_hex_literal() { - printf "x'%s'" "$(printf '%s' "$1" | od -An -v -tx1 | tr -d ' \n')" + printf "'%s'" "$(printf '%s' "$1" | od -An -v -tx1 | tr -d ' \n')" } -# SQL for the §10.2 count, fed to sqlite3 on stdin. The SELECT text is a -# constant; the pubkey arrives as a bound parameter. -# -# Note the nested cast rather than `.parameter set :pubkey ''`: -# dot-command arguments are split on whitespace, so a value containing a space -# (e.g. "' OR 1=1 --") makes sqlite3 print the .parameter help to STDOUT, exit -# 0, and leave :pubkey unbound. COUNT(*) then returns 0 — which reads exactly -# like a passing security fix. -bail does not catch it either. -# -# The column is transmissions.from_pubkey (cmd/ingestor/db.go CREATE TABLE and -# the from_pubkey_v1 migration; asserted by internal/dbschema). The ingestor -# fills it only for ADVERTs, with hex.EncodeToString output — lowercase. The -# script's hex gate and the server's nodeBlacklist both accept any case, so the -# bound value is lowercased inside SQL; the parameter itself is still bound. -# There is no from_node column: querying it errors on every real database. +postgres_read_guard() { + cat <<'SQL' +BEGIN READ ONLY; +DO $qa$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname=current_user AND + (rolsuper OR rolcreaterole OR rolcreatedb OR rolreplication OR rolbypassrls)) + OR has_database_privilege(current_database(),'CREATE') + OR EXISTS (SELECT 1 FROM pg_namespace WHERE nspname !~ '^pg_' AND nspname <> 'information_schema' + AND (has_schema_privilege(oid,'CREATE') OR pg_has_role(current_user,nspowner,'MEMBER'))) + OR EXISTS (SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema' + AND CASE WHEN c.relkind IN ('r','p','v','m','f') + THEN has_table_privilege(c.oid,'INSERT,UPDATE,DELETE,TRUNCATE,TRIGGER') + OR pg_has_role(current_user,c.relowner,'MEMBER') ELSE false END) + OR EXISTS (SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema' + AND CASE WHEN c.relkind='S' THEN has_sequence_privilege(c.oid,'USAGE,UPDATE') ELSE false END) + THEN RAISE EXCEPTION 'QA requires a restricted PostgreSQL reader role' USING ERRCODE='42501'; END IF; + IF (SELECT count(*) FROM corescope_schema WHERE kind='telemetry' AND version=1 AND ready) <> 1 + THEN RAISE EXCEPTION 'telemetry schema is not ready' USING ERRCODE='55000'; END IF; +END $qa$; +SQL +} transmission_count_sql() { - printf '.parameter init\n' - printf '.parameter set :pubkey "cast(%s as text)"\n' "$(sql_hex_literal "$1")" - printf 'SELECT COUNT(*) FROM transmissions WHERE from_pubkey = lower(:pubkey);\n' + postgres_read_guard + printf '%s\n' "SELECT COUNT(*) FROM transmissions WHERE from_pubkey = lower(convert_from(decode(\$1,'hex'),'UTF8'))" + printf '\\bind %s\n\\g\nCOMMIT;\n' "$(sql_hex_literal "$1")" } - -# Capability probe: bind a known value and read it back. A version number only -# implies that .parameter works; binding something and getting it back proves it -# on the binary actually in front of us, which is the operator's, not ours. -sqlite_probe_sql() { - printf '.parameter init\n' - printf '.parameter set :probe "cast(%s as text)"\n' "$(sql_hex_literal "$SQLITE_PROBE_TOKEN")" - printf 'SELECT :probe;\n' +postgres_probe_sql() { + postgres_read_guard + printf '%s\n' "SELECT convert_from(decode(\$1,'hex'),'UTF8')" + printf '\\bind %s\n\\g\nCOMMIT;\n' "$(sql_hex_literal "$POSTGRES_PROBE_TOKEN")" } -# Find a sqlite3 that can bind a parameter — in the container first, then on the -# host. Sets SQLITE_RUNNER; returns 1 if neither qualifies. There is deliberately -# no interpolating fallback: that would leave the vulnerable path in place under -# a nicer name. -# -# Probe stderr is collected rather than discarded, but only printed if BOTH -# probes fail. The container miss is the known-normal case — the app image has -# no sqlite3 (pure-Go driver, no CGO; Dockerfile:15) — so surfacing it on every -# run would be noise. -resolve_sqlite_runner() { - local probe out - probe=$(sqlite_probe_sql) - SQLITE_RUNNER="" - out=$(ssh_t "docker exec -i $(printf %q "$TARGET_CONTAINER") sqlite3 ${SQLITE_ARGS[*]} :memory:" \ - <<<"$probe" 2>>"$TMP/sqlite-probe.err") - if [[ "$out" == "$SQLITE_PROBE_TOKEN" ]]; then SQLITE_RUNNER="container"; return 0; fi - out=$(ssh_t "sqlite3 ${SQLITE_ARGS[*]} :memory:" <<<"$probe" 2>>"$TMP/sqlite-probe.err") - if [[ "$out" == "$SQLITE_PROBE_TOKEN" ]]; then SQLITE_RUNNER="host"; return 0; fi +# POSIX shell quoting also works inside the image's /bin/sh. These arguments +# contain only public launcher code/container names, never a URL or password. +postgres_shell_quote() { + local value="$1" + value=${value//\'/\'\\\'\'} + printf "'%s'" "$value" +} +postgres_command() { + local python + python=$(postgres_shell_quote "$POSTGRES_CONNECT_PY") + printf '%s' "set +x; if [ -n \"\${CORESCOPE_READER_DATABASE_URL:-}\" ]; then exec \"\${CORESCOPE_QA_PYTHON:-python3}\" -c $python; elif [ -n \"\${PGSERVICE:-}\" ] || [ -n \"\${PGDATABASE:-}\" ]; then case \"\${PGDATABASE:-}\" in *://*) echo 'put connection URLs in CORESCOPE_READER_DATABASE_URL' >&2; exit 2;; esac; export PGCONNECT_TIMEOUT=10 PGCLIENTENCODING=UTF8; exec psql -X -qAt -w -v ON_ERROR_STOP=1 -v VERBOSITY=sqlstate; else echo 'configure a private PostgreSQL reader connection' >&2; exit 2; fi" +} +run_postgres() { + local cmd + cmd=$(postgres_command) + case "$POSTGRES_RUNNER" in + local) sh -c "$cmd" ;; + container) ssh_t "docker exec -i $(printf %q "$TARGET_CONTAINER") sh -c $(postgres_shell_quote "$cmd")" ;; + host) ssh_t "sh -c $(postgres_shell_quote "$cmd")" ;; + *) echo 'run_postgres: no reader runner resolved' >&2; return 127 ;; + esac | tr -d '\r' +} +resolve_postgres_runner() { + local runner out + POSTGRES_RUNNER="" + # An explicit local connection is authoritative: never silently fall through + # to a different database after a bad URL, elevated role, or failed import. + if [[ -n "${CORESCOPE_READER_DATABASE_URL:-}${PGSERVICE:-}${PGDATABASE:-}" ]]; then + local candidates=(local) + else + local candidates=(container host) + fi + for runner in "${candidates[@]}"; do + POSTGRES_RUNNER="$runner" + if out=$(postgres_probe_sql | run_postgres 2>>"$TMP/postgres-probe.err") && [[ "$out" == "$POSTGRES_PROBE_TOKEN" ]]; then return 0; fi + done + POSTGRES_RUNNER="" return 1 } - -# Run SQL from stdin against TARGET_DB_PATH via the resolved runner. Stderr is -# left alone so the caller can capture it, and the exit status is sqlite3's. -# The SQL crosses on stdin, so only the container name and db path still need -# printf %q for the remote shell. docker exec needs -i to attach stdin. -run_sqlite() { - case "$SQLITE_RUNNER" in - container) ssh_t "docker exec -i $(printf %q "$TARGET_CONTAINER") sqlite3 ${SQLITE_ARGS[*]} $(printf %q "$TARGET_DB_PATH")" ;; - host) ssh_t "sqlite3 ${SQLITE_ARGS[*]} $(printf %q "$TARGET_DB_PATH")" ;; - *) echo "run_sqlite: no runner resolved" >&2; return 127 ;; - esac +postgres_error_summary() { + # libpq connection errors can contain caller-supplied strings. Publish only + # SQLSTATE classes, retaining the complete diagnostic in the private temp file. + local summary + summary=$(sed -nE 's/.*(ERROR|FATAL):[[:space:]]+([0-9A-Z]{5}).*/PostgreSQL error \2/p' "$1" | tail -3) + if [[ -n "$summary" ]]; then printf '%s\n' "$summary" >&2; else + echo 'Check PostgreSQL 18 client availability and private reader connection settings.' >&2 + fi } - -# Read the retained-transmission count into RETAIN_COUNT. Prints a classified -# "retain-failed" line and returns 1 on failure, so §10.2 has exactly one place -# that increments $fails. read_retain_count() { RETAIN_COUNT="" local code if [[ -n "$ADMIN_API_TOKEN" ]]; then - # Read auth header from stdin so the token never enters argv (ps-safe). code=$(printf 'header = "Authorization: Bearer %s"\n' "$ADMIN_API_TOKEN" | \ curl -s -m "$CURL_TIMEOUT" -K - -o "$TMP/admin.json" -w "%{http_code}" \ - "$TARGET_URL/api/admin/transmissions?from_node=$TEST_PUBKEY&count=1" 2>/dev/null || echo "000") - if [[ "$code" == "200" ]]; then - RETAIN_COUNT=$(jq -r '.count // ((.transmissions // []) | length)' "$TMP/admin.json" 2>/dev/null || echo "") + "$TARGET_URL/api/admin/transmissions?from_node=$TEST_PUBKEY&count=1" 2>/dev/null || echo '000') + if [[ "$code" == 200 ]]; then + RETAIN_COUNT=$(jq -r '.count // ((.transmissions // []) | length)' "$TMP/admin.json" 2>/dev/null || echo '') fi - if [[ -n "$RETAIN_COUNT" ]]; then return 0; fi + if [[ "$RETAIN_COUNT" =~ ^[0-9]+$ ]]; then return 0; fi + RETAIN_COUNT="" fi - - if [[ -z "$TARGET_DB_PATH" ]]; then - echo " ❌ retain-failed: TARGET_DB_PATH unset and no ADMIN_API_TOKEN — cannot probe" + if ! resolve_postgres_runner; then + echo ' ❌ retain-failed: no ready, restricted PostgreSQL reader with native parameter binding' + postgres_error_summary "$TMP/postgres-probe.err" return 1 fi - if ! resolve_sqlite_runner; then - echo " ❌ retain-failed: no sqlite3 able to bind a parameter on the target" - echo " tried: docker exec -i $TARGET_CONTAINER sqlite3, then sqlite3 on $TARGET_SSH_HOST" - echo " need: the sqlite3 CLI reachable over ssh, supporting '.parameter set'" - cat "$TMP/sqlite-probe.err" >&2 + echo " PostgreSQL reader runner: $POSTGRES_RUNNER" + if ! RETAIN_COUNT=$(transmission_count_sql "$TEST_PUBKEY" | run_postgres 2>"$TMP/postgres.err"); then + echo " ❌ retain-failed: PostgreSQL query failed via $POSTGRES_RUNNER" + postgres_error_summary "$TMP/postgres.err" + RETAIN_COUNT="" return 1 fi - echo " sqlite3 runner: $SQLITE_RUNNER" - if ! RETAIN_COUNT=$(run_sqlite <<<"$(transmission_count_sql "$TEST_PUBKEY")" 2>"$TMP/sqlite.err"); then - echo " ❌ retain-failed: sqlite3 query failed via $SQLITE_RUNNER" - cat "$TMP/sqlite.err" >&2 + if ! [[ "$RETAIN_COUNT" =~ ^[0-9]+$ ]]; then + echo ' ❌ retain-failed: PostgreSQL query did not return one numeric count' RETAIN_COUNT="" return 1 fi @@ -299,7 +306,6 @@ main() { TARGET_SSH_KEY="${TARGET_SSH_KEY:-/root/.ssh/id_ed25519}" TARGET_CONFIG_PATH="${TARGET_CONFIG_PATH:-}" TARGET_CONTAINER="${TARGET_CONTAINER:-}" - TARGET_DB_PATH="${TARGET_DB_PATH:-}" ADMIN_API_TOKEN="${ADMIN_API_TOKEN:-}" if [[ -z "$TEST_PUBKEY" || -z "$TARGET_SSH_HOST" || -z "$TARGET_CONFIG_PATH" || -z "$TARGET_CONTAINER" ]]; then @@ -326,8 +332,8 @@ main() { echo "error: TARGET_CONFIG_PATH must be a sane absolute path" >&2 exit 2 fi - if [[ -n "$TARGET_DB_PATH" ]] && ! [[ "$TARGET_DB_PATH" =~ ^/[A-Za-z0-9_./-]+$ ]]; then - echo "error: TARGET_DB_PATH must be a sane absolute path" >&2 + if [[ -n "${TARGET_DB_PATH:-}" ]]; then + echo "error: TARGET_DB_PATH is obsolete; configure a private PostgreSQL reader connection" >&2 exit 2 fi @@ -378,8 +384,7 @@ main() { echo "=== §10.2 verify packets retained in DB ===" if ! read_retain_count; then # read_retain_count already printed the classified reason. Counting here and - # nowhere else: the old code incremented $fails for the "TARGET_DB_PATH - # unset" case and then again for the empty count it left behind. + # nowhere else, so one failed reader probe remains one classified failure. fails=$((fails+1)) elif [[ "$RETAIN_COUNT" =~ ^[0-9]+$ ]] && (( RETAIN_COUNT > 0 )); then echo " ✅ DB retains $RETAIN_COUNT packets from $TEST_PUBKEY" diff --git a/qa/scripts/postgres-connect.py b/qa/scripts/postgres-connect.py new file mode 100644 index 000000000..74713dcef --- /dev/null +++ b/qa/scripts/postgres-connect.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Run psql with connection credentials exclusively in private libpq variables. + +PGDATABASE does not expand a URL. This QA launcher accepts the application's +reader URL, including search_path, or preconfigured native libpq settings. +Queries arrive on stdin. Their caller is responsible for checking reader grants. +""" +import os +import re +import subprocess +import sys +import urllib.parse + + +def connection_env(source): + env = dict(source) + url = env.get("CORESCOPE_READER_DATABASE_URL", "") + if url: + try: + if re.search(r"%(?![0-9a-fA-F]{2})|[\x00-\x20\x7f]", url): + raise ValueError() + parsed = urllib.parse.urlsplit(url) + if ";" in parsed.query: + raise ValueError() + host = urllib.parse.unquote(parsed.hostname or "") + if parsed.scheme not in ("postgres", "postgresql") or not host or "," in host or not parsed.username or not parsed.path.strip("/") or parsed.fragment: + raise ValueError() + port = parsed.port + if port is None: + port = env.get("PGPORT") or "5432" + if not re.fullmatch(r"[0-9]+", str(port)) or not 1 <= int(port) <= 65535: + raise ValueError() + env.update(PGHOST=host, PGPORT=str(int(port)), + PGUSER=urllib.parse.unquote(parsed.username), + PGDATABASE=urllib.parse.unquote(parsed.path[1:])) + if parsed.password is not None: + env["PGPASSWORD"] = urllib.parse.unquote(parsed.password) + env.pop("PGSERVICE", None) + env.pop("PGHOSTADDR", None) + env.pop("PGSERVICEFILE", None) + env.pop("PGOPTIONS", None) + settings = {"sslmode": "PGSSLMODE", "sslrootcert": "PGSSLROOTCERT", "sslcert": "PGSSLCERT", + "sslkey": "PGSSLKEY", "sslpassword": "PGSSLPASSWORD", "connect_timeout": "PGCONNECT_TIMEOUT", + "target_session_attrs": "PGTARGETSESSIONATTRS", "application_name": "PGAPPNAME"} + seen = set() + for key, value in urllib.parse.parse_qsl(parsed.query, keep_blank_values=True, strict_parsing=True): + if key in seen: + raise ValueError() + seen.add(key) + if key == "search_path": + env["PGOPTIONS"] = "-c search_path=" + re.sub(r"([\\\s])", r"\\\1", value) + elif key in settings: + if key == "sslmode" and value not in ("disable", "allow", "prefer", "require", "verify-ca", "verify-full"): + raise ValueError() + env[settings[key]] = value + else: + raise ValueError() + except (ValueError, UnicodeError): + raise ValueError("invalid PostgreSQL reader URL; check its private configuration") from None + elif not env.get("PGSERVICE") and not env.get("PGDATABASE"): + raise ValueError("configure CORESCOPE_READER_DATABASE_URL or private libpq database/service settings") + elif "://" in env.get("PGDATABASE", ""): + raise ValueError("PGDATABASE requires a database name; put the URL in CORESCOPE_READER_DATABASE_URL") + try: + env["PGCONNECT_TIMEOUT"] = str(min(10, max(1, int(env.get("PGCONNECT_TIMEOUT", "10"))))) + except ValueError: + raise ValueError("invalid PostgreSQL connection timeout") from None + env["PGCLIENTENCODING"] = "UTF8" + env.setdefault("PGAPPNAME", "corescope-qa") + return env + + +if __name__ == "__main__": + try: + env = connection_env(os.environ) + result = subprocess.run(["psql", "-X", "-qAt", "-w", "-v", "ON_ERROR_STOP=1", "-v", "VERBOSITY=sqlstate"], + env=env, stderr=subprocess.PIPE, text=True, encoding="utf-8", errors="replace") + if result.returncode: + # libpq parse/connection errors can quote an invalid setting value. + # Keep server error classes; never relay those arbitrary strings. + codes = re.findall(r"^(?:ERROR|FATAL):\s+([0-9A-Z]{5})\s*$", result.stderr, re.MULTILINE) + if codes: + for code in codes[-3:]: + print("ERROR: " + code, file=sys.stderr) + else: + print("PostgreSQL client failed; check private connection settings and PostgreSQL 18 client availability", file=sys.stderr) + sys.exit(result.returncode) + except ValueError as error: + print(str(error), file=sys.stderr) + sys.exit(2) + except OSError: + print("psql is unavailable; install the PostgreSQL 18 client", file=sys.stderr) + sys.exit(127) diff --git a/qa/scripts/test-blacklist-sql.sh b/qa/scripts/test-blacklist-sql.sh index 4dfdc8b4b..bec4f04df 100755 --- a/qa/scripts/test-blacklist-sql.sh +++ b/qa/scripts/test-blacklist-sql.sh @@ -1,287 +1,246 @@ #!/usr/bin/env bash -# test-blacklist-sql.sh — unit tests for the §10.2 SQL construction in -# qa/scripts/blacklist-test.sh (issue #1977). Sources the script and exercises -# its pure helpers, plus a real local sqlite3 against throwaway fixture DBs. -# -# Run: bash qa/scripts/test-blacklist-sql.sh -# Exits non-zero if any case fails. -# -# The point of the sqlite3 group is that BOTH directions are asserted. A test -# that only checks "the injection payload returns 0" passes just as happily when -# the query is silently broken and returns 0 for everything, so the legitimate -# pubkey must be shown to still return the row it should. -# -# The fixture schema is NOT hand-written: it is the transmissions CREATE TABLE -# extracted from cmd/ingestor/db.go, and the query is also run against the -# committed staging-captured test-fixtures/e2e-fixture.db. An earlier version -# invented a `from_node` column that no CoreScope database has, and so passed -# while the real probe could never succeed. - +# Native PostgreSQL regressions for the blacklist retention probe. No external +# endpoints are contacted: SSH is stubbed and all schemas/roles are disposable. set -uo pipefail - +set +x SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" -INGESTOR_DB_GO="$REPO_ROOT/cmd/ingestor/db.go" -REAL_FIXTURE="$REPO_ROOT/test-fixtures/e2e-fixture.db" # shellcheck source=blacklist-test.sh . "$SCRIPT_DIR/blacklist-test.sh" - -PASS=0 -FAIL=0 - +PYTHON="${CORESCOPE_QA_PYTHON:-python3}" +CONNECT="$SCRIPT_DIR/postgres-connect.py" +export QA_CONNECT="$CONNECT" +PASS=0; FAIL=0 assert_eq() { - local label="$1" expected="$2" actual="$3" - if [ "$expected" = "$actual" ]; then - PASS=$((PASS + 1)) - else - FAIL=$((FAIL + 1)) - echo "FAIL: $label — expected '$expected' got '$actual'" >&2 - fi + if [[ "$2" == "$3" ]]; then PASS=$((PASS+1)); else + FAIL=$((FAIL+1)); printf 'FAIL: %s — expected %q got %q\n' "$1" "$2" "$3" >&2 + fi } - -assert_match() { - local label="$1" pattern="$2" actual="$3" - if [[ "$actual" =~ $pattern ]]; then - PASS=$((PASS + 1)) - else - FAIL=$((FAIL + 1)) - echo "FAIL: $label — '$actual' does not match /$pattern/" >&2 - fi +assert_true() { local label="$1"; shift; if "$@"; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); echo "FAIL: $label" >&2; fi; } +contains() { [[ "$1" == *"$2"* ]]; } +lacks() { [[ "$1" != *"$2"* ]]; } +hex_only() { [[ "$1" =~ ^\'[0-9a-f]*\'$ ]]; } +assert_eq 'hex of deadbeef' "'6465616462656566'" "$(sql_hex_literal deadbeef)" +assert_eq 'hex of empty value' "''" "$(sql_hex_literal '')" +for payload in "' OR 1=1 --" '"; DROP TABLE transmissions; --' 'a\b' '$(id) `id`' "$(printf 'a\nb')" 'héllo'; do + assert_true 'bound parameter alphabet is only hexadecimal' hex_only "$(sql_hex_literal "$payload")" +done +LONG=$(printf 'x%.0s' $(seq 1 4096)); LONG_HEX=$(sql_hex_literal "$LONG") +assert_true 'long bound parameter alphabet is hexadecimal' hex_only "$LONG_HEX" +assert_eq 'repeated lines are not collapsed by od' 8192 "$(( ${#LONG_HEX} - 2 ))" + +command -v "$PYTHON" >/dev/null || { echo 'FAIL: Python 3 is required' >&2; exit 1; } +"$PYTHON" "$SCRIPT_DIR/test-postgres-connect.py" || exit 1 +command -v psql >/dev/null || { echo 'FAIL: PostgreSQL 18 psql is required' >&2; exit 1; } +[[ -n "${CORESCOPE_TEST_POSTGRES_URL:-}" ]] || { echo 'FAIL: set CORESCOPE_TEST_POSTGRES_URL to a disposable PostgreSQL administrator URL' >&2; exit 1; } +FIXTURE_ROOT="$(cd "${TMPDIR:-/tmp}" && pwd -P)" +FIXTURE_DIR=$(mktemp -d "$FIXTURE_ROOT/corescope-qa-XXXXXX") +FIXTURE_DIR="$(cd "$FIXTURE_DIR" && pwd -P)" +SCHEMA="corescope_qa_${BASHPID}_${RANDOM}" +OWNER="${SCHEMA}_owner"; READER="${SCHEMA}_reader" +FIXTURE_PASSWORD=$(od -An -N24 -v -tx1 /dev/urandom | tr -d ' \n') +CREATED=0 +admin_sql() { CORESCOPE_READER_DATABASE_URL="$CORESCOPE_TEST_POSTGRES_URL" "$PYTHON" "$CONNECT" | tr -d '\r'; } +cleanup() { + local rc=$? + if [[ "$CREATED" == 1 && "$SCHEMA" =~ ^corescope_qa_[0-9]+_[0-9]+$ ]]; then + if ! admin_sql >/dev/null <&2; rc=1; fi + fi + case "$FIXTURE_DIR" in "$FIXTURE_ROOT"/corescope-qa-*) rm -rf -- "$FIXTURE_DIR" ;; *) echo 'FAIL: refusing unsafe temporary cleanup' >&2; rc=1 ;; esac + exit "$rc" } - -assert_true() { - local label="$1"; shift - if "$@"; then PASS=$((PASS + 1)); else - FAIL=$((FAIL + 1)); echo "FAIL: $label" >&2 - fi +trap cleanup EXIT +# Build private role URLs without putting the original URL/password in argv or +# printing either. The connection launcher validates the URL before psql runs. +set_role_url() { + local base="$CORESCOPE_TEST_POSTGRES_URL" authority tail query part rest="" + authority=${base#*://}; tail=${authority#*/}; authority=${authority%%/*} + QA_ROLE_URL="${base%%://*}://$1:$FIXTURE_PASSWORD@${authority##*@}/${tail%%\?*}" + if [[ "$tail" == *\?* ]]; then + query=${tail#*\?} + while [[ -n "$query" ]]; do + part=${query%%&*}; if [[ "$query" == *'&'* ]]; then query=${query#*&}; else query=""; fi + [[ "${part%%=*}" == search_path ]] || rest+="${part}&" + done + fi + QA_ROLE_URL+="?${rest}search_path=$2" } - -contains() { [[ "$1" == *"$2"* ]]; } -lacks() { [[ "$1" != *"$2"* ]]; } - -# ----- sql_hex_literal ------------------------------------------------------ -# The security property: whatever goes in, the SQL text it produces is drawn -# from [0-9a-f] only. No caller-supplied byte can close a string literal or add -# a dot-command argument. Needs no sqlite3, so this group always runs. -assert_eq "hex of deadbeef" "x'6465616462656566'" "$(sql_hex_literal deadbeef)" -assert_eq "hex of empty" "x''" "$(sql_hex_literal "")" - -HEX_ONLY="^x'[0-9a-f]*'\$" -assert_match "alphabet: sql quote payload" "$HEX_ONLY" "$(sql_hex_literal "' OR 1=1 --")" -assert_match "alphabet: drop table" "$HEX_ONLY" "$(sql_hex_literal '"; DROP TABLE transmissions; --')" -assert_match "alphabet: backslash" "$HEX_ONLY" "$(sql_hex_literal 'a\b')" -assert_match "alphabet: dollar and backtick" "$HEX_ONLY" "$(sql_hex_literal '$(id) `id`')" -assert_match "alphabet: embedded newline" "$HEX_ONLY" "$(sql_hex_literal "$(printf 'a\nb')")" -assert_match "alphabet: multibyte" "$HEX_ONLY" "$(sql_hex_literal 'héllo')" - -# `od` without -v collapses runs of identical lines to '*'. A long repetitive -# value is the case that catches losing the flag. -LONG=$(printf 'x%.0s' $(seq 1 4096)) -LONG_HEX=$(sql_hex_literal "$LONG") -assert_match "alphabet: 4096 repeated bytes" "$HEX_ONLY" "$LONG_HEX" -# 4096 bytes → 8192 hex digits, plus the 3 chars of x''. A collapsed run would -# be far shorter and would also fail the alphabet check on '*'. -assert_eq "no od line-collapse in 4096-byte value" "8192" "$(( ${#LONG_HEX} - 3 ))" - -# ----- query text ------------------------------------------------------------- -# Synthetic 32-byte pubkeys in the ingestor's form (hex.EncodeToString → lowercase). +if ! admin_sql >/dev/null <&2; exit 1; fi +CREATED=1 +set_role_url "$OWNER" "$SCHEMA"; OWNER_URL="$QA_ROLE_URL" +set_role_url "$READER" "$SCHEMA"; READER_URL="$QA_ROLE_URL" +set_role_url "$READER" "${SCHEMA}_bad"; BAD_URL="$QA_ROLE_URL" +set_role_url "$READER" "${SCHEMA}_empty"; EMPTY_URL="$QA_ROLE_URL" +set_role_url "$OWNER" "${SCHEMA}_import"; IMPORT_OWNER_URL="$QA_ROLE_URL" +set_role_url "$READER" "${SCHEMA}_import"; IMPORT_READER_URL="$QA_ROLE_URL" +MIGRATOR="${CORESCOPE_QA_MIGRATOR:-$FIXTURE_DIR/migrate}" +PREPARER="${CORESCOPE_QA_PREPARER:-$FIXTURE_DIR/prepare-fixture}" +if [[ -z "${CORESCOPE_QA_MIGRATOR:-}" ]]; then + (cd "$REPO_ROOT/cmd/migrate" && CGO_ENABLED=1 go build -o "$MIGRATOR" .) || exit 1 +fi +if [[ -z "${CORESCOPE_QA_PREPARER:-}" ]]; then + (cd "$REPO_ROOT/cmd/migrate" && CGO_ENABLED=1 go build -o "$PREPARER" ../../scripts/prepare-postgres-fixture.go) || exit 1 +fi +if ! CORESCOPE_DATABASE_URL="$OWNER_URL" CORESCOPE_USERS_DATABASE_URL="" "$MIGRATOR" >"$FIXTURE_DIR/bootstrap.out" 2>"$FIXTURE_DIR/bootstrap.err"; then + echo 'FAIL: shipping migration tool could not bootstrap the owner schema' >&2; exit 1 +fi PK_A="0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" PK_B="fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210" PK_ABSENT="00000000000000000000000000000000000000000000000000000000000000ff" PK_A_UPPER=$(printf '%s' "$PK_A" | tr 'a-f' 'A-F') - -COUNT_SQL=$(transmission_count_sql "$PK_A") -assert_true "query targets transmissions.from_pubkey" contains "$COUNT_SQL" "WHERE from_pubkey = lower(:pubkey)" -assert_true "query does not reference from_node" lacks "$COUNT_SQL" "from_node" -assert_true "query text does not carry the raw pubkey" lacks "$COUNT_SQL" "$PK_A" - -# ----- schema taken from the ingestor ------------------------------------------ -# Pull the transmissions DDL out of cmd/ingestor/db.go instead of restating it. -transmissions_ddl() { - awk '/CREATE TABLE IF NOT EXISTS transmissions \(/{p=1} p{print} p&&/^[[:space:]]*\);/{exit}' "$INGESTOR_DB_GO" -} -DDL=$(transmissions_ddl) -assert_match "ingestor DDL extracted" "CREATE TABLE IF NOT EXISTS transmissions \\(" "$DDL" -assert_match "ingestor DDL has from_pubkey" "from_pubkey[[:space:]]+TEXT" "$DDL" -assert_true "ingestor DDL has no from_node" lacks "$DDL" "from_node" - -# ----- against a real sqlite3 ---------------------------------------------- -if ! command -v sqlite3 >/dev/null 2>&1; then - echo "SKIP: sqlite3 not on PATH — skipping the ${#SQLITE_ARGS[@]}-flag query group" >&2 - echo " (the alphabet, query-text and schema assertions above still ran)" >&2 -else - FIXTURE_DIR=$(mktemp -d) - trap 'rm -rf "$FIXTURE_DIR"' EXIT - DB="$FIXTURE_DIR/fixture.db" - LEGACY_DB="$FIXTURE_DIR/legacy-from-node.db" - EMPTY_DB="$FIXTURE_DIR/no-table.db" - - # Schema-realistic fixture: the ingestor's own DDL. Three ADVERTs from A, one - # from B, and two non-ADVERT rows whose from_pubkey is NULL (the ingestor only - # attributes ADVERTs), so a structural injection has 6 rows to leak. - printf '%s\n' "$DDL" | sqlite3 "$DB" - sqlite3 "$DB" </dev/null <&1); rc=$? - assert_eq "payload binds literally: $(printf %q "$payload")" "0" "$out" - assert_eq "payload exit 0: $(printf %q "$payload")" "0" "$rc" - done - - # Interpolating the same payload the old way returns the whole table. This is - # the behaviour the change removes; asserting it keeps the test honest about - # what "0" above is worth. - legacy="SELECT COUNT(*) FROM transmissions WHERE from_pubkey = '' OR 1=1 --';" - assert_eq "interpolated form leaks the table" "6" "$(run_local "$DB" <<<"$legacy")" - - # Multibyte, whitespace, empty and long values bind as themselves. - LONG_PK=$(printf 'ab%.0s' $(seq 1 5000)) - sqlite3 "$DB" "INSERT INTO transmissions(raw_hex,hash,first_seen,from_pubkey) VALUES - ('00','m1','t','héllo wörld'),('00','m2','t',''),('00','m3','t',' '),('00','m4','t','$LONG_PK');" - assert_eq "multibyte value with a space binds" "1" "$(count 'héllo wörld' "$DB")" - assert_eq "empty value binds as ''" "1" "$(count '' "$DB")" - assert_eq "whitespace value binds as itself" "1" "$(count ' ' "$DB")" - assert_eq "10000-byte value binds" "1" "$(count "$LONG_PK" "$DB")" - - # REAL DATA: the committed staging-captured fixture. Its most-attributed - # pubkey, counted by the script's query, must equal a direct count. - if [ -f "$REAL_FIXTURE" ]; then - cp "$REAL_FIXTURE" "$FIXTURE_DIR/real.db" - real_pk=$(run_local "$FIXTURE_DIR/real.db" <<<"SELECT from_pubkey FROM transmissions WHERE from_pubkey IS NOT NULL GROUP BY from_pubkey ORDER BY COUNT(*) DESC, from_pubkey LIMIT 1;") - real_n=$(run_local "$FIXTURE_DIR/real.db" <<<"SELECT COUNT(*) FROM transmissions WHERE from_pubkey = '$real_pk';") - assert_match "real fixture has an attributed pubkey" '^[0-9a-f]{64}$' "$real_pk" - assert_match "real fixture count > 0" '^[1-9][0-9]*$' "$real_n" - assert_eq "script query on real fixture" "$real_n" "$(count "$real_pk" "$FIXTURE_DIR/real.db")" - else - FAIL=$((FAIL + 1)); echo "FAIL: $REAL_FIXTURE missing" >&2 - fi - - # ERROR SURFACING: a broken query must be distinguishable from an empty - # result — non-zero exit and something on stderr, not a silent "" or "0". - for bad in "$LEGACY_DB:no such column: from_pubkey" "$EMPTY_DB:no such table: transmissions"; do - bad_db=${bad%%:*}; want=${bad#*:} - err_file="$FIXTURE_DIR/err" - out=$(count "$PK_A" "$bad_db" 2>"$err_file"); rc=$? - assert_true "$want → non-zero exit (got $rc)" test "$rc" -ne 0 - assert_true "$want → named on stderr" contains "$(cat "$err_file")" "$want" - assert_eq "$want → no count on stdout" "" "$out" - done - - # ----- runner selection and transport, with ssh stubbed --------------------- - # ssh_t is replaced by a stub that logs its argv, captures stdin, and runs the - # remote command string through a real `bash -c`, so the script's own quoting - # is what gets parsed. "docker exec -i NAME" is peeled off when the stub - # container is said to have sqlite3; otherwise it fails like the app image. - STUB_ARGV="$FIXTURE_DIR/ssh.argv"; STUB_STDIN="$FIXTURE_DIR/ssh.stdin" - STUB_CONTAINER_SQLITE=0 - STUB_PATH="$PATH" - ssh_t() { - local cmd="$1" in - printf '%s\n' "$*" >>"$STUB_ARGV" - in=$(cat); printf '%s\n' "$in" >>"$STUB_STDIN" - case "$cmd" in - "docker exec -i "*) - if [ "$STUB_CONTAINER_SQLITE" != 1 ]; then - echo 'exec: "sqlite3": executable file not found in $PATH' >&2; return 126 - fi - cmd=${cmd#docker exec -i }; cmd=${cmd#* } ;; - esac - printf '%s\n' "$in" | PATH="$STUB_PATH" bash -c "$cmd" - } - reset_stub() { : >"$STUB_ARGV"; : >"$STUB_STDIN"; } - TMP="$FIXTURE_DIR"; ADMIN_API_TOKEN=""; TARGET_CONTAINER="corescope-stub" - TARGET_SSH_HOST="stub-host"; TARGET_DB_PATH="$DB"; TEST_PUBKEY="$PK_A" - - # Container has no sqlite3 → host runner, correct count. - reset_stub - read_retain_count >/dev/null 2>&1; rc=$? - assert_eq "fallback: rc" "0" "$rc" - assert_eq "fallback: runner" "host" "$SQLITE_RUNNER" - assert_eq "fallback: count" "3" "$RETAIN_COUNT" - argv=$(cat "$STUB_ARGV"); stdin=$(cat "$STUB_STDIN") - assert_true "transport: container tried first" contains "$(head -1 "$STUB_ARGV")" "docker exec -i corescope-stub sqlite3" - assert_true "transport: pubkey not in remote argv" lacks "$argv" "$PK_A" - assert_true "transport: SQL not in remote argv" lacks "$argv" "SELECT" - assert_true "transport: column not in remote argv" lacks "$argv" "from_pubkey" - assert_true "transport: SQL arrives on stdin" contains "$stdin" "SELECT COUNT(*) FROM transmissions WHERE from_pubkey = lower(:pubkey);" - assert_true "transport: pubkey arrives hex-encoded on stdin" contains "$stdin" "$(sql_hex_literal "$PK_A")" - assert_true "transport: raw pubkey never on stdin" lacks "$stdin" "$PK_A" - - # Container has sqlite3 → container runner. - reset_stub; STUB_CONTAINER_SQLITE=1 - read_retain_count >/dev/null 2>&1; rc=$? - assert_eq "container: rc" "0" "$rc" - assert_eq "container: runner" "container" "$SQLITE_RUNNER" - assert_eq "container: count" "3" "$RETAIN_COUNT" - STUB_CONTAINER_SQLITE=0 - - # A sqlite3 that cannot bind (.parameter prints help to stdout, exit 0, - # parameter left NULL) must be rejected by the probe, not trusted. - FAKE_BIN="$FIXTURE_DIR/fakebin"; mkdir -p "$FAKE_BIN" - cat >"$FAKE_BIN/sqlite3" <<'FAKE' -#!/usr/bin/env bash -cat >/dev/null -echo ".parameter CMD ... Manage SQL parameter bindings" -echo "0" -FAKE - chmod +x "$FAKE_BIN/sqlite3" - reset_stub; STUB_PATH="$FAKE_BIN:$PATH" - read_retain_count >/dev/null 2>&1; rc=$? - assert_eq "unbindable sqlite3: rejected" "1" "$rc" - assert_eq "unbindable sqlite3: no runner" "" "$SQLITE_RUNNER" - assert_eq "unbindable sqlite3: no count" "" "$RETAIN_COUNT" - STUB_PATH="$PATH" - - # A query error on the target is a failure with no count, never "0". - reset_stub; TARGET_DB_PATH="$LEGACY_DB" - read_retain_count >/dev/null 2>&1; rc=$? - assert_eq "remote schema error: rc" "1" "$rc" - assert_eq "remote schema error: count" "" "$RETAIN_COUNT" - assert_true "remote schema error: stderr kept" contains "$(cat "$TMP/sqlite.err")" "no such column: from_pubkey" - TARGET_DB_PATH="$DB" - - # run_sqlite with no resolved runner must refuse rather than guess. - SQLITE_RUNNER="" - if run_sqlite /dev/null 2>&1; then - FAIL=$((FAIL + 1)); echo "FAIL: run_sqlite with no runner — expected non-zero exit" >&2 - else - PASS=$((PASS + 1)) - fi +then echo 'FAIL: native fixture seeding failed' >&2; exit 1; fi +run_local() { CORESCOPE_READER_DATABASE_URL="$1" "$PYTHON" "$CONNECT" | tr -d '\r'; } +count() { transmission_count_sql "$1" | run_local "$2"; } + +# Positive controls must succeed, otherwise zero-valued injection results prove +# nothing. These exercise the actual canonical schema and restricted LOGIN role. +assert_eq 'fixture holds six rows' 6 "$(run_local "$READER_URL" <<<'SELECT count(*) FROM transmissions;')" +out=$(count "$PK_A" "$READER_URL"); rc=$? +assert_eq 'legitimate pubkey has three rows' 3 "$out" +assert_eq 'legitimate pubkey returns success' 0 "$rc" +assert_eq 'native parameter capability probe round-trips' "$POSTGRES_PROBE_TOKEN" "$(postgres_probe_sql | run_local "$READER_URL")" +COUNT_SQL=$(transmission_count_sql "$PK_A") +assert_true 'query uses the real attribution column' contains "$COUNT_SQL" 'WHERE from_pubkey = lower(convert_from(decode($1' +assert_true 'query has no invented from_node column' lacks "$COUNT_SQL" from_node +assert_true 'query contains no raw pubkey' lacks "$COUNT_SQL" "$PK_A" +assert_eq 'other legitimate pubkey has one row' 1 "$(count "$PK_B" "$READER_URL")" +assert_eq 'upper-case key matches' 3 "$(count "$PK_A_UPPER" "$READER_URL")" +assert_eq 'absent key has no rows' 0 "$(count "$PK_ABSENT" "$READER_URL")" +assert_eq 'prefix is not an exact key' 0 "$(count "${PK_A:0:16}" "$READER_URL")" +for payload in "' OR 1=1 --" "') OR 1=1 --" '" OR 1=1 --' "x' OR from_pubkey IS NOT NULL --" "$PK_A' OR '1'='1" \ + '"); .shell id; --' "$(printf 'a\n.shell id\nSELECT 99;')" "$(printf 'a\n\\! id\nSELECT 99;')" '$(id) `id`' 'a\b'; do + out=$(count "$payload" "$READER_URL" 2>"$FIXTURE_DIR/payload.err"); rc=$? + assert_eq 'hostile pubkey is a literal, not SQL/meta-command/shell code' 0 "$out" + assert_eq 'hostile literal query succeeds' 0 "$rc" +done +assert_eq 'unsafe interpolated control would leak all rows' 6 "$(run_local "$READER_URL" <<<"SELECT count(*) FROM transmissions WHERE from_pubkey = '' OR 1=1 --';")" +admin_sql >/dev/null <"$FIXTURE_DIR/schema.err"); rc=$? + assert_true 'bad schema returns nonzero' test "$rc" -ne 0 + assert_true 'bad schema exposes its PostgreSQL error class' contains "$(cat "$FIXTURE_DIR/schema.err")" "${spec##*|}" + assert_eq 'bad schema produces no count' '' "$out" +done +# Prove actual grants, independently of the read-only transaction setting. +out=$(run_local "$READER_URL" <<<'BEGIN READ WRITE; DELETE FROM transmissions;' 2>"$FIXTURE_DIR/write.err"); rc=$? +assert_true 'restricted reader cannot delete even in read-write transaction' test "$rc" -ne 0 +assert_true 'delete fails for privilege, not transaction mode' contains "$(cat "$FIXTURE_DIR/write.err")" '42501' +out=$(count "$PK_A" "$OWNER_URL" 2>"$FIXTURE_DIR/owner.err"); rc=$? +assert_true 'owner credential refused by runtime probe' test "$rc" -ne 0 +assert_eq 'owner refusal has no count' '' "$out" +admin_sql >/dev/null <<<"UPDATE $SCHEMA.corescope_schema SET ready=false;" +out=$(count "$PK_A" "$READER_URL" 2>"$FIXTURE_DIR/ready.err"); rc=$? +assert_true 'unfinished import refused' test "$rc" -ne 0 +assert_eq 'unfinished import produces no count' '' "$out" +admin_sql >/dev/null <<<"UPDATE $SCHEMA.corescope_schema SET ready=true;" + +# Preserve the committed fixture comparison through its explicit offline repair +# and shipping importer. The original SQLite file is never modified or queried +# as a runtime backend. +REAL_FIXTURE="$REPO_ROOT/test-fixtures/e2e-fixture.db" +if ! "$PREPARER" -source "$REAL_FIXTURE" -destination "$FIXTURE_DIR/prepared.sqlite" >"$FIXTURE_DIR/prepare.out" 2>"$FIXTURE_DIR/prepare.err"; then + echo 'FAIL: offline historical fixture preparation failed' >&2; exit 1 fi - +if ! CORESCOPE_DATABASE_URL="$IMPORT_OWNER_URL" CORESCOPE_USERS_DATABASE_URL="" "$MIGRATOR" -offline -from-sqlite "$FIXTURE_DIR/prepared.sqlite" -state-dir "$FIXTURE_DIR/import-state" >"$FIXTURE_DIR/import.out" 2>"$FIXTURE_DIR/import.err"; then + echo 'FAIL: shipping importer failed for the historical fixture' >&2; exit 1 +fi +admin_sql >/dev/null <<<"GRANT SELECT ON ALL TABLES IN SCHEMA ${SCHEMA}_import TO $READER;" +real_row=$(run_local "$IMPORT_READER_URL" <<<'SELECT from_pubkey,count(*) FROM transmissions WHERE from_pubkey IS NOT NULL GROUP BY from_pubkey ORDER BY count(*) DESC,from_pubkey LIMIT 1;') +real_pk=${real_row%|*}; real_n=${real_row##*|} +assert_true 'imported historical fixture has attributed pubkey' bash -c '[[ "$1" =~ ^[0-9a-f]{64}$ ]]' _ "$real_pk" +assert_true 'imported fixture has positive retained count' bash -c '[[ "$1" =~ ^[1-9][0-9]*$ ]]' _ "$real_n" +assert_eq 'bound query matches direct count on imported fixture' "$real_n" "$(count "$real_pk" "$IMPORT_READER_URL")" + +# Transport uses the real query/client but stubs all SSH/container activity. +STUB_ARGV="$FIXTURE_DIR/ssh.argv"; STUB_STDIN="$FIXTURE_DIR/ssh.stdin" +STUB_CONTAINER=0 +ssh_t() { + local cmd="$1" input + printf '%s\n' "$cmd" >>"$STUB_ARGV" + input=$(cat); printf '%s\n' "$input" >>"$STUB_STDIN" + case "$cmd" in + 'docker exec -i '*) + if [[ "$STUB_CONTAINER" != 1 ]]; then echo 'client unavailable in stub container' >&2; return 127; fi + cmd=${cmd#docker exec -i }; cmd=${cmd#* } ;; + esac + printf '%s\n' "$input" | CORESCOPE_READER_DATABASE_URL="$READER_URL" bash -c "$cmd" +} +reset_stub() { : >"$STUB_ARGV"; : >"$STUB_STDIN"; } +TMP="$FIXTURE_DIR"; ADMIN_API_TOKEN=''; TEST_PUBKEY="$PK_A" +TARGET_CONTAINER='corescope-stub'; TARGET_SSH_HOST='stub-host' +unset CORESCOPE_READER_DATABASE_URL PGSERVICE PGDATABASE +for runner in host container; do + reset_stub; [[ "$runner" == container ]] && STUB_CONTAINER=1 + read_retain_count >/dev/null 2>"$FIXTURE_DIR/transport.err"; rc=$? + assert_eq 'remote reader count succeeds' 0 "$rc" + assert_eq 'resolved runner is expected target' "$runner" "${POSTGRES_RUNNER:-}" + assert_eq 'transport preserves retained count' 3 "$RETAIN_COUNT" + argv=$(cat "$STUB_ARGV"); input=$(cat "$STUB_STDIN") + assert_true 'pubkey is absent from remote argv' lacks "$argv" "$PK_A" + assert_true 'credentials are absent from remote argv' lacks "$argv" "$FIXTURE_PASSWORD" + assert_true 'SQL is absent from remote argv' lacks "$argv" 'SELECT' + assert_true 'native parameter query arrives on stdin' contains "$input" 'decode($1' + assert_true 'raw pubkey is absent from stdin' lacks "$input" "$PK_A" +done +reset_stub +CORESCOPE_READER_DATABASE_URL="$READER_URL" read_retain_count >/dev/null 2>"$FIXTURE_DIR/local.err"; rc=$? +assert_eq 'explicit local reader URL succeeds' 0 "$rc" +assert_eq 'explicit URL selects local client' local "${POSTGRES_RUNNER:-}" +assert_eq 'explicit URL does not contact SSH' '' "$(cat "$STUB_ARGV")" +# Missing/unusable binding capability and query errors must fail closed. +ssh_t() { cat >/dev/null; printf '\\bind unsupported\n0\n'; } +read_retain_count >/dev/null 2>&1; rc=$? +assert_eq 'client that does not bind is rejected' 1 "$rc" +assert_eq 'failed capability leaves no count' '' "$RETAIN_COUNT" +POSTGRES_RUNNER='' +if run_postgres /dev/null 2>&1; then FAIL=$((FAIL+1)); echo 'FAIL: unresolved runner accepted' >&2; else PASS=$((PASS+1)); fi +for malformed in 'postgresql://test:secret-sentinel@/db' 'postgresql://test:secret-sentinel@localhost/db?unsupported=yes' 'postgresql://test:secret-sentinel%xx@localhost/db' "${READER_URL}&target_session_attrs=secret-sentinel"; do + out=$(CORESCOPE_READER_DATABASE_URL="$malformed" "$PYTHON" "$CONNECT" &1); rc=$? + assert_true 'malformed reader URL fails' test "$rc" -ne 0 + assert_true 'malformed URL never reveals credentials' lacks "$out" 'secret-sentinel' +done +# Native libpq configuration is also accepted without putting values in argv. +out=$(CORESCOPE_READER_DATABASE_URL="$READER_URL" "$PYTHON" -c 'import os,runpy,subprocess,sys; m=runpy.run_path(os.environ["QA_CONNECT"]); e=m["connection_env"](os.environ); e.pop("CORESCOPE_READER_DATABASE_URL",None); sys.exit(subprocess.run(["psql","-X","-qAt","-w","-v","ON_ERROR_STOP=1"],env=e).returncode)' <<<'SELECT 1977;' 2>"$FIXTURE_DIR/libpq.err" | tr -d '\r') +assert_eq 'private libpq fields reach the real database' 1977 "$out" +space_url=${READER_URL/search_path=$SCHEMA/search_path=$SCHEMA%2C%20pg_catalog} +assert_eq 'search_path option whitespace is escaped on the real client' "$SCHEMA, pg_catalog" "$(run_local "$space_url" <<<"SELECT current_setting('search_path');")" echo "test-blacklist-sql.sh: $PASS passed, $FAIL failed" -[ "$FAIL" -eq 0 ] +[[ "$FAIL" == 0 ]] diff --git a/qa/scripts/test-postgres-connect.py b/qa/scripts/test-postgres-connect.py new file mode 100644 index 000000000..5c2919af0 --- /dev/null +++ b/qa/scripts/test-postgres-connect.py @@ -0,0 +1,136 @@ +import contextlib +import importlib.util +import io +import os +from pathlib import Path +import runpy +import subprocess +import sys +import unittest +from unittest import mock + +sys.dont_write_bytecode = True +PATH = Path(__file__).with_name("postgres-connect.py") +SPEC = importlib.util.spec_from_file_location("postgres_connect", PATH) +CONNECT = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(CONNECT) + + +class ConnectionTests(unittest.TestCase): + def test_url_decodes_credentials_into_environment_and_retains_tls(self): + env = CONNECT.connection_env({"CORESCOPE_READER_DATABASE_URL": + "postgresql://qa_reader:p%40ss%2Fword@[::1]:5444/measurements?sslmode=verify-full&sslrootcert=%2Fprivate%2Fca.pem", + "PGSERVICE": "unrelated", "PGOPTIONS": "-c role=owner"}) + self.assertEqual((env["PGHOST"], env["PGPORT"], env["PGUSER"], env["PGDATABASE"]), ("::1", "5444", "qa_reader", "measurements")) + self.assertEqual(env["PGPASSWORD"], "p@ss/word") + self.assertEqual((env["PGSSLMODE"], env["PGSSLROOTCERT"]), ("verify-full", "/private/ca.pem")) + self.assertNotIn("PGSERVICE", env) + self.assertNotIn("PGOPTIONS", env) + + def test_url_without_tls_override_preserves_private_tls_setting(self): + env = CONNECT.connection_env({"CORESCOPE_READER_DATABASE_URL": "postgres://reader@localhost/data", "PGSSLMODE": "verify-full"}) + self.assertEqual(env["PGSSLMODE"], "verify-full") + + def test_raw_query_semicolons_cannot_hide_tls_parameters(self): + for query in ("application_name=qa;sslmode=verify-full", + "application_name=qa&sslrootcert=ca;sslmode=verify-full"): + with self.subTest(query=query), self.assertRaises(ValueError): + CONNECT.connection_env({"CORESCOPE_READER_DATABASE_URL": "postgres://reader@localhost/data?" + query}) + + def test_encoded_semicolon_is_a_value_and_preserves_explicit_tls(self): + env = CONNECT.connection_env({"CORESCOPE_READER_DATABASE_URL": + "postgres://reader@localhost/data?application_name=qa%3Bsslmode%3Ddisable&sslmode=verify-full"}) + self.assertEqual(env["PGAPPNAME"], "qa;sslmode=disable") + self.assertEqual(env["PGSSLMODE"], "verify-full") + + def test_explicit_url_clears_inherited_host_address(self): + source = {"CORESCOPE_READER_DATABASE_URL": "postgres://reader@localhost:5444/data", + "PGHOST": "unrelated-profile", "PGHOSTADDR": "stale-address"} + env = CONNECT.connection_env(source) + self.assertEqual((env["PGHOST"], env["PGPORT"]), ("localhost", "5444")) + self.assertNotIn("PGHOSTADDR", env) + self.assertEqual(source["PGHOSTADDR"], "stale-address") + self.assertEqual(CONNECT.connection_env({"PGDATABASE": "data", "PGHOSTADDR": "private-address"})["PGHOSTADDR"], "private-address") + + def test_omitted_url_port_uses_environment_or_default(self): + for inherited, expected in ((None, "5432"), ("", "5432"), ("25432", "25432"), ("1", "1"), ("65535", "65535")): + source = {"CORESCOPE_READER_DATABASE_URL": "postgres://reader@localhost/data"} + if inherited is not None: + source["PGPORT"] = inherited + with self.subTest(inherited=inherited): + self.assertEqual(CONNECT.connection_env(source)["PGPORT"], expected) + + def test_explicit_url_port_overrides_even_invalid_inherited_port(self): + for port in (1, 5444, 65535): + env = CONNECT.connection_env({"CORESCOPE_READER_DATABASE_URL": f"postgres://reader@localhost:{port}/data", "PGPORT": "invalid-inherited"}) + self.assertEqual(env["PGPORT"], str(port)) + + def test_invalid_effective_url_port_is_rejected_safely(self): + for port in ("0", "65536", "-1", "invalid-secret", " 5432", "5432.0"): + for explicit in (False, True): + url = f"postgres://reader:secret-sentinel@localhost{':' + port if explicit else ''}/data" + with self.subTest(explicit=explicit, port=port): + with self.assertRaises(ValueError) as raised: + CONNECT.connection_env({"CORESCOPE_READER_DATABASE_URL": url, "PGPORT": "5432" if explicit else port}) + self.assertNotIn("secret", str(raised.exception)) + self.assertNotIn(port, str(raised.exception)) + + def test_search_path_escapes_option_delimiters(self): + env = CONNECT.connection_env({"CORESCOPE_READER_DATABASE_URL": + "postgres://reader@localhost/data?search_path=qa%2C%20%22space%20%5C%20-crole%3Downer%22"}) + self.assertEqual(env["PGOPTIONS"], '-c search_path=qa,\\ "space\\ \\\\\\ -crole=owner"') + + def test_native_private_settings_need_no_url(self): + self.assertEqual(CONNECT.connection_env({"PGSERVICE": "qa-reader", "PGPASSFILE": "/private/pass"})["PGPASSFILE"], "/private/pass") + self.assertEqual(CONNECT.connection_env({"PGDATABASE": "data", "PGUSER": "reader"})["PGDATABASE"], "data") + with self.assertRaises(ValueError): + CONNECT.connection_env({"PGDATABASE": "postgres://reader:secret@localhost/data"}) + + def test_invalid_urls_reject_without_exposing_values(self): + for url in ("postgres://u:secret@/db", "postgres://u:secret@a,b/db", "postgres://u:secret@localhost/", + "postgres://u:secret@localhost/db?unsupported=yes", "postgres://u:secret@localhost/db?sslmode=require&sslmode=disable", + "postgres://u:secret@localhost/db?sslmode=", "postgres://u:secret@localhost/db?sslmode=invalid", + "postgres://u:secret%xx@localhost/db", "postgres://u:secret@localhost/db\n", "postgres://u:secret@localhost:bad/db"): + with self.subTest(url_kind=url.split("?")[0].split(":")[-1]): + with self.assertRaises(ValueError) as raised: + CONNECT.connection_env({"CORESCOPE_READER_DATABASE_URL": url}) + self.assertNotIn("secret", str(raised.exception)) + self.assertNotIn(url, str(raised.exception)) + + def test_psql_argv_never_contains_credentials(self): + url = "postgres://reader:secret-sentinel@localhost/data?sslmode=verify-full" + with mock.patch.dict(os.environ, {"CORESCOPE_READER_DATABASE_URL": url}, clear=True), \ + mock.patch.object(subprocess, "run", return_value=mock.Mock(returncode=0)) as run: + with self.assertRaises(SystemExit) as exited: + runpy.run_path(str(PATH), run_name="__main__") + self.assertEqual(exited.exception.code, 0) + argv = run.call_args.args[0] + self.assertNotIn(url, argv) + self.assertNotIn("secret-sentinel", " ".join(argv)) + self.assertEqual(run.call_args.kwargs["env"]["PGPASSWORD"], "secret-sentinel") + + def test_malformed_url_error_is_safe_for_job_output(self): + stream = io.StringIO() + with mock.patch.dict(os.environ, {"CORESCOPE_READER_DATABASE_URL": "postgres://u:secret-sentinel@/db"}, clear=True), contextlib.redirect_stderr(stream): + with self.assertRaises(SystemExit): + runpy.run_path(str(PATH), run_name="__main__") + self.assertNotIn("secret-sentinel", stream.getvalue()) + self.assertIn("invalid PostgreSQL reader URL", stream.getvalue()) + + def test_libpq_errors_cannot_echo_connection_values(self): + stream = io.StringIO() + def fail(*args, **kwargs): + diagnostic = 'psql: invalid connection setting "secret-sentinel"\n' + if kwargs.get("stderr") != subprocess.PIPE: + print(diagnostic, file=sys.stderr) + return mock.Mock(returncode=2, stderr=diagnostic) + with mock.patch.dict(os.environ, {"PGDATABASE": "data"}, clear=True), \ + mock.patch.object(subprocess, "run", side_effect=fail), contextlib.redirect_stderr(stream): + with self.assertRaises(SystemExit): + runpy.run_path(str(PATH), run_name="__main__") + self.assertNotIn("secret-sentinel", stream.getvalue()) + self.assertIn("private connection settings", stream.getvalue()) + + +if __name__ == "__main__": + unittest.main() From 6eda1e6aa61a9eb30000e9dad02709e31c0bb844 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 01:00:38 -0400 Subject: [PATCH 05/67] test: wait for initial theme refresh before scope assertions --- tests/e2e/test-issue-1648-m2-icons-e2e.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/e2e/test-issue-1648-m2-icons-e2e.js b/tests/e2e/test-issue-1648-m2-icons-e2e.js index c31ce2069..80ff1540c 100644 --- a/tests/e2e/test-issue-1648-m2-icons-e2e.js +++ b/tests/e2e/test-issue-1648-m2-icons-e2e.js @@ -94,7 +94,12 @@ async function checkConfiguredScope(page) { await page.goto('about:blank'); const route = live ? '/live?lat=1&lon=1&zoom=10' : '/nodes/' + key; await page.goto(`${BASE}/#${route}`, { waitUntil: 'domcontentloaded' }); + // The initial theme refresh redraws an open detail as Loading…. + // Let it finish before opening Live details or asserting row absence. + await page.waitForFunction(() => window.__configuredScopeThemeReady, null, { timeout: 8000 }); if (live) await page.locator('#liveMap .live-node-marker').click(); + await page.locator(live ? '#nodeDetailContent code' : '#nodeFullBody .node-detail-key') + .filter({ hasText: key }).waitFor({ state: 'visible', timeout: 8000 }); await page.locator(live ? '#nodeDetailContent table' : '#node-stats').waitFor({ state: 'visible', timeout: 8000 }); const row = live ? page.locator('#nodeDetailContent tr').filter({ hasText: 'Configured scope' }) @@ -141,6 +146,9 @@ async function main() { } const ctx = await browser.newContext({ viewport: { width: 1280, height: 900 } }); + await ctx.addInitScript(() => { + window.addEventListener('theme-refresh', () => { window.__configuredScopeThemeReady = true; }, { once: true }); + }); const page = await ctx.newPage(); await checkConfiguredScope(page); From b5941a8761dc8a73e3fa41b050f7645d48ccab03 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 01:03:26 -0400 Subject: [PATCH 06/67] fix: enable and verify benchmark I/O resource sampling --- .github/workflows/postgres-benchmark.yml | 5 +- scripts/postgres-benchmark/README.md | 5 +- scripts/postgres-benchmark/run.py | 58 ++++++++++--- scripts/postgres-benchmark/test_harness.py | 99 ++++++++++++++++++++++ 4 files changed, 153 insertions(+), 14 deletions(-) diff --git a/.github/workflows/postgres-benchmark.yml b/.github/workflows/postgres-benchmark.yml index bd3dea97c..3f9990193 100644 --- a/.github/workflows/postgres-benchmark.yml +++ b/.github/workflows/postgres-benchmark.yml @@ -84,7 +84,10 @@ jobs: delegated="/sys/fs/cgroup/corescope-benchmark-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" test ! -e "$delegated" sudo mkdir "$delegated" - printf '+cpu +memory\n' | sudo tee "$delegated/cgroup.subtree_control" >/dev/null + for controller in cpu memory io; do + grep -qw "$controller" "$delegated/cgroup.controllers" || { echo "Required cgroup controller unavailable: $controller" >&2; exit 1; } + done + printf '+cpu +memory +io\n' | sudo tee "$delegated/cgroup.subtree_control" >/dev/null sudo mkdir "$delegated/coordinator" sudo chown "$(id -u):$(id -g)" "$delegated" "$delegated/cgroup.procs" "$delegated/cgroup.subtree_control" "$delegated/coordinator" "$delegated/coordinator/cgroup.procs" # The coordinator/generator live in a sibling of the measured app+DB diff --git a/scripts/postgres-benchmark/README.md b/scripts/postgres-benchmark/README.md index 3995442a9..acf92004d 100644 --- a/scripts/postgres-benchmark/README.md +++ b/scripts/postgres-benchmark/README.md @@ -21,8 +21,9 @@ pilot. **No empirical database comparison is included in this change.** `pg_isready`, and never contacts an existing PostgreSQL service. - Run as a non-root user. PostgreSQL starts on a newly selected loopback port with a private data directory and synthetic, local roles. -- Primary runs require a delegated cgroup v2 parent with CPU and memory - controllers enabled. The runner creates its own child and writes its limits +- Primary runs require a delegated cgroup v2 parent with CPU, memory and I/O + controllers enabled. Required resource counters are checked before builds. + The runner creates its own child and writes its limits before any application/database child executes. All descendants inherit the same **3 CPU / 6 GiB** application budget. The coordinator and event generator stay outside that child. diff --git a/scripts/postgres-benchmark/run.py b/scripts/postgres-benchmark/run.py index 59dc8ca40..b2c6a1ea1 100644 --- a/scripts/postgres-benchmark/run.py +++ b/scripts/postgres-benchmark/run.py @@ -33,6 +33,7 @@ FIRMWARE = "a366955cb2f67b8e6842d4f00d2b6a554dddd88a" HERE = Path(__file__).resolve().parent PROCESSES = [] +RESOURCE_COUNTERS = ("cgroup.procs", "memory.current", "memory.events", "cpu.stat", "io.stat") TABLES = ["nodes", "inactive_nodes", "observers", "transmissions", "observations", "observer_metrics", "neighbor_edges", "dropped_packets", "client_receptions", "client_observers", "client_rx_observations", "client_rf_samples", "node_declared_regions", "scope_match_totals", "_migrations", "_async_migrations", "advert_route_evidence", "advert_evidence_backfill"] # These are operational receipt/creation times, not packet timestamps. Full # pre-run parity includes them; only post-replay logical parity excludes them. @@ -192,12 +193,28 @@ def record_failure(log, exit_code): return result +def record_resource_failure(path, error, **context): + filename = getattr(error, "filename", None) + name = Path(filename).name if filename else None + known = (*RESOURCE_COUNTERS, "memory.peak", "smaps_rollup", "comm", "resources.csv", "processes.csv", "resource-final.json", "resource-error.json") + detail = dict(error_type=type(error).__name__, errno=getattr(error, "errno", None), + counter=name if name in known else None, **context) + try: + write_json(path, detail) + except OSError as write_error: + # A full/unwritable output filesystem must not replace the original + # workload exception with another cleanup exception. + detail["evidence_write_error_type"] = type(write_error).__name__ + detail["evidence_write_errno"] = write_error.errno + return detail + + def public_bundle(output, repository=None): """Explicit whitelist: never export databases, event bodies or controls.""" output = Path(output).resolve() public = output / "public" public.mkdir(exist_ok=True) - patterns = ["manifest.json", "summary*.csv", "paired-summary.json", "report.md", "logs/failure-*.json", "corpus/*.json", "pair-*/corpus-offset.json", "pair-*/events.json", "pair-*/*/failure-*.json", "pair-*/*/*.jsonl", "pair-*/*/resources.csv", "pair-*/*/processes.csv", "pair-*/*/resource-final.json", "pair-*/*/*validation*.json", "pair-*/*/startup*.json", "pair-*/*/migration.json", "pair-*/*/retention*.json", "pair-*/*/handler.json", "pair-*/*/database-settings.json", "pair-*/*/reader-settings.json", "pair-*/*/http-workload.json", "pair-*/*/visibility.json", "pair-*/*/plans/*"] + patterns = ["manifest.json", "resource-preflight.json", "summary*.csv", "paired-summary.json", "report.md", "logs/failure-*.json", "corpus/*.json", "pair-*/corpus-offset.json", "pair-*/events.json", "pair-*/*/failure-*.json", "pair-*/*/*.jsonl", "pair-*/*/resources.csv", "pair-*/*/processes.csv", "pair-*/*/resource-final.json", "pair-*/*/resource-error.json", "pair-*/*/*validation*.json", "pair-*/*/startup*.json", "pair-*/*/migration.json", "pair-*/*/retention*.json", "pair-*/*/handler.json", "pair-*/*/database-settings.json", "pair-*/*/reader-settings.json", "pair-*/*/http-workload.json", "pair-*/*/visibility.json", "pair-*/*/plans/*"] for pattern in patterns: for source in sorted(output.glob(pattern)): if not source.is_file() or source.is_symlink(): @@ -318,6 +335,12 @@ def __init__(self, parent, required): self.path = None raise + def preflight(self): + if self.path: + for name in RESOURCE_COUNTERS: + (self.path / name).read_text() + self.info["resource_counters"] = list(RESOURCE_COUNTERS) + def arguments(self, args): if not self.path: return args @@ -566,16 +589,23 @@ def __enter__(self): self.thread.start() return self - def __exit__(self, *args): - self.done.set() - self.thread.join(timeout=10) - if self.thread.is_alive(): - raise RuntimeError("resource sampler failed to stop") - if self.error: - raise RuntimeError("resource sampler failed") from self.error - if self.budget.path: - c=self.budget.path - write_json(self.path.parent/"resource-final.json",dict(memory_peak_bytes=int((c/"memory.peak").read_text()) if self.budget.info.get("memory_peak_reset") else None,cpu_stat=(c/"cpu.stat").read_text(),io_stat=(c/"io.stat").read_text(),memory_events=(c/"memory.events").read_text(),budget=self.budget.info)) + def __exit__(self, exc_type, exc_value, traceback): + try: + self.done.set() + self.thread.join(timeout=10) + if self.thread.is_alive(): + raise TimeoutError("resource sampler failed to stop") + if self.error: + raise self.error + if self.budget.path: + c=self.budget.path + write_json(self.path.parent/"resource-final.json",dict(memory_peak_bytes=int((c/"memory.peak").read_text()) if self.budget.info.get("memory_peak_reset") else None,cpu_stat=(c/"cpu.stat").read_text(),io_stat=(c/"io.stat").read_text(),memory_events=(c/"memory.events").read_text(),budget=self.budget.info)) + except BaseException as error: + detail = record_resource_failure(self.path.parent/"resource-error.json", error, phase=self.phase, + workload_error_type=exc_type.__name__ if exc_type else None) + if exc_value is None: + raise RuntimeError("resource sampler failed: " + json.dumps(detail, sort_keys=True)) from error + return False def wait_file(path, processes, seconds=120): @@ -812,6 +842,12 @@ def main(argv=None): manifest = dict(status="incomplete", baseline_sha=BASELINE, candidate_sha=args.candidate_sha, seed=args.seed, corpus=args.corpus, shape=corpus(args.corpus), firmware_reference=FIRMWARE, harness_sha256=hashlib.sha256(b"".join(bytes.fromhex(sha256(p)) for p in sorted(HERE.rglob("*")) if p.is_file() and "__pycache__" not in p.parts)).hexdigest(), host=dict(system=platform.system(), kernel=platform.release(), architecture=platform.machine(), cpus=os.cpu_count(), ram_bytes=os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE")), postgres_version=version, go_version=command(["go", "version"]).strip(), compiler=command(["cc", "--version"]).splitlines()[0], resource_budget=budget.info, cache_policy=dict(startup="new application process after full validation; OS cache warm", mixed="warmup then application-warm measurement", sql="channel application cache explicitly cleared; OS cache warm"), os_cold="not measured", comparison_eligible=False, extended_120s_600s_profile="not run unless explicitly requested by duration flags", order=pair_order(args.pairs)) write_json(output / "manifest.json", manifest) try: + try: + budget.preflight() + except OSError as error: + detail = record_resource_failure(output / "resource-preflight.json", error, status="failed") + raise RuntimeError("resource preflight failed: " + json.dumps(detail, sort_keys=True)) from error + write_json(output / "resource-preflight.json", dict(status="passed", enforced=bool(budget.path), counters=list(RESOURCE_COUNTERS) if budget.path else [])) roots = {} for backend, revision in (("sqlite", BASELINE), ("postgres", args.candidate_sha)): root = runtime / backend diff --git a/scripts/postgres-benchmark/test_harness.py b/scripts/postgres-benchmark/test_harness.py index 6fb66888c..a6004c995 100644 --- a/scripts/postgres-benchmark/test_harness.py +++ b/scripts/postgres-benchmark/test_harness.py @@ -1,8 +1,13 @@ import importlib.util +import csv +import errno +import json import pathlib +import re import tempfile import types import unittest +from unittest import mock HERE = pathlib.Path(__file__).resolve().parent spec = importlib.util.spec_from_file_location("bench", HERE / "run.py") @@ -11,6 +16,100 @@ class HarnessTests(unittest.TestCase): + def resource_fixture(self, root, missing=None): + group = root / "private-cgroup-path" + group.mkdir() + for name, value in {"memory.current": "0", "memory.events": "oom 0\n", "cpu.stat": "usage_usec 0\n", + "io.stat": "8:0 rbytes=1024 wbytes=2048\n", "cgroup.procs": ""}.items(): + if name != missing: + (group / name).write_text(value) + budget = bench.Budget.__new__(bench.Budget) + budget.path = group + budget.info = {"enforced": True, "memory_peak_reset": False} + budget.close = mock.Mock() + run = root / "pair-00" / "sqlite" + run.mkdir(parents=True) + return budget, run + + def test_sampler_failure_identifies_counter_without_private_path(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + budget, run = self.resource_fixture(root, missing="io.stat") + with self.assertRaisesRegex(RuntimeError, "io.stat"): + with bench.Resources(run / "resources.csv", budget, []) as resource: + resource.thread.join(timeout=2) + evidence = json.loads((run / "resource-error.json").read_text()) + self.assertEqual(evidence["error_type"], "FileNotFoundError") + self.assertEqual(evidence["errno"], errno.ENOENT) + self.assertEqual(evidence["counter"], "io.stat") + self.assertNotIn("private-cgroup-path", json.dumps(evidence)) + bench.public_bundle(root) + self.assertTrue((root / "public/pair-00/sqlite/resource-error.json").is_file()) + + def test_sampler_failure_does_not_replace_workload_exception(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + budget, run = self.resource_fixture(root, missing="io.stat") + with self.assertRaisesRegex(ValueError, "independent workload failure"): + with bench.Resources(run / "resources.csv", budget, []) as resource: + resource.thread.join(timeout=2) + raise ValueError("independent workload failure") + self.assertEqual(json.loads((run / "resource-error.json").read_text())["counter"], "io.stat") + + def test_successful_resource_sample_keeps_io_metrics(self): + with tempfile.TemporaryDirectory() as directory: + budget, run = self.resource_fixture(pathlib.Path(directory)) + resource = bench.Resources(run / "resources.csv", budget, []) + with mock.patch.object(resource.done, "wait", side_effect=lambda _: resource.done.set()): + resource.sample() + self.assertIsNone(resource.error) + with open(run / "resources.csv", newline="") as stream: + rows = list(csv.DictReader(stream)) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["io_stat"], "8:0 rbytes=1024 wbytes=2048") + + def test_final_resource_write_failure_does_not_replace_workload_exception(self): + with tempfile.TemporaryDirectory() as directory: + budget, run = self.resource_fixture(pathlib.Path(directory)) + with mock.patch.object(bench, "write_json", side_effect=PermissionError(errno.EACCES, "private detail", str(run / "resource-final.json"))): + with self.assertRaisesRegex(ValueError, "independent workload failure"): + with bench.Resources(run / "resources.csv", budget, []): + raise ValueError("independent workload failure") + + def test_missing_counter_fails_before_archiving_or_building(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + budget, _ = self.resource_fixture(root, missing="io.stat") + postgres_bin = root / "pg-bin" + postgres_bin.mkdir() + for tool in ("postgres", "initdb", "psql", "pg_isready"): + (postgres_bin / tool).touch() + output = root / "corescope-bench-preflight" + def version(args, **kwargs): + return "go version go1.27.2" if args[0] == "go" else "PostgreSQL 18.6" + with mock.patch.object(bench.sys, "platform", "linux"), \ + mock.patch.object(bench.os, "geteuid", return_value=1000, create=True), \ + mock.patch.object(bench.os, "sysconf", return_value=4096, create=True), \ + mock.patch.object(bench.shutil, "which", return_value="tool"), \ + mock.patch.object(bench.shutil, "disk_usage", return_value=types.SimpleNamespace(free=32 * 1024**3)), \ + mock.patch.object(bench, "command", side_effect=version), \ + mock.patch.object(bench, "Budget", return_value=budget), \ + mock.patch.object(bench, "archive") as archive, \ + mock.patch.object(bench, "build", side_effect=AssertionError("must preflight before building")) as build: + with self.assertRaisesRegex(RuntimeError, "io.stat"): + bench.main(["--candidate-sha", "1" * 40, "--profile", "smoke", "--pairs", "1", "--corpus", "S", + "--output", str(output), "--postgres-bin", str(postgres_bin)]) + archive.assert_not_called() + build.assert_not_called() + evidence = json.loads((output / "public/resource-preflight.json").read_text()) + self.assertEqual(evidence["counter"], "io.stat") + self.assertFalse(json.loads((output / "public/manifest.json").read_text())["comparison_eligible"]) + + def test_workflow_delegates_every_required_resource_controller(self): + workflow = (HERE.parents[1] / ".github/workflows/postgres-benchmark.yml").read_text() + command = next(line for line in workflow.splitlines() if "printf '+" in line and '"$delegated/cgroup.subtree_control"' in line) + self.assertTrue({"cpu", "memory", "io"}.issubset(set(re.findall(r"\+([a-z]+)", command)))) + def test_corpus_counts_are_exact(self): for name, transmissions, observations in [("S", 30_000, 90_000), ("B", 128_000, 2_048_000), ("L", 1_000_000, 16_000_000)]: shape = bench.corpus(name) From e07bf46c43ac138da3f9bf0912062ea57448496e Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 01:25:19 -0400 Subject: [PATCH 07/67] fix: validate benchmark startup and measured samples --- scripts/postgres-benchmark/run.py | 75 ++++++++++++--- scripts/postgres-benchmark/test_harness.py | 102 ++++++++++++++++++++- 2 files changed, 164 insertions(+), 13 deletions(-) diff --git a/scripts/postgres-benchmark/run.py b/scripts/postgres-benchmark/run.py index b2c6a1ea1..d1ef5f6b1 100644 --- a/scripts/postgres-benchmark/run.py +++ b/scripts/postgres-benchmark/run.py @@ -93,18 +93,26 @@ def q(p): def visibility(ingest, messages): - sent = {e["hash"]: e["completed_ns"] for e in ingest if e.get("new_transmission") and not e.get("error") and not e.get("dropped")} - received, duplicate_receipts = {}, 0 + sent = {e["hash"]: e for e in ingest if e.get("new_transmission") and not e.get("error") and not e.get("dropped")} + measured = {key: event for key, event in sent.items() if event.get("measured")} + received, duplicate_receipts, duplicate_receipts_all = {}, 0, 0 for message in messages: key = message.get("hash") if key not in sent: continue if key in received: - duplicate_receipts += 1 + duplicate_receipts_all += 1 + duplicate_receipts += int(key in measured) received[key] = min(received.get(key, message["received_ns"]), message["received_ns"]) missing = sorted(set(sent) - set(received)) - lags = [received[key] - committed for key, committed in sent.items() if key in received] - return dict(verified=not missing, sentinels=len(sent), received=len(received), missing=len(missing), missing_hashes=missing[:100], duplicate_hash_receipts=duplicate_receipts, visible_before_ingest_return=sum(x < 0 for x in lags), lag_ns=percentiles(lags), samples_ns=lags) + lags = [received[key] - event["completed_ns"] for key, event in measured.items() if key in received] + return dict(verified=not missing, verification_scope="full stream including warmup", sample_scope="measured events only", + sentinels=len(sent), received=len(received), missing=len(missing), missing_hashes=missing[:100], + measured_sentinels=len(measured), measured_received=len(measured.keys() & received.keys()), measured_missing=len(measured.keys() - received.keys()), + duplicate_hash_receipts=duplicate_receipts, duplicate_hash_receipts_all=duplicate_receipts_all, + visible_before_ingest_return=sum(x < 0 for x in lags), + visible_before_ingest_return_all=sum(received[key] < event["completed_ns"] for key, event in sent.items() if key in received), + lag_ns=percentiles(lags), samples_ns=lags) def json_lines(path): @@ -183,12 +191,47 @@ def safe_diagnostics(text): return out[-10:] +def postgres_startup_errors(text): + # Only fixed error kinds and known SQLSTATE codes are public. Server log + # text, directories, connection strings and arbitrary values stay private. + codes = {"08001", "08006", "22023", "42501", "53100", "53200", "53300", "53400", + "55000", "58000", "58030", "58P01", "58P02", "F0000", "F0001", "XX000", "XX001", "XX002"} + errors = [] + for line in text.splitlines(): + match = re.search(r"(?:^|\]\s+)(FATAL|PANIC|ERROR):\s*(?:([0-9A-Z]{5}):\s*)?(.*)", line) + if not match: + continue + severity, code, message = match.groups() + message = message.lower() + kind = "unclassified_postgres_startup_error" + if "could not create lock file" in message and ".s.pgsql." in message and "permission denied" in message: + kind = "unix_socket_lock_permission_denied" + elif "could not bind" in message and "address already in use" in message: + kind = "socket_address_in_use" + elif "could not create any tcp/ip sockets" in message: + kind = "tcp_listener_unavailable" + elif "could not create any unix-domain sockets" in message: + kind = "unix_socket_unavailable" + elif "shared memory" in message: + kind = "shared_memory_unavailable" + elif "invalid value for parameter" in message or "configuration file" in message: + kind = "invalid_server_configuration" + elif "data directory" in message: + kind = "data_directory_unavailable" + errors.append(dict(severity=severity, sqlstate=code if code in codes else None, kind=kind)) + return errors[-10:] or [dict(severity="UNKNOWN", sqlstate=None, kind="unclassified_postgres_startup_error")] + + def record_failure(log, exit_code): log = Path(log) with open(log, "rb") as stream: stream.seek(max(0, log.stat().st_size - 8192)) tail = stream.read(8192).decode("utf-8", errors="replace") result = dict(log=log.name, exit_code=exit_code, diagnostics=safe_diagnostics(tail), policy="At most 10 sanitized diagnostic lines from the final 8 KiB; raw logs and event bodies excluded") + if log.name == "postgres.log": + result["postgres_errors"] = postgres_startup_errors(tail) + result["diagnostics"] = ["{severity}: {sqlstate} {kind}".format(**error) for error in result["postgres_errors"]] + result["policy"] = "Only allowlisted PostgreSQL startup error kinds and SQLSTATE codes; raw messages excluded" write_json(log.with_name("failure-" + log.stem + ".json"), result) return result @@ -214,7 +257,7 @@ def public_bundle(output, repository=None): output = Path(output).resolve() public = output / "public" public.mkdir(exist_ok=True) - patterns = ["manifest.json", "resource-preflight.json", "summary*.csv", "paired-summary.json", "report.md", "logs/failure-*.json", "corpus/*.json", "pair-*/corpus-offset.json", "pair-*/events.json", "pair-*/*/failure-*.json", "pair-*/*/*.jsonl", "pair-*/*/resources.csv", "pair-*/*/processes.csv", "pair-*/*/resource-final.json", "pair-*/*/resource-error.json", "pair-*/*/*validation*.json", "pair-*/*/startup*.json", "pair-*/*/migration.json", "pair-*/*/retention*.json", "pair-*/*/handler.json", "pair-*/*/database-settings.json", "pair-*/*/reader-settings.json", "pair-*/*/http-workload.json", "pair-*/*/visibility.json", "pair-*/*/plans/*"] + patterns = ["manifest.json", "resource-preflight.json", "postgres-preflight.json", "summary*.csv", "paired-summary.json", "report.md", "logs/failure-*.json", "corpus/*.json", "pair-*/corpus-offset.json", "pair-*/events.json", "pair-*/*/failure-*.json", "pair-*/*/*.jsonl", "pair-*/*/resources.csv", "pair-*/*/processes.csv", "pair-*/*/resource-final.json", "pair-*/*/resource-error.json", "pair-*/*/*validation*.json", "pair-*/*/startup*.json", "pair-*/*/migration.json", "pair-*/*/retention*.json", "pair-*/*/handler.json", "pair-*/*/database-settings.json", "pair-*/*/reader-settings.json", "pair-*/*/http-workload.json", "pair-*/*/visibility.json", "pair-*/*/plans/*"] for pattern in patterns: for source in sorted(output.glob(pattern)): if not source.is_file() or source.is_symlink(): @@ -443,16 +486,17 @@ def __init__(self, runtime, bin_dir, budget, logs): self.env = dict({k:v for k,v in os.environ.items() if not k.startswith("PG")}, PGHOST="127.0.0.1", PGPORT=str(self.port), PGUSER="bench_admin", PGDATABASE="postgres") def start(self): - self.proc = spawn([self.bin / "postgres", "-D", self.data, "-h", "127.0.0.1", "-p", self.port, "-c", "fsync=on", "-c", "synchronous_commit=on", "-c", "full_page_writes=on", "-c", "shared_buffers=256MB", "-c", "max_connections=32"], cwd=self.root, env=self.env, log=self.logs / "postgres.log", budget=self.budget) + self.proc = spawn([self.bin / "postgres", "-D", self.data, "-h", "127.0.0.1", "-p", self.port, "-c", "unix_socket_directories=", "-c", "log_line_prefix=", "-c", "log_error_verbosity=verbose", "-c", "fsync=on", "-c", "synchronous_commit=on", "-c", "full_page_writes=on", "-c", "shared_buffers=256MB", "-c", "max_connections=32"], cwd=self.root, env=self.env, log=self.logs / "postgres.log", budget=self.budget) for _ in range(200): if self.proc.poll() is not None: - record_failure(self.proc.bench_log.name, self.proc.returncode) - raise RuntimeError("private PostgreSQL failed to start") + failure = record_failure(self.proc.bench_log.name, self.proc.returncode) + raise RuntimeError("private PostgreSQL failed to start: " + "; ".join(failure["diagnostics"])) if subprocess.run([str(self.bin / "pg_isready"), "-q"], env=self.env).returncode == 0: self.sql("SELECT 1") return time.sleep(.1) - raise TimeoutError("private PostgreSQL readiness timeout") + record_failure(self.proc.bench_log.name, None) + raise TimeoutError("private PostgreSQL readiness timeout; see failure-postgres.json") def stop(self): stop(self.proc) @@ -728,7 +772,7 @@ def add(pair,backend,scenario,name,values,errors=0,drops=0,throughput=None): if (directory/"handler.json").exists(): obj=json.loads((directory/"handler.json").read_text());add(pair,backend,"handler-control","200-valid-envelopes",obj["samples_ns"]) if (directory/"visibility.json").exists(): - obj=json.loads((directory/"visibility.json").read_text());add(pair,backend,"websocket","new-transmission-sentinels",obj["samples_ns"],obj["missing"]) + obj=json.loads((directory/"visibility.json").read_text());add(pair,backend,"websocket","new-transmission-sentinels",obj["samples_ns"],obj["measured_missing"]) if rows: with open(output/"summary.csv","w",newline="") as stream: writer=csv.DictWriter(stream,fieldnames=list(rows[0]));writer.writeheader();writer.writerows(rows) @@ -848,6 +892,14 @@ def main(argv=None): detail = record_resource_failure(output / "resource-preflight.json", error, status="failed") raise RuntimeError("resource preflight failed: " + json.dumps(detail, sort_keys=True)) from error write_json(output / "resource-preflight.json", dict(status="passed", enforced=bool(budget.path), counters=list(RESOURCE_COUNTERS) if budget.path else [])) + try: + pg = Postgres(runtime, args.postgres_bin, budget, logs) + pg.start() + pg.stop() + except BaseException as error: + write_json(output / "postgres-preflight.json", dict(status="failed", error_type=type(error).__name__)) + raise + write_json(output / "postgres-preflight.json", dict(status="passed", transport="loopback_tcp", readiness="SELECT 1", stopped=True)) roots = {} for backend, revision in (("sqlite", BASELINE), ("postgres", args.candidate_sha)): root = runtime / backend @@ -867,7 +919,6 @@ def main(argv=None): write_json(prepare_dir / "validation.json", canonical_validation) manifest.update(epoch=epoch, canonical_file_sha256=sha256(canonical), table_manifest=tables) manifest["binary_sha256"] = {p.name: sha256(p) for p in binaries.iterdir() if p.is_file() and not p.name.endswith(".json")} - pg = Postgres(runtime, args.postgres_bin, budget, logs) warmup = args.warmup if args.warmup is not None else (60 if args.profile == "primary" else 2) seconds = args.seconds if args.seconds is not None else (180 if args.profile == "primary" else 10) if warmup < 0 or seconds < 1 or args.ingest_rate < 1 or args.http_rate < 1 or (warmup+seconds)*max(args.ingest_rate,args.http_rate)>1_000_000: diff --git a/scripts/postgres-benchmark/test_harness.py b/scripts/postgres-benchmark/test_harness.py index a6004c995..91558afa6 100644 --- a/scripts/postgres-benchmark/test_harness.py +++ b/scripts/postgres-benchmark/test_harness.py @@ -110,6 +110,76 @@ def test_workflow_delegates_every_required_resource_controller(self): command = next(line for line in workflow.splitlines() if "printf '+" in line and '"$delegated/cgroup.subtree_control"' in line) self.assertTrue({"cpu", "memory", "io"}.issubset(set(re.findall(r"\+([a-z]+)", command)))) + def test_private_postgres_launch_uses_only_loopback_tcp_and_durable_settings(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + pg = bench.Postgres.__new__(bench.Postgres) + pg.root, pg.bin, pg.data, pg.logs = root, root / "bin", root / "pgdata", root + pg.port, pg.env, pg.budget = 25433, {"PGHOST": "127.0.0.1"}, object() + pg.sql = mock.Mock(return_value="1\n") + with mock.patch.object(bench, "spawn", return_value=types.SimpleNamespace(poll=lambda: None)) as spawn, \ + mock.patch.object(bench.subprocess, "run", return_value=types.SimpleNamespace(returncode=0)): + pg.start() + args = spawn.call_args.args[0] + settings = dict(args[i+1].split("=", 1) for i, arg in enumerate(args[:-1]) if arg == "-c") + self.assertEqual(settings.get("unix_socket_directories"), "") + self.assertEqual(args[args.index("-h")+1], "127.0.0.1") + for setting in ("fsync", "synchronous_commit", "full_page_writes"): + self.assertEqual(settings[setting], "on") + self.assertEqual(settings["shared_buffers"], "256MB") + self.assertEqual(settings["max_connections"], "32") + self.assertEqual(settings.get("log_error_verbosity"), "verbose") + pg.sql.assert_called_once_with("SELECT 1") + + def test_postgres_timestamped_failure_keeps_only_allowlisted_diagnostics(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + logs = root / "logs" + logs.mkdir() + path = logs / "postgres.log" + path.write_text('2026-10-09 05:08:18.001 UTC [123] FATAL: 42501: could not create lock file "/private/socket/.s.PGSQL.5432.lock": Permission denied\n' + '2026-10-09 05:08:18.002 UTC [123] DETAIL: postgres://private:secret@private.example/data\n' + '2026-10-09 05:08:18.003 UTC [123] FATAL: arbitrary private body\n') + result = bench.record_failure(path, 1) + self.assertIn("42501", json.dumps(result)) + self.assertEqual(result["postgres_errors"][0], {"severity": "FATAL", "sqlstate": "42501", "kind": "unix_socket_lock_permission_denied"}) + for private in ("/private", "secret", "private.example", "arbitrary private body"): + self.assertNotIn(private, json.dumps(result)) + bench.public_bundle(root) + self.assertTrue((root / "public/logs/failure-postgres.json").is_file()) + + def test_failed_postgres_launch_precedes_archiving_and_building(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + budget, _ = self.resource_fixture(root) + postgres_bin = root / "pg-bin" + postgres_bin.mkdir() + for tool in ("postgres", "initdb", "psql", "pg_isready"): + (postgres_bin / tool).touch() + output = root / "corescope-bench-pg-preflight" + pg = mock.Mock() + pg.start.side_effect = RuntimeError("private PostgreSQL failed to start") + def version(args, **kwargs): + return "go version go1.27.2" if args[0] == "go" else "PostgreSQL 18.6" + with mock.patch.object(bench.sys, "platform", "linux"), \ + mock.patch.object(bench.os, "geteuid", return_value=1000, create=True), \ + mock.patch.object(bench.os, "sysconf", return_value=4096, create=True), \ + mock.patch.object(bench.shutil, "which", return_value="tool"), \ + mock.patch.object(bench.shutil, "disk_usage", return_value=types.SimpleNamespace(free=32 * 1024**3)), \ + mock.patch.object(bench, "command", side_effect=version), \ + mock.patch.object(bench, "Budget", return_value=budget), \ + mock.patch.object(bench, "Postgres", return_value=pg), \ + mock.patch.object(bench, "archive") as archive, \ + mock.patch.object(bench, "build", side_effect=AssertionError("must launch PostgreSQL before building")) as build: + with self.assertRaisesRegex(RuntimeError, "private PostgreSQL failed"): + bench.main(["--candidate-sha", "1" * 40, "--profile", "smoke", "--pairs", "1", "--corpus", "S", + "--output", str(output), "--postgres-bin", str(postgres_bin)]) + archive.assert_not_called() + build.assert_not_called() + pg.start.assert_called_once() + pg.stop.assert_called() + self.assertEqual(json.loads((output / "public/postgres-preflight.json").read_text())["status"], "failed") + def test_corpus_counts_are_exact(self): for name, transmissions, observations in [("S", 30_000, 90_000), ("B", 128_000, 2_048_000), ("L", 1_000_000, 16_000_000)]: shape = bench.corpus(name) @@ -175,7 +245,7 @@ def test_public_bundle_excludes_databases_payloads_and_controls(self): self.assertEqual(files, {"manifest.json", "pair-00/sqlite/requests.jsonl"}) def test_visibility_is_bound_to_durable_sent_hashes(self): - sent = [{"hash":"a", "completed_ns":100, "new_transmission":True}, {"hash":"b", "completed_ns":200, "new_transmission":True}] + sent = [{"hash":"a", "completed_ns":100, "new_transmission":True, "measured":True}, {"hash":"b", "completed_ns":200, "new_transmission":True, "measured":True}] result = bench.visibility(sent,[{"hash":"a","received_ns":90},{"hash":"a","received_ns":110},{"hash":"unrelated","received_ns":300}]) self.assertFalse(result["verified"]) self.assertEqual(result["missing_hashes"],["b"]) @@ -183,6 +253,36 @@ def test_visibility_is_bound_to_durable_sent_hashes(self): self.assertEqual(result["visible_before_ingest_return"],1) self.assertIsNone(result["lag_ns"]["p99"]) + def test_visibility_headline_excludes_warmup_but_keeps_earliest_negative_lags(self): + sent = [{"hash": "warm", "completed_ns": 100, "new_transmission": True, "measured": False}, + {"hash": "measured", "completed_ns": 200, "new_transmission": True, "measured": True}] + received = [{"hash": "warm", "received_ns": 90}, {"hash": "warm", "received_ns": 95}, + {"hash": "warm", "received_ns": 99}, {"hash": "measured", "received_ns": 190}, + {"hash": "measured", "received_ns": 220}] + result = bench.visibility(sent, received) + self.assertTrue(result["verified"]) + self.assertEqual(result["samples_ns"], [-10]) + self.assertEqual(result["lag_ns"]["samples"], 1) + self.assertEqual(result["duplicate_hash_receipts"], 1) + self.assertEqual(result["duplicate_hash_receipts_all"], 3) + self.assertEqual(result["visible_before_ingest_return"], 1) + self.assertEqual(result["visible_before_ingest_return_all"], 2) + + def test_missing_warmup_sentinel_still_fails_verification(self): + sent = [{"hash": "warm", "completed_ns": 100, "new_transmission": True, "measured": False}, + {"hash": "measured", "completed_ns": 200, "new_transmission": True, "measured": True}] + result = bench.visibility(sent, [{"hash": "measured", "received_ns": 230}]) + self.assertFalse(result["verified"]) + self.assertEqual(result["missing_hashes"], ["warm"]) + self.assertEqual(result["measured_missing"], 0) + self.assertEqual(result["samples_ns"], [30]) + + def test_warmup_does_not_make_sparse_visibility_p99_reportable(self): + sent = [{"hash": str(i), "completed_ns": 100, "new_transmission": True, "measured": i == 1000} for i in range(1001)] + result = bench.visibility(sent, [{"hash": str(i), "received_ns": 110} for i in range(1001)]) + self.assertEqual(result["lag_ns"]["samples"], 1) + self.assertIsNone(result["lag_ns"]["p99"]) + def test_cgroup_launch_preserves_arguments_without_shell_interpolation(self): budget = bench.Budget.__new__(bench.Budget) budget.path = pathlib.Path("/sys/fs/cgroup/bench-test") From 8e7a11b641efa12e5a3d470ea3eeb92992af8612 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 01:48:41 -0400 Subject: [PATCH 08/67] fix: preserve legacy date text during PostgreSQL imports --- cmd/migrate/datetime_test.go | 153 +++++++++++++++++++++++++++++++++++ cmd/migrate/import.go | 6 ++ 2 files changed, 159 insertions(+) create mode 100644 cmd/migrate/datetime_test.go diff --git a/cmd/migrate/datetime_test.go b/cmd/migrate/datetime_test.go new file mode 100644 index 000000000..f713e6a76 --- /dev/null +++ b/cmd/migrate/datetime_test.go @@ -0,0 +1,153 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +func dateSource(t *testing.T, declared string, values []any) string { + t.Helper() + source := telemetrySource(t) + db, err := sql.Open(importSQLiteDriver, sqliteURL(source, "rw")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if declared != "DATETIME" { + var ddl string + if err := db.QueryRow(`SELECT sql FROM sqlite_master WHERE name='dropped_packets'`).Scan(&ddl); err != nil { + t.Fatal(err) + } + if !strings.Contains(ddl, "dropped_at DATETIME") { + t.Fatal("legacy date declaration changed") + } + if _, err := db.Exec(`DROP TABLE dropped_packets`); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(strings.Replace(ddl, "dropped_at DATETIME", "dropped_at "+declared, 1)); err != nil { + t.Fatal(err) + } + } + for i, value := range values { + if _, err := db.Exec(`INSERT INTO dropped_packets(id,reason,dropped_at) VALUES(?,?,?)`, i, "synthetic", value); err != nil { + t.Fatal(err) + } + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + return source +} + +func TestImportPreservesDeclaredDateTextAndResume(t *testing.T) { + values := []any{ + nil, + "2026-10-09T01:02:03.123456789-04:30", + "2026-10-09 01:02:03.1200+00:00", + "2026-10-09T01:02:03Z", + "2026-10-09", + "", + "not a timestamp", + } + for _, declared := range []string{"DATETIME", "DATE", "TIMESTAMP"} { + t.Run(declared, func(t *testing.T) { + source := dateSource(t, declared, values) + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + dsn := pgtest.NewSchema(t) + target := openImportDB(t, dsn) + options := importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 2} + options.afterBatch = func() error { + var rows int + if err := target.QueryRow(`SELECT count(*) FROM dropped_packets`).Scan(&rows); err != nil { + return err + } + if rows == 2 { + return context.Canceled + } + return nil + } + if _, err := importSQLite(context.Background(), options); !errors.Is(err, context.Canceled) { + t.Fatalf("expected interruption after preserving two date rows: %v", err) + } + if err := dbschema.AssertReady(target); err == nil { + t.Fatal("interrupted date import became ready") + } + options.Resume, options.afterBatch = true, nil + report, err := importSQLite(context.Background(), options) + if err != nil || !report.Verified { + t.Fatalf("resume date import: verified=%v error=%v", report.Verified, err) + } + if err := finalizeImport(context.Background(), dsn, "telemetry"); err != nil { + t.Fatal(err) + } + if err := dbschema.AssertReady(target); err != nil { + t.Fatal(err) + } + rows, err := target.Query(`SELECT id,dropped_at FROM dropped_packets ORDER BY id`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + count := 0 + for rows.Next() { + var id int + var value sql.NullString + if err := rows.Scan(&id, &value); err != nil { + t.Fatal(err) + } + if id != count || id >= len(values) { + t.Fatalf("date row identity changed: %d", id) + } + if want, ok := values[id].(string); ok { + if !value.Valid || value.String != want { + t.Errorf("date text changed at row %d: got %q want %q", id, value.String, want) + } + } else if value.Valid { + t.Errorf("NULL date became non-NULL at row %d", id) + } + count++ + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + if count != len(values) { + t.Fatalf("date rows duplicated or lost: %d", count) + } + after, err := sourceFingerprint(source) + if err != nil || before != after { + t.Fatal("date import changed the original SQLite source", err) + } + }) + } +} + +func TestImportRefusesNumericStorageInDeclaredDateColumn(t *testing.T) { + for _, value := range []any{int64(1700000000), 2.5, "12345"} { + source := dateSource(t, "DATETIME", []any{value}) + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + dsn := pgtest.NewSchema(t) + report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry"}) + if err == nil || !strings.Contains(err.Error(), "non-text value") || report.Verified { + t.Fatalf("non-text date storage was silently coerced: verified=%v error=%v", report.Verified, err) + } + if err := dbschema.AssertReady(openImportDB(t, dsn)); err == nil { + t.Fatal("invalid date source became ready") + } + after, err := sourceFingerprint(source) + if err != nil || before != after { + t.Fatal("refused date import changed the source", err) + } + } +} diff --git a/cmd/migrate/import.go b/cmd/migrate/import.go index eaac3c0ee..c43aae21b 100644 --- a/cmd/migrate/import.go +++ b/cmd/migrate/import.go @@ -564,6 +564,12 @@ func copyTable(ctx context.Context, source, owner *sql.DB, conn *pgx.Conn, table if o.Kind == "accounts" && table.Name == "mail_events" { expressions[0] = "rowid" } + // SQLite's unary + preserves the stored value/type while removing declared + // column metadata. Otherwise the driver parses DATE/DATETIME/TIMESTAMP + // values into time.Time before copy/resume hashing can preserve their text. + for i, expression := range expressions { + expressions[i] = "+" + expression + } rows, err := source.QueryContext(ctx, `SELECT `+strings.Join(expressions, ",")+` FROM `+quote(table.Name)+` ORDER BY `+order) if err != nil { return report, err From caac986ed467f2a9f942271ed964c63119e55a5d Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 01:49:37 -0400 Subject: [PATCH 09/67] fix: retain sanitized importer errors in benchmark results --- scripts/postgres-benchmark/run.py | 2 ++ scripts/postgres-benchmark/test_harness.py | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/scripts/postgres-benchmark/run.py b/scripts/postgres-benchmark/run.py index d1ef5f6b1..c4b470bbd 100644 --- a/scripts/postgres-benchmark/run.py +++ b/scripts/postgres-benchmark/run.py @@ -171,6 +171,8 @@ def safe_diagnostics(text): out = [] for line in text.splitlines(): line = line.strip() + if line.startswith("[migrate] "): + line = "migrate: " + line[len("[migrate] "):] if not re.match(r"(?:--- FAIL:|FAIL\b|panic:|fatal error:|[^\s]*\.go:\d+(?::\d+)?:|ERROR:|FATAL:|error:|benchmark:|migrate:)", line): codes = re.findall(r"SQLSTATE [0-9A-Z]{5}", line) if codes: diff --git a/scripts/postgres-benchmark/test_harness.py b/scripts/postgres-benchmark/test_harness.py index 91558afa6..88cff59a5 100644 --- a/scripts/postgres-benchmark/test_harness.py +++ b/scripts/postgres-benchmark/test_harness.py @@ -336,6 +336,17 @@ def test_failure_diagnostic_bounds_long_logs(self): self.assertLessEqual(sum(len(x) for x in result["diagnostics"]), 2400) self.assertIn("--- FAIL: TestCoreScopeBenchmark (1s)", result["diagnostics"]) + def test_shipping_migrate_prefix_retains_safe_failure_reason(self): + with tempfile.TemporaryDirectory() as directory: + log = pathlib.Path(directory) / "migration.log" + log.write_text("[migrate] import dropped_packets: column dropped_at: non-text value in a text column\n" + "[migrate] connection failed postgres://owner:private-secret@private.example/db at /private/state/file\n") + result = bench.record_failure(log, 1) + diagnostic = json.dumps(result) + self.assertIn("import dropped_packets: column dropped_at: non-text value in a text column", diagnostic) + for private in ("private-secret", "private.example", "/private/state"): + self.assertNotIn(private, diagnostic) + def test_finish_records_failed_child_before_raising(self): with tempfile.TemporaryDirectory() as directory: log = pathlib.Path(directory) / "replay.log" From 7b2c19a1eaf0050113c1ce8ae73cc69b6e541893 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 03:12:46 -0400 Subject: [PATCH 10/67] test: qualify the full benchmark corpus under common memory limits --- scripts/postgres-benchmark/README.md | 27 ++++++ .../overlays/ingestor_test.go | 92 +++++++++++++++++++ .../overlays/sqlite_adapter_test.go | 43 +++++++-- scripts/postgres-benchmark/run.py | 21 ++++- scripts/postgres-benchmark/test_harness.py | 28 ++++++ 5 files changed, 200 insertions(+), 11 deletions(-) diff --git a/scripts/postgres-benchmark/README.md b/scripts/postgres-benchmark/README.md index acf92004d..0a985cd10 100644 --- a/scripts/postgres-benchmark/README.md +++ b/scripts/postgres-benchmark/README.md @@ -104,6 +104,22 @@ validation, startup and drain time. The longer design profile is explicit: `--warmup 120 --seconds 600`. Record and compare identical locked settings; changing rates after a pilot produces a separately identified experiment. +Both engines use a 2,048 MiB packet-store allowance and a 3,072 MiB server Go +memory limit. The ingestor remains at 512 MiB and the whole application/database +cgroup remains capped at 3 CPUs / 6 GiB. These are common settings for full +startup, hot startup and replay; the manifest records them. + +The earlier 1,024 MiB store / 1,536 MiB server profile failed B capacity +qualification: the unchanged baseline loaded all 112,000 retained transmissions +and 1,792,000 observations, then its own memory accounting triggered a 25% +eviction. The corrected corpus accounts for about 1,222 MiB before replay. +A conservative check using the baseline's actual estimators bounds all 24,000 +primary replay events at another 190 MiB, even counting every event as a new +transmission plus observation with padded field widths. The new common store +allowance leaves more than 630 MiB above that bound. This is capacity evidence, +not a performance result; measurements from the earlier profile must not be +pooled with this one. Rates, corpus sizes, durability and count gates stay fixed. + ## Data and correctness The immutable baseline creates the complete canonical SQLite schema and seeds @@ -114,6 +130,17 @@ nullable and fractional signal values, integral scores, resolved/unresolved path one-, two-, and three-byte hop hashes. S/B have seven retained days and one expired day. The current day includes a real hot-startup window. +Every generated observation also has a unique normalized observer/path key +within its transmission, matching the unchanged server's in-memory identity. +NULL observer positions and fanout counts stay fixed. If short path prefixes +collide for two NULL observations, the generator deterministically tries the next +valid node/path variant, up to 64 attempts, preserving the wire payload/hash and +route/hop/hash-size shape. Exhaustion fails generation. This avoids accidental +duplicate rows that SQLite's nullable unique index permits but the loader merges. +Compiled generator controls, including the complete repeating B/L path key space, +run during both builds before corpus generation. The full retained-count gate +continues to compare every eligible row. + Scores in the timed corpus and replay are integral values supported by both revisions. The unchanged SQLite server silently drops fractional-score rows; including those would compare different loaded datasets after the PostgreSQL diff --git a/scripts/postgres-benchmark/overlays/ingestor_test.go b/scripts/postgres-benchmark/overlays/ingestor_test.go index da9fa0334..9e92eb6b1 100644 --- a/scripts/postgres-benchmark/overlays/ingestor_test.go +++ b/scripts/postgres-benchmark/overlays/ingestor_test.go @@ -119,6 +119,98 @@ func (f *benchFixture) packet(index, variant int, stamp int64) (*PacketData, err return p, nil } +type benchGeneratedObservation struct { + Packet *PacketData + ObserverIndex any + PathVariant int +} + +func (f *benchFixture) observation(index, ordinal int, stamp int64, used map[string]bool) (benchGeneratedObservation, error) { + observerIndex := ordinal%f.c.Shape.Observers + 1 + result := benchGeneratedObservation{ObserverIndex: observerIndex} + observerID := f.public[observerIndex-1] + if ordinal%17 == 16 { + result.ObserverIndex, observerID = nil, "" + } + // Keep every observation and NULL identity. A path-prefix collision for + // the same identity needs a different valid wire path, not different JSON + // spelling. The bound stays below packet()'s special replay variants. + for attempt := 0; attempt < 64; attempt++ { + result.PathVariant = ordinal + attempt + packet, err := f.packet(index, result.PathVariant, stamp) + if err != nil { + return result, err + } + key := observerID + "|" + packet.PathJSON + if !used[key] { + used[key] = true + result.Packet = packet + return result, nil + } + } + return result, fmt.Errorf("no unique observer/path identity after 64 valid path variants") +} + +func TestCoreScopeBenchmarkObservationKeyspace(t *testing.T) { + for _, shape := range []struct { + name string + nodes, observers int + }{{"B", 2000, 128}, {"L", 5000, 512}} { + t.Run(shape.name, func(t *testing.T) { + c := benchConfig{Corpus: shape.name, Seed: 20261008, Epoch: 1791451200, Shape: benchShape{Nodes: shape.nodes, Observers: shape.observers, Days: 8}} + f := benchFixtureFor(c) + // The path recipe depends on index modulo Nodes and modulo 3; + // route/fanout periods 5/10 divide both node counts. Thus these + // 6,000/15,000 positions cover the complete B/L path key space. + corrections, missing := 0, 0 + for index := 0; index < 3*shape.nodes; index++ { + used := make(map[string]bool) + count := benchFanout(c, index) + if count > shape.observers { + t.Fatal("known observer identities would repeat within a transmission") + } + // Known observer IDs are distinct. Only NULL identities can + // share a key, so exercise every such pair with real decoding. + for ordinal := 16; ordinal < count; ordinal += 17 { + original, err := f.packet(index, ordinal, c.Epoch) + if err != nil { + t.Fatal(err) + } + got, err := f.observation(index, ordinal, c.Epoch, used) + if err != nil { + t.Fatal(err) + } + if got.ObserverIndex != nil || got.Packet.Hash != original.Hash || got.Packet.RawHex[:4] != original.RawHex[:4] { + t.Fatal("path correction changed observer, payload identity, route or hop/hash shape") + } + if got.PathVariant != ordinal { + corrections++ + if got.Packet.PathJSON == original.PathJSON { + t.Fatal("retry did not change the real decoded path") + } + } + missing++ + } + } + if corrections == 0 || missing != 3*shape.nodes*6/10 { + t.Fatalf("NULL collision coverage: corrections=%d NULLs=%d", corrections, missing) + } + }) + } +} + +func TestCoreScopeBenchmarkObservationRetryBound(t *testing.T) { + c := benchConfig{Corpus: "B", Seed: 20261008, Epoch: 1791451200, Shape: benchShape{Nodes: 2000, Observers: 128}} + f := benchFixtureFor(c) + used := make(map[string]bool) + for prefix := 0; prefix < 256; prefix++ { + used[fmt.Sprintf(`|["%02X"]`, prefix)] = true + } + if _, err := f.observation(9, 16, c.Epoch, used); err == nil { + t.Fatal("exhausted path space must fail rather than drop an observation or loop forever") + } +} + func TestCoreScopeBenchmarkProtocol(t *testing.T) { c := benchConfig{Seed: 20261008, Epoch: 1791451200, Shape: benchShape{Nodes: 80, Observers: 8, Transmissions: 100, Days: 8}} f := benchFixtureFor(c) diff --git a/scripts/postgres-benchmark/overlays/sqlite_adapter_test.go b/scripts/postgres-benchmark/overlays/sqlite_adapter_test.go index ab9a3b60a..e84cede6a 100644 --- a/scripts/postgres-benchmark/overlays/sqlite_adapter_test.go +++ b/scripts/postgres-benchmark/overlays/sqlite_adapter_test.go @@ -70,6 +70,39 @@ func TestCoreScopeBenchmarkSmallCorpus(t *testing.T) { } } +func TestCoreScopeBenchmarkNullableObservationIdentity(t *testing.T) { + dir := t.TempDir() + // This includes the first collision in the real B key space without + // requiring a two-million-row fixture for the regression itself. + c := benchConfig{Corpus: "B", Seed: 20261008, Epoch: 1791451200, SQLite: filepath.Join(dir, "corpus.sqlite"), Output: dir, + Shape: benchShape{Transmissions: 610, Observations: 9760, Nodes: 2000, Observers: 128, Days: 8}} + if err := benchPrepare(c); err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite3", c.SQLite+"?mode=ro") + if err != nil { + t.Fatal(err) + } + defer db.Close() + var rows, missing, duplicateKeys int + if err := db.QueryRow(`SELECT count(*),sum(observer_idx IS NULL) FROM observations`).Scan(&rows, &missing); err != nil { + t.Fatal(err) + } + if rows != 9760 || missing != 366 { + t.Fatalf("generator changed fanout/NULL coverage: observations=%d, NULL observers=%d", rows, missing) + } + if err := db.QueryRow(`SELECT count(*) FROM ( + SELECT o.transmission_id,coalesce(obs.id,''),o.path_json FROM observations o + LEFT JOIN observers obs ON obs.rowid=o.observer_idx + GROUP BY o.transmission_id,coalesce(obs.id,''),o.path_json HAVING count(*)>1 + )`).Scan(&duplicateKeys); err != nil { + t.Fatal(err) + } + if duplicateKeys != 0 { + t.Fatalf("%d generated observer/path keys would be dropped by the unchanged upstream loader", duplicateKeys) + } +} + func benchOpen(c benchConfig) (*Store, error) { return OpenStore(c.SQLite) } func benchSettings(s *Store) (map[string]string, error) { out := map[string]string{} @@ -172,22 +205,20 @@ func benchPrepare(c benchConfig) error { return e } bits := packetpath.AdvertRouteEvidence(p.RawHex) + observationKeys := make(map[string]bool, count) for o := 0; o < count; o++ { - seen, e := f.packet(i, o, at) + generated, e := f.observation(i, o, at, observationKeys) if e != nil { return e } - var observer any = (o % c.Shape.Observers) + 1 - if o%17 == 16 { - observer = nil - } + seen, observer := generated.Packet, generated.ObserverIndex var snr any = float64((i+o)%20) - 10.5 if o%11 == 10 { snr = nil } var resolved any if i%3 != 0 && seen.PathJSON != "[]" { - b, _ := json.Marshal([]string{f.public[(i+o*7+1)%len(f.public)]}) + b, _ := json.Marshal([]string{f.public[(i+generated.PathVariant*7+1)%len(f.public)]}) resolved = string(b) } if _, e = observation.Exec(i+1, observer, "rx", snr, -100.5+float64(o%25), p.Score, seen.PathJSON, at+int64(o), seen.RawHex, resolved); e != nil { diff --git a/scripts/postgres-benchmark/run.py b/scripts/postgres-benchmark/run.py index c4b470bbd..aa1e82397 100644 --- a/scripts/postgres-benchmark/run.py +++ b/scripts/postgres-benchmark/run.py @@ -34,6 +34,9 @@ HERE = Path(__file__).resolve().parent PROCESSES = [] RESOURCE_COUNTERS = ("cgroup.procs", "memory.current", "memory.events", "cpu.stat", "io.stat") +# Common to both engines: B itself accounts for about 1,222 MiB before replay. +SERVER_MEMORY_MIB = 3072 +PACKET_STORE_MIB = 2048 TABLES = ["nodes", "inactive_nodes", "observers", "transmissions", "observations", "observer_metrics", "neighbor_edges", "dropped_packets", "client_receptions", "client_observers", "client_rx_observations", "client_rf_samples", "node_declared_regions", "scope_match_totals", "_migrations", "_async_migrations", "advert_route_evidence", "advert_evidence_backfill"] # These are operational receipt/creation times, not packet timestamps. Full # pre-run parity includes them; only post-replay logical parity excludes them. @@ -668,6 +671,14 @@ def wait_file(path, processes, seconds=120): raise TimeoutError("benchmark start barrier timeout") +def server_settings(backend, sqlite_path, state_dir, run_env): + config = dict(hashChannels=[f"#bench-{i:02d}" for i in range(20)], dbPath=str(sqlite_path), stateDir=str(state_dir), port=free_port(), packetStore=dict(retentionHours=168, maxMemoryMB=PACKET_STORE_MIB, hotStartupHours=0), runtime=dict(maxMemoryMB=SERVER_MEMORY_MIB), clientRxCoverage=dict(enabled=True), userManagement=dict(enabled=False)) + if backend == "postgres": + config.pop("dbPath") + env = dict(run_env, GOMEMLIMIT=f"{SERVER_MEMORY_MIB}MiB", CORESCOPE_DATABASE_URL=run_env.get("CORESCOPE_READER_DATABASE_URL", "")) + return config, env + + def retained_counts(source, since_epoch): db = sqlite3.connect(f"file:{source}?mode=ro", uri=True) try: @@ -817,6 +828,9 @@ def build(root, backend, binaries, env, logs): write_json(overlay, {"Replace": replacements}) for package in ("ingestor", "server"): command(["go", "test", "-overlay", overlay, "-c", "-o", binaries / (backend + "-" + package + ".test"), "."], cwd=root / "cmd" / package, env=env, log=logs / (backend + "-" + package + "-build.log")) + if package == "ingestor": + command([binaries / (backend + "-ingestor.test"), "-test.run=^TestCoreScopeBenchmark.+", "-test.count=1", "-test.timeout=3m"], + cwd=root / "cmd" / package, env=env, log=logs / (backend + "-generator-controls.log")) command(["go", "build", "-trimpath", "-o", binaries / (backend + "-server"), "."], cwd=root / "cmd/server", env=env, log=logs / (backend + "-server-binary.log")) if backend == "postgres": command(["go", "build", "-trimpath", "-o", binaries / "migrate", "."], cwd=root / "cmd/migrate", env=env, log=logs / "migrate-build.log") @@ -925,7 +939,7 @@ def main(argv=None): seconds = args.seconds if args.seconds is not None else (180 if args.profile == "primary" else 10) if warmup < 0 or seconds < 1 or args.ingest_rate < 1 or args.http_rate < 1 or (warmup+seconds)*max(args.ingest_rate,args.http_rate)>1_000_000: raise ValueError("invalid workload duration or offered rate") - manifest["workload"] = dict(warmup_seconds=warmup, measured_seconds=seconds, ingest_rate=args.ingest_rate, http_rate=args.http_rate, queue_limit=1024, http_concurrency=64, server_memory_mib=1536, ingestor_memory_mib=512, packet_store_mib=1024, retention_hours=168, sql_reader_pool=4, sql_writer_pool=1) + manifest["workload"] = dict(warmup_seconds=warmup, measured_seconds=seconds, ingest_rate=args.ingest_rate, http_rate=args.http_rate, queue_limit=1024, http_concurrency=64, server_memory_mib=SERVER_MEMORY_MIB, ingestor_memory_mib=512, packet_store_mib=PACKET_STORE_MIB, retention_hours=168, sql_reader_pool=4, sql_writer_pool=1) write_json(output / "manifest.json", manifest) after_by_pair = {} paired_sources = {} @@ -984,11 +998,8 @@ def main(argv=None): raise RuntimeError("initial backend state differs from the canonical corpus") write_json(run_dir / "validation-before.json", pre) retained_expected = retained_counts(sqlite_path, int(time.time())-168*3600) if args.corpus in ("S", "B") else None - app_config = dict(hashChannels=[f"#bench-{i:02d}" for i in range(20)], dbPath=str(sqlite_path), stateDir=str(state_dir), port=free_port(), packetStore=dict(retentionHours=168, maxMemoryMB=1024, hotStartupHours=0), runtime=dict(maxMemoryMB=1536), clientRxCoverage=dict(enabled=True), userManagement=dict(enabled=False)) - if backend == "postgres": - app_config.pop("dbPath") + app_config, server_env = server_settings(backend, sqlite_path, state_dir, run_env) write_json(config_dir / "config.json", app_config) - server_env = dict(run_env, GOMEMLIMIT="1536MiB", CORESCOPE_DATABASE_URL=run_env.get("CORESCOPE_READER_DATABASE_URL", "")) resources.phase="startup_full" launched_ns=time.monotonic_ns() server = spawn([binaries / (backend + "-server"), "-config-dir", config_dir, "-public", roots[backend] / "public", "-port", app_config["port"], "-poll-ms", 1000], cwd=roots[backend], env=server_env, log=run_dir / "server.log", budget=budget) diff --git a/scripts/postgres-benchmark/test_harness.py b/scripts/postgres-benchmark/test_harness.py index 88cff59a5..ec4d4037d 100644 --- a/scripts/postgres-benchmark/test_harness.py +++ b/scripts/postgres-benchmark/test_harness.py @@ -16,6 +16,34 @@ class HarnessTests(unittest.TestCase): + def test_common_server_memory_profile_retains_b_and_replay_headroom(self): + # Native baseline B qualification tracked 1221.7 MiB. A conservative + # actual-estimator bound counts every primary event as a new tx + obs. + required_store_mib = 1221.7 + 24000 * 8279 / 1048576 + settings = [] + for backend in ("sqlite", "postgres"): + with mock.patch.object(bench, "free_port", return_value=12345): + config, env = bench.server_settings(backend, pathlib.Path("corpus.sqlite"), pathlib.Path("state"), {}) + self.assertGreater(config["packetStore"]["maxMemoryMB"], required_store_mib) + self.assertEqual(config["packetStore"], {"retentionHours": 168, "maxMemoryMB": 2048, "hotStartupHours": 0}) + self.assertEqual(config["runtime"]["maxMemoryMB"], 3072) + self.assertEqual(env["GOMEMLIMIT"], "3072MiB") + self.assertEqual("dbPath" in config, backend == "sqlite") + config.pop("dbPath", None) + settings.append((config, env)) + self.assertEqual(settings[0], settings[1], "capacity settings must be common to both engines") + + def test_build_runs_compiled_generator_controls(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + binaries, logs = root / "binaries", root / "logs" + binaries.mkdir(); logs.mkdir() + with mock.patch.object(bench, "production_hash", return_value="stable"), mock.patch.object(bench, "command") as command: + bench.build(root, "sqlite", binaries, {}, logs) + controls = [call.args[0] for call in command.call_args_list if pathlib.Path(call.args[0][0]).name == "sqlite-ingestor.test"] + self.assertEqual(len(controls), 1, "new Go generator regressions must execute, not only compile") + self.assertIn("-test.run=^TestCoreScopeBenchmark.+", controls[0]) + def resource_fixture(self, root, missing=None): group = root / "private-cgroup-path" group.mkdir() From c315d7d1926d1ae470d63af34bd68e48e6d63613 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 03:50:12 -0400 Subject: [PATCH 11/67] test: separate the lower-rate comparison from overload qualification --- .github/workflows/postgres-benchmark.yml | 6 ++- scripts/postgres-benchmark/README.md | 34 +++++++++++----- scripts/postgres-benchmark/run.py | 2 +- scripts/postgres-benchmark/test_harness.py | 46 +++++++++++++++++++++- 4 files changed, 74 insertions(+), 14 deletions(-) diff --git a/.github/workflows/postgres-benchmark.yml b/.github/workflows/postgres-benchmark.yml index 3f9990193..8a4280742 100644 --- a/.github/workflows/postgres-benchmark.yml +++ b/.github/workflows/postgres-benchmark.yml @@ -22,7 +22,7 @@ concurrency: jobs: benchmark: - name: Five paired runs (128K transmissions / 2.048M observations) + name: Five supplemental pairs (50 events/s, 128K transmissions / 2.048M observations) runs-on: ubuntu-24.04 timeout-minutes: 240 env: @@ -78,7 +78,7 @@ jobs: - name: Install exact PostgreSQL 18.6 binaries run: bash scripts/install-postgres-ci.sh server - - name: Provision owned cgroup and run primary comparison + - name: Provision owned cgroup and run supplemental comparison run: | set -euo pipefail delegated="/sys/fs/cgroup/corescope-benchmark-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" @@ -97,11 +97,13 @@ jobs: # Prove the controller and Linux prerequisites before preparing corpus B. python3 scripts/postgres-benchmark/run.py --repo . \ --candidate-sha "$CANDIDATE_SHA" --corpus S --pairs 1 --profile smoke \ + --ingest-rate 50 --http-rate 20 \ --warmup 2 --seconds 10 --postgres-version 18.6 --go-version 1.27.2 \ --postgres-bin /usr/lib/postgresql/18/bin \ --output "$SMOKE_OUTPUT" --cgroup "$delegated" python3 scripts/postgres-benchmark/run.py --repo . \ --candidate-sha "$CANDIDATE_SHA" --corpus B --pairs 5 --profile primary \ + --ingest-rate 50 --http-rate 20 \ --postgres-version 18.6 --go-version 1.27.2 \ --postgres-bin /usr/lib/postgresql/18/bin \ --output "$BENCH_OUTPUT" --cgroup "$delegated" diff --git a/scripts/postgres-benchmark/README.md b/scripts/postgres-benchmark/README.md index 0a985cd10..834ccb5d7 100644 --- a/scripts/postgres-benchmark/README.md +++ b/scripts/postgres-benchmark/README.md @@ -67,7 +67,7 @@ Manual dispatch remains available: use `deploy.yml` with `postgres_benchmark=true` and the full `candidate_sha`, from the matching candidate branch/commit. That dispatch runs the comparison alone and skips normal publishing/deployment jobs. Both entry points run the small pilot first and then -five primary pairs; upload only the sanitized `public/` outputs. +five supplemental pairs; upload only the sanitized `public/` outputs. ## Commands @@ -82,28 +82,43 @@ Pilot on the same Linux host intended for the comparison: ```sh python3 scripts/postgres-benchmark/run.py --repo . \ --candidate-sha --corpus S --pairs 1 \ - --profile smoke --ingest-rate 10 --http-rate 5 \ + --profile smoke --ingest-rate 50 --http-rate 20 \ --output /tmp/corescope-bench-pilot ``` -Smoke results are explicitly ineligible for a primary comparison. Primary: +Smoke results are explicitly ineligible for a primary comparison. The current +supplemental comparison uses the five-pair `primary` execution mode: ```sh python3 scripts/postgres-benchmark/run.py --repo . \ --candidate-sha --corpus B --pairs 5 \ + --ingest-rate 50 --http-rate 20 \ --profile primary --cgroup "$CORESCOPE_BENCH_CGROUP" \ - --output /tmp/corescope-bench-primary + --output /tmp/corescope-bench-supplemental-50eps ``` The baseline is fixed to `9dbc287579a237ffa744dd0c91fa7227d09763ac`. Floating revisions and existing output directories are refused. Primary defaults to -60 seconds of warmup and 180 measured seconds per backend, with 100 offered -ingest events/second and 20 HTTP requests/second. Five alternating A/B pairs +60 seconds of warmup and 180 measured seconds per backend, with 50 offered +ingest events/second and 20 HTTP requests/second: 12,000 scheduled ingest events +per backend. Five alternating A/B pairs therefore spend 40 minutes in load windows, plus preparation, imports, validation, startup and drain time. The longer design profile is explicit: `--warmup 120 --seconds 600`. Record and compare identical locked settings; changing rates after a pilot produces a separately identified experiment. +The initial 100-event/s B qualification at candidate `7b2c19a` +([Actions run 37897747657](https://github.com/Kpa-clawbot/CoreScope/actions/runs/37897747657)) +failed its SQLite leg with 82 dropped schedules. All drops fell +at 205.92–206.76 seconds, after retention was scheduled at 195 seconds; that pass took +11.7446 seconds and ingestion queue wait reached 11.054 seconds. The bounded +1,024-event queue was full. Durable effects matched the 23,918 completed events, +but the missing events keep that run failed. Its artifacts remain separate. +The supplemental 50-event/s rate gives the same queue 20.48 seconds of offered +work, versus 10.24 seconds at 100/s. This gives the observed maintenance burst +headroom; the Linux run must still prove zero drops, including during warmup. +The manifest records the actual offered rate. Do not pool the two experiments. + Both engines use a 2,048 MiB packet-store allowance and a 3,072 MiB server Go memory limit. The ingestor remains at 512 MiB and the whole application/database cgroup remains capped at 3 CPUs / 6 GiB. These are common settings for full @@ -114,11 +129,12 @@ qualification: the unchanged baseline loaded all 112,000 retained transmissions and 1,792,000 observations, then its own memory accounting triggered a 25% eviction. The corrected corpus accounts for about 1,222 MiB before replay. A conservative check using the baseline's actual estimators bounds all 24,000 -primary replay events at another 190 MiB, even counting every event as a new -transmission plus observation with padded field widths. The new common store +events from the original 100-event/s profile at another 190 MiB, even counting +every event as a new transmission plus observation with padded field widths. +That bound also covers the current 12,000-event profile. The common store allowance leaves more than 630 MiB above that bound. This is capacity evidence, not a performance result; measurements from the earlier profile must not be -pooled with this one. Rates, corpus sizes, durability and count gates stay fixed. +pooled with this one. Corpus sizes, durability and count gates stay fixed. ## Data and correctness diff --git a/scripts/postgres-benchmark/run.py b/scripts/postgres-benchmark/run.py index aa1e82397..1df110d8c 100644 --- a/scripts/postgres-benchmark/run.py +++ b/scripts/postgres-benchmark/run.py @@ -862,7 +862,7 @@ def main(argv=None): parser.add_argument("--seed", type=int, default=20261008) parser.add_argument("--warmup", type=int) parser.add_argument("--seconds", type=int) - parser.add_argument("--ingest-rate", type=int, default=100) + parser.add_argument("--ingest-rate", type=int, default=50) parser.add_argument("--http-rate", type=int, default=20) args = parser.parse_args(argv) check_revisions(args.baseline_sha, args.candidate_sha) diff --git a/scripts/postgres-benchmark/test_harness.py b/scripts/postgres-benchmark/test_harness.py index ec4d4037d..bd711e6f0 100644 --- a/scripts/postgres-benchmark/test_harness.py +++ b/scripts/postgres-benchmark/test_harness.py @@ -16,9 +16,51 @@ class HarnessTests(unittest.TestCase): + def test_default_common_rate_and_emitted_worker_configs(self): + parse = bench.argparse.ArgumentParser.parse_args + for extra, expected_rate in (([], 50), (["--ingest-rate", "100"], 100)): + parsed = [] + def capture(parser, argv): + parsed.append(parse(parser, argv)) + raise RuntimeError("stop after parsing") + with mock.patch.object(bench.argparse.ArgumentParser, "parse_args", capture): + with self.assertRaisesRegex(RuntimeError, "stop after parsing"): + bench.main(["--candidate-sha", "1" * 40, "--output", "corescope-bench-test", *extra]) + args = parsed[0] + self.assertEqual((args.ingest_rate, args.http_rate, args.pairs), (expected_rate, 20, 5)) + self.assertEqual((args.warmup, args.seconds), (None, None)) + # Exercise the actual JSON controls consumed by both compiled workers. + emitted = [] + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + for backend in ("sqlite", "postgres"): + controls = root / backend + controls.mkdir() + config = dict(control_dir=str(controls), warmup=60, seconds=180, + ingest_rate=args.ingest_rate, http_rate=args.http_rate) + with mock.patch.object(bench, "spawn") as spawn: + bench.worker(root / (backend + "-ingestor.test"), config, "replay", {}, root / (backend + ".log")) + written = json.loads(pathlib.Path(spawn.call_args.kwargs["env"]["CORESCOPE_BENCH_CONFIG"]).read_text()) + written.pop("control_dir") + emitted.append(written) + self.assertEqual(emitted[0], emitted[1]) + self.assertEqual(emitted[0], dict(mode="replay", warmup=60, seconds=180, + ingest_rate=expected_rate, http_rate=20)) + self.assertEqual((emitted[0]["warmup"] + emitted[0]["seconds"]) * emitted[0]["ingest_rate"], + 12000 if expected_rate == 50 else 24000) + + def test_workflow_declares_the_common_supplemental_rate(self): + workflow = (HERE.parent.parent / ".github/workflows/postgres-benchmark.yml").read_text() + commands = re.findall(r"(?m)^\s*python3 scripts/postgres-benchmark/run\.py (.+)", workflow.replace("\\\n", " ")) + self.assertEqual(len(commands), 2) + for command in commands: + self.assertIn("--ingest-rate 50", command) + self.assertIn("--http-rate 20", command) + self.assertIn("supplemental", workflow.lower()) + def test_common_server_memory_profile_retains_b_and_replay_headroom(self): - # Native baseline B qualification tracked 1221.7 MiB. A conservative - # actual-estimator bound counts every primary event as a new tx + obs. + # Native baseline B qualification tracked 1221.7 MiB. The original + # 100/s envelope conservatively bounds the current 50/s experiment too. required_store_mib = 1221.7 + 24000 * 8279 / 1048576 settings = [] for backend in ("sqlite", "postgres"): From 1c0281aac03eaa184965cd1fb9f335c8079d25dd Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 05:59:31 -0400 Subject: [PATCH 12/67] fix: make interrupted PostgreSQL upgrades safe to resume --- .env.example | 13 +- DEPLOY.md | 24 +- cmd/migrate/cli_test.go | 2 +- cmd/migrate/datetime_test.go | 2 +- cmd/migrate/destination_empty_test.go | 39 +++ cmd/migrate/import.go | 172 +++++++++--- cmd/migrate/import_safety_test.go | 2 +- cmd/migrate/import_test.go | 4 +- cmd/migrate/main.go | 20 +- cmd/migrate/main_test.go | 2 +- cmd/migrate/operator_paths_test.go | 83 ++++++ cmd/migrate/resume_finalized_test.go | 359 ++++++++++++++++++++++++++ cmd/migrate/target_identity_test.go | 63 +++++ docker-compose.example.yml | 7 +- docs/postgresql-upgrade.md | 125 +++++++-- scripts/test-postgres-compose.sh | 16 +- tests/unit/test-postgres-ops.js | 22 +- 17 files changed, 859 insertions(+), 96 deletions(-) create mode 100644 cmd/migrate/destination_empty_test.go create mode 100644 cmd/migrate/operator_paths_test.go create mode 100644 cmd/migrate/resume_finalized_test.go create mode 100644 cmd/migrate/target_identity_test.go diff --git a/.env.example b/.env.example index e7b4a327e..50c485f89 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,6 @@ # MeshCore Analyzer — Environment Configuration -# Copy to .env and customize. Generate the required database passwords below. +# New installation: copy to .env and customize. Existing installation: merge +# missing keys into your private .env; do not replace paths or credentials. # # This file is read by BOTH docker compose AND manage.sh — one source of truth. # manage.sh setup negotiates and updates only these production managed keys: @@ -14,6 +15,7 @@ # Keep .env private. Hexadecimal passwords can be safely embedded in the # generated container URLs; the initializer enforces at least 32 hex digits. # These roles are created only when PostgreSQL's data directory is empty. +# Editing these values afterward does not change existing database passwords. POSTGRES_ADMIN_PASSWORD= CORESCOPE_OWNER_PASSWORD= CORESCOPE_READER_PASSWORD= @@ -27,6 +29,15 @@ POSTGRES_DATA_DIR=./postgres-data # Existing instances: stop both processes and follow docs/postgresql-upgrade.md # before starting the PostgreSQL image. Never mount two servers on one directory. +# --- Example deployment (docker-compose.example.yml only) --- +# For upgrades, point DATA_DIR at the existing state directory BEFORE bootstrap. +# PROD_DATA_DIR and STAGING_DATA_DIR below do not control the example variant. +DATA_DIR=./data +HTTP_PORT=80 +# CORESCOPE_IMAGE= +# Bundled Caddy forwards port 80 to the Go server on 3000 by default. If you +# explicitly set DISABLE_CADDY=true, your proxy/port mapping must use port 3000. + # --- Production --- # State directory (config, theme, queues and backups) # Default: ~/meshcore-data diff --git a/DEPLOY.md b/DEPLOY.md index e65ed6e42..6fc60b8b7 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -14,9 +14,10 @@ Set `CORESCOPE_REF` to the reviewed application revision, then retain the comple git clone https://github.com/Kpa-clawbot/CoreScope.git cd CoreScope git checkout --detach "$CORESCOPE_REF" -cp .env.example .env +test -f .env || cp .env.example .env chmod 600 .env -# Fill every PostgreSQL password field with a different openssl rand -hex 32 value. +# New PostgreSQL storage only: fill missing password fields with distinct +# openssl rand -hex 32 values. Preserve an existing .env and merge missing keys. # Set CORESCOPE_IMAGE in .env to the matching reviewed image tag or digest. docker compose -f docker-compose.example.yml up -d ``` @@ -37,7 +38,7 @@ Settings can be overridden via environment variables: | Variable | Default | Description | |----------|---------|-------------| -| `DISABLE_CADDY` | Variant-specific; `true` in the example | Skip internal Caddy (set `true` behind a reverse proxy) | +| `DISABLE_CADDY` | `false` in the example | Caddy forwards published port 80 to the Go server on 3000. If disabled, also change your proxy/port mapping to target 3000. | | `DISABLE_MOSQUITTO` | `true` in `docker-compose.staging.yml`; `false` elsewhere | Skip internal MQTT broker. Default flipped to `true` for the staging deploy in v3.7+ because a standalone `mqtt-broker` container owns MQTT on that host — see "Standalone MQTT broker (staging)" below. | | `HTTP_PORT` | `80` | Host port mapping | | `DATA_DIR` | `./data` | Host path for persistent data | @@ -55,11 +56,11 @@ docker compose -f docker-compose.example.yml up -d PostgreSQL persists telemetry and accounts in its own host directory (`POSTGRES_DATA_DIR`, or the production/staging equivalent). `/app/data` holds config, theme, queues, statistics and account backup files. Keep these locations separate and preserve the private `.env` with your recovery material. -Use native `pg_dump` archives, the authenticated backup endpoints, or `./manage.sh backup `. Do not copy live PostgreSQL storage or a legacy SQLite file as a current backup. `./manage.sh restore ` accepts native archives only, refuses nonempty targets and leaves the application stopped for validation. See the [backup and account recovery boundaries](docs/postgresql-upgrade.md#native-backups-and-restores). +Use native `pg_dump` archives or the authenticated backup endpoints. `./manage.sh backup ` and `restore` operate the production Compose variant managed by that script; they do not select `docker-compose.example.yml`. The [backup and restore instructions](docs/postgresql-upgrade.md#native-backups-and-restores) include the example variant. Do not copy live PostgreSQL storage or a legacy SQLite file as a current backup. ## TLS -Option A — **External reverse proxy** (recommended): Run with `DISABLE_CADDY=true`, put nginx/traefik/Cloudflare in front. +Option A — **External reverse proxy**: Keep the default internal Caddy and proxy to the published HTTP port. Alternatively, set `DISABLE_CADDY=true` and have the proxy reach container port 3000; change the Compose port mapping from `:80` to `:3000` if the proxy runs on the host. Setting the flag alone leaves the existing port-80 mapping without a listener. Option B — **Built-in Caddy**: Mount a custom Caddyfile at `/etc/caddy/Caddyfile` and expose ports 80+443. @@ -169,18 +170,19 @@ Pre-built images skip the build step entirely — faster updates, no Go toolchai **Updates after migration:** ```bash -docker compose pull && docker compose up -d +docker compose -f docker-compose.example.yml pull +docker compose -f docker-compose.example.yml up -d ``` ### What about manage.sh features? | manage.sh command | Pre-built equivalent | |---|---| -| `./manage.sh update` | `docker compose pull && docker compose up -d` | -| `./manage.sh stop` | `docker compose down` | -| `./manage.sh start` | `docker compose up -d` | -| `./manage.sh logs` | `docker compose logs -f` | -| `./manage.sh status` | `docker compose ps` | +| `./manage.sh update` | `docker compose -f docker-compose.example.yml pull` then `up -d` with the same file | +| `./manage.sh stop` | `docker compose -f docker-compose.example.yml stop corescope` | +| `./manage.sh start` | `docker compose -f docker-compose.example.yml up -d` | +| `./manage.sh logs` | `docker compose -f docker-compose.example.yml logs -f` | +| `./manage.sh status` | `docker compose -f docker-compose.example.yml ps` | | `./manage.sh setup` | Retain the full checkout, configure private credentials and select the Compose variant | `manage.sh` remains available for advanced use cases (building from source, custom patches, development). Pre-built images are recommended for most production deployments. diff --git a/cmd/migrate/cli_test.go b/cmd/migrate/cli_test.go index 6bf0e0d9c..e9bfbf3e6 100644 --- a/cmd/migrate/cli_test.go +++ b/cmd/migrate/cli_test.go @@ -71,7 +71,7 @@ func TestCheckImportMarkerBindsExactSource(t *testing.T) { if err := runCommand(context.Background(), args, &output); err == nil { t.Fatal("unfinalized import passed bootstrap guard") } - if err := finalizeImport(context.Background(), dsn, "telemetry"); err != nil { + if _, err := finalizeImport(context.Background(), dsn, "telemetry"); err != nil { t.Fatal(err) } if err := runCommand(context.Background(), args, &output); err != nil { diff --git a/cmd/migrate/datetime_test.go b/cmd/migrate/datetime_test.go index f713e6a76..9f0054d3a 100644 --- a/cmd/migrate/datetime_test.go +++ b/cmd/migrate/datetime_test.go @@ -86,7 +86,7 @@ func TestImportPreservesDeclaredDateTextAndResume(t *testing.T) { if err != nil || !report.Verified { t.Fatalf("resume date import: verified=%v error=%v", report.Verified, err) } - if err := finalizeImport(context.Background(), dsn, "telemetry"); err != nil { + if _, err := finalizeImport(context.Background(), dsn, "telemetry"); err != nil { t.Fatal(err) } if err := dbschema.AssertReady(target); err != nil { diff --git a/cmd/migrate/destination_empty_test.go b/cmd/migrate/destination_empty_test.go new file mode 100644 index 000000000..4b7aa91c9 --- /dev/null +++ b/cmd/migrate/destination_empty_test.go @@ -0,0 +1,39 @@ +package main + +import ( + "context" + "strings" + "testing" + + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +func TestFreshImportRefusesStandaloneSequence(t *testing.T) { + dsn := pgtest.NewSchema(t) + db := openImportDB(t, dsn) + if _, err := db.Exec(`CREATE SEQUENCE retained_sequence START WITH 77`); err != nil { + t.Fatal(err) + } + source := accountSource(t, 5) + before, err := fingerprint(context.Background(), source) + if err != nil { + t.Fatal(err) + } + _, err = importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "accounts"}) + if err == nil || !strings.Contains(err.Error(), "not empty") { + t.Fatalf("fresh import accepted a sequence-only occupied target: %v", err) + } + var value int + var called bool + if err := db.QueryRow(`SELECT last_value,is_called FROM retained_sequence`).Scan(&value, &called); err != nil || value != 77 || called { + t.Fatal("fresh import refusal changed the existing sequence", err) + } + var tables int + if err := db.QueryRow(`SELECT count(*) FROM information_schema.tables WHERE table_schema=current_schema()`).Scan(&tables); err != nil || tables != 0 { + t.Fatal("fresh import refusal created destination tables", err) + } + after, err := fingerprint(context.Background(), source) + if err != nil || before != after { + t.Fatal("fresh import refusal changed the retained source", err) + } +} diff --git a/cmd/migrate/import.go b/cmd/migrate/import.go index c43aae21b..34a9dc659 100644 --- a/cmd/migrate/import.go +++ b/cmd/migrate/import.go @@ -14,6 +14,7 @@ import ( "math" "os" "path/filepath" + "slices" "strings" "unicode/utf8" @@ -52,6 +53,8 @@ type sourceManifest struct { const importerLock int64 = 7289041410 +const importDestinationObjects = `SELECT count(*) FROM pg_class WHERE relnamespace=current_schema()::regnamespace AND relkind IN ('r','p','v','m','S','f')` + func fingerprint(ctx context.Context, path string) (string, error) { fp, err := sourceFingerprintContext(ctx, path) if err != nil { @@ -60,7 +63,7 @@ func fingerprint(ctx context.Context, path string) (string, error) { return fmt.Sprintf("%x:%x", fp[0], fp[1]), nil } -func prepareSource(ctx context.Context, o importOptions, tables []importTable) (sourceManifest, string, error) { +func prepareSource(ctx context.Context, o importOptions, tables []importTable, allowNew bool) (sourceManifest, string, error) { var manifest sourceManifest if o.Source == "" || o.StateDir == "" { return manifest, "", errors.New("source and state directory are required for import") @@ -98,7 +101,14 @@ func prepareSource(ctx context.Context, o importOptions, tables []importTable) ( if !errors.Is(err, os.ErrNotExist) { return manifest, "", err } - if o.Resume { + entries, err := os.ReadDir(dir) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return manifest, "", err + } + if len(entries) != 0 { + return manifest, "", errors.New("preparation stopped before a complete manifest; preserve this state directory and the original SQLite files, then use a new state directory for this store only after confirming its PostgreSQL destination is empty") + } + if !allowNew { return manifest, "", errors.New("no complete migration state manifest exists to resume") } if err := os.MkdirAll(dir, 0o700); err != nil { @@ -176,7 +186,8 @@ func importSQLite(ctx context.Context, o importOptions) (importReport, error) { if err != nil { return report, err } - if _, err := pgutil.ParseConfig(o.DatabaseURL); err != nil { + config, err := pgutil.ParseConfig(o.DatabaseURL) + if err != nil { return report, err } owner, err := pgutil.Open(o.DatabaseURL, false) @@ -185,22 +196,19 @@ func importSQLite(ctx context.Context, o importOptions) (importReport, error) { } defer owner.Close() owner.SetMaxOpenConns(1) - var database, schema, address string + var database, schema string var databaseOID, schemaOID int64 - var port int if err := owner.QueryRowContext(ctx, `SELECT current_database(),current_schema(), (SELECT oid::bigint FROM pg_database WHERE datname=current_database()), - (SELECT oid::bigint FROM pg_namespace WHERE nspname=current_schema()), - COALESCE(inet_server_addr()::text,'local'),COALESCE(inet_server_port(),0)`).Scan(&database, &schema, &databaseOID, &schemaOID, &address, &port); err != nil { + (SELECT oid::bigint FROM pg_namespace WHERE nspname=current_schema())`).Scan(&database, &schema, &databaseOID, &schemaOID); err != nil { return report, err } - identity, _ := json.Marshal([]any{database, schema, databaseOID, schemaOID, address, port}) + // Container IPs can change while their persistent volume and configured DNS + // endpoint remain the same. Bind resume to that endpoint and logical OIDs; + // the destination's random import run ID is verified separately below. + identity, _ := json.Marshal([]any{database, schema, databaseOID, schemaOID, config.Host, config.Port}) identityHash := sha256.Sum256(identity) o.targetDigest = hex.EncodeToString(identityHash[:]) - config, err := pgutil.ParseConfig(o.DatabaseURL) - if err != nil { - return report, err - } conn, err := pgx.ConnectConfig(ctx, config) if err != nil { return report, errors.New("connect to PostgreSQL import destination failed") @@ -211,7 +219,24 @@ func importSQLite(ctx context.Context, o importOptions) (importReport, error) { if err := lockImport(ctx, conn); err != nil { return report, err } - manifest, path, err := prepareSource(ctx, o, tables) + allowNew := !o.Resume + if o.Resume { + var hasImport bool + if err := conn.QueryRow(ctx, `SELECT to_regclass('corescope_import') IS NOT NULL`).Scan(&hasImport); err != nil { + return report, err + } + if !hasImport { + var occupied int + if err := conn.QueryRow(ctx, importDestinationObjects).Scan(&occupied); err != nil { + return report, err + } + if occupied != 0 { + return report, errors.New("resume destination has no import marker and is not empty; existing data was not changed") + } + allowNew = true + } + } + manifest, path, err := prepareSource(ctx, o, tables, allowNew) if err != nil { return report, err } @@ -231,7 +256,7 @@ func importSQLite(ctx context.Context, o importOptions) (importReport, error) { } if !exists { var occupied int - if err := conn.QueryRow(ctx, `SELECT count(*) FROM information_schema.tables WHERE table_schema=current_schema()`).Scan(&occupied); err != nil { + if err := conn.QueryRow(ctx, importDestinationObjects).Scan(&occupied); err != nil { return report, err } if occupied != 0 { @@ -278,25 +303,49 @@ func importSQLite(ctx context.Context, o importOptions) (importReport, error) { if version != 1 { return report, errors.New("unsupported PostgreSQL schema version for import") } + var recordedReport importReport if ready { - return report, errors.New("import destination is already live; do not resume after cutover") + // A process can stop between the two databases' readiness commits. Keep + // this completed store unchanged, but never trust readiness by itself. + tx, err := conn.Begin(ctx) + if err != nil { + return report, err + } + defer tx.Rollback(context.Background()) + lockTables := []string{"corescope_import", "corescope_import_progress", "corescope_schema"} + for _, table := range tables { + lockTables = append(lockTables, quote(table.Name)) + } + if _, err := tx.Exec(ctx, `LOCK TABLE `+strings.Join(lockTables, ",")+` IN SHARE MODE`); err != nil { + return report, err + } + var verified bool + var reportText string + if err := tx.QueryRow(ctx, `SELECT verified,report FROM corescope_import WHERE id=1`).Scan(&verified, &reportText); err != nil { + return report, err + } + if !verified || json.Unmarshal([]byte(reportText), &recordedReport) != nil || !recordedReport.Verified || recordedReport.Kind != o.Kind || len(recordedReport.Tables) != len(tables) { + return report, errors.New("ready destination has no complete verified import report; resume refused") + } } for _, table := range tables { - got, err := copyTable(ctx, source, owner, conn, table, o) + got, err := copyTable(ctx, source, owner, conn, table, o, ready) if err != nil { return report, fmt.Errorf("import %s: %w", table.Name, err) } report.Tables = append(report.Tables, got) } - if err := reseed(ctx, source, conn, tables, o.Kind); err != nil { + if err := reseed(ctx, source, conn, tables, o.Kind, ready); err != nil { return report, err } // COPY does not populate planner statistics. Prepare only this store's // imported tables before declaring verification complete; the cost belongs // to migration timing, never to the steady-state ingestion benchmark. - for _, table := range tables { - if _, err := conn.Exec(ctx, `ANALYZE `+quote(table.Name)); err != nil { - return report, fmt.Errorf("analyze imported %s: %w", table.Name, err) + if !ready { + for _, table := range tables { + if _, err := conn.Exec(ctx, `ANALYZE `+quote(table.Name)); err != nil { + return report, fmt.Errorf("analyze imported %s: %w", table.Name, err) + } } } // Source stays frozen through verification, not merely while snapshotting. @@ -305,7 +354,7 @@ func importSQLite(ctx context.Context, o importOptions) (importReport, error) { return report, err } if current != manifest.SourceDigest { - return report, errors.New("original SQLite source changed during import; target remains unready") + return report, errors.New("original SQLite source changed during import; keep services stopped") } normalized, err := fingerprint(ctx, path) if err != nil { @@ -315,6 +364,12 @@ func importSQLite(ctx context.Context, o importOptions) (importReport, error) { return report, errors.New("normalized snapshot changed during import") } report.Verified = true + if ready { + if !slices.Equal(report.Tables, recordedReport.Tables) { + return report, errors.New("ready destination differs from its verified import report; resume refused") + } + return report, nil + } data, err := json.Marshal(report) if err != nil { return report, err @@ -495,7 +550,7 @@ func digestTarget(ctx context.Context, db *sql.DB, table importTable) (tableRepo return report, rows.Err() } -func copyTable(ctx context.Context, source, owner *sql.DB, conn *pgx.Conn, table importTable, o importOptions) (tableReport, error) { +func copyTable(ctx context.Context, source, owner *sql.DB, conn *pgx.Conn, table importTable, o importOptions, verifyOnly bool) (tableReport, error) { report := tableReport{Table: table.Name} var copied int64 var storedDigest string @@ -504,6 +559,9 @@ func copyTable(ctx context.Context, source, owner *sql.DB, conn *pgx.Conn, table if err != nil && !errors.Is(err, pgx.ErrNoRows) { return report, err } + if verifyOnly && !complete { + return report, errors.New("ready destination lacks complete import progress; resume refused") + } existing, err := digestTarget(ctx, owner, table) if err != nil { return report, err @@ -626,8 +684,10 @@ func copyTable(ctx context.Context, source, owner *sql.DB, conn *pgx.Conn, table if report.Rows < copied { return report, errors.New("source table shorter than committed progress") } - if err := flush(true); err != nil { - return report, err + if !verifyOnly { + if err := flush(true); err != nil { + return report, err + } } report.SHA256 = hex.EncodeToString(h.Sum(nil)) final, err := digestTarget(ctx, owner, table) @@ -640,7 +700,7 @@ func copyTable(ctx context.Context, source, owner *sql.DB, conn *pgx.Conn, table return report, nil } -func reseed(ctx context.Context, source *sql.DB, conn *pgx.Conn, tables []importTable, kind string) error { +func reseed(ctx context.Context, source *sql.DB, conn *pgx.Conn, tables []importTable, kind string, verifyOnly bool) error { highWater := map[string]int64{} var hasSequence int if err := source.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name='sqlite_sequence'`).Scan(&hasSequence); err != nil { @@ -700,7 +760,21 @@ func reseed(ctx context.Context, source *sql.DB, conn *pgx.Conn, tables []import if !called { high = 1 } - if _, err := conn.Exec(ctx, `SELECT setval(pg_get_serial_sequence($1,$2),$3,$4)`, table.Name, col.Name, high, called); err != nil { + if verifyOnly { + var sequence string + if err := conn.QueryRow(ctx, `SELECT pg_get_serial_sequence($1,$2)`, table.Name, col.Name).Scan(&sequence); err != nil { + return err + } + var actual int64 + var wasCalled bool + // pg_get_serial_sequence returns a safely quoted qualified identifier. + if err := conn.QueryRow(ctx, `SELECT last_value,is_called FROM `+sequence).Scan(&actual, &wasCalled); err != nil { + return err + } + if actual != high || wasCalled != called { + return fmt.Errorf("ready destination sequence for %s changed since import; resume refused", table.Name) + } + } else if _, err := conn.Exec(ctx, `SELECT setval(pg_get_serial_sequence($1,$2),$3,$4)`, table.Name, col.Name, high, called); err != nil { return err } } @@ -708,46 +782,48 @@ func reseed(ctx context.Context, source *sql.DB, conn *pgx.Conn, tables []import return nil } -func finalizeImport(ctx context.Context, databaseURL, kind string) error { +// finalizeImport reports whether this call committed a closed-to-ready transition. +// The caller may roll back only transitions it owns if another store fails. +func finalizeImport(ctx context.Context, databaseURL, kind string) (bool, error) { tables, err := layouts(kind) if err != nil { - return err + return false, err } db, err := pgutil.Open(databaseURL, false) if err != nil { - return err + return false, err } defer db.Close() config, err := pgutil.ParseConfig(databaseURL) if err != nil { - return err + return false, err } conn, err := pgx.ConnectConfig(ctx, config) if err != nil { - return errors.New("connect to PostgreSQL finalization destination failed") + return false, errors.New("connect to PostgreSQL finalization destination failed") } defer conn.Close(context.Background()) if err := lockImport(ctx, conn); err != nil { - return err + return false, err } var verified bool var recordedKind, reportText string if err := conn.QueryRow(ctx, `SELECT kind,verified,report FROM corescope_import WHERE id=1`).Scan(&recordedKind, &verified, &reportText); err != nil { - return err + return false, err } if !verified || recordedKind != kind { - return errors.New("cannot finalize an unverified or mismatched import") + return false, errors.New("cannot finalize an unverified or mismatched import") } var report importReport if err := json.Unmarshal([]byte(reportText), &report); err != nil { - return err + return false, err } if !report.Verified || report.Kind != kind || len(report.Tables) != len(tables) { - return errors.New("incomplete verification manifest") + return false, errors.New("incomplete verification manifest") } tx, err := conn.Begin(ctx) if err != nil { - return err + return false, err } defer tx.Rollback(context.Background()) var lockTables []string @@ -755,25 +831,35 @@ func finalizeImport(ctx context.Context, databaseURL, kind string) error { lockTables = append(lockTables, quote(table.Name)) } if _, err := tx.Exec(ctx, `LOCK TABLE `+strings.Join(lockTables, ",")+` IN SHARE MODE`); err != nil { - return err + return false, err } for i, table := range tables { got, err := digestTarget(ctx, db, table) if err != nil { - return err + return false, err } if got != report.Tables[i] { - return fmt.Errorf("final verification failed for %s; target remains unready", table.Name) + return false, fmt.Errorf("final verification failed for %s; target remains unready", table.Name) } } + var ready bool + if err := tx.QueryRow(ctx, `SELECT ready FROM corescope_schema WHERE kind=$1 AND version=1`, kind).Scan(&ready); err != nil { + return false, errors.New("missing or unsupported readiness marker") + } + if ready { + return false, nil + } result, err := tx.Exec(ctx, `UPDATE corescope_schema SET ready=true WHERE kind=$1 AND version=1`, kind) if err != nil { - return err + return false, err } if result.RowsAffected() != 1 { - return errors.New("missing or unsupported readiness marker") + return false, errors.New("missing or unsupported readiness marker") + } + if err := tx.Commit(ctx); err != nil { + return false, err } - return tx.Commit(ctx) + return true, nil } func lockImport(ctx context.Context, conn *pgx.Conn) error { diff --git a/cmd/migrate/import_safety_test.go b/cmd/migrate/import_safety_test.go index c8d8f823c..422a10c2a 100644 --- a/cmd/migrate/import_safety_test.go +++ b/cmd/migrate/import_safety_test.go @@ -50,7 +50,7 @@ func TestResumeAndFinalizationRejectTargetCorruption(t *testing.T) { o.afterBatch = nil _, err = importSQLite(context.Background(), o) } else { - err = finalizeImport(context.Background(), o.DatabaseURL, o.Kind) + _, err = finalizeImport(context.Background(), o.DatabaseURL, o.Kind) } if err == nil { t.Fatal("corrupted target was accepted") diff --git a/cmd/migrate/import_test.go b/cmd/migrate/import_test.go index de0905001..fbea42345 100644 --- a/cmd/migrate/import_test.go +++ b/cmd/migrate/import_test.go @@ -69,7 +69,7 @@ func TestImportTelemetryPreservesIDsNullsAndHighWater(t *testing.T) { if err := dbschema.AssertReady(db); err == nil { t.Fatal("import accepted before explicit cutover finalization") } - if err := finalizeImport(context.Background(), dsn, "telemetry"); err != nil { + if _, err := finalizeImport(context.Background(), dsn, "telemetry"); err != nil { t.Fatal(err) } if err := dbschema.AssertReady(db); err != nil { @@ -233,7 +233,7 @@ func TestImportLegacyAccountsPreservesAuthenticationAndEventOrder(t *testing.T) if err := users.AssertReady(db); err == nil { t.Fatal("accounts accepted before finalization") } - if err := finalizeImport(context.Background(), dsn, "accounts"); err != nil { + if _, err := finalizeImport(context.Background(), dsn, "accounts"); err != nil { t.Fatal(err) } if err := users.AssertReady(db); err != nil { diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go index 7778cb18e..e987de673 100644 --- a/cmd/migrate/main.go +++ b/cmd/migrate/main.go @@ -68,8 +68,10 @@ func runCommand(ctx context.Context, args []string, out io.Writer) error { return errors.New("telemetry and accounts require separate PostgreSQL databases") } } - type store struct{ kind, dsn, source string } - stores := []store{{"telemetry", *databaseURL, *source}, {"accounts", *usersURL, *accountSource}} + type store struct { + kind, dsn, source string + } + stores := []store{{kind: "telemetry", dsn: *databaseURL, source: *source}, {kind: "accounts", dsn: *usersURL, source: *accountSource}} if *checkImportKind != "" { if *checkReady || *resume { return errors.New("-check-import-kind cannot be combined with -check-ready or -resume") @@ -161,11 +163,14 @@ func runCommand(ctx context.Context, args []string, out io.Writer) error { } fmt.Fprintf(out, "%s PostgreSQL schema initialized\n", s.kind) } - for i, s := range imported { - if err := finalizeImport(ctx, s.dsn, s.kind); err != nil { + var opened []store + for _, s := range imported { + didOpen, err := finalizeImport(ctx, s.dsn, s.kind) + if err != nil { // There is no cross-database transaction. Writers stay offline throughout - // cutover; close any earlier readiness gate if a later finalization fails. - for _, prior := range imported[:i] { + // cutover; close only gates this invocation actually opened. Another + // finalizer can open a gate after our earlier import verification. + for _, prior := range opened { db, openErr := pgutil.Open(prior.dsn, false) if openErr == nil { _, resetErr := db.Exec(`UPDATE corescope_schema SET ready=false WHERE kind=$1`, prior.kind) @@ -179,6 +184,9 @@ func runCommand(ctx context.Context, args []string, out io.Writer) error { } return err } + if didOpen { + opened = append(opened, s) + } } for _, s := range imported { fmt.Fprintf(out, "%s import verified and finalized; keep the recovery snapshot for rollback\n", s.kind) diff --git a/cmd/migrate/main_test.go b/cmd/migrate/main_test.go index 19dfec26a..3bb4f8161 100644 --- a/cmd/migrate/main_test.go +++ b/cmd/migrate/main_test.go @@ -87,7 +87,7 @@ func TestPostgresImportCommittedFixture(t *testing.T) { if !report.Verified { t.Fatal("committed fixture import was not verified") } - if err := finalizeImport(context.Background(), dsn, "telemetry"); err != nil { + if _, err := finalizeImport(context.Background(), dsn, "telemetry"); err != nil { t.Fatal(err) } db := openImportDB(t, dsn) diff --git a/cmd/migrate/operator_paths_test.go b/cmd/migrate/operator_paths_test.go new file mode 100644 index 000000000..f10b97208 --- /dev/null +++ b/cmd/migrate/operator_paths_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +func TestOperatorCustomSourcePathsAndOptionalAccounts(t *testing.T) { + for _, existingAccounts := range []bool{false, true} { + t.Run(map[bool]string{false: "no previous accounts", true: "preserve existing accounts"}[existingAccounts], func(t *testing.T) { + t.Setenv("CORESCOPE_DATABASE_URL", pgtest.NewDatabase(t)) + accounts := pgtest.NewDatabase(t) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", accounts) + dir := filepath.Join(t.TempDir(), "custom state and sources") + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatal(err) + } + source := filepath.Join(dir, "telemetry retained.db") + if err := os.Rename(telemetrySource(t), source); err != nil { + t.Fatal(err) + } + args := []string{"-offline", "-from-sqlite", source, "-state-dir", filepath.Join(dir, "migration snapshots")} + paths := []string{source} + if existingAccounts { + users := filepath.Join(dir, "existing accounts.db") + if err := os.Rename(accountSource(t, 5), users); err != nil { + t.Fatal(err) + } + args = append(args, "-users-from-sqlite", users) + paths = append(paths, users) + } + before := make([]string, len(paths)) + for i, path := range paths { + var err error + before[i], err = fingerprint(context.Background(), path) + if err != nil { + t.Fatal(err) + } + } + var output bytes.Buffer + if err := runCommand(context.Background(), args, &output); err != nil { + t.Fatal(err) + } + if err := runCommand(context.Background(), []string{"-check-ready"}, &output); err != nil { + t.Fatal(err) + } + for i, path := range paths { + kind, flag := "telemetry", "-from-sqlite" + if i == 1 { + kind, flag = "accounts", "-users-from-sqlite" + } + if err := runCommand(context.Background(), []string{"-check-import-kind=" + kind, flag, path}, &output); err != nil { + t.Fatal(err) + } + after, err := fingerprint(context.Background(), path) + if err != nil || before[i] != after { + t.Fatal("operator import/check changed the retained source", err) + } + } + var n int + if err := openImportDB(t, accounts).QueryRow(`SELECT count(*) FROM users`).Scan(&n); err != nil { + t.Fatal(err) + } + if (existingAccounts && n != 1) || (!existingAccounts && n != 0) { + t.Fatal("optional account initialization lost or invented accounts") + } + for _, kind := range []string{"telemetry", "accounts"} { + if !strings.Contains(output.String(), kind+" PostgreSQL schema is ready") { + t.Fatal("documented readiness message is missing") + } + } + if !existingAccounts && !strings.Contains(output.String(), "accounts PostgreSQL schema initialized") { + t.Fatal("documented absent-account initialization message is missing") + } + }) + } +} diff --git a/cmd/migrate/resume_finalized_test.go b/cmd/migrate/resume_finalized_test.go new file mode 100644 index 000000000..57b60dbbe --- /dev/null +++ b/cmd/migrate/resume_finalized_test.go @@ -0,0 +1,359 @@ +package main + +import ( + "bytes" + "context" + "database/sql" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil/pgtest" + "github.com/meshcore-analyzer/users" +) + +// Include tuple versions and sequence state so a refusal or a verified ready +// store cannot conceal rewrites of identical values, progress, or readiness. +func importDatabaseState(t *testing.T, db *sql.DB, kind string) string { + t.Helper() + tables, err := layouts(kind) + if err != nil { + t.Fatal(err) + } + var state strings.Builder + for _, table := range append(tables, importTable{Name: "corescope_import"}, importTable{Name: "corescope_import_progress"}, importTable{Name: "corescope_schema"}) { + rows, err := db.Query(`SELECT row_to_json(t)::text,t.xmin::text,t.ctid::text FROM ` + quote(table.Name) + ` t ORDER BY row_to_json(t)::text COLLATE "C"`) + if err != nil { + t.Fatal(err) + } + for rows.Next() { + var values, version, position string + if err := rows.Scan(&values, &version, &position); err != nil { + t.Fatal(err) + } + fmt.Fprintln(&state, table.Name, values, version, position) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + rows.Close() + for _, column := range table.Columns { + if !column.Identity { + continue + } + var sequence string + if err := db.QueryRow(`SELECT pg_get_serial_sequence($1,$2)`, table.Name, column.Name).Scan(&sequence); err != nil { + t.Fatal(err) + } + var value int64 + var called bool + if err := db.QueryRow(`SELECT last_value,is_called FROM `+sequence).Scan(&value, &called); err != nil { + t.Fatal(err) + } + fmt.Fprintln(&state, sequence, value, called) + } + } + return state.String() +} + +func mixedImport(t *testing.T, startAccounts, readyTelemetry bool) [2]importOptions { + t.Helper() + state := filepath.Join(t.TempDir(), "migration state") + options := [2]importOptions{ + {Source: telemetrySource(t), DatabaseURL: pgtest.NewDatabase(t), StateDir: state, Kind: "telemetry"}, + {Source: accountSource(t, 5), DatabaseURL: pgtest.NewDatabase(t), StateDir: state, Kind: "accounts"}, + } + for i, o := range options { + if i == 1 && !startAccounts { + continue + } + if _, err := importSQLite(context.Background(), o); err != nil { + t.Fatal(err) + } + } + if readyTelemetry { + if _, err := finalizeImport(context.Background(), options[0].DatabaseURL, "telemetry"); err != nil { + t.Fatal(err) + } + } + t.Setenv("CORESCOPE_DATABASE_URL", options[0].DatabaseURL) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", options[1].DatabaseURL) + return options +} + +func resumeMixedCommand(options [2]importOptions, out io.Writer) error { + return runCommand(context.Background(), []string{"-offline", "-resume", "-from-sqlite", options[0].Source, "-users-from-sqlite", options[1].Source, "-state-dir", options[0].StateDir}, out) +} + +func TestCLIResumeAfterFirstStoreFinalized(t *testing.T) { + options := mixedImport(t, true, true) + telemetry := openImportDB(t, options[0].DatabaseURL) + accounts := openImportDB(t, options[1].DatabaseURL) + before := importDatabaseState(t, telemetry, "telemetry") + beforeSources := make(map[string]string) + for _, o := range options { + for _, path := range []string{o.Source, filepath.Join(o.StateDir, o.Kind, "recovery.sqlite"), filepath.Join(o.StateDir, o.Kind, "normalized.sqlite")} { + digest, err := fingerprint(context.Background(), path) + if err != nil { + t.Fatal(err) + } + beforeSources[path] = digest + } + } + if err := users.AssertReady(accounts); err == nil { + t.Fatal("fixture must reproduce the crash between the two readiness commits") + } + var output bytes.Buffer + if err := resumeMixedCommand(options, &output); err != nil { + t.Fatalf("resume after telemetry finalized but accounts unready: %v", err) + } + if err := dbschema.AssertReady(telemetry); err != nil { + t.Fatal(err) + } + if err := users.AssertReady(accounts); err != nil { + t.Fatal(err) + } + if got := importDatabaseState(t, telemetry, "telemetry"); got != before { + t.Fatal("resume wrote already-ready telemetry data, progress, sequence, or readiness") + } + for _, o := range options { + if !strings.Contains(output.String(), o.Kind+" import verified and finalized") || strings.Contains(output.String(), o.DatabaseURL) { + t.Fatal("missing success evidence or leaked database URL") + } + } + for path, before := range beforeSources { + after, err := fingerprint(context.Background(), path) + if err != nil || after != before { + t.Fatal("resume changed original source or immutable snapshot", err) + } + } +} + +type importOutputFunc func([]byte) (int, error) + +func (f importOutputFunc) Write(data []byte) (int, error) { return f(data) } + +func TestCLIResumeFailureDoesNotResetAlreadyReadyStore(t *testing.T) { + options := mixedImport(t, true, true) + telemetry := openImportDB(t, options[0].DatabaseURL) + accounts := openImportDB(t, options[1].DatabaseURL) + before := importDatabaseState(t, telemetry, "telemetry") + // Fail the second finalization after both imports have been verified, so + // the actual CLI rollback path must distinguish previously ready stores. + out := importOutputFunc(func(data []byte) (int, error) { + if bytes.Contains(data, []byte(`"kind":"accounts"`)) { + if _, err := accounts.Exec(`UPDATE users SET display_name='post-verification change'`); err != nil { + return 0, err + } + } + return len(data), nil + }) + if err := resumeMixedCommand(options, out); err == nil || !strings.Contains(err.Error(), "final verification failed") { + t.Fatalf("second finalization corruption should be refused: %v", err) + } + if got := importDatabaseState(t, telemetry, "telemetry"); got != before { + t.Fatal("failed second finalization reset or rewrote the previously ready store") + } + if err := users.AssertReady(accounts); err == nil { + t.Fatal("changed accounts became ready") + } +} + +func TestCLIRollbackDoesNotCloseGateOpenedByAnotherFinalizer(t *testing.T) { + options := mixedImport(t, true, false) + telemetry := openImportDB(t, options[0].DatabaseURL) + accounts := openImportDB(t, options[1].DatabaseURL) + var finalizedState string + out := importOutputFunc(func(data []byte) (int, error) { + if bytes.Contains(data, []byte(`"kind":"accounts"`)) { + // The importer observed telemetry unready. A separate allowed + // finalizer opens it before this CLI reaches its finalization loop. + if _, err := finalizeImport(context.Background(), options[0].DatabaseURL, "telemetry"); err != nil { + return 0, err + } + finalizedState = importDatabaseState(t, telemetry, "telemetry") + if _, err := accounts.Exec(`UPDATE users SET display_name='fail second finalization'`); err != nil { + return 0, err + } + } + return len(data), nil + }) + if err := resumeMixedCommand(options, out); err == nil || !strings.Contains(err.Error(), "final verification failed") { + t.Fatalf("second finalization should fail after the separate finalizer ran: %v", err) + } + if finalizedState == "" { + t.Fatal("fixture did not execute the intervening finalization") + } + if err := dbschema.AssertReady(telemetry); err != nil { + t.Error("CLI reset a gate it did not open", err) + } + if got := importDatabaseState(t, telemetry, "telemetry"); got != finalizedState { + t.Error("CLI changed tuples or readiness owned by the separate finalizer") + } + if err := users.AssertReady(accounts); err == nil { + t.Fatal("corrupted accounts became ready") + } +} + +func TestCLIRollbackClosesGateOpenedByThisInvocation(t *testing.T) { + options := mixedImport(t, true, false) + telemetry := openImportDB(t, options[0].DatabaseURL) + accounts := openImportDB(t, options[1].DatabaseURL) + out := importOutputFunc(func(data []byte) (int, error) { + if bytes.Contains(data, []byte(`"kind":"accounts"`)) { + if _, err := accounts.Exec(`UPDATE users SET display_name='fail second finalization'`); err != nil { + return 0, err + } + } + return len(data), nil + }) + if err := resumeMixedCommand(options, out); err == nil || !strings.Contains(err.Error(), "final verification failed") { + t.Fatalf("second finalization should fail: %v", err) + } + if err := dbschema.AssertReady(telemetry); err == nil { + t.Fatal("CLI did not close the telemetry gate it opened before account failure") + } + if err := users.AssertReady(accounts); err == nil { + t.Fatal("corrupted accounts became ready") + } +} + +func TestCLIResumeDoesNotStartOccupiedSecondStore(t *testing.T) { + options := mixedImport(t, false, true) + telemetry := openImportDB(t, options[0].DatabaseURL) + accounts := openImportDB(t, options[1].DatabaseURL) + if _, err := accounts.Exec(`CREATE SEQUENCE keep_me START WITH 77`); err != nil { + t.Fatal(err) + } + before := importDatabaseState(t, telemetry, "telemetry") + if err := resumeMixedCommand(options, io.Discard); err == nil || !strings.Contains(err.Error(), "not empty") { + t.Fatalf("resume accepted an occupied unmarked destination: %v", err) + } + if got := importDatabaseState(t, telemetry, "telemetry"); got != before { + t.Fatal("refusal changed already-ready telemetry") + } + var value int + var called bool + if err := accounts.QueryRow(`SELECT last_value,is_called FROM keep_me`).Scan(&value, &called); err != nil || value != 77 || called { + t.Fatal("refusal changed occupied account destination", err) + } + if _, err := os.Stat(filepath.Join(options[1].StateDir, "accounts")); !os.IsNotExist(err) { + t.Fatal("occupied destination should be refused before source preparation") + } +} + +func TestCLIResumeStartsUntouchedSecondStore(t *testing.T) { + options := mixedImport(t, false, false) + if err := resumeMixedCommand(options, io.Discard); err != nil { + t.Fatalf("resume must finish the verified first store and start untouched accounts: %v", err) + } + if err := dbschema.AssertReady(openImportDB(t, options[0].DatabaseURL)); err != nil { + t.Fatal(err) + } + if err := users.AssertReady(openImportDB(t, options[1].DatabaseURL)); err != nil { + t.Fatal(err) + } +} + +func TestResumePreparationArtifactsArePreserved(t *testing.T) { + o := importOptions{Source: telemetrySource(t), DatabaseURL: pgtest.NewSchema(t), StateDir: t.TempDir(), Kind: "telemetry", Resume: true} + recovery := filepath.Join(o.StateDir, o.Kind, "recovery.sqlite") + if err := os.MkdirAll(filepath.Dir(recovery), 0o700); err != nil { + t.Fatal(err) + } + if err := snapshotSource(context.Background(), o.Source, recovery); err != nil { + t.Fatal(err) + } + before, err := os.ReadFile(recovery) + if err != nil { + t.Fatal(err) + } + _, err = importSQLite(context.Background(), o) + if err == nil || !strings.Contains(err.Error(), "preserve") || !strings.Contains(err.Error(), "state directory") { + t.Fatalf("interrupted preparation needs preserving recovery guidance: %v", err) + } + after, err := os.ReadFile(recovery) + if err != nil || !bytes.Equal(before, after) { + t.Fatal("refusal changed the only recovery snapshot", err) + } + var n int + if err := openImportDB(t, o.DatabaseURL).QueryRow(`SELECT count(*) FROM information_schema.tables WHERE table_schema=current_schema()`).Scan(&n); err != nil || n != 0 { + t.Fatal("preparation refusal changed the empty destination", err) + } + // Exercise the documented recovery: archive the incomplete per-store state + // without deleting it, then let resume prepare the still-empty store anew. + archived := filepath.Join(t.TempDir(), "preserved incomplete preparation") + if err := os.Rename(filepath.Dir(recovery), archived); err != nil { + t.Fatal(err) + } + if report, err := importSQLite(context.Background(), o); err != nil || !report.Verified { + t.Fatal("resume after preserving incomplete preparation failed", err) + } + retained, err := os.ReadFile(filepath.Join(archived, "recovery.sqlite")) + if err != nil || !bytes.Equal(before, retained) { + t.Fatal("retry changed archived recovery evidence", err) + } +} + +func TestReadyResumeRefusesChangedEvidenceWithoutWrites(t *testing.T) { + changes := map[string]string{ + "new PostgreSQL row": `INSERT INTO observers(id) VALUES('post-cutover')`, + "changed PostgreSQL row": `UPDATE observers SET name='changed' WHERE rowid=0`, + "changed sequence only": `SELECT nextval(pg_get_serial_sequence('observers','rowid'))`, + "incomplete progress": `UPDATE corescope_import_progress SET complete=false WHERE table_name='observers'`, + "changed progress digest": `UPDATE corescope_import_progress SET sha256=repeat('0',64) WHERE table_name='observers'`, + "unverified marker": `UPDATE corescope_import SET verified=false`, + "changed report": `UPDATE corescope_import SET report=replace(report,'"rows":2','"rows":200')`, + "changed run identity": `UPDATE corescope_import SET run_id='different'`, + "source": "", + "recovery snapshot": "", + "normalized snapshot": "", + } + for name, statement := range changes { + t.Run(name, func(t *testing.T) { + o := importOptions{Source: telemetrySource(t), DatabaseURL: pgtest.NewSchema(t), StateDir: t.TempDir(), Kind: "telemetry"} + if _, err := importSQLite(context.Background(), o); err != nil { + t.Fatal(err) + } + if _, err := finalizeImport(context.Background(), o.DatabaseURL, o.Kind); err != nil { + t.Fatal(err) + } + db := openImportDB(t, o.DatabaseURL) + if statement != "" { + if _, err := db.Exec(statement); err != nil { + t.Fatal(err) + } + } else { + path := o.Source + if name != "source" { + file := "normalized.sqlite" + if name == "recovery snapshot" { + file = "recovery.sqlite" + } + path = filepath.Join(o.StateDir, o.Kind, file) + } + changed, err := sql.Open("sqlite3", path) + if err != nil { + t.Fatal(err) + } + _, err = changed.Exec(`UPDATE observers SET name='changed' WHERE rowid=0`) + changed.Close() + if err != nil { + t.Fatal(err) + } + } + before := importDatabaseState(t, db, o.Kind) + o.Resume = true + if _, err := importSQLite(context.Background(), o); err == nil { + t.Fatal("changed evidence accepted for an already-ready destination") + } + if after := importDatabaseState(t, db, o.Kind); after != before { + t.Fatal("refusal changed data, progress, sequence, or readiness") + } + }) + } +} diff --git a/cmd/migrate/target_identity_test.go b/cmd/migrate/target_identity_test.go new file mode 100644 index 000000000..4175b81a1 --- /dev/null +++ b/cmd/migrate/target_identity_test.go @@ -0,0 +1,63 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "net/url" + "os" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +func TestImportIdentityUsesConfiguredEndpoint(t *testing.T) { + dsn := pgtest.NewSchema(t) + u, err := url.Parse(dsn) + if err != nil { + t.Fatal("parse test URL") + } + // Connect through the stable name instead of its transient resolved address. + u.Host = "localhost:" + u.Port() + o := importOptions{Source: accountSource(t, 5), DatabaseURL: u.String(), StateDir: t.TempDir(), Kind: "accounts"} + if _, err := importSQLite(context.Background(), o); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(o.StateDir, o.Kind, "manifest.json")) + if err != nil { + t.Fatal(err) + } + var manifest sourceManifest + if err := json.Unmarshal(data, &manifest); err != nil { + t.Fatal(err) + } + db := openImportDB(t, o.DatabaseURL) + var database, schema string + var databaseOID, schemaOID int64 + if err := db.QueryRow(`SELECT current_database(),current_schema(),(SELECT oid::bigint FROM pg_database WHERE datname=current_database()),(SELECT oid::bigint FROM pg_namespace WHERE nspname=current_schema())`).Scan(&database, &schema, &databaseOID, &schemaOID); err != nil { + t.Fatal(err) + } + config, err := pgutil.ParseConfig(o.DatabaseURL) + if err != nil { + t.Fatal(err) + } + identity, _ := json.Marshal([]any{database, schema, databaseOID, schemaOID, config.Host, config.Port}) + want := sha256.Sum256(identity) + if manifest.TargetDigest != hex.EncodeToString(want[:]) { + t.Fatal("manifest identity used a transient resolved address instead of the configured endpoint") + } + o.Resume = true + if _, err := importSQLite(context.Background(), o); err != nil { + t.Fatalf("same configured endpoint did not resume: %v", err) + } + // The IP alias reaches the same database/OIDs but is a different configured + // endpoint. Operators must retain their original endpoint when resuming. + u.Host = "127.0.0.1:" + u.Port() + o.DatabaseURL = u.String() + if _, err := importSQLite(context.Background(), o); err == nil { + t.Fatal("changed configured endpoint accepted") + } +} diff --git a/docker-compose.example.yml b/docker-compose.example.yml index d5bf80136..fba94a6fa 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -2,9 +2,8 @@ # application image from GHCR. # # AUDIENCE: third-party operators running a self-hosted CoreScope on -# ONE host. Ships with the built-in Mosquitto broker DISABLED by -# default (DISABLE_MOSQUITTO defaults to `false` here, meaning -# mosquitto RUNS in-container — the historical shape); flip to `true` +# ONE host. Ships with the built-in Mosquitto broker enabled +# (DISABLE_MOSQUITTO defaults to `false`); flip to `true` # if you already have an external MQTT broker to point CoreScope at. # # For the CoreScope maintainers' internal staging deployment (with a @@ -45,7 +44,7 @@ services: - CORESCOPE_WRITER_DATABASE_URL=postgresql://corescope_writer:${CORESCOPE_WRITER_PASSWORD:?Set CORESCOPE_WRITER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable - CORESCOPE_USERS_DATABASE_URL=postgresql://corescope_accounts:${CORESCOPE_ACCOUNTS_PASSWORD:?Set CORESCOPE_ACCOUNTS_PASSWORD}@postgres/corescope_accounts?sslmode=disable - CORESCOPE_APPROVED_CHANNELS_DATABASE_URL=postgresql://corescope_channels:${CORESCOPE_CHANNELS_PASSWORD:?Set CORESCOPE_CHANNELS_PASSWORD}@postgres/corescope_accounts?sslmode=disable - - DISABLE_CADDY=${DISABLE_CADDY:-true} + - DISABLE_CADDY=${DISABLE_CADDY:-false} - DISABLE_MOSQUITTO=${DISABLE_MOSQUITTO:-false} restart: unless-stopped healthcheck: diff --git a/docs/postgresql-upgrade.md b/docs/postgresql-upgrade.md index 99a4e9d6d..f31f6dd2e 100644 --- a/docs/postgresql-upgrade.md +++ b/docs/postgresql-upgrade.md @@ -16,7 +16,7 @@ The supplied deployment creates `corescope_telemetry` and `corescope_accounts` o | `corescope_accounts` | Account data writes; no schema or readiness-marker writes. | | `corescope_channels` | Ingestor reads of the approved-channel view; no access to passwords or sessions. | -Copy `.env.example` to a private `.env` and fill every PostgreSQL password field with a different value from `openssl rand -hex 32`. Keep file permissions at `0600`. The Compose files supply runtime URLs; credentials do not belong in Git, screenshots, issue reports or command arguments. On native installations, use private environment configuration with `CORESCOPE_READER_DATABASE_URL`, `CORESCOPE_WRITER_DATABASE_URL`, `CORESCOPE_USERS_DATABASE_URL` and `CORESCOPE_APPROVED_CHANNELS_DATABASE_URL`. +Preserve an existing private `.env` and merge only missing settings from `.env.example`; do not replace its paths, ports, broker settings or passwords. For a new PostgreSQL data directory, fill each missing password field with a different value from `openssl rand -hex 32` and keep `.env` permissions at `0600`. PostgreSQL creates roles only on first initialization: editing `.env` later does **not** rotate their passwords. If an interrupted installation has lost its matching `.env`, recover that file or perform a coordinated administrator password rotation with all application services stopped; do not delete the PostgreSQL volume to fix authentication. The Compose files supply runtime URLs; credentials do not belong in Git, screenshots, issue reports or command arguments. On native installations, use private environment configuration with `CORESCOPE_READER_DATABASE_URL`, `CORESCOPE_WRITER_DATABASE_URL`, `CORESCOPE_USERS_DATABASE_URL` and `CORESCOPE_APPROVED_CHANNELS_DATABASE_URL`. `databaseURL` replaces `dbPath`; `CORESCOPE_DATABASE_URL` is the per-process fallback. Remove legacy `dbPath`, `userManagement.dbPath` and `DB_PATH` settings after preserving their old values for recovery. `stateDir` / `CORESCOPE_STATE_DIR` is a separate filesystem directory for queues, statistics and account backups. A database URL is never a directory name. @@ -29,7 +29,8 @@ Main runtime pool limits are four server telemetry connections, one ingestor con | Telemetry v3 with `observations.observer_idx` and the known optional columns | Supported. Missing supported columns/tables are added only to a working copy. | | Telemetry v2 with `observer_id`, or an older Node-era layout | Refused. Upgrade a recovery copy using the last SQLite release, review the result, then import it. | | Account `schema_version` 0–5 | Supported, including empty version 0. Each declared version must have its complete historical shape. | -| Accounts absent/disabled | Omit `-users-from-sqlite`; an explicitly configured account destination is initialized empty. | +| No account database has ever existed | Omit `-users-from-sqlite`; the configured account destination is initialized empty. | +| Accounts currently disabled, but a users database exists | Import it with `-users-from-sqlite` so accounts, audit records and tokens remain available if re-enabled. | | Unknown/newer columns, tables, views, triggers or virtual tables | Refused, with no silent field or record loss. | | Duplicate observation identity, empty required transmission fields or case-colliding node keys | Refused. Perform an explicit, reviewed repair on a disposable recovery copy with the last SQLite release. | @@ -39,23 +40,39 @@ Identity sequences are reseeded above imported IDs, available `sqlite_sequence` ## Preflight and recovery -1. Record the exact application revision/image, broker settings, channel keys, account mail settings, config/theme files and state-directory contents. Keep the private `.env` and PostgreSQL credentials separately recoverable. -2. Arrange the ingestion gap and stop every instance writer. For Compose, stop the application service (`prod`, `staging-go` or `corescope`); leave the new PostgreSQL service available to the importer. -3. Retain the original SQLite directories, including any `-wal` and `-shm` siblings. Prefer a clean writer shutdown. Mount complete source directories, not just a `.db` file, so committed WAL data is available. -4. Use new, empty PostgreSQL destinations. The importer refuses occupied destinations, future schemas and a different source or destination on resume. -5. Allocate a private migration-state directory. It holds a WAL-aware immutable recovery snapshot, a normalized working snapshot and a source/destination manifest for each store. Snapshot creation uses SQLite's backup API so hidden observer/mail-event row IDs do not change. -6. Measure available disk space against the actual source, both snapshot copies, PostgreSQL data/indexes, WAL and temporary work. The required space and duration depend on the data; no fixed multiplier or downtime estimate is guaranteed. +1. While still using the **old checkout and its original `.env`**, record the Compose file, project name, application service/container, image tag **and image ID**, and the actual telemetry/account SQLite paths. Read the old `dbPath`, `userManagement.dbPath` and `DB_PATH` values before removing them. A custom `stateDir` does not prove where either database lives. +2. Stop the old deployment with its own command, before switching checkout or configuration. For example, from that checkout, use `docker compose -f "$OLD_COMPOSE" -p "$OLD_PROJECT" stop "$OLD_SERVICE"` with the recorded values; old `manage.sh` installations can use their existing `./manage.sh stop`. Verify the old application container is stopped, and stop any separately launched server, ingestor or maintenance writer. The new Compose project may have a different name and may require settings the old one never used. +3. Make a private, quiescent recovery bundle containing the old Compose files/scripts, `.env`, image identity, configuration/theme, keys and **whole state/source directories**, including SQLite `-wal` and `-shm` siblings. Keep an untouched original. Do not use the new PostgreSQL-only `manage.sh backup` to back up SQLite. Restore the preserved old configuration and image together if rolling back. +4. Select new, empty PostgreSQL storage and allocate a private migration-state directory. It will hold a WAL-aware recovery snapshot, normalized working snapshot and source/destination manifest for each store. Snapshot creation uses SQLite's backup API so hidden observer/mail-event row IDs do not change. +5. Measure free space against the actual source, both snapshot copies, PostgreSQL data/indexes, WAL and temporary work. The required space and duration depend on the data; no fixed multiplier or downtime estimate is guaranteed. -If an uncheckpointed WAL cannot be opened from a read-only mount, keep the original untouched and make a quiescent filesystem recovery copy of the whole source directory in private writable storage. Import that identical copy. Never omit a nonempty WAL or run `VACUUM` on the only recovery copy. +For a host-side copy, after stopping every writer, choose a new private directory outside the active state tree and copy each complete source/state directory with `cp -a`. Include custom account directories too. Never omit a nonempty WAL or run `VACUUM` on the only recovery copy. If an uncheckpointed WAL cannot be read from a read-only mount, import a quiescent copy of the whole source directory from private writable storage, while retaining the original unchanged. ## Compose cutover -Clone and pin the complete repository. A downloaded Compose YAML alone is insufficient: the variants extend `docker/postgres.compose.yml` and mount `docker/postgres-init.sh`. Select the matching image through `CORESCOPE_IMAGE` when using `docker-compose.example.yml`. +The common path below uses `docker-compose.example.yml`. Keep the complete pinned checkout and matching `CORESCOPE_IMAGE`; a downloaded YAML alone is insufficient because it extends `docker/postgres.compose.yml` and mounts the initialization scripts. For production/staging variants, retain their existing project name, Compose file and service names throughout. -The following uses the example deployment. Substitute the production/staging Compose variant and the actual source filenames for your instance. Preserve the original data directories first. +### Point the new deployment at the right files + +Preserve/merge `.env`; do not regenerate passwords after PostgreSQL has initialized. In the example variant, set `DATA_DIR` to the **existing absolute host state directory**, `POSTGRES_DATA_DIR` to the selected PostgreSQL storage and `CORESCOPE_IMAGE` to the matching reviewed image. `PROD_DATA_DIR` and `STAGING_DATA_DIR` do not control this variant. Set these before running bootstrap. Keep PostgreSQL storage separate from state/recovery directories. + +Inside the supplied container, the selected host `DATA_DIR` is always `/app/data` and the runtime state directory is `/app/data`. Keep config/theme/queues there; remove legacy database-path settings from the active config and any `/app/data/.env` only after saving their original values in the recovery bundle. Preserve unrelated configuration and broker/mail credentials. + +Confirm the exact source mounts before initializing schemas. This check does not start the application or PostgreSQL: ```sh -docker compose -f docker-compose.example.yml stop corescope +set -eu +docker compose -f docker-compose.example.yml run --rm --no-deps \ + --entrypoint /bin/sh bootstrap -eu -c \ + 'test -f /app/data/config.json; test -f /app/data/meshcore.db; test -f /app/data/users.db' +``` + +Remove the `users.db` check and import flag only when no account source exists, even if account features are currently disabled. If the old installation intentionally had no config file, omit that one file check. For custom SQLite filenames or directories, add a read-only mount such as `-v "$SOURCE_DIR:/legacy:ro"` to **every import and source-check command** and use `/legacy/the-actual-name.db`. Verify those filenames with the same mount first. Keep both sources mounted as directories so their WAL siblings remain visible. + +### Import with the application stopped + +```sh +set -eu docker compose -f docker-compose.example.yml up -d --wait postgres install -d -m 700 migration-state @@ -66,15 +83,29 @@ docker compose -f docker-compose.example.yml run --rm \ -users-from-sqlite /app/data/users.db -state-dir /migration ``` -Omit the users source flag only when there is no account source to preserve. For sources outside `/app/data`, add a read-only directory mount and pass its exact file path. The bootstrap service supplies owner URLs through its environment. +The bootstrap service supplies owner URLs through its environment. Its PostgreSQL health check only proves the server is accepting connections; successful importer authentication is still required. The importer copies in bounded batches, commits progress with each batch, verifies counts and logical digests in primary-key order, reseeds identities and analyzes the imported tables. Both requested imports must verify before readiness finalization begins. -The importer copies in bounded batches, commits progress with each batch, verifies counts and logical digests in primary-key order, reseeds identities and analyzes only the imported tables. Normalization restores original values only in the columns historical migrations can change; it does not rewrite the observation corpus. Both requested stores must verify before cutover finalization. Until then their readiness markers remain false and runtime services refuse them. +Success is exit status zero, a JSON report with `"verified":true` for each imported store, followed by: -For an interruption, keep the same source files, PostgreSQL destinations and migration-state directory, then repeat the command with `-resume`. A changed source, changed snapshot, unexpected target rows or corrupted progress refuses resume. Do not delete the state directory or manually set readiness true. +```text +telemetry import verified and finalized; keep the recovery snapshot for rollback +accounts import verified and finalized; keep the recovery snapshot for rollback +``` + +Without an account source, the account line is `accounts PostgreSQL schema initialized`. **On any error, keep all application services stopped.** A successful report for telemetry alone is not a successful two-store cutover. + +### Interruption and resume + +Keep the same source files, configured PostgreSQL host/port, database/schema and migration-state directory, then repeat the import command with `-resume` using the same pinned importer image. A container IP change behind the same configured DNS endpoint does not change import identity. Resume also handles a store not yet started and an interruption between the two readiness commits. An already-ready store is accepted only after its source/snapshot/target identity, saved verification report, completed progress, every table digest and identity sequence are rechecked without rewriting it. New PostgreSQL data or changed evidence refuses resume; it is not a way to re-import after cutover. + +If preparation stopped **before `/manifest.json` exists**, preserve that incomplete `` directory and the originals. Do not delete snapshots or guess a manifest. Confirm that store's PostgreSQL destination is still empty, move its incomplete per-store directory to a separate private recovery location, and retry with `-resume`; keep completed stores' state directories in place. This creates new preparation state only for the untouched store. If the destination has data but its manifest is missing, restore the matching state from recovery or investigate before proceeding. Never manually set readiness true. -After a successful import: +### Check the retained source, then grant and start + +If an old source needed an explicit repair/upgrade, the verified import belongs to the **repaired copy**, not the untouched original. Keep the original database and its WAL/SHM siblings together in the private rollback bundle outside `/app/data`. Put the exact repaired source used for import at the active default guard path, with its matching sidecars, or retain it at its custom path and run the explicit matching source check. Never leave the unrepaired original at `/app/data/meshcore.db` or `/app/data/users.db` and then bypass a fingerprint refusal. Do not repair or rewrite the accepted source after import. ```sh +set -eu docker compose -f docker-compose.example.yml run --rm \ --entrypoint /app/corescope-migrate bootstrap \ -check-import-kind=telemetry -from-sqlite /app/data/meshcore.db @@ -83,11 +114,16 @@ docker compose -f docker-compose.example.yml run --rm \ --entrypoint /app/corescope-migrate bootstrap \ -check-import-kind=accounts -users-from-sqlite /app/data/users.db +docker compose -f docker-compose.example.yml run --rm \ + --entrypoint /app/corescope-migrate bootstrap -check-ready + docker compose -f docker-compose.example.yml run --rm bootstrap docker compose -f docker-compose.example.yml up -d corescope ``` -Skip the account-source check when accounts were absent. The normal bootstrap verifies any retained SQLite files against the completed import marker and source fingerprint, then grants runtime privileges. A manually initialized empty PostgreSQL database does not satisfy this upgrade guard. Absent and zero-length WAL sidecars have the same fingerprint; every nonempty WAL byte remains significant. +Skip the account-source check only when there was no account source. The source checks print ` verified import matches the retained SQLite source`; readiness prints ` PostgreSQL schema is ready` for both configured destinations. Normal bootstrap checks retained `/app/data/meshcore.db` and `/app/data/users.db`, then grants restricted runtime privileges. A manually initialized empty database does not satisfy this upgrade guard. Custom source paths require the explicit checks above because automatic bootstrap only checks those two default paths. Absent and zero-length WAL sidecars share a fingerprint; every nonempty WAL byte remains significant. + +The example publishes HTTP port 80 through bundled Caddy. Open the configured host port and verify `/api/healthz` plus actual packet ingestion. If you explicitly disable Caddy, your proxy or port mapping must instead reach container port 3000. For native installations, invoke `corescope-migrate` with owner URLs in `CORESCOPE_DATABASE_URL` and, when applicable, `CORESCOPE_USERS_DATABASE_URL`. The same `-offline`, source, state-directory and resume flags apply. `-check-ready` performs read-only schema checks. Run `docker/postgres-grants.sql` as the owner once in each database after bootstrap/import, then start the services with their restricted runtime URLs and a shared state directory. @@ -95,7 +131,7 @@ For native installations, invoke `corescope-migrate` with owner URLs in `CORESCO Verify packet and observation counts, representative raw/null values, observers and paths, live ingestion/WebSocket visibility, retention, account login, existing sessions/CSRF, proposals, notifications and account export. Test accounts enabled and disabled as applicable. Restore a native backup into fresh isolated databases and exercise the API/login there before declaring recovery proven. -Rollback is straightforward **before PostgreSQL accepts new writes**: stop the new services and restart the pinned SQLite application with the retained original files, configuration and keys. Once PostgreSQL has accepted new telemetry or account changes, the old SQLite copy is stale. There is no automatic lossless reverse converter; stop and decide explicitly how to handle the new data before reverting. +Rollback **before PostgreSQL accepts new writes**: stop the new application with its new Compose project, leave its PostgreSQL storage and migration evidence intact, then return to the preserved old checkout/Compose project, original `.env`, pinned SQLite image and untouched source/state directories. Restore the old database-path settings from that bundle; the PostgreSQL configuration cannot be passed to the SQLite release. Confirm the new writers are stopped before starting the old application. Once PostgreSQL has accepted new telemetry or account changes, the old SQLite copy is stale. There is no automatic lossless reverse converter; stop and decide explicitly how to handle the new data before reverting. Keep at least one verified recovery copy. If you later archive the old SQLite files outside the active state directory, move their WAL siblings with them and preserve the import report and original fingerprint evidence. Do not remove the only rollback copy to reclaim space. @@ -103,10 +139,61 @@ Keep at least one verified recovery copy. If you later archive the old SQLite fi `GET /api/backup` and `/api/admin/users-backup` download PostgreSQL custom-format `.dump` archives (`application/octet-stream`). Account snapshots remain subject to their configured schedule and rotation. `pg_dump` 18 must be available to the application; the supplied image includes it. -`./manage.sh backup ` creates `telemetry.dump`, `accounts.dump` and available config/theme/Caddy files. Each database archive is transactionally consistent; two sequential dumps are not one cross-database snapshot. Stop both writers when you need a coordinated pair. Preserve private credentials, queues/state and TLS material separately as part of instance recovery. Never copy a live PostgreSQL data directory as a logical backup. +For the production variant selected by `manage.sh` (not `docker-compose.example.yml`), `./manage.sh backup ` creates `telemetry.dump`, `accounts.dump` and available config/theme/Caddy files. Each database archive is transactionally consistent; two sequential dumps are not one cross-database snapshot. Stop both writers when you need a coordinated pair. Preserve private credentials, queues/state and TLS material separately as part of instance recovery. Never copy a live PostgreSQL data directory as a logical backup. `./manage.sh restore ` requires both databases to be empty and the application stopped. It refuses SQLite files and nonempty destinations, uses transactional `pg_restore`, reapplies runtime grants and retains previous configuration copies. Services remain stopped for validation. For a single downloaded archive, use `pg_restore --no-owner --no-privileges --exit-on-error --single-transaction --dbname=` with connection credentials supplied through private libpq environment settings; then apply the appropriate grants and validation. +For the **example Compose variant**, use its PostgreSQL container directly. This helper reads the owner password already present inside that container; it does not put a URL or password in command arguments. Run it from the matching checkout with the matching private `.env`: + +```sh +pg_owner() { + docker compose -f docker-compose.example.yml exec -T postgres sh -eu -c ' + export PGHOST=127.0.0.1 PGUSER=corescope_owner PGSSLMODE=disable + export PGPASSWORD="$CORESCOPE_OWNER_PASSWORD" + exec "$@" + ' sh "$@" +} +``` + +For a coordinated backup pair, stop the application, create a new private backup directory, then dump both stores. Keep the application stopped until both commands succeed; a failed `.partial` file is not a backup. + +```sh +set -eu +umask 077 +docker compose -f docker-compose.example.yml stop corescope +BACKUP_DIR=$(mktemp -d "$PWD/corescope-backup.XXXXXX") +for store in telemetry accounts; do + pg_owner pg_dump --no-password --format=custom --no-owner --no-privileges \ + --dbname="corescope_$store" > "$BACKUP_DIR/$store.dump.partial" + mv "$BACKUP_DIR/$store.dump.partial" "$BACKUP_DIR/$store.dump" +done +# Preserve matching .env, config/theme, keys and remaining state privately too. +docker compose -f docker-compose.example.yml up -d corescope +``` + +Restore into **new empty PostgreSQL storage**, keeping the previous storage intact. With the original Compose file/project and `.env`, stop `corescope` and then `postgres` before changing any storage path. Record the old `POSTGRES_DATA_DIR`, select a new empty path in `.env`, use matching recovered credentials, and keep `corescope` stopped. Restore the matching config/theme and remaining state from the private bundle before bootstrap; preserve their previous copies. Set `BACKUP_DIR` to the verified backup bundle. Before either restore, check that both destinations are empty: + +```sh +set -eu +docker compose -f docker-compose.example.yml up -d --wait postgres +for store in telemetry accounts; do + test -f "$BACKUP_DIR/$store.dump" + objects=$(pg_owner psql -X -v ON_ERROR_STOP=1 -At --dbname="corescope_$store" \ + -c "SELECT count(*) FROM pg_class WHERE relnamespace='public'::regnamespace AND relkind IN ('r','p','v','m','S','f')") + test "$objects" = 0 || { echo 'Restore destination is not empty; stop here.' >&2; exit 1; } +done +for store in telemetry accounts; do + pg_owner pg_restore --no-password --no-owner --no-privileges \ + --exit-on-error --single-transaction --dbname="corescope_$store" \ + < "$BACKUP_DIR/$store.dump" +done +docker compose -f docker-compose.example.yml run --rm bootstrap +docker compose -f docker-compose.example.yml run --rm \ + --entrypoint /app/corescope-migrate bootstrap -check-ready +``` + +If one restore fails, the other database may already have restored successfully: keep services stopped and preserve both the failure evidence and old storage. Do not retry into an occupied database. After successful grants/readiness and the account checks below, validate the restored instance in isolation before starting it for normal traffic. + Account restoration can revive deleted accounts, old passwords, revoked sessions and already-used links. Record relevant deletions from the current audit log before replacement. With the restored server still stopped, disable those accounts and invalidate sessions/tokens as required, then repeat account deletion through the application so mail addresses are anonymized and the deletion is audited. See [account recovery](user-guide/accounts.md#backups). ## Test fixtures and benchmark evidence diff --git a/scripts/test-postgres-compose.sh b/scripts/test-postgres-compose.sh index 5a099af0b..b5124f8e7 100755 --- a/scripts/test-postgres-compose.sh +++ b/scripts/test-postgres-compose.sh @@ -57,7 +57,7 @@ PY if [[ $(realpath -e "$work") == "$temp_root"/corescope-pg-smoke-* && ! -L "$work" ]]; then if $cleanup_ok; then sudo rm -rf -- "$work/state" "$work/postgres" || result=1 - rm -f -- "$work/.env" "$work/compose.json" "$work/override.yml" "$work/packet.json" "$work/stats.json" "$work/states.json" "$work/health.json" "$work/actions.raw.log" "$work/container.raw.log" || result=1 + rm -f -- "$work/.env" "$work/compose.json" "$work/override.yml" "$work/packet.json" "$work/stats.json" "$work/states.json" "$work/health.json" "$work/proxy-health.json" "$work/actions.raw.log" "$work/container.raw.log" || result=1 fi if [ "$result" -eq 0 ]; then rm -f -- "$work/failure.log" @@ -83,7 +83,10 @@ for key in POSTGRES_ADMIN_PASSWORD CORESCOPE_OWNER_PASSWORD CORESCOPE_READER_PAS done unset password export CORESCOPE_IMAGE="$image" DATA_DIR="$work/state" POSTGRES_DATA_DIR="$work/postgres" -export DISABLE_MOSQUITTO=false DISABLE_CADDY=true +export DISABLE_MOSQUITTO=false +# Exercise the example's default Caddy/port-80 path, including when the runner +# environment has its own Caddy preference. The owned .env contains only secrets. +unset DISABLE_CADDY mkdir "$work/state" python3 - "$repo" "$work" <<'PY' import json,re,sys @@ -128,6 +131,7 @@ assert config["name"]==work.name.lower() assert set(config["services"])=={"postgres","bootstrap","corescope"} assert set(config["networks"])=={"default"} assert config["networks"]["default"]["internal"] is True +assert config["services"]["corescope"]["environment"]["DISABLE_CADDY"] == "false", "smoke must use the default Caddy listener" allowed={(work/"state").resolve(),(repo/"docker/postgres-init.sh").resolve()} for service in config["services"].values(): assert not service.get("ports"), "smoke must publish no ports" @@ -178,14 +182,16 @@ phase=mqtt-ingestion timeout 10s "${dc[@]}" exec -T corescope mosquitto_pub -h 127.0.0.1 -q 1 -r -t meshcore/SJC/ci-smoke/packets -s < "$work/packet.json" >> "$work/actions.raw.log" 2>&1 for attempt in $(seq 1 30); do if timeout 5s "${dc[@]}" exec -T corescope wget -qO- http://127.0.0.1:3000/api/healthz > "$work/health.json" 2>> "$work/actions.raw.log" && - timeout 5s "${dc[@]}" exec -T corescope wget -qO- http://127.0.0.1:3000/api/stats > "$work/stats.json" 2>> "$work/actions.raw.log" && + timeout 5s "${dc[@]}" exec -T corescope wget -qO- http://127.0.0.1:80/api/healthz > "$work/proxy-health.json" 2>> "$work/actions.raw.log" && + timeout 5s "${dc[@]}" exec -T corescope wget -qO- http://127.0.0.1:80/api/stats > "$work/stats.json" 2>> "$work/actions.raw.log" && python3 - "$work" "$GITHUB_SHA" <<'PY' import json,sys from pathlib import Path root=Path(sys.argv[1]) health=json.loads((root/"health.json").read_text()) +proxy_health=json.loads((root/"proxy-health.json").read_text()) stats=json.loads((root/"stats.json").read_text()) -sys.exit(0 if health.get("ready") is True and stats.get("totalTransmissions")==1 and stats.get("totalObservations")==1 and stats.get("commit")==sys.argv[2][:7] else 1) +sys.exit(0 if health.get("ready") is True and proxy_health.get("ready") is True and stats.get("totalTransmissions")==1 and stats.get("totalObservations")==1 and stats.get("commit")==sys.argv[2][:7] else 1) PY then break; fi [ "$attempt" -lt 30 ] || { echo 'Published fixture did not become visible through the packaged API.' >&2; exit 1; } @@ -202,4 +208,4 @@ SELECT (SELECT count(*)=1 FROM transmissions) AND NOT has_schema_privilege('corescope_writer','public','CREATE'); SQL ) == t ]] -echo 'Packaged Compose bootstrap, restricted PostgreSQL runtime, MQTT ingestion, API visibility and pg_dump availability passed.' +echo 'Packaged Compose bootstrap, restricted PostgreSQL runtime, MQTT ingestion, direct and Caddy HTTP readiness, API visibility and pg_dump availability passed.' diff --git a/tests/unit/test-postgres-ops.js b/tests/unit/test-postgres-ops.js index 87adf7e84..b1579dc95 100644 --- a/tests/unit/test-postgres-ops.js +++ b/tests/unit/test-postgres-ops.js @@ -11,9 +11,29 @@ const source = fs.readFileSync(path.join(root, 'manage.sh'), 'utf8').replace(/\r const bash = require('../../scripts/bash-path')(); const functions = ['pg_exec', 'pg_empty', 'pg_dump_file', 'pg_restore_file', 'cmd_backup', 'cmd_restore', 'prepare_staging_db', 'prepare_staging_config', 'write_env_managed_values', 'is_true', 'cmd_start'] .map(name => source.match(new RegExp(`^${name}\\(\\)\\s*\\{[\\s\\S]*?^}`, 'm'))?.[0] || '').join('\n'); -const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'corescope-pg-ops-')); const quote = value => "'" + value.replace(/'/g, "'\\''") + "'"; const posix = value => value.replace(/\\/g, '/'); +// The example's published HTTP port must reach a listener in the default +// supervisor selection. A healthy private port 3000 alone does not prove this. +const example = fs.readFileSync(path.join(root, 'docker-compose.example.yml'), 'utf8'); +const disableCaddy = example.match(/DISABLE_CADDY=\$\{DISABLE_CADDY:-(true|false)\}/)?.[1]; +assert(disableCaddy, 'example must declare its Caddy default'); +const supervisor = fs.readFileSync(path.join(root, 'docker', disableCaddy === 'true' ? 'supervisord-go-no-caddy.conf' : 'supervisord-go.conf'), 'utf8'); +const listeners = new Set([Number(supervisor.match(/corescope-server[^\n]* -port (\d+)/)?.[1])]); +if (supervisor.includes('[program:caddy]')) { + const caddy = fs.readFileSync(path.join(root, 'docker/Caddyfile'), 'utf8'); + listeners.add(Number(caddy.match(/^:(\d+)\s*\{/m)?.[1])); + assert(listeners.has(Number(caddy.match(/^\s*reverse_proxy localhost:(\d+)/m)?.[1])), 'Caddy upstream does not reach the application'); +} +const publishedPort = Number(example.match(/HTTP_PORT:-\d+\}:(\d+)/)?.[1]); +assert(listeners.has(publishedPort), 'default example publishes HTTP to a port with no running listener'); +const smoke = fs.readFileSync(path.join(root, 'scripts/test-postgres-compose.sh'), 'utf8'); +assert(!/\bDISABLE_CADDY=true\b/.test(smoke), 'packaged smoke bypasses the example default HTTP listener'); +for (const endpoint of ['http://127.0.0.1:3000/api/healthz', 'http://127.0.0.1:80/api/healthz', 'http://127.0.0.1:80/api/stats']) { + assert(smoke.includes(endpoint), `packaged smoke does not exercise ${endpoint}`); +} +assert(smoke.includes('proxy_health.get("ready") is True'), 'packaged smoke does not validate readiness through Caddy'); +const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'corescope-pg-ops-')); function run(command, occupied = '0', failRestore = '0', prodReady = '1') { const setup = ` set -e From 0a24721a44100a54167f0bfb67d98123d3e36865 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 05:59:32 -0400 Subject: [PATCH 13/67] fix: retain selective PostgreSQL coverage query plans --- cmd/server/rx_coverage.go | 6 +- cmd/server/rx_coverage_postgres_plan_test.go | 117 +++++++++++++++++++ cmd/server/rx_dashboard.go | 5 +- 3 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 cmd/server/rx_coverage_postgres_plan_test.go diff --git a/cmd/server/rx_coverage.go b/cmd/server/rx_coverage.go index 667a385e3..6d768fcba 100644 --- a/cmd/server/rx_coverage.go +++ b/cmd/server/rx_coverage.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/gorilla/mux" + "github.com/jackc/pgx/v5" ) // coverageRow is one raw reception read from client_receptions. @@ -282,11 +283,14 @@ func (s *Server) queryCoverageRows(pubkey string, b bbox) ([]coverageRow, error) args = append(args, c) } args = append(args, b.MinLat, b.MaxLat, b.MinLon, b.MaxLon) + // A cached generic plan can treat a wide bbox as selective and scan the + // entire geo index before filtering the heard key. Plan this SELECT with + // its actual bounds; Exec still uses server-bound extended-protocol values. rows, err := s.db.conn.Query(` SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions WHERE heard_key IN (`+sqlPlaceholders(len(cands))+`) - AND lat BETWEEN `+fmt.Sprintf("$%d AND $%d AND lon BETWEEN $%d AND $%d", len(cands)+1, len(cands)+2, len(cands)+3, len(cands)+4), args...) + AND lat BETWEEN `+fmt.Sprintf("$%d AND $%d AND lon BETWEEN $%d AND $%d", len(cands)+1, len(cands)+2, len(cands)+3, len(cands)+4), append([]any{pgx.QueryExecModeExec}, args...)...) if err != nil { return nil, err } diff --git a/cmd/server/rx_coverage_postgres_plan_test.go b/cmd/server/rx_coverage_postgres_plan_test.go new file mode 100644 index 000000000..688c81fb7 --- /dev/null +++ b/cmd/server/rx_coverage_postgres_plan_test.go @@ -0,0 +1,117 @@ +package main + +import ( + "crypto/ed25519" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "reflect" + "sort" + "testing" + "time" + + "github.com/meshcore-analyzer/dbschema" +) + +// A generic plan estimates an unknown bbox as tiny. With a wide viewport it +// instead visits all 128K receptions through the geo index, rejecting 127936 +// rows after fetching them. The first five custom plans hide this regression. +func TestCoverageQueriesKeepValueSpecificPlans(t *testing.T) { + dsn := postgresTestDSN(t) + writer, err := openFixtureSQL(dsn) + if err != nil { + t.Fatal(err) + } + defer writer.Close() + if err := dbschema.Apply(writer, nil); err != nil { + t.Fatal(err) + } + keys := make([]string, 2000) + for i := range keys { + seed := sha256.Sum256([]byte(fmt.Sprintf("corescope-bench/%d/node/%d", 20261008, i))) + key := ed25519.NewKeyFromSeed(seed[:]) + keys[i] = hex.EncodeToString(key.Public().(ed25519.PublicKey)) + } + copyTestRows(t, writer, "nodes", []string{"public_key", "name"}, len(keys), func(i int) []any { + return []any{keys[i], fmt.Sprintf("Synthetic-%04d", i)} + }) + stamp := time.Now().UTC().Format("2006-01-02T15:04:05") + copyTestRows(t, writer, "client_receptions", []string{"rx_pubkey", "heard_key", "heard_keylen", "rssi", "snr", "lat", "lon", "rx_at", "ingested_at", "src"}, 128000, func(i int) []any { + size := []int{32, 8, 3, 2}[i%4] + var snr, rssi any = 6.5, -95 + if i == 0 { + snr, rssi = nil, nil + } + lat, lon := 20+float64(i%2000)/10000, 30+float64(i%2000)/10000 + if i == 2000 { + lat, lon = 20.0123456789, 30.07654321 + } + return []any{keys[(i+100)%len(keys)], keys[i%len(keys)][:size*2], size, rssi, snr, lat, lon, fmt.Sprintf("%s.%09dZ", stamp, i), stamp + "Z", "synthetic"} + }) + if _, err := writer.Exec("ANALYZE nodes; ANALYZE client_receptions"); err != nil { + t.Fatal(err) + } + db, err := openFixtureReader(t, dsn) + if err != nil { + t.Fatal(err) + } + defer db.Close() + // Keep this test on one session so pg_prepared_statements observes every + // execution. Production pools retain the same per-connection plan behavior. + db.conn.SetMaxOpenConns(1) + srv := &Server{db: db, cfg: &Config{ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}}} + for _, tc := range []struct { + name, pattern string + query func(bbox) ([]coverageRow, error) + }{ + {"node", "%WHERE heard_key IN%", func(b bbox) ([]coverageRow, error) { return srv.queryCoverageRows(keys[0], b) }}, + {"dashboard", "%WHERE lat BETWEEN%", func(b bbox) ([]coverageRow, error) { return srv.queryCoverageFiltered(keys[0], "", 0, b) }}, + } { + t.Run(tc.name, func(t *testing.T) { + var first []coverageRow + for n := 0; n < 10; n++ { + rows, err := tc.query(bbox{19, 29, 22, 32}) + if err != nil || len(rows) != 64 { + t.Fatalf("pass %d: rows=%d, err=%v", n, len(rows), err) + } + sort.Slice(rows, func(i, j int) bool { return rows[i].RxAt < rows[j].RxAt }) + if n == 0 { + first = rows + } else if !reflect.DeepEqual(rows, first) { + t.Fatal("repeated coverage query changed its rows") + } + } + if first[0].SNR != nil || first[0].RSSI != nil || first[1].SNR == nil || *first[1].SNR != 6.5 || first[1].RSSI == nil || *first[1].RSSI != -95 || first[1].Lat != 20.0123456789 || first[1].Lon != 30.07654321 { + t.Fatalf("coverage null/numeric values changed: %+v", first[:2]) + } + if rows, err := tc.query(bbox{20, 30, 20, 30}); err != nil || len(rows) != 63 { + t.Fatalf("inclusive narrow bbox: rows=%d, err=%v", len(rows), err) + } + if rows, err := tc.query(bbox{0, 0, 1, 1}); err != nil || len(rows) != 0 { + t.Fatalf("empty bbox: rows=%d, err=%v", len(rows), err) + } + var generic int64 + if err := db.conn.QueryRow(`SELECT COALESCE(SUM(generic_plans),0)::bigint FROM pg_prepared_statements + WHERE statement LIKE $1 AND statement LIKE $2 AND statement LIKE $3`, + "%SELECT lat, lon, snr, rssi, heard_key, rx_at%", "%FROM client_receptions%", tc.pattern).Scan(&generic); err != nil { + t.Fatal(err) + } + if generic != 0 { + t.Fatalf("bbox-dependent coverage SELECT reused %d generic plans; broad views must retain value-specific planning", generic) + } + }) + } + rr := serveRxCoverage(srv, "/api/nodes/"+keys[0]+"/rx-coverage?bbox=19,29,22,32&z=14") + var fc CoverageFeatureCollection + if rr.Code != 200 || json.Unmarshal(rr.Body.Bytes(), &fc) != nil { + t.Fatalf("coverage API status=%d", rr.Code) + } + count := 0 + for _, feature := range fc.Features { + count += feature.Properties.Count + } + if count != 64 || fc.MobileReceptions != 64 || fc.MobileClients != 1 { + t.Fatalf("coverage API totals changed: cells=%d node=%d clients=%d", count, fc.MobileReceptions, fc.MobileClients) + } +} diff --git a/cmd/server/rx_dashboard.go b/cmd/server/rx_dashboard.go index 8148fe573..9a1120e68 100644 --- a/cmd/server/rx_dashboard.go +++ b/cmd/server/rx_dashboard.go @@ -11,6 +11,8 @@ import ( "strconv" "strings" "time" + + "github.com/jackc/pgx/v5" ) // scanCoverageRows reads (lat,lon,snr,rssi,heard_key,rx_at) rows into coverageRow values. @@ -179,7 +181,8 @@ func (s *Server) queryCoverageFiltered(node, rx string, days int, b bbox) ([]cov where = append(where, fmt.Sprintf("rx_at >= $%d", len(args)+1)) args = append(args, since) } - rows, err := s.db.conn.Query("SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions WHERE "+strings.Join(where, " AND "), args...) + // As in queryCoverageRows, choose the plan using the actual bbox and filters. + rows, err := s.db.conn.Query("SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions WHERE "+strings.Join(where, " AND "), append([]any{pgx.QueryExecModeExec}, args...)...) if err != nil { return nil, err } From ce023e0d623116fe66987761d06016590f98fb11 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 05:59:32 -0400 Subject: [PATCH 14/67] test: read cgroup memory peaks through their reset descriptor --- scripts/postgres-benchmark/README.md | 14 ++-- scripts/postgres-benchmark/run.py | 31 ++++++-- scripts/postgres-benchmark/test_harness.py | 90 +++++++++++++++++++++- 3 files changed, 124 insertions(+), 11 deletions(-) diff --git a/scripts/postgres-benchmark/README.md b/scripts/postgres-benchmark/README.md index 834ccb5d7..1220af267 100644 --- a/scripts/postgres-benchmark/README.md +++ b/scripts/postgres-benchmark/README.md @@ -7,10 +7,10 @@ drivers. Fixture construction happens outside measured ingestion. ## Status -The generator, protocol vectors, bounded fixture/handler test, and harness unit -checks have been exercised locally. The Linux controller, full S/B corpora, -paired runtime profiles, and resource enforcement still need an isolated Linux -pilot. **No empirical database comparison is included in this change.** +The generator, protocol vectors, fixture/handler controls and Linux S/B runs +have been exercised. The [performance report](../../docs/postgresql-performance.md) +identifies measured revisions, results and limitations. A completed run qualifies +only its exact candidate and workload; source changes require remeasurement. ## Prerequisites @@ -116,7 +116,7 @@ at 205.92–206.76 seconds, after retention was scheduled at 195 seconds; that p but the missing events keep that run failed. Its artifacts remain separate. The supplemental 50-event/s rate gives the same queue 20.48 seconds of offered work, versus 10.24 seconds at 100/s. This gives the observed maintenance burst -headroom; the Linux run must still prove zero drops, including during warmup. +headroom; every comparison must still prove zero drops, including during warmup. The manifest records the actual offered rate. Do not pool the two experiments. Both engines use a 2,048 MiB packet-store allowance and a 3,072 MiB server Go @@ -221,6 +221,10 @@ effects, rather than trusting the void callback return. - Whole application/database cgroup CPU, memory and I/O; per-process RSS/PSS, database/WAL storage, and remaining filesystem headroom. RSS is not used as a substitute for shared-memory-aware PSS/cgroup memory. + The optional kernel `memory.peak` counter is reset and read through one held + descriptor per backend leg, including preparation and migration. It is not a + measured-window-only peak; use windowed `memory.current` samples for that + comparison. Kernels without reset support leave this counter unmeasured. - Actual compiled retention and advert-preservation query plans in a separate diagnostic phase. PostgreSQL uses `EXPLAIN (ANALYZE, BUFFERS, SETTINGS, FORMAT JSON)`; SQLite uses `EXPLAIN QUERY PLAN`. Plan collection is outside headlines. diff --git a/scripts/postgres-benchmark/run.py b/scripts/postgres-benchmark/run.py index 1df110d8c..0d59077df 100644 --- a/scripts/postgres-benchmark/run.py +++ b/scripts/postgres-benchmark/run.py @@ -339,6 +339,7 @@ def archive(repo, revision, destination): class Budget: def __init__(self, parent, required): self.path = None + self._memory_peak_file = None self.required = required self.info = {"enforced": False, "cpu_limit": None, "memory_limit_bytes": None} if not parent: @@ -405,6 +406,8 @@ def verify(self, pid): raise RuntimeError("application/database process failed to enter the benchmark cgroup") def reset(self): + self.close_memory_peak() + self.info["memory_peak_reset"] = False if not self.path: return if (self.path / "cgroup.procs").read_text().strip(): @@ -424,12 +427,27 @@ def reset(self): raise RuntimeError("more than 64 MiB of inactive prior-backend cgroup memory remains") self.info["inactive_charge_reset_bytes"] = remaining try: - (self.path / "memory.peak").write_text("0") + # Linux resets memory.peak only for subsequent reads through this + # same open file description, not later opens of the counter. + self._memory_peak_file = (self.path / "memory.peak").open("r+b", buffering=0) + self._memory_peak_file.write(b"0") self.info["memory_peak_reset"] = True except OSError: - self.info["memory_peak_reset"] = False + self.close_memory_peak() + + def memory_peak(self): + if self._memory_peak_file is None: + return None + self._memory_peak_file.seek(0) + return int(self._memory_peak_file.read()) + + def close_memory_peak(self): + if self._memory_peak_file is not None: + self._memory_peak_file.close() + self._memory_peak_file = None def close(self): + self.close_memory_peak() if self.path: if (self.path / "cgroup.procs").read_text().strip(): raise RuntimeError("owned benchmark processes remain; refusing cgroup removal") @@ -599,8 +617,9 @@ def sample_rows(self): if self.budget.path: c = self.budget.path row["cgroup_memory_bytes"] = (c / "memory.current").read_text().strip() - if self.budget.info.get("memory_peak_reset"): - row["cgroup_memory_peak_bytes"]=(c/"memory.peak").read_text().strip() + peak = self.budget.memory_peak() + if peak is not None: + row["cgroup_memory_peak_bytes"] = peak cpu = dict(line.split() for line in (c / "cpu.stat").read_text().splitlines()) row["cgroup_cpu_usec"] = cpu.get("usage_usec", "not measured") row["io_stat"] = (c / "io.stat").read_text().strip() @@ -648,12 +667,14 @@ def __exit__(self, exc_type, exc_value, traceback): raise self.error if self.budget.path: c=self.budget.path - write_json(self.path.parent/"resource-final.json",dict(memory_peak_bytes=int((c/"memory.peak").read_text()) if self.budget.info.get("memory_peak_reset") else None,cpu_stat=(c/"cpu.stat").read_text(),io_stat=(c/"io.stat").read_text(),memory_events=(c/"memory.events").read_text(),budget=self.budget.info)) + write_json(self.path.parent/"resource-final.json",dict(memory_peak_bytes=self.budget.memory_peak(),cpu_stat=(c/"cpu.stat").read_text(),io_stat=(c/"io.stat").read_text(),memory_events=(c/"memory.events").read_text(),budget=self.budget.info)) except BaseException as error: detail = record_resource_failure(self.path.parent/"resource-error.json", error, phase=self.phase, workload_error_type=exc_type.__name__ if exc_type else None) if exc_value is None: raise RuntimeError("resource sampler failed: " + json.dumps(detail, sort_keys=True)) from error + finally: + self.budget.close_memory_peak() return False diff --git a/scripts/postgres-benchmark/test_harness.py b/scripts/postgres-benchmark/test_harness.py index bd711e6f0..dd05292d1 100644 --- a/scripts/postgres-benchmark/test_harness.py +++ b/scripts/postgres-benchmark/test_harness.py @@ -1,6 +1,7 @@ import importlib.util import csv import errno +import io import json import pathlib import re @@ -15,6 +16,25 @@ spec.loader.exec_module(bench) +class MemoryPeakFile(io.BytesIO): + """Model cgroup v2's reset scope: one open file description.""" + def __init__(self, state, mode): + super().__init__() + self.state, self.binary, self.peak = state, "b" in mode, state.lifetime + state.handles.append(self) + + def write(self, value): + if self.state.reject_reset: + raise OSError(errno.EINVAL, "reset unsupported") + self.peak = self.state.current + return len(value) + + def read(self, size=-1): + value = b"" if self.tell() else str(self.peak).encode() + b"\n" + self.seek(len(value), io.SEEK_CUR) + return value if self.binary else value.decode() + + class HarnessTests(unittest.TestCase): def test_default_common_rate_and_emitted_worker_configs(self): parse = bench.argparse.ArgumentParser.parse_args @@ -93,7 +113,7 @@ def resource_fixture(self, root, missing=None): "io.stat": "8:0 rbytes=1024 wbytes=2048\n", "cgroup.procs": ""}.items(): if name != missing: (group / name).write_text(value) - budget = bench.Budget.__new__(bench.Budget) + budget = bench.Budget(None, False) budget.path = group budget.info = {"enforced": True, "memory_peak_reset": False} budget.close = mock.Mock() @@ -101,6 +121,74 @@ def resource_fixture(self, root, missing=None): run.mkdir(parents=True) return budget, run + def memory_peak_counter(self, budget, reject_reset=False): + (budget.path / "memory.reclaim").touch() + state = types.SimpleNamespace(current=0, lifetime=9000, handles=[], reject_reset=reject_reset) + real_open = pathlib.Path.open + def open_file(path, mode="r", *args, **kwargs): + if path == budget.path / "memory.peak": + return MemoryPeakFile(state, mode) + return real_open(path, mode, *args, **kwargs) + return state, mock.patch.object(pathlib.Path, "open", open_file) + + def test_memory_peak_is_reset_per_leg_for_samples_and_final_report(self): + with tempfile.TemporaryDirectory() as directory: + budget, run = self.resource_fixture(pathlib.Path(directory)) + state, counter = self.memory_peak_counter(budget) + def sample_peak(value): + state.current = value + state.lifetime = max(state.lifetime, value) + for handle in state.handles: + if not handle.closed: + handle.peak = max(handle.peak, value) + resource = bench.Resources(run / "resources.csv", budget, []) + with mock.patch.object(resource.done, "wait", side_effect=lambda _: resource.done.set()): + resource.sample() + self.assertIsNone(resource.error) + with open(run / "resources.csv", newline="") as stream: + row = next(csv.DictReader(stream)) + # Use the real final-write path after the sampler has stopped. + with mock.patch.object(resource.thread, "join"): + resource.__exit__(None, None, None) + final = json.loads((run / "resource-final.json").read_text()) + self.assertTrue(final["budget"]["memory_peak_reset"]) + self.assertEqual(int(row["cgroup_memory_peak_bytes"]), value) + self.assertEqual(final["memory_peak_bytes"], value) + self.assertTrue(all(handle.closed for handle in state.handles)) + with counter: + budget.reset() + sample_peak(3000) + state.current = 0 + budget.reset() + sample_peak(1000) + + def test_memory_peak_descriptor_closes_on_failure_and_unsupported_reset(self): + for unsupported in (False, True): + with self.subTest(unsupported=unsupported), tempfile.TemporaryDirectory() as directory: + budget, run = self.resource_fixture(pathlib.Path(directory)) + state, counter = self.memory_peak_counter(budget, reject_reset=unsupported) + with counter: + budget.reset() + self.assertEqual(budget.info["memory_peak_reset"], not unsupported) + with self.assertRaisesRegex(ValueError, "workload failure"): + with bench.Resources(run / "resources.csv", budget, []): + raise ValueError("workload failure") + self.assertTrue(all(handle.closed for handle in state.handles)) + final = json.loads((run / "resource-final.json").read_text()) + self.assertEqual(final["memory_peak_bytes"], None if unsupported else 0) + + def test_memory_peak_descriptor_closes_if_no_resource_context_is_entered(self): + with tempfile.TemporaryDirectory() as directory: + budget, _ = self.resource_fixture(pathlib.Path(directory)) + state, counter = self.memory_peak_counter(budget) + with counter, mock.patch.object(pathlib.Path, "rmdir"): + budget.reset() + first = state.handles[0] + budget.reset() + self.assertTrue(first.closed) + bench.Budget.close(budget) + self.assertTrue(all(handle.closed for handle in state.handles)) + def test_sampler_failure_identifies_counter_without_private_path(self): with tempfile.TemporaryDirectory() as directory: root = pathlib.Path(directory) From fde34bfac56f375dd7d5d05bf2ee03dd1660943f Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 07:52:21 -0400 Subject: [PATCH 15/67] docs: publish verified PostgreSQL upgrade and paired performance results --- .../postgresql-ce023e0-details.json | 16883 ++++++++++++++++ .../performance/postgresql-ce023e0-details.md | 273 + docs/postgresql-migration-plan.md | 2 +- docs/postgresql-performance.md | 158 +- docs/postgresql-upgrade.md | 8 + 5 files changed, 17274 insertions(+), 50 deletions(-) create mode 100644 docs/performance/postgresql-ce023e0-details.json create mode 100644 docs/performance/postgresql-ce023e0-details.md diff --git a/docs/performance/postgresql-ce023e0-details.json b/docs/performance/postgresql-ce023e0-details.json new file mode 100644 index 000000000..e8139dc48 --- /dev/null +++ b/docs/performance/postgresql-ce023e0-details.json @@ -0,0 +1,16883 @@ +{ + "status": "eligible B public evidence validated", + "analysis_reference": "c315d7d1926d1ae470d63af34bd68e48e6d63613", + "source_bundle": "corescope-bench-37914893623-1/public", + "provenance": { + "baseline_sha": "9dbc287579a237ffa744dd0c91fa7227d09763ac", + "candidate_sha": "ce023e0d623116fe66987761d06016590f98fb11", + "harness_sha256": "207f4317b3dcf0395ca8f76f3a8d4398e2520d2aff429a3381b502becf856f8a", + "binary_sha256": { + "migrate": "c55d7c8622a1fc2fb29bbb82d71e67c5dbd284328c37d6a453a111b3d73c0082", + "postgres-ingestor.test": "86eb1ae7b138e5f287f744de9dccdf0728748c892649ea89680391b3710f71df", + "postgres-server": "18df625586a957adcf27a51cf694123069605b957cb30ba6a36dcca64738cb17", + "postgres-server.test": "11ebf70e5aabf0625e2166dc8ce36bfa5551d4ae5d97db95fa9571fcbc870b9d", + "sqlite-ingestor.test": "a9fa5d702931123634315534485c3ce1df17c6b21fd9d0077d8fa8d1d13ec308", + "sqlite-server": "799fbc8bbf0ceffeafb2b718485f46047b88b6c334326255affef7d91dfcc867", + "sqlite-server.test": "18137f5cb34ad710a9428971b821ba9caae6cb64946a4afc7928662f53c911e7" + }, + "canonical_file_sha256": "a9cfe8d97782ff748e90f155e0d6c70cb6626d84f087fddc986803af6d187304", + "firmware_reference": "a366955cb2f67b8e6842d4f00d2b6a554dddd88a", + "host": { + "architecture": "x86_64", + "cpus": 4, + "kernel": "6.17.0-1022-azure", + "ram_bytes": 16765378560, + "system": "Linux" + }, + "go_version": "go version go1.27.2 linux/amd64", + "postgres_version": "postgres (PostgreSQL) 18.6 (Ubuntu 18.6-1.pgdg24.04+2)", + "compiler": "cc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0", + "resource_budget": { + "allowed_cpus": [ + 0, + 1, + 2, + 3 + ], + "cpu_limit": 3.0, + "enforced": true, + "inactive_charge_reset_bytes": 5099520, + "memory_limit_bytes": 6442450944, + "memory_peak_reset": true, + "resource_counters": [ + "cgroup.procs", + "memory.current", + "memory.events", + "cpu.stat", + "io.stat" + ] + }, + "workload": { + "http_concurrency": 64, + "http_rate": 20, + "ingest_rate": 50, + "ingestor_memory_mib": 512, + "measured_seconds": 180, + "packet_store_mib": 2048, + "queue_limit": 1024, + "retention_hours": 168, + "server_memory_mib": 3072, + "sql_reader_pool": 4, + "sql_writer_pool": 1, + "warmup_seconds": 60 + }, + "cache_policy": { + "mixed": "warmup then application-warm measurement", + "sql": "channel application cache explicitly cleared; OS cache warm", + "startup": "new application process after full validation; OS cache warm" + }, + "shape": { + "days": 8, + "nodes": 2000, + "observations": 2048000, + "observers": 128, + "transmissions": 128000 + }, + "order": [ + [ + 0, + "sqlite" + ], + [ + 0, + "postgres" + ], + [ + 1, + "postgres" + ], + [ + 1, + "sqlite" + ], + [ + 2, + "sqlite" + ], + [ + 2, + "postgres" + ], + [ + 3, + "postgres" + ], + [ + 3, + "sqlite" + ], + [ + 4, + "sqlite" + ], + [ + 4, + "postgres" + ] + ], + "postgres_production_hash_before": "74f4bdc613e1311f65dbbb27d216c186d499d82c05fbb7c9337e239a07b5ae5c", + "postgres_production_hash_after": "74f4bdc613e1311f65dbbb27d216c186d499d82c05fbb7c9337e239a07b5ae5c", + "sqlite_production_hash_before": "7b786814eec6adc6c322a331727d9a4c2bd51a63456b6bb233f2a3327b39381c", + "sqlite_production_hash_after": "7b786814eec6adc6c322a331727d9a4c2bd51a63456b6bb233f2a3327b39381c" + }, + "input_sha256": { + "corpus\\validation.json": "0abc5c5043aa0838cb23f502cc8585351bc739c41ba45492c27efe24f112988d", + "manifest.json": "450e2d6615a024864a113d60f80276eee2f1288c2b771e156ed1ebdef9128287", + "pair-00\\corpus-offset.json": "4e6d379b0ccc41e21fac35c070142c5b010144dbdf089c60853c49056077a830", + "pair-00\\events.json": "1776fa550db2b988f260f001e67869eb578c3a5ecf3568b03556abf1b65595ea", + "pair-00\\postgres\\client-validation.json": "4f21868c5782ae50fed6d276fdab52cfc8e473d525385852e973e85ddc2989cf", + "pair-00\\postgres\\database-settings.json": "999ed1c72e72ba5c038102ccb63b21014bb009c78c16cdf7b4b036ab34a09664", + "pair-00\\postgres\\handler.json": "501bb9fceaab112b9c2d651b5ff878164d20f2f7e9ec0be5cb8ec4ce1c6140cc", + "pair-00\\postgres\\ingest-validation.json": "5b8505ff19af48ba82e349d64ac7d0895ff3d4d3081dd1fd7785caec631af6ee", + "pair-00\\postgres\\ingest.jsonl": "47ca0c9a6904ccc8461444619d81f5620606e98c231a96497e137c1e6412f8bc", + "pair-00\\postgres\\migration.json": "0bdd011cdc9c316775638ab24253272a494578aa7f55ee933b7d27e902e17105", + "pair-00\\postgres\\requests.jsonl": "c7395f4f099bfb37f2a93756eeed52baf61cbccb51203403e1229f7487962d17", + "pair-00\\postgres\\resource-final.json": "82fbfb035d3a36bd1ceaa27f19b4328169a323d07df0666d7e01286d2dc762db", + "pair-00\\postgres\\resources.csv": "5c1ff08394ba4372df6447c1159933827c294149eaf3689172267b1f8fe6f3e7", + "pair-00\\postgres\\retention-empty.json": "c5fe536c768842e23680d26a328166ce9ff69a4c2e1836b8bb8e504676829cf4", + "pair-00\\postgres\\retention.json": "9dc9a4fe31c4a1cf6ddf172312387061f8afeafe4887ee115b7abb5376602754", + "pair-00\\postgres\\sql-queries.jsonl": "2e251f7913842ba1800f1c052d74e57b79fa29d484c16c07598a36b0366dd6e9", + "pair-00\\postgres\\startup-hot1.json": "d5ed69944702fe18933ff37b0236fccf505baca81b22911f4f365d8eb3178a40", + "pair-00\\postgres\\startup.json": "aefe0d6883417e9ea8e49ad1ab05f74ccfb1d5850b2a8fe05906f144ea8f5943", + "pair-00\\postgres\\validation-after.json": "d1274e0e8414cb04fb061b7894419ed113e42222fe5c1f7e46a5f37b4fd492c9", + "pair-00\\postgres\\validation-before.json": "b0b10f8fb4a6ef3c4e84de55bf583bfa008af44f2f75c9f7e609e627ee604b98", + "pair-00\\postgres\\visibility.json": "6e017eedc097af1ccabd57ec6aa4f3134f9a76e65f07d391f7847cbf4596d183", + "pair-00\\postgres\\websocket.jsonl": "2adac189f0b81a841bfb92b2732838186b2dbb1ef96df45b016bf1a7ac38155c", + "pair-00\\sqlite\\client-validation.json": "01d0b74e2a3ba8f07f97857ce162c625d5b30b207af05f833dd864206e56fb22", + "pair-00\\sqlite\\database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-00\\sqlite\\handler.json": "3bbb508a1fb62a648d04bf6d8bc106c1e4a9043503491495cf4c815a92adf618", + "pair-00\\sqlite\\ingest-validation.json": "4e52b809e1f1e3a90c426604cdfbcb5a4e6aabf0b284a919b68539b7c4cd465f", + "pair-00\\sqlite\\ingest.jsonl": "98d8dd336d6f6d69bce71a8575778416e536c9cbcf9cc45f3ac49e12bea74f49", + "pair-00\\sqlite\\requests.jsonl": "3729415f906975aebaac1b7a83cc2414ffba78c464b7dee38732cd45d2faf222", + "pair-00\\sqlite\\resource-final.json": "ce019c5ef4b6de72b9234e774242e75ac84fdaacc426288cab43acf727e48877", + "pair-00\\sqlite\\resources.csv": "26fe5eb8a4e84a7973cd643a0bd023efd5fabc553ec93b4a99d20f8ba8dcd2f2", + "pair-00\\sqlite\\retention-empty.json": "4cf6b91a4e3f2c6910dd3487431fe9db6dbce2bc3887cf95f77917267aad02c0", + "pair-00\\sqlite\\retention.json": "2bad6e7e7b148fedf2ad8f3f4be492efb6e253e7503dba2805fecb936ed5f7a2", + "pair-00\\sqlite\\sql-queries.jsonl": "5f088471df5e053e4744e2f379df2ddfc87fcc197b7f32ab66032ecfb671e9ab", + "pair-00\\sqlite\\startup-hot1.json": "57cab422f21ff14c809775b3db99f7759899d7c7a28f9002a43ba564ee2724be", + "pair-00\\sqlite\\startup.json": "da6b1fbe052114cedc5d69282dba95e5da6c9bb8c65f050520661ef4ce4e74b3", + "pair-00\\sqlite\\validation-after.json": "d1274e0e8414cb04fb061b7894419ed113e42222fe5c1f7e46a5f37b4fd492c9", + "pair-00\\sqlite\\validation-before.json": "b0b10f8fb4a6ef3c4e84de55bf583bfa008af44f2f75c9f7e609e627ee604b98", + "pair-00\\sqlite\\visibility.json": "b2d3424d60c30ccd6453565d1bc97d91d9b4e3c34f6bbeb0925a36cbc6be53f4", + "pair-00\\sqlite\\websocket.jsonl": "7ee02bdcb3aba3cc1f14dbb878e478dd7f60c34603b93ffb36bac292fd3a9d84", + "pair-01\\corpus-offset.json": "c6e0201b0bc92a639780e8ff72c0714558870528b36d9b4fabef8ed8f49aaeca", + "pair-01\\events.json": "aed562b2329d8be75af386e4fd372d3944aabf5bbd22841fb3e3099390cb447a", + "pair-01\\postgres\\client-validation.json": "ceebe158c841c2ba144654b72740de229fb09a4cd92922313668a36695467a64", + "pair-01\\postgres\\database-settings.json": "999ed1c72e72ba5c038102ccb63b21014bb009c78c16cdf7b4b036ab34a09664", + "pair-01\\postgres\\handler.json": "dcce886f69b460a8df0320c91a3e2c9518eecd9c9d97be4bbfab25e20f5cfdfd", + "pair-01\\postgres\\ingest-validation.json": "130ca1161307ffb37208ccdddc8fbe2341ce1f0abb612fc4a8858b431a7b50ea", + "pair-01\\postgres\\ingest.jsonl": "4bda1be778d92124c9c39971c2a0fdabffccf0acd9fda638d156f03b86b6f3b0", + "pair-01\\postgres\\migration.json": "e17434bf06c320275d9d8c21e4ee43587f40e2d0fe4ff44ca9fa95b2cea44315", + "pair-01\\postgres\\requests.jsonl": "4818b890fbaae28b9d37d410c494a406619da3fe4e70483be37e5b9cf7e34bdd", + "pair-01\\postgres\\resource-final.json": "a874a98ceecab0dc127694e0ba35d29a28f9ec1b6803a121d640cafef5c02934", + "pair-01\\postgres\\resources.csv": "3875af349caea857075284456057850fb9f95d262de9ac25b471e4e3d5a71470", + "pair-01\\postgres\\retention-empty.json": "3f9b12a4409ffb86ee95f2fb9b02aa2643104a88415828d070a167eca77c9bd5", + "pair-01\\postgres\\retention.json": "0041627d467372c7450ccecd59b2ff745a53dfc8e6257e58afb4dbd919b0da13", + "pair-01\\postgres\\sql-queries.jsonl": "388715a65a18362ff14d907928f25707c1852642b581c4d5b36a4dea2b3ed174", + "pair-01\\postgres\\startup-hot1.json": "6d44d3271ee05ae67c6c37cb04b0130f5b570789edb7ce8235a1e22b83ed9077", + "pair-01\\postgres\\startup.json": "263172f50597135e3284bc5075b2c80cb88a491394d01c457dab926b5bf0fac8", + "pair-01\\postgres\\validation-after.json": "18595b6a29a8630af687f91045e502a115ab343ae95ea5bc6b782fd402003e2f", + "pair-01\\postgres\\validation-before.json": "69bc3ffd7afdb1a7fe18c3b1a8697cdc7f8a985875d5e23b7a817a34910a73fd", + "pair-01\\postgres\\visibility.json": "489be81a615155c1c62108570956b033f65ded548936def47c28d589f2e046c6", + "pair-01\\postgres\\websocket.jsonl": "39a02187e37d5cf267aa0e36cfb6716d0a8a85faae63724551148872b40efef5", + "pair-01\\sqlite\\client-validation.json": "da337d7268c5cfb96c7632db4f8ed28c2f49ddb4a614abca68f04fcdcc075588", + "pair-01\\sqlite\\database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-01\\sqlite\\handler.json": "12008e674715f3020900c81ac819b8d5d01e189ccf8501deb9536c25f29f86ca", + "pair-01\\sqlite\\ingest-validation.json": "0e2503a8051acb7965aa43cacbdb056d04fc497ef60cdf57d1c576fa64422514", + "pair-01\\sqlite\\ingest.jsonl": "fba318afedd7b32152a5b04110c1c63865a5bbb4968a2eefcea7d4308eb44b85", + "pair-01\\sqlite\\requests.jsonl": "456686d3d8efb45962bc6347931de76974df1b64b3e9d2ac9335b92d041e6b89", + "pair-01\\sqlite\\resource-final.json": "fdea49d1d7fb6fead53066e1ac7f9d1eb5ad81d09332b391b7aa66f3e4620943", + "pair-01\\sqlite\\resources.csv": "a87af66fda01415e5626dcc7507f1acef534e6c3205a9546b1ded745b6a804d2", + "pair-01\\sqlite\\retention-empty.json": "e31d6c286e25e8e08adffbc81db46d5ab8f804b5b81ac0ab17bc3ca3e144e6e7", + "pair-01\\sqlite\\retention.json": "1b1bbd5d5bcfe252e639705be189e80615526084aac4b6de785db8f41fc2c4d3", + "pair-01\\sqlite\\sql-queries.jsonl": "63f71f61308cd6514e0f403bb492ab95268690c35b406cc42aaa2558dd2ca2ec", + "pair-01\\sqlite\\startup-hot1.json": "77e4e1c5dc628cfcebdc554b7d960cbdd38596a7ebe48319db117a84013ce615", + "pair-01\\sqlite\\startup.json": "ceda0453c31505dca24f0d3ee7d7095ee00c907776e23d8215952045ec8a8846", + "pair-01\\sqlite\\validation-after.json": "18595b6a29a8630af687f91045e502a115ab343ae95ea5bc6b782fd402003e2f", + "pair-01\\sqlite\\validation-before.json": "69bc3ffd7afdb1a7fe18c3b1a8697cdc7f8a985875d5e23b7a817a34910a73fd", + "pair-01\\sqlite\\visibility.json": "ce4b8739da93457dddcadc075a07e7492c78fae859f0dfaaa358d22f6e8025ad", + "pair-01\\sqlite\\websocket.jsonl": "07c5ed8cca9e9ee3cfa839111c2f77065accd242a4f2fe7adba5b2bfd15934ca", + "pair-02\\corpus-offset.json": "713960cb7a99533467984a71176934e09f3b3fc24e9dd32c03e58668898c8fb9", + "pair-02\\events.json": "7ffd82a6928e6b5470206d270ef2d0db4b549ab03c2973072ff8a8dfb6e4e885", + "pair-02\\postgres\\client-validation.json": "67b5c187bbfae0164da5694f44a2b6e31f05fadd1c5e1293c29d6eff6ecb8985", + "pair-02\\postgres\\database-settings.json": "999ed1c72e72ba5c038102ccb63b21014bb009c78c16cdf7b4b036ab34a09664", + "pair-02\\postgres\\handler.json": "63a948b9fd21acbc1ad4bcd5751d85627fe2220f5a39c9d7edb190857911dd09", + "pair-02\\postgres\\ingest-validation.json": "c6f4e8df23a8565ff733c09cdcb1ecaf77e20e3aa9811c5470d9e50c83d46d3b", + "pair-02\\postgres\\ingest.jsonl": "c2f2e3ec30390c7043f5f9d7b1583fc1c03e70d43ded8dc390e7f4036ba31f0d", + "pair-02\\postgres\\migration.json": "bb7b863bec4b68817f134edaefd874853b0a3153263939cea3516643e69e3e7e", + "pair-02\\postgres\\requests.jsonl": "54de521f3148e35c47e5c8fc2e3f708812f85582d201cbbd7e7cfb1e029f0d32", + "pair-02\\postgres\\resource-final.json": "5570b84bfb8764d056edbe772767529ff34a04eea7303a55feaa7d6650c0d8de", + "pair-02\\postgres\\resources.csv": "4c9cb7d053c8e6555f2e7b841e266cdb28289957db929b75a54a7489d73240ea", + "pair-02\\postgres\\retention-empty.json": "61188a4140302beae57dbdb70724349163880a01e4669d56bef63d5da04cd3b9", + "pair-02\\postgres\\retention.json": "5db1813af9139f734d1b3521670c7bd7a0ea5f9b10ed5342341f1fd4d55b37ed", + "pair-02\\postgres\\sql-queries.jsonl": "0b4b4ea4c8ecf6b86042453a5f368cb29abc4a4d896939cddf719155ffc605e7", + "pair-02\\postgres\\startup-hot1.json": "d88951b1b35ba20f65764d0db6b2933cec4034a4cce63bbde7adb09bdd06c616", + "pair-02\\postgres\\startup.json": "b391fa0e7b935995cc3055bd8d8d82790ba3476a1bace9639e831f249e57056b", + "pair-02\\postgres\\validation-after.json": "78a43982b62110118710f21c391c1bee23a52592029b2086730c488523db2dc6", + "pair-02\\postgres\\validation-before.json": "9d3be9ab788221f2af31bbec37c551df7a58a3a4b8af3d75482a952fc72e0f8b", + "pair-02\\postgres\\visibility.json": "a33adfcb17f218af3e1d1c71aaa745e8e142df193c1ecbe40da0c4fe7b8c75a3", + "pair-02\\postgres\\websocket.jsonl": "aa693c64045713a9ecda6a7c6a3f2f9856d0a79c9377120ec7d7352278522463", + "pair-02\\sqlite\\client-validation.json": "8f695bfc01fb13430272b55b5d771901a408ac22c5df8545c0d6072554506e8a", + "pair-02\\sqlite\\database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-02\\sqlite\\handler.json": "c3d1d7aa421b1dd72c3f73e25a91b8fbbf766e9c26775ff0a56069e92cdd7c39", + "pair-02\\sqlite\\ingest-validation.json": "aea4736f3b2c2d181280e4287d105fcbfd61ff19f3d80d59cf5126b4eeca9dcc", + "pair-02\\sqlite\\ingest.jsonl": "346b619dceb5fab18edba6cc4181682510c0c5afc97459395743c99c003448b2", + "pair-02\\sqlite\\requests.jsonl": "190c6fd3d83a190be3c94e2f6a37cfe829839067e06b7515e441871bc4272c62", + "pair-02\\sqlite\\resource-final.json": "b4d1fa3770cda9a60b7506fc4abfd0030907b75e09d80dfb9e80a074f18c03bf", + "pair-02\\sqlite\\resources.csv": "f4f6fd866fd3606ca1429f299abf394a52871b23aad84f0ef4b97b8e9a4f9980", + "pair-02\\sqlite\\retention-empty.json": "f779520b58d52c02815ced58f33262bd2b1c1a989804ba0bcbd1c9f511c35599", + "pair-02\\sqlite\\retention.json": "94a9171e33a0d992c54ed10d0045a2f6c3fd2c142ffa8fcc8e9fc45bdd4cfa09", + "pair-02\\sqlite\\sql-queries.jsonl": "94aec8a4eeb3d83e3d877f4e1182b2384cf6aa4be2f79d038ca86cc5f2e28a40", + "pair-02\\sqlite\\startup-hot1.json": "c3ef3aac5067fdda84643380e4e9f950dc95a239f490bb294aa74af1c309b105", + "pair-02\\sqlite\\startup.json": "8472b0bb73512e199ffb70a97d9d1ece16955de42525b9872df0a6fce40017bf", + "pair-02\\sqlite\\validation-after.json": "78a43982b62110118710f21c391c1bee23a52592029b2086730c488523db2dc6", + "pair-02\\sqlite\\validation-before.json": "9d3be9ab788221f2af31bbec37c551df7a58a3a4b8af3d75482a952fc72e0f8b", + "pair-02\\sqlite\\visibility.json": "6cfff0c60b4803145af5a42bebc7765286f2568c3716e61d95baaaac97af45c9", + "pair-02\\sqlite\\websocket.jsonl": "6411d6486441278b367e3d5672b7c63a40c97a6a244642ab3493d65996e51183", + "pair-03\\corpus-offset.json": "888c4841603856b39675e1d4dd32ed83c9dee6ff579c738668e01e8f14ad82be", + "pair-03\\events.json": "9c0ea88bbd8fe2f98a3da66cfec4e2669432f8d656053c548c5ef8c62f62503f", + "pair-03\\postgres\\client-validation.json": "368e030f46115c84f5e3529f52bfd4e216fe09727ff463be51880592a51bbe04", + "pair-03\\postgres\\database-settings.json": "999ed1c72e72ba5c038102ccb63b21014bb009c78c16cdf7b4b036ab34a09664", + "pair-03\\postgres\\handler.json": "e8e559a5ba00d2773e70a4005d1e3e85e29c80ed38e5fba1f87ea9c806e61b3a", + "pair-03\\postgres\\ingest-validation.json": "2acb50519c27cd76d8cd0ecaf9f4f896763eb94f9eb3d7d05ec8e3d4e50dab1f", + "pair-03\\postgres\\ingest.jsonl": "90c5a02dd272171a0827ad28277210811caf41cf34eb8ffc5a9b35a880e0ca05", + "pair-03\\postgres\\migration.json": "e97f3e7eef0dc31645c8475aac76ebb6f4700d80088275ebd1f713f439a5c0f8", + "pair-03\\postgres\\requests.jsonl": "6b5eb423fbdb39a30006684f7c6ece37d1553c4bbf478ef64208c6c5307a8df6", + "pair-03\\postgres\\resource-final.json": "c42a13843cd06a4c76cda9be6cc0e2a28f22789830ba79a2c0af8197f31e2154", + "pair-03\\postgres\\resources.csv": "736b271a69ba26fca7b2167a81756fd1623aceb015bf4b6a26ae5f4515ee47e4", + "pair-03\\postgres\\retention-empty.json": "e052d1f9082cd839e56e584444f007f7a88ab865d302d7a0149b7ca53fe0ee11", + "pair-03\\postgres\\retention.json": "5404316d243e5dc1ea66d04c3233222c10fff3fbe8e67d3d24ca6147ec40ce1c", + "pair-03\\postgres\\sql-queries.jsonl": "fd408a8b30165707981c8b90564f34e96a284de9b0129f48051eeaea928c7dac", + "pair-03\\postgres\\startup-hot1.json": "3bd7ed46bbe73f287aadab8d976e85fb56276418ff4c5e8f1ab150008a5331ca", + "pair-03\\postgres\\startup.json": "4ee1c80117b371d83c9e2de0ee46ce18038e303d1d7afe79ad45c36c9e23c9f8", + "pair-03\\postgres\\validation-after.json": "6308f35294d4ea20c7a97c2720bda6697c228589d673824565334637c0389201", + "pair-03\\postgres\\validation-before.json": "6c08926231682baccd5dcdfd6f040804ca5abf735763972a3cfc6700abce5fa0", + "pair-03\\postgres\\visibility.json": "6bb896334965d388446c55350511c4322ce39ab8807caefa7df4de2a2278d76f", + "pair-03\\postgres\\websocket.jsonl": "f623a3e9d94cf577796a6453c6d38b55c3a18a904dad01e5280ba26d78971da5", + "pair-03\\sqlite\\client-validation.json": "ae1eb7ae7a767bc38f6c87554f5d8d7891f251fe5cb4da6d51941ec7c127e9c1", + "pair-03\\sqlite\\database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-03\\sqlite\\handler.json": "9185c17b293290c0e806105d9480aa07aeb5c4876ece5891024af1b6ff0613b6", + "pair-03\\sqlite\\ingest-validation.json": "e3374a41394a9fa5c221f9ee63c8e075bb64bd400e663c1f7ea19b56b4ab98f7", + "pair-03\\sqlite\\ingest.jsonl": "d3bf52444ce16ecb3f7c274ced696e8233f779c9b1e860fd2e716e1920f52f26", + "pair-03\\sqlite\\requests.jsonl": "8370d9d03aa31a22290f327cb8b8330a84bc35d577c4ca61b15825878c21715e", + "pair-03\\sqlite\\resource-final.json": "15c5739d1c6956eccedee6c7115ec53304a161ee0e4c2c449065670fa6f243c6", + "pair-03\\sqlite\\resources.csv": "706a4ee33daac7dbf44d4e02c8084a0334a580f37506edb419c40a2811c0343c", + "pair-03\\sqlite\\retention-empty.json": "63c5a3193bba62e31aaed2b8048bd0e2db8fd68b480ea216ed694f7d3821c2e6", + "pair-03\\sqlite\\retention.json": "20f84d3a0a106ad24860e6794f37e4f2d4d7ec0295118ef9062b6c6c9b511407", + "pair-03\\sqlite\\sql-queries.jsonl": "0b4f70e0b0c4fe7e1b5d6c28a9c1b3778fe588e0bcd9cbacc61cca78e541b44d", + "pair-03\\sqlite\\startup-hot1.json": "b6651e65e938d3cefddb13a84e42f94e86d74b6565bfd0d993bdc7d67867b4c9", + "pair-03\\sqlite\\startup.json": "073f14784c895ed22996e393f3ad3c3fd8dc2b47b9a6b74307345622f4e12d32", + "pair-03\\sqlite\\validation-after.json": "6308f35294d4ea20c7a97c2720bda6697c228589d673824565334637c0389201", + "pair-03\\sqlite\\validation-before.json": "6c08926231682baccd5dcdfd6f040804ca5abf735763972a3cfc6700abce5fa0", + "pair-03\\sqlite\\visibility.json": "a17e93da356f31d766d32ab2602f927c4903b0e1883e38f54b49961e63ae46da", + "pair-03\\sqlite\\websocket.jsonl": "5f8c9e8173b954a82d218748e445dbfdc90f5ff7408ead9843a75a61ed761b1e", + "pair-04\\corpus-offset.json": "35a5aa0fa301f779f6b02de68dbdf84d0da682224cb231ca3d26b7b49c96a492", + "pair-04\\events.json": "960a37a1f289865678039833f3413298b6f438254799520698d6d951b4a527cd", + "pair-04\\postgres\\client-validation.json": "446b6da6130714b5bfe905b5d027af472a6d74a90e064515a6599daea107c09f", + "pair-04\\postgres\\database-settings.json": "999ed1c72e72ba5c038102ccb63b21014bb009c78c16cdf7b4b036ab34a09664", + "pair-04\\postgres\\handler.json": "aec29036e41e4ecfa06f84884929baf1c28d141fda4e1b5f5174bc603dec4008", + "pair-04\\postgres\\ingest-validation.json": "12f320c63cd74aa96750ea54b7115cf3c120a2d71ef55c00406a1df0e49247fb", + "pair-04\\postgres\\ingest.jsonl": "1b38d22ad709e132709bad18009e617cb3be208f92f3bb2b3daf55a9c4c3ed6c", + "pair-04\\postgres\\migration.json": "983ad67a725c788da9de482713d8dd2a95977aa00137bb5ed11220cacb5d6d07", + "pair-04\\postgres\\requests.jsonl": "884abfead88783486087822ffe436241482e10f06a6852e0889a97d3a59bf2f2", + "pair-04\\postgres\\resource-final.json": "010bf57d1bdec28409aed9db276576d55805207983ad5cc85e4fd5d60e484e41", + "pair-04\\postgres\\resources.csv": "76522f4a1734b84139fb9d82cc59d8c3d6920cb7159e5fea04067cb0b755ba33", + "pair-04\\postgres\\retention-empty.json": "4f4bf8efd53a7aced68436cac89db77b5aea6e41ddc30af73086956220f495fc", + "pair-04\\postgres\\retention.json": "1f3c7623672f21a67ea3d52e2310db81279b9090d36b0b2bd52a4be051654ff9", + "pair-04\\postgres\\sql-queries.jsonl": "36dc854b18462fdcbe3a47c23eefef7259dbf2dac024c84f394af139326367f3", + "pair-04\\postgres\\startup-hot1.json": "51fbfffb6897b7dabfb26decade26c41a9bf0bbcb2da7a2a1a6ebb73901acf98", + "pair-04\\postgres\\startup.json": "fb52717efee3d11b89f1284b9a1423d4ecbb7a9de15dd152df03e1f3bb8e520f", + "pair-04\\postgres\\validation-after.json": "15c98a35f0749b54aca57c405c5b3e76f22a6c79984555c0261cf0b591b5bfa1", + "pair-04\\postgres\\validation-before.json": "6ced39b5ca12b6572139e718a0da9f1615e25cf1ee150c1bd6b9312da5e1bbf0", + "pair-04\\postgres\\visibility.json": "a4ec199404d4589e6608d9c94c927d4b0b74f3e929218aa644cde43047fef5c0", + "pair-04\\postgres\\websocket.jsonl": "2d07f99f3f0dbc29e1ee5984e430710574b79b2e16c89f6880ff68da2a83ec42", + "pair-04\\sqlite\\client-validation.json": "0d53a788f043402864112d7e69866b6e07fe3a13e9f52a9cdb2a0d6ca7fdcb84", + "pair-04\\sqlite\\database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-04\\sqlite\\handler.json": "253275e5151c4af032ef144deb303d6b258851333f2820b6c895f7de5b958c26", + "pair-04\\sqlite\\ingest-validation.json": "e713cca1e7929201ef8db53d861b2ffcbd7c3f73be49fb8e97bd92f67abab11d", + "pair-04\\sqlite\\ingest.jsonl": "086cd8e4ded2e3075522e4792ca2b0532c4fb4f4f6d0080c7520f529be3e085e", + "pair-04\\sqlite\\requests.jsonl": "ca95825e22cc2d4f6a3331f56f38769971e741747998ab27e3f39bd150cb8b6b", + "pair-04\\sqlite\\resource-final.json": "849f28df6e4dae3c3faf74707cf842fe1cd4a505d18966998fb532f84ee2502b", + "pair-04\\sqlite\\resources.csv": "4d4cb00c24877d162307fa2a7f318272053391f81663b62c9be0b7a8f7689ab1", + "pair-04\\sqlite\\retention-empty.json": "15e932b27564fb3c8d64511d3165c7c831b6006d06294aa886c3409883abf9b8", + "pair-04\\sqlite\\retention.json": "0f75951fac5a70c3efb30430032f7cccc50ea08b1837a32066123475ccf63ef3", + "pair-04\\sqlite\\sql-queries.jsonl": "b1bf7fff01d0b7f5dd0fe26316dd6232bae3a37d192b2944d3e139a580e128be", + "pair-04\\sqlite\\startup-hot1.json": "233a42581b48fdecfab686f5ff269da362eba19a7484734720781704b6474799", + "pair-04\\sqlite\\startup.json": "7f5592051cd1c1b2991f8055735e352bd6c2607df635f5849c188c0da654a161", + "pair-04\\sqlite\\validation-after.json": "15c98a35f0749b54aca57c405c5b3e76f22a6c79984555c0261cf0b591b5bfa1", + "pair-04\\sqlite\\validation-before.json": "6ced39b5ca12b6572139e718a0da9f1615e25cf1ee150c1bd6b9312da5e1bbf0", + "pair-04\\sqlite\\visibility.json": "b030e3b0da40de2ab6bbc536505d712912e17497bd3bef2133e725279bd2bc65", + "pair-04\\sqlite\\websocket.jsonl": "c26dbd9b65dc48b25bb227f7fc1206f1d7f492ae5f6a2d73b7af6e8e182ea3e4", + "postgres-preflight.json": "e926795faf9c07ff13185a7914625c659ad3a825a513fb7dd1006af59ca700ed", + "resource-preflight.json": "16e6006274973fdff80b18b2afcd4a3ebb02de9fad49588ce88e77bdc443218e" + }, + "latency": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1461095, + "minimum": 1375210, + "maximum": 1516862, + "runs": 5 + }, + "postgres": { + "median": 2049150, + "minimum": 2006288, + "maximum": 2149712, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.446314532385475, + "minimum": 1.3348966484756029, + "maximum": 1.4588957322881597, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1453320040, + "minimum": 1394052328, + "maximum": 1630047018, + "runs": 5 + }, + "postgres": { + "median": 2844395, + "minimum": 2787618, + "maximum": 2956499, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.0019650051381739993, + "minimum": 0.001721807388993978, + "maximum": 0.002045758213460693, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": { + "median": 7115326735, + "minimum": 6890210177, + "maximum": 7476023526, + "runs": 5 + }, + "postgres": { + "median": 4160408, + "minimum": 3517210, + "maximum": 4914909, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.000593051655551445, + "minimum": 0.0004829165600164888, + "maximum": 0.0006907495864980822, + "runs": 5 + }, + "note": "" + }, + "samples_per_run": { + "sqlite": [ + 2700, + 2700, + 2700, + 2700, + 2700 + ], + "postgres": [ + 2700, + 2700, + 2700, + 2700, + 2700 + ] + } + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 639691, + "minimum": 622789, + "maximum": 644889, + "runs": 5 + }, + "postgres": { + "median": 620986, + "minimum": 614822, + "maximum": 629399, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.97630056067439, + "minimum": 0.9611234173999634, + "maximum": 1.0048058010016234, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1433325433, + "minimum": 1393818642, + "maximum": 1610055353, + "runs": 5 + }, + "postgres": { + "median": 1075628, + "minimum": 1068853, + "maximum": 1079794, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.0007504422758679955, + "minimum": 0.000668867065963539, + "maximum": 0.0007668522774715478, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": { + "median": 7075714964, + "minimum": 6889936133, + "maximum": 7307703962, + "runs": 5 + }, + "postgres": { + "median": 1123515, + "minimum": 1117584, + "maximum": 1137461, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.00015877490906797358, + "minimum": 0.00015345117375485683, + "maximum": 0.0001650902095524548, + "runs": 5 + }, + "note": "" + }, + "samples_per_run": { + "sqlite": [ + 2700, + 2700, + 2700, + 2700, + 2700 + ], + "postgres": [ + 2700, + 2700, + 2700, + 2700, + 2700 + ] + } + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 761548, + "minimum": 703363, + "maximum": 822717, + "runs": 5 + }, + "postgres": { + "median": 1366552, + "minimum": 1333597, + "maximum": 1450290, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.8595610519625814, + "minimum": 1.7354924884307843, + "maximum": 1.89602950396879, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1267221, + "minimum": 1159387, + "maximum": 1353189, + "runs": 5 + }, + "postgres": { + "median": 2040614, + "minimum": 1987759, + "maximum": 2185353, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.6149650935678608, + "minimum": 1.5018697270153305, + "maximum": 1.7600801113001958, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": { + "median": 3273407, + "minimum": 3075125, + "maximum": 3536701, + "runs": 5 + }, + "postgres": { + "median": 3024104, + "minimum": 2628122, + "maximum": 3568311, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.8550635182335177, + "minimum": 0.8028705260299132, + "maximum": 1.026559036959809, + "runs": 5 + }, + "note": "" + }, + "samples_per_run": { + "sqlite": [ + 2700, + 2700, + 2700, + 2700, + 2700 + ], + "postgres": [ + 2700, + 2700, + 2700, + 2700, + 2700 + ] + } + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1327799, + "minimum": 1228897, + "maximum": 1419332, + "runs": 5 + }, + "postgres": { + "median": 1897170, + "minimum": 1850921, + "maximum": 1978513, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.4807365399294743, + "minimum": 1.3040789610887376, + "maximum": 1.5086325379588363, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1394158178, + "minimum": 1315047968, + "maximum": 1570732361, + "runs": 5 + }, + "postgres": { + "median": 2489061, + "minimum": 2425550, + "maximum": 2628900, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.0018498189034752161, + "minimum": 0.0015442159722588156, + "maximum": 0.0018800226761005877, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": { + "median": 7204380610, + "minimum": 6831905094, + "maximum": 7436618017, + "runs": 5 + }, + "postgres": { + "median": 3075857, + "minimum": 2793261, + "maximum": 4712941, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.0004278146553282684, + "minimum": 0.00037560904615709, + "maximum": 0.0006898428674220105, + "runs": 5 + }, + "note": "" + }, + "samples_per_run": { + "sqlite": [ + 1350, + 1350, + 1350, + 1350, + 1350 + ], + "postgres": [ + 1350, + 1350, + 1350, + 1350, + 1350 + ] + } + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 634966, + "minimum": 622088, + "maximum": 653777, + "runs": 5 + }, + "postgres": { + "median": 605116, + "minimum": 598808, + "maximum": 629875, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9709259413769423, + "minimum": 0.9194312433750346, + "maximum": 1.0125175216368103, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1393778079, + "minimum": 1314878153, + "maximum": 1570563908, + "runs": 5 + }, + "postgres": { + "median": 1073323, + "minimum": 1065135, + "maximum": 1080537, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.0007642070255289185, + "minimum": 0.0006879930160727977, + "maximum": 0.0008152588112854591, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": { + "median": 7196012542, + "minimum": 6811915680, + "maximum": 7436413274, + "runs": 5 + }, + "postgres": { + "median": 1118114, + "minimum": 1112423, + "maximum": 1139867, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.00015534895103021904, + "minimum": 0.0001503566247332262, + "maximum": 0.00016733427915831161, + "runs": 5 + }, + "note": "" + }, + "samples_per_run": { + "sqlite": [ + 1350, + 1350, + 1350, + 1350, + 1350 + ], + "postgres": [ + 1350, + 1350, + 1350, + 1350, + 1350 + ] + } + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 643791, + "minimum": 576349, + "maximum": 699461, + "runs": 5 + }, + "postgres": { + "median": 1251706, + "minimum": 1210811, + "maximum": 1314432, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.9931996564102326, + "minimum": 1.839549691767708, + "maximum": 2.1008295321064145, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1005604, + "minimum": 930528, + "maximum": 1108514, + "runs": 5 + }, + "postgres": { + "median": 1673785, + "minimum": 1612256, + "maximum": 1761448, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.6816585252150569, + "minimum": 1.5890173691987652, + "maximum": 1.7326249183259397, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": { + "median": 1602756, + "minimum": 1484077, + "maximum": 2023963, + "runs": 5 + }, + "postgres": { + "median": 2027802, + "minimum": 1935544, + "maximum": 3129539, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.3095668216676088, + "minimum": 1.153177585906947, + "maximum": 1.5462431872519409, + "runs": 5 + }, + "note": "" + }, + "samples_per_run": { + "sqlite": [ + 1350, + 1350, + 1350, + 1350, + 1350 + ], + "postgres": [ + 1350, + 1350, + 1350, + 1350, + 1350 + ] + } + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1624934, + "minimum": 1534555, + "maximum": 1814254, + "runs": 5 + }, + "postgres": { + "median": 2188247, + "minimum": 2120271, + "maximum": 2265928, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.346668233909808, + "minimum": 1.2489585251017774, + "maximum": 1.4146133569666777, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1354740231, + "minimum": 1275692851, + "maximum": 1531316113, + "runs": 5 + }, + "postgres": { + "median": 2903790, + "minimum": 2752027, + "maximum": 2952505, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.0021434293701136863, + "minimum": 0.0018537589828129758, + "maximum": 0.0022435007149761606, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 450, + 450, + 450, + 450, + 450 + ], + "postgres": [ + 450, + 450, + 450, + 450, + 450 + ] + } + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 634351, + "minimum": 594151, + "maximum": 703181, + "runs": 5 + }, + "postgres": { + "median": 637320, + "minimum": 599580, + "maximum": 643936, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.955318112527607, + "minimum": 0.8889870754522554, + "maximum": 1.0726566142276963, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1354374580, + "minimum": 1275228598, + "maximum": 1530905355, + "runs": 5 + }, + "postgres": { + "median": 1072073, + "minimum": 1058335, + "maximum": 1084884, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.0007965935096035249, + "minimum": 0.0007086551735263869, + "maximum": 0.0008380669957340464, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 450, + 450, + 450, + 450, + 450 + ], + "postgres": [ + 450, + 450, + 450, + 450, + 450 + ] + } + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 937704, + "minimum": 848965, + "maximum": 1004443, + "runs": 5 + }, + "postgres": { + "median": 1545967, + "minimum": 1505178, + "maximum": 1630812, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.6786907489118152, + "minimum": 1.615520462747306, + "maximum": 1.7729564823049242, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1463580, + "minimum": 1361402, + "maximum": 1519725, + "runs": 5 + }, + "postgres": { + "median": 2070698, + "minimum": 1918817, + "maximum": 2122728, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.407658479847018, + "minimum": 1.3502678274870097, + "maximum": 1.4643220738620921, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 450, + 450, + 450, + 450, + 450 + ], + "postgres": [ + 450, + 450, + 450, + 450, + 450 + ] + } + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1671092, + "minimum": 1602181, + "maximum": 1830858, + "runs": 5 + }, + "postgres": { + "median": 2180463, + "minimum": 2130649, + "maximum": 2310597, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.2750040093543622, + "minimum": 1.2502622660952383, + "maximum": 1.3609342514984262, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1303820971, + "minimum": 1236594667, + "maximum": 1516178442, + "runs": 5 + }, + "postgres": { + "median": 2857915, + "minimum": 2831329, + "maximum": 2997415, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.0022896176698455712, + "minimum": 0.0018849463366792812, + "maximum": 0.0023085654133262945, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": { + "median": 7113311177, + "minimum": 7046454983, + "maximum": 7302409744, + "runs": 5 + }, + "postgres": { + "median": 3861511, + "minimum": 3415905, + "maximum": 4424840, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.0005295559954374446, + "minimum": 0.00047391287551941017, + "maximum": 0.0006279526386921075, + "runs": 5 + }, + "note": "" + }, + "samples_per_run": { + "sqlite": [ + 4500, + 4500, + 4500, + 4500, + 4500 + ], + "postgres": [ + 4500, + 4500, + 4500, + 4500, + 4500 + ] + } + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 630744, + "minimum": 629344, + "maximum": 644579, + "runs": 5 + }, + "postgres": { + "median": 621090, + "minimum": 619166, + "maximum": 645825, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9844646665989538, + "minimum": 0.9778878927175723, + "maximum": 1.011660748048579, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1283830060, + "minimum": 1216603038, + "maximum": 1496191553, + "runs": 5 + }, + "postgres": { + "median": 1074208, + "minimum": 1070358, + "maximum": 1084325, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.0008446016601293788, + "minimum": 0.0007179615456631307, + "maximum": 0.0008797923123384474, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": { + "median": 7112861352, + "minimum": 7034943960, + "maximum": 7282426629, + "runs": 5 + }, + "postgres": { + "median": 1125895, + "minimum": 1115403, + "maximum": 1152250, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.0001582900248271281, + "minimum": 0.00015316364404664983, + "maximum": 0.00016353465667013806, + "runs": 5 + }, + "note": "" + }, + "samples_per_run": { + "sqlite": [ + 4500, + 4500, + 4500, + 4500, + 4500 + ], + "postgres": [ + 4500, + 4500, + 4500, + 4500, + 4500 + ] + } + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1002572, + "minimum": 939502, + "maximum": 1073497, + "runs": 5 + }, + "postgres": { + "median": 1514924, + "minimum": 1479998, + "maximum": 1606693, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.5424867241454978, + "minimum": 1.469949761518654, + "maximum": 1.5753005315582085, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1633878, + "minimum": 1525241, + "maximum": 1803359, + "runs": 5 + }, + "postgres": { + "median": 2084132, + "minimum": 2027637, + "maximum": 2211119, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.2409965737955955, + "minimum": 1.222893233555336, + "maximum": 1.3588914801005219, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": { + "median": 3689803, + "minimum": 3642435, + "maximum": 4040757, + "runs": 5 + }, + "postgres": { + "median": 2785475, + "minimum": 2578411, + "maximum": 2993042, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.7308026689058511, + "minimum": 0.6876756391083011, + "maximum": 0.8111657993665244, + "runs": 5 + }, + "note": "" + }, + "samples_per_run": { + "sqlite": [ + 4500, + 4500, + 4500, + 4500, + 4500 + ], + "postgres": [ + 4500, + 4500, + 4500, + 4500, + 4500 + ] + } + }, + { + "scenario": "handler-control", + "event_class": "200_valid_envelopes", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 812859, + "minimum": 802914, + "maximum": 875069, + "runs": 5 + }, + "postgres": { + "median": 1576230, + "minimum": 1545732, + "maximum": 1727646, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.9391185925234267, + "minimum": 1.903299654067572, + "maximum": 1.974296884017146, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2036319, + "minimum": 1798700, + "maximum": 2168709, + "runs": 5 + }, + "postgres": { + "median": 2274351, + "minimum": 2112537, + "maximum": 2395291, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1475475886941728, + "minimum": 1.0539127195027087, + "maximum": 1.264441541113026, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 200, + 200, + 200, + 200, + 200 + ], + "postgres": [ + 200, + 200, + 200, + 200, + 200 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 30108886, + "minimum": 29536575, + "maximum": 30578375, + "runs": 5 + }, + "postgres": { + "median": 31003092, + "minimum": 30712876, + "maximum": 31105091, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.030300622879239, + "minimum": 1.0043985659800432, + "maximum": 1.0531041936988292, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 34076747, + "minimum": 33489189, + "maximum": 35526494, + "runs": 5 + }, + "postgres": { + "median": 36573822, + "minimum": 34500193, + "maximum": 41080871, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0301889663556798, + "minimum": 1.0250547976307716, + "maximum": 1.2011004454795287, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 72, + 72, + 72, + 72, + 72 + ], + "postgres": [ + 72, + 72, + 72, + 72, + 72 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 355485, + "minimum": 299402, + "maximum": 426083, + "runs": 5 + }, + "postgres": { + "median": 356897, + "minimum": 315689, + "maximum": 511851, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0621683630260381, + "minimum": 0.9061772405160353, + "maximum": 1.4398666610405502, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1032801, + "minimum": 897122, + "maximum": 1077937, + "runs": 5 + }, + "postgres": { + "median": 1039591, + "minimum": 972809, + "maximum": 1064886, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0212010610120275, + "minimum": 0.9279512624578246, + "maximum": 1.1572570544864464, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 72, + 72, + 72, + 72, + 72 + ], + "postgres": [ + 72, + 72, + 72, + 72, + 72 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 29513239, + "minimum": 29127123, + "maximum": 30106300, + "runs": 5 + }, + "postgres": { + "median": 30488073, + "minimum": 30081857, + "maximum": 30607540, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0330303969686283, + "minimum": 1.014358821907707, + "maximum": 1.0508260633911561, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 33765859, + "minimum": 32457613, + "maximum": 34817275, + "runs": 5 + }, + "postgres": { + "median": 36474216, + "minimum": 33855412, + "maximum": 40572855, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0475896232545483, + "minimum": 1.005022736131191, + "maximum": 1.190145007952613, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 72, + 72, + 72, + 72, + 72 + ], + "postgres": [ + 72, + 72, + 72, + 72, + 72 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 2564913, + "minimum": 2541082, + "maximum": 2608613, + "runs": 5 + }, + "postgres": { + "median": 2337254, + "minimum": 2258955, + "maximum": 2373943, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.8959757541651444, + "minimum": 0.880714082699881, + "maximum": 0.9297369846425224, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 3277188, + "minimum": 3126130, + "maximum": 3432011, + "runs": 5 + }, + "postgres": { + "median": 3175122, + "minimum": 3014884, + "maximum": 3336811, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9342306310536922, + "minimum": 0.9182161712185655, + "maximum": 1.0371759971594272, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 108, + 108, + 108, + 108, + 108 + ], + "postgres": [ + 108, + 108, + 108, + 108, + 108 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 823685, + "minimum": 609677, + "maximum": 844082, + "runs": 5 + }, + "postgres": { + "median": 605844, + "minimum": 571153, + "maximum": 650844, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.7696065450473817, + "minimum": 0.7355287518893752, + "maximum": 0.9645550020748691, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1068985, + "minimum": 1051723, + "maximum": 1107672, + "runs": 5 + }, + "postgres": { + "median": 1057577, + "minimum": 1020589, + "maximum": 1074081, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9893281945022615, + "minimum": 0.9417697657790393, + "maximum": 1.0152036230071986, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 108, + 108, + 108, + 108, + 108 + ], + "postgres": [ + 108, + 108, + 108, + 108, + 108 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1748543, + "minimum": 1741725, + "maximum": 1783152, + "runs": 5 + }, + "postgres": { + "median": 1595196, + "minimum": 1576282, + "maximum": 1666121, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9115250685383743, + "minimum": 0.89880949702318, + "maximum": 0.9343684666253914, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2581029, + "minimum": 2472415, + "maximum": 2659179, + "runs": 5 + }, + "postgres": { + "median": 2588219, + "minimum": 2394004, + "maximum": 2618167, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0037694268448747, + "minimum": 0.911647630881574, + "maximum": 1.0213370328201374, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 108, + 108, + 108, + 108, + 108 + ], + "postgres": [ + 108, + 108, + 108, + 108, + 108 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1897638, + "minimum": 1862673, + "maximum": 2016137, + "runs": 5 + }, + "postgres": { + "median": 2224535, + "minimum": 2163585, + "maximum": 2325365, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1583135916482634, + "minimum": 1.139839651243761, + "maximum": 1.2002247003906963, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 582522495, + "minimum": 482539009, + "maximum": 783293577, + "runs": 5 + }, + "postgres": { + "median": 576142647, + "minimum": 549166657, + "maximum": 622329680, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0615178220845272, + "minimum": 0.7365079734338228, + "maximum": 1.1408725589685953, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 72, + 72, + 72, + 72, + 72 + ], + "postgres": [ + 72, + 72, + 72, + 72, + 72 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 119520, + "minimum": 115940, + "maximum": 146997, + "runs": 5 + }, + "postgres": { + "median": 137712, + "minimum": 126975, + "maximum": 207353, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1877867862687597, + "minimum": 1.0327478607733385, + "maximum": 1.4105934134710232, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 862300, + "minimum": 808610, + "maximum": 934107, + "runs": 5 + }, + "postgres": { + "median": 984944, + "minimum": 946777, + "maximum": 1054308, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.14222892264873, + "minimum": 1.11926992698797, + "maximum": 1.1896674540260448, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 72, + 72, + 72, + 72, + 72 + ], + "postgres": [ + 72, + 72, + 72, + 72, + 72 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1711493, + "minimum": 1652674, + "maximum": 1774758, + "runs": 5 + }, + "postgres": { + "median": 1965424, + "minimum": 1897057, + "maximum": 1995707, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1198473256635553, + "minimum": 1.1136224751262254, + "maximum": 1.2075624109776035, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 582035908, + "minimum": 482259186, + "maximum": 783197789, + "runs": 5 + }, + "postgres": { + "median": 576027263, + "minimum": 548415278, + "maximum": 621344736, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0615363002349036, + "minimum": 0.7364431962154071, + "maximum": 1.141298216764294, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 72, + 72, + 72, + 72, + 72 + ], + "postgres": [ + 72, + 72, + 72, + 72, + 72 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 30333498, + "minimum": 30175679, + "maximum": 31061956, + "runs": 5 + }, + "postgres": { + "median": 30593146, + "minimum": 30548049, + "maximum": 31153570, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0070730714934362, + "minimum": 0.9930744541650887, + "maximum": 1.0289316969370215, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 33416628, + "minimum": 32160768, + "maximum": 37017366, + "runs": 5 + }, + "postgres": { + "median": 39207753, + "minimum": 33590024, + "maximum": 48289233, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0881747896167842, + "minimum": 0.907412591160592, + "maximum": 1.445066001273378, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 108, + 108, + 108, + 108, + 108 + ], + "postgres": [ + 108, + 108, + 108, + 108, + 108 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 745307, + "minimum": 715768, + "maximum": 808375, + "runs": 5 + }, + "postgres": { + "median": 650104, + "minimum": 525352, + "maximum": 738119, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.8722633760316219, + "minimum": 0.6498865006958404, + "maximum": 1.0157414542852425, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1022097, + "minimum": 977377, + "maximum": 1086638, + "runs": 5 + }, + "postgres": { + "median": 1036273, + "minimum": 1003407, + "maximum": 1074273, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0178349204166524, + "minimum": 0.9234050346113425, + "maximum": 1.0716284504341722, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 108, + 108, + 108, + 108, + 108 + ], + "postgres": [ + 108, + 108, + 108, + 108, + 108 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 29635707, + "minimum": 29567465, + "maximum": 30390158, + "runs": 5 + }, + "postgres": { + "median": 30081038, + "minimum": 29975967, + "maximum": 30584877, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0125216854114532, + "minimum": 0.999360055976017, + "maximum": 1.0322251979182075, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 32519796, + "minimum": 31248067, + "maximum": 36502620, + "runs": 5 + }, + "postgres": { + "median": 38553942, + "minimum": 32516371, + "maximum": 47812178, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1028613886016156, + "minimum": 0.8907955374162183, + "maximum": 1.4702483988521946, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 108, + 108, + 108, + 108, + 108 + ], + "postgres": [ + 108, + 108, + 108, + 108, + 108 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1296687, + "minimum": 1262361, + "maximum": 1321596, + "runs": 5 + }, + "postgres": { + "median": 1307488, + "minimum": 1265774, + "maximum": 1385291, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0091118036678752, + "minimum": 0.9577616760341284, + "maximum": 1.097381018583432, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1768156, + "minimum": 1747663, + "maximum": 1822537, + "runs": 5 + }, + "postgres": { + "median": 1852827, + "minimum": 1794376, + "maximum": 2155784, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0497294130711163, + "minimum": 1.0148290083001725, + "maximum": 1.1828478653656962, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 587628, + "minimum": 536300, + "maximum": 632933, + "runs": 5 + }, + "postgres": { + "median": 586968, + "minimum": 573890, + "maximum": 637022, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0357624981393303, + "minimum": 0.9067152447415445, + "maximum": 1.0840565800132056, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1046275, + "minimum": 1027657, + "maximum": 1076397, + "runs": 5 + }, + "postgres": { + "median": 1057938, + "minimum": 1030172, + "maximum": 1063279, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0016440247763871, + "minimum": 0.9846092088600034, + "maximum": 1.0134670651799058, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 669180, + "minimum": 652985, + "maximum": 697135, + "runs": 5 + }, + "postgres": { + "median": 672386, + "minimum": 658119, + "maximum": 705143, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0081457899762687, + "minimum": 0.9901514296675769, + "maximum": 1.0207722995168342, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 929269, + "minimum": 874879, + "maximum": 1063634, + "runs": 5 + }, + "postgres": { + "median": 1058122, + "minimum": 931447, + "maximum": 1458218, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.2094495353071681, + "minimum": 0.9345337650258805, + "maximum": 1.370977234650265, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1951684, + "minimum": 1936174, + "maximum": 2063745, + "runs": 5 + }, + "postgres": { + "median": 2016844, + "minimum": 1963119, + "maximum": 2117237, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0259198689760605, + "minimum": 1.0139166211301256, + "maximum": 1.0570035927947352, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 60085376, + "minimum": 58820296, + "maximum": 64019267, + "runs": 5 + }, + "postgres": { + "median": 113275690, + "minimum": 108292762, + "maximum": 178983058, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.9066549410087974, + "minimum": 1.7823108847931548, + "maximum": 2.795768623842569, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 549013, + "minimum": 497273, + "maximum": 650462, + "runs": 5 + }, + "postgres": { + "median": 570097, + "minimum": 547257, + "maximum": 650513, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0117694767959278, + "minimum": 0.9968015329327357, + "maximum": 1.2058044575112665, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1038010, + "minimum": 1015061, + "maximum": 1053613, + "runs": 5 + }, + "postgres": { + "median": 1048862, + "minimum": 1031272, + "maximum": 1063140, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0159704687698572, + "minimum": 0.9920910239338353, + "maximum": 1.0242097860328898, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1338143, + "minimum": 1319800, + "maximum": 1365008, + "runs": 5 + }, + "postgres": { + "median": 1410775, + "minimum": 1354168, + "maximum": 1477472, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0542782049452113, + "minimum": 1.0084591330561532, + "maximum": 1.0823907259151595, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 59905684, + "minimum": 58731213, + "maximum": 63539454, + "runs": 5 + }, + "postgres": { + "median": 112703993, + "minimum": 107712355, + "maximum": 178265731, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.9003210269128956, + "minimum": 1.7850266238415478, + "maximum": 2.805591168598962, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 2011405, + "minimum": 1928511, + "maximum": 2142243, + "runs": 5 + }, + "postgres": { + "median": 2089331, + "minimum": 2065070, + "maximum": 2199043, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0309619235486691, + "minimum": 0.9738057490788062, + "maximum": 1.100215140074389, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 83206815, + "minimum": 82649912, + "maximum": 86906543, + "runs": 5 + }, + "postgres": { + "median": 114909699, + "minimum": 111890610, + "maximum": 165416750, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.375957122520424, + "minimum": 1.3447289143323176, + "maximum": 1.9033866069209542, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 216, + 216, + 216, + 216, + 216 + ], + "postgres": [ + 216, + 216, + 216, + 216, + 216 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 622575, + "minimum": 585777, + "maximum": 726902, + "runs": 5 + }, + "postgres": { + "median": 633636, + "minimum": 573203, + "maximum": 661902, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.920697104766494, + "minimum": 0.8867165037377803, + "maximum": 1.1180984781811825, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1054293, + "minimum": 1038778, + "maximum": 1087887, + "runs": 5 + }, + "postgres": { + "median": 1051472, + "minimum": 1039658, + "maximum": 1074946, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0019553658211202, + "minimum": 0.9652840782176826, + "maximum": 1.021356825038651, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 216, + 216, + 216, + 216, + 216 + ], + "postgres": [ + 216, + 216, + 216, + 216, + 216 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1377416, + "minimum": 1343552, + "maximum": 1408771, + "runs": 5 + }, + "postgres": { + "median": 1411343, + "minimum": 1390344, + "maximum": 1455159, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.036034865283981, + "minimum": 1.0018257048164676, + "maximum": 1.0444754363348732, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 82786779, + "minimum": 81992900, + "maximum": 86201479, + "runs": 5 + }, + "postgres": { + "median": 114125189, + "minimum": 111369213, + "maximum": 165207339, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.3785436561072149, + "minimum": 1.3437824489043837, + "maximum": 1.9165255737665476, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 216, + 216, + 216, + 216, + 216 + ], + "postgres": [ + 216, + 216, + 216, + 216, + 216 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 3906297, + "minimum": 3844893, + "maximum": 3950121, + "runs": 5 + }, + "postgres": { + "median": 4404624, + "minimum": 4334900, + "maximum": 4495293, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.138014000077466, + "minimum": 1.1016482664713558, + "maximum": 1.1455777833089242, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 4845487, + "minimum": 4579403, + "maximum": 4921916, + "runs": 5 + }, + "postgres": { + "median": 5237076, + "minimum": 5151929, + "maximum": 7075678, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0730576545244104, + "minimum": 1.0525236043633697, + "maximum": 1.5451092642425224, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 566798, + "minimum": 482327, + "maximum": 636269, + "runs": 5 + }, + "postgres": { + "median": 566823, + "minimum": 512560, + "maximum": 585372, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0104569882039103, + "minimum": 0.8887090208701037, + "maximum": 1.2136413677857554, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1048472, + "minimum": 1039131, + "maximum": 1057474, + "runs": 5 + }, + "postgres": { + "median": 1038893, + "minimum": 1010156, + "maximum": 1071728, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9975016682749317, + "minimum": 0.9634553903203901, + "maximum": 1.0134792912166162, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 3327173, + "minimum": 3283949, + "maximum": 3395092, + "runs": 5 + }, + "postgres": { + "median": 3850807, + "minimum": 3795591, + "maximum": 3870696, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.143696765993232, + "minimum": 1.1400857473081731, + "maximum": 1.1726147391448527, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 4125651, + "minimum": 4075310, + "maximum": 4210769, + "runs": 5 + }, + "postgres": { + "median": 4688150, + "minimum": 4503436, + "maximum": 6617901, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1296057798468642, + "minimum": 1.069513042259843, + "maximum": 1.62390124923012, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 2043880, + "minimum": 1948411, + "maximum": 2160152, + "runs": 5 + }, + "postgres": { + "median": 2283101, + "minimum": 2239556, + "maximum": 2337098, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1149963054866818, + "minimum": 1.081913680148434, + "maximum": 1.1877201473405765, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2502633, + "minimum": 2451622, + "maximum": 2898488, + "runs": 5 + }, + "postgres": { + "median": 2819113, + "minimum": 2774734, + "maximum": 2986641, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1317951951809864, + "minimum": 0.9758556875170779, + "maximum": 1.1717225096628705, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 625627, + "minimum": 590603, + "maximum": 678346, + "runs": 5 + }, + "postgres": { + "median": 536064, + "minimum": 501337, + "maximum": 576003, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.801335300426612, + "minimum": 0.760476217151719, + "maximum": 0.9752795024745895, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1053972, + "minimum": 1036853, + "maximum": 1063729, + "runs": 5 + }, + "postgres": { + "median": 1017079, + "minimum": 963861, + "maximum": 1052802, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9680943918659736, + "minimum": 0.9145034213432615, + "maximum": 0.9952798037426912, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1369154, + "minimum": 1353340, + "maximum": 1429911, + "runs": 5 + }, + "postgres": { + "median": 1735658, + "minimum": 1708666, + "maximum": 1787384, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.2607673035665434, + "minimum": 1.2499966781149316, + "maximum": 1.2677198020915654, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1606237, + "minimum": 1554201, + "maximum": 2146589, + "runs": 5 + }, + "postgres": { + "median": 1961007, + "minimum": 1920762, + "maximum": 2118583, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.2555943536260754, + "minimum": 0.9135456298341229, + "maximum": 1.2980463589954734, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 62495104, + "minimum": 61842826, + "maximum": 63601574, + "runs": 5 + }, + "postgres": { + "median": 64354893, + "minimum": 63354364, + "maximum": 64695942, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0244416062099102, + "minimum": 1.0138817114377472, + "maximum": 1.0319408968438384, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 119552875, + "minimum": 69925563, + "maximum": 122082210, + "runs": 5 + }, + "postgres": { + "median": 104746733, + "minimum": 72775318, + "maximum": 126961061, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.872295851608779, + "minimum": 0.6087291334482755, + "maximum": 1.727049505486284, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 599957, + "minimum": 486503, + "maximum": 618306, + "runs": 5 + }, + "postgres": { + "median": 510944, + "minimum": 475594, + "maximum": 557085, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.8476529384928041, + "minimum": 0.7691887188544183, + "maximum": 1.1322109010633028, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1033180, + "minimum": 986390, + "maximum": 1080958, + "runs": 5 + }, + "postgres": { + "median": 1045438, + "minimum": 1019702, + "maximum": 1060228, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9911433277679204, + "minimum": 0.9755809198877292, + "maximum": 1.0598627317795193, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 61854758, + "minimum": 61200154, + "maximum": 62987426, + "runs": 5 + }, + "postgres": { + "median": 63765017, + "minimum": 62763292, + "maximum": 64361304, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0265258646244584, + "minimum": 1.0146881829203827, + "maximum": 1.032829220806567, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 119434458, + "minimum": 69454249, + "maximum": 121738785, + "runs": 5 + }, + "postgres": { + "median": 103797969, + "minimum": 72346081, + "maximum": 126902611, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.8690029959590068, + "minimum": 0.6057387642685162, + "maximum": 1.7301921873779098, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 3661213, + "minimum": 3536827, + "maximum": 3765067, + "runs": 5 + }, + "postgres": { + "median": 5525959, + "minimum": 5443298, + "maximum": 5645859, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.5093246418604982, + "minimum": 1.4902130105525384, + "maximum": 1.5518830861673472, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 8125197, + "minimum": 5561475, + "maximum": 8895935, + "runs": 5 + }, + "postgres": { + "median": 9891781, + "minimum": 7728956, + "maximum": 9995458, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.2174204514672076, + "minimum": 0.8688188481592998, + "maximum": 1.7971169518877637, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 229739, + "minimum": 134430, + "maximum": 279939, + "runs": 5 + }, + "postgres": { + "median": 175403, + "minimum": 138223, + "maximum": 195652, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.7634881321847836, + "minimum": 0.6016671025656168, + "maximum": 1.308971211783084, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1029170, + "minimum": 986360, + "maximum": 1050931, + "runs": 5 + }, + "postgres": { + "median": 1016385, + "minimum": 982448, + "maximum": 1066486, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0002846954341849, + "minimum": 0.9348358740963965, + "maximum": 1.06028123421736, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 3331942, + "minimum": 3237866, + "maximum": 3413817, + "runs": 5 + }, + "postgres": { + "median": 5230810, + "minimum": 5067802, + "maximum": 5322495, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.5662611684157428, + "minimum": 1.5393257198329882, + "maximum": 1.5832440255402787, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 7375630, + "minimum": 4911310, + "maximum": 8104951, + "runs": 5 + }, + "postgres": { + "median": 9456650, + "minimum": 6928485, + "maximum": 9643424, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.2944864370907976, + "minimum": 0.8548460070887536, + "maximum": 1.925484239439172, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 5921280, + "minimum": 5746777, + "maximum": 6172381, + "runs": 5 + }, + "postgres": { + "median": 8267731, + "minimum": 7987993, + "maximum": 8484868, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.3763196992907674, + "minimum": 1.3631014217872692, + "maximum": 1.4386726681755704, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 12196883, + "minimum": 9102632, + "maximum": 14106060, + "runs": 5 + }, + "postgres": { + "median": 15574976, + "minimum": 14593040, + "maximum": 16300579, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.2769636307899321, + "minimum": 1.0345227512147261, + "maximum": 1.7907544762877374, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 216, + 216, + 216, + 216, + 216 + ], + "postgres": [ + 216, + 216, + 216, + 216, + 216 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 481205, + "minimum": 429079, + "maximum": 525179, + "runs": 5 + }, + "postgres": { + "median": 494180, + "minimum": 474799, + "maximum": 554883, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0790202487471785, + "minimum": 0.9040708025263767, + "maximum": 1.1517226431496297, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1051484, + "minimum": 1035773, + "maximum": 1065376, + "runs": 5 + }, + "postgres": { + "median": 1046342, + "minimum": 1024145, + "maximum": 1067326, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9877254341125884, + "minimum": 0.9702367990268225, + "maximum": 1.0304632385667516, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 216, + 216, + 216, + 216, + 216 + ], + "postgres": [ + 216, + 216, + 216, + 216, + 216 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 5476232, + "minimum": 5335928, + "maximum": 5637300, + "runs": 5 + }, + "postgres": { + "median": 7723789, + "minimum": 7528155, + "maximum": 7812237, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.401719094643305, + "minimum": 1.3858118248097493, + "maximum": 1.4311714850725121, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 11606900, + "minimum": 8665244, + "maximum": 13741682, + "runs": 5 + }, + "postgres": { + "median": 15279315, + "minimum": 14147839, + "maximum": 15747938, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.3163992969699059, + "minimum": 1.0295565710223828, + "maximum": 1.8173680971938009, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 216, + 216, + 216, + 216, + 216 + ], + "postgres": [ + 216, + 216, + 216, + 216, + 216 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 3475660, + "minimum": 3440043, + "maximum": 3605090, + "runs": 5 + }, + "postgres": { + "median": 3518928, + "minimum": 3462715, + "maximum": 3543096, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0016135707317184, + "minimum": 0.9828037580199107, + "maximum": 1.022931399404019, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 4572379, + "minimum": 4434308, + "maximum": 4899550, + "runs": 5 + }, + "postgres": { + "median": 4884393, + "minimum": 4819054, + "maximum": 5226672, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.074320784323041, + "minimum": 1.053948939928208, + "maximum": 1.0959852134763757, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 360, + 360, + 360, + 360, + 360 + ], + "postgres": [ + 360, + 360, + 360, + 360, + 360 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 564717, + "minimum": 539879, + "maximum": 577949, + "runs": 5 + }, + "postgres": { + "median": 557758, + "minimum": 522037, + "maximum": 581177, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0146330637385985, + "minimum": 0.9032578999185049, + "maximum": 1.0648626821936027, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1033286, + "minimum": 1018988, + "maximum": 1055795, + "runs": 5 + }, + "postgres": { + "median": 1043002, + "minimum": 1033918, + "maximum": 1063925, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0062511253096122, + "minimum": 0.9923574178699464, + "maximum": 1.02965200341435, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 360, + 360, + 360, + 360, + 360 + ], + "postgres": [ + 360, + 360, + 360, + 360, + 360 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 2846359, + "minimum": 2814772, + "maximum": 2967040, + "runs": 5 + }, + "postgres": { + "median": 2931575, + "minimum": 2867017, + "maximum": 2990849, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0270834607390162, + "minimum": 1.0030083854616048, + "maximum": 1.0507630976977957, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 3911868, + "minimum": 3757436, + "maximum": 4060831, + "runs": 5 + }, + "postgres": { + "median": 4266545, + "minimum": 4180163, + "maximum": 4617742, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.135493724976287, + "minimum": 1.066225994616703, + "maximum": 1.1498557722295333, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 360, + 360, + 360, + 360, + 360 + ], + "postgres": [ + 360, + 360, + 360, + 360, + 360 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1867204, + "minimum": 1846237, + "maximum": 1919097, + "runs": 5 + }, + "postgres": { + "median": 3145195, + "minimum": 3068597, + "maximum": 3347848, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.679306064040137, + "minimum": 1.6501276381548198, + "maximum": 1.792323069658576, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2407444, + "minimum": 2345033, + "maximum": 2501474, + "runs": 5 + }, + "postgres": { + "median": 3763013, + "minimum": 3731687, + "maximum": 4101247, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.5500618082912831, + "minimum": 1.5379568206585397, + "maximum": 1.7489080111026156, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 560073, + "minimum": 531046, + "maximum": 586605, + "runs": 5 + }, + "postgres": { + "median": 571591, + "minimum": 509408, + "maximum": 641479, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0128340412767622, + "minimum": 0.9592540005950521, + "maximum": 1.0935450601341619, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1027984, + "minimum": 990454, + "maximum": 1057173, + "runs": 5 + }, + "postgres": { + "median": 1031314, + "minimum": 1011553, + "maximum": 1039489, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0017452427438955, + "minimum": 0.978736687372833, + "maximum": 1.0386903379662256, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1263712, + "minimum": 1212069, + "maximum": 1368605, + "runs": 5 + }, + "postgres": { + "median": 2581827, + "minimum": 2495053, + "maximum": 2778915, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 2.043050156997797, + "minimum": 1.961384037030407, + "maximum": 2.170249105984651, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1585619, + "minimum": 1505468, + "maximum": 1954966, + "runs": 5 + }, + "postgres": { + "median": 3093005, + "minimum": 3043911, + "maximum": 3304445, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.9360370206234867, + "minimum": 1.6902825931499577, + "maximum": 2.188864857971076, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1275193, + "minimum": 1235647, + "maximum": 1430893, + "runs": 5 + }, + "postgres": { + "median": 1254152, + "minimum": 1217322, + "maximum": 1356879, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9834997525864713, + "minimum": 0.8647269798039855, + "maximum": 1.0741489281475867, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1734242, + "minimum": 1724933, + "maximum": 1790719, + "runs": 5 + }, + "postgres": { + "median": 1823438, + "minimum": 1693827, + "maximum": 2102929, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0571065658782108, + "minimum": 0.9802404091272365, + "maximum": 1.1743489626233932, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 45, + 45, + 45, + 45, + 45 + ], + "postgres": [ + 45, + 45, + 45, + 45, + 45 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 544909, + "minimum": 480343, + "maximum": 714878, + "runs": 5 + }, + "postgres": { + "median": 500297, + "minimum": 424549, + "maximum": 619487, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9181294491373789, + "minimum": 0.6685504379768296, + "maximum": 0.9497876084601706, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1062243, + "minimum": 1021301, + "maximum": 1071692, + "runs": 5 + }, + "postgres": { + "median": 1036393, + "minimum": 1008922, + "maximum": 1050543, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9743928649094417, + "minimum": 0.9431231286812923, + "maximum": 1.0286321074785985, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 45, + 45, + 45, + 45, + 45 + ], + "postgres": [ + 45, + 45, + 45, + 45, + 45 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 680745, + "minimum": 652325, + "maximum": 693450, + "runs": 5 + }, + "postgres": { + "median": 715544, + "minimum": 677662, + "maximum": 732517, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0511189946308823, + "minimum": 0.9830726981242368, + "maximum": 1.108462806116583, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 792570, + "minimum": 772831, + "maximum": 837369, + "runs": 5 + }, + "postgres": { + "median": 1051024, + "minimum": 869329, + "maximum": 1373714, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.3329308844828067, + "minimum": 1.096848227916777, + "maximum": 1.7775089249784235, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 45, + 45, + 45, + 45, + 45 + ], + "postgres": [ + 45, + 45, + 45, + 45, + 45 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1726965, + "minimum": 1680197, + "maximum": 1756956, + "runs": 5 + }, + "postgres": { + "median": 2145375, + "minimum": 2105552, + "maximum": 2315509, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.2325380074292183, + "minimum": 1.2173691830211981, + "maximum": 1.3655873329661115, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2196763, + "minimum": 2151266, + "maximum": 2224465, + "runs": 5 + }, + "postgres": { + "median": 2728989, + "minimum": 2661574, + "maximum": 3427111, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.2610750003003672, + "minimum": 1.211589051709265, + "maximum": 1.5406450539792713, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 135, + 135, + 135, + 135, + 135 + ], + "postgres": [ + 135, + 135, + 135, + 135, + 135 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 570114, + "minimum": 556303, + "maximum": 682154, + "runs": 5 + }, + "postgres": { + "median": 620332, + "minimum": 566948, + "maximum": 634898, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0880841375584531, + "minimum": 0.8995205290363935, + "maximum": 1.1215198593900424, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1045903, + "minimum": 1032975, + "maximum": 1057630, + "runs": 5 + }, + "postgres": { + "median": 1046120, + "minimum": 1000333, + "maximum": 1068505, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.999120378285091, + "minimum": 0.956429993986058, + "maximum": 1.0146857377961713, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 135, + 135, + 135, + 135, + 135 + ], + "postgres": [ + 135, + 135, + 135, + 135, + 135 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1090014, + "minimum": 1065890, + "maximum": 1155457, + "runs": 5 + }, + "postgres": { + "median": 1523597, + "minimum": 1508968, + "maximum": 1625972, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.407211172722135, + "minimum": 1.38847479807567, + "maximum": 1.4301003350669486, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1330433, + "minimum": 1292966, + "maximum": 1351718, + "runs": 5 + }, + "postgres": { + "median": 1933827, + "minimum": 1790464, + "maximum": 2641595, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.446268271614633, + "minimum": 1.3543733788607604, + "maximum": 1.9542500728702288, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 135, + 135, + 135, + 135, + 135 + ], + "postgres": [ + 135, + 135, + 135, + 135, + 135 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 30327771, + "minimum": 30066971, + "maximum": 33274626, + "runs": 5 + }, + "postgres": { + "median": 35339795, + "minimum": 34692184, + "maximum": 36850675, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1601198574245788, + "minimum": 1.1074707496336698, + "maximum": 1.1950532401474543, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 32370208, + "minimum": 31874787, + "maximum": 35972527, + "runs": 5 + }, + "postgres": { + "median": 38324784, + "minimum": 37207627, + "maximum": 40126398, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1673059023108139, + "minimum": 1.1154734278189575, + "maximum": 1.2294649759433118, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 621192, + "minimum": 547346, + "maximum": 672579, + "runs": 5 + }, + "postgres": { + "median": 580773, + "minimum": 508948, + "maximum": 587488, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.8832100068026266, + "minimum": 0.82597246793874, + "maximum": 1.073339350246462, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1055509, + "minimum": 1040615, + "maximum": 1060265, + "runs": 5 + }, + "postgres": { + "median": 1015080, + "minimum": 1011785, + "maximum": 1055496, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9623637709087519, + "minimum": 0.9551366875262316, + "maximum": 0.9999876836673112, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 29796902, + "minimum": 29351432, + "maximum": 32670356, + "runs": 5 + }, + "postgres": { + "median": 34739808, + "minimum": 34078962, + "maximum": 36264010, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1614427753246408, + "minimum": 1.1099973933556158, + "maximum": 1.1966940724240394, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 31859822, + "minimum": 31499654, + "maximum": 35263850, + "runs": 5 + }, + "postgres": { + "median": 37451081, + "minimum": 36809693, + "maximum": 39238667, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1684145735809899, + "minimum": 1.109726391190979, + "maximum": 1.2456856510233414, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1393898, + "minimum": 1291446, + "maximum": 1424358, + "runs": 5 + }, + "postgres": { + "median": 1413174, + "minimum": 1392669, + "maximum": 1475761, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0104132440106808, + "minimum": 0.9777520819906231, + "maximum": 1.1065782076834805, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1811044, + "minimum": 1793966, + "maximum": 1885075, + "runs": 5 + }, + "postgres": { + "median": 1877009, + "minimum": 1842015, + "maximum": 1927129, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0335049838656598, + "minimum": 0.9877772939627029, + "maximum": 1.0686456710996752, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 587119, + "minimum": 554532, + "maximum": 611834, + "runs": 5 + }, + "postgres": { + "median": 589940, + "minimum": 565897, + "maximum": 632978, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0345583932896831, + "minimum": 0.9638540057467055, + "maximum": 1.0678863450899716, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1031534, + "minimum": 973458, + "maximum": 1056347, + "runs": 5 + }, + "postgres": { + "median": 1053007, + "minimum": 1011119, + "maximum": 1066532, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0054976374991034, + "minimum": 0.9690282873123028, + "maximum": 1.095611726443257, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 802021, + "minimum": 776938, + "maximum": 807890, + "runs": 5 + }, + "postgres": { + "median": 828113, + "minimum": 818668, + "maximum": 841431, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0438388360658588, + "minimum": 1.0133409251259453, + "maximum": 1.0658675467025682, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 931987, + "minimum": 912227, + "maximum": 949553, + "runs": 5 + }, + "postgres": { + "median": 960232, + "minimum": 946870, + "maximum": 983246, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0284714271765594, + "minimum": 1.0041497120229237, + "maximum": 1.076771108139689, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 2163966, + "minimum": 2137228, + "maximum": 2200032, + "runs": 5 + }, + "postgres": { + "median": 2331168, + "minimum": 2276651, + "maximum": 2420877, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0873264006306131, + "minimum": 1.05082391777854, + "maximum": 1.1125387651668424, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2688784, + "minimum": 2623178, + "maximum": 2722840, + "runs": 5 + }, + "postgres": { + "median": 2872466, + "minimum": 2744902, + "maximum": 3055676, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0756396327893245, + "minimum": 1.0208711447256456, + "maximum": 1.1222385450485521, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 590583, + "minimum": 568150, + "maximum": 606329, + "runs": 5 + }, + "postgres": { + "median": 567655, + "minimum": 527788, + "maximum": 593967, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9442142539916466, + "minimum": 0.8936728622395159, + "maximum": 1.0283344384791577, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1045129, + "minimum": 1004059, + "maximum": 1053979, + "runs": 5 + }, + "postgres": { + "median": 1037542, + "minimum": 1002055, + "maximum": 1047818, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.992740609054002, + "minimum": 0.9507352613287362, + "maximum": 1.033533885956901, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1555822, + "minimum": 1539357, + "maximum": 1603798, + "runs": 5 + }, + "postgres": { + "median": 1799312, + "minimum": 1680802, + "maximum": 1845767, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1548007345111224, + "minimum": 1.0918857678888003, + "maximum": 1.1566839307688948, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1769381, + "minimum": 1764734, + "maximum": 1830097, + "runs": 5 + }, + "postgres": { + "median": 1958771, + "minimum": 1841533, + "maximum": 2229272, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1070374328649397, + "minimum": 1.0425174179092396, + "maximum": 1.2181168539153935, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1110411, + "minimum": 1068535, + "maximum": 1144805, + "runs": 5 + }, + "postgres": { + "median": 1075203, + "minimum": 1034430, + "maximum": 1142754, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9914914387555599, + "minimum": 0.9413150529158181, + "maximum": 1.006240319690043, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2746312, + "minimum": 2703840, + "maximum": 3130427, + "runs": 5 + }, + "postgres": { + "median": 4480686, + "minimum": 4162587, + "maximum": 5315999, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.5157006924195067, + "minimum": 1.431333808454885, + "maximum": 1.9660922983608498, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 585749, + "minimum": 519204, + "maximum": 615901, + "runs": 5 + }, + "postgres": { + "median": 561443, + "minimum": 512707, + "maximum": 588746, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9519646081678916, + "minimum": 0.8324503451041645, + "maximum": 1.133939646073605, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1035433, + "minimum": 999447, + "maximum": 1054275, + "runs": 5 + }, + "postgres": { + "median": 1031617, + "minimum": 1013361, + "maximum": 1051609, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.991946364467812, + "minimum": 0.9627501854985279, + "maximum": 1.0514804686991908, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 469071, + "minimum": 454118, + "maximum": 487079, + "runs": 5 + }, + "postgres": { + "median": 460190, + "minimum": 442406, + "maximum": 462567, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9791441871936369, + "minimum": 0.9269278958770706, + "maximum": 1.0016967674516117, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2278827, + "minimum": 2099687, + "maximum": 2711671, + "runs": 5 + }, + "postgres": { + "median": 3787246, + "minimum": 3708109, + "maximum": 4775971, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.66192782514864, + "minimum": 1.3966779155730913, + "maximum": 2.0746816065500564, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 76110000, + "minimum": 66715720, + "maximum": 81735095, + "runs": 5 + }, + "postgres": { + "median": 106304598, + "minimum": 102155105, + "maximum": 118295581, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.4334956410560244, + "minimum": 1.3679033898305084, + "maximum": 1.5311999181002618, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 85136438, + "minimum": 79008480, + "maximum": 102979309, + "runs": 5 + }, + "postgres": { + "median": 122686767, + "minimum": 111946799, + "maximum": 183925783, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.3867284987296424, + "minimum": 1.2923999057615598, + "maximum": 1.7862597027559572, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 100, + 100, + 100, + 100, + 100 + ], + "postgres": [ + 100, + 100, + 100, + 100, + 100 + ] + } + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 765374, + "minimum": 729945, + "maximum": 845636, + "runs": 5 + }, + "postgres": { + "median": 847299, + "minimum": 814669, + "maximum": 892350, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0710309739167017, + "minimum": 1.006590913294655, + "maximum": 1.1659005923901256, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1313757, + "minimum": 1304741, + "maximum": 1466828, + "runs": 5 + }, + "postgres": { + "median": 1382099, + "minimum": 1210486, + "maximum": 1408799, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0132654680124256, + "minimum": 0.8741157644503369, + "maximum": 1.0723436678168032, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 100, + 100, + 100, + 100, + 100 + ], + "postgres": [ + 100, + 100, + 100, + 100, + 100 + ] + } + }, + { + "scenario": "websocket", + "event_class": "new_transmission", + "timing": "receipt_after_ingest_return", + "unit": "ns", + "p50": { + "sqlite": { + "median": 507054442, + "minimum": 506091765, + "maximum": 534605779, + "runs": 5 + }, + "postgres": { + "median": 511175986, + "minimum": 505540967, + "maximum": 519671849, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0035750994413632, + "minimum": 0.9697640081066164, + "maximum": 1.0268332443623145, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 985455250, + "minimum": 969220451, + "maximum": 991805005, + "runs": 5 + }, + "postgres": { + "median": 959560523, + "minimum": 946833676, + "maximum": 971019472, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9689722122100164, + "minimum": 0.9608083938869878, + "maximum": 0.9916422832476943, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": { + "median": 1389020095, + "minimum": 1330045457, + "maximum": 1892262507, + "runs": 5 + }, + "postgres": { + "median": 998959615, + "minimum": 986662688, + "maximum": 1569019521, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.733397643646307, + "minimum": 0.5279180934487543, + "maximum": 1.119658148000995, + "runs": 5 + }, + "note": "" + }, + "samples_per_run": { + "sqlite": [ + 4500, + 4500, + 4500, + 4500, + 4500 + ], + "postgres": [ + 4500, + 4500, + 4500, + 4500, + 4500 + ] + } + } + ], + "scalar_metrics": [ + { + "metric": "durable-ingest/additional/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 15.024899342488451, + "minimum": 15.024764325650635, + "maximum": 15.024982037215919, + "runs": 5 + }, + "postgres": { + "median": 15.02486346818685, + "minimum": 15.024829572423004, + "maximum": 15.024894453131463, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999976123432986, + "minimum": 0.999993047660823, + "maximum": 1.000004342615362, + "runs": 5 + } + }, + { + "metric": "durable-ingest/duplicate/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 7.51497417095111, + "minimum": 7.514958763057562, + "maximum": 7.514981423623983, + "runs": 5 + }, + "postgres": { + "median": 7.514955154081024, + "minimum": 7.514925769106568, + "maximum": 7.514973916479687, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999995197609658, + "minimum": 0.9999928865605316, + "maximum": 1.0000007668168052, + "runs": 5 + } + }, + { + "metric": "durable-ingest/late/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 2.5055425541970617, + "minimum": 2.505538498823182, + "maximum": 2.505553877065402, + "runs": 5 + }, + "postgres": { + "median": 2.5055338903411295, + "minimum": 2.505532175372296, + "maximum": 2.505542867763413, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999978579028009, + "minimum": 0.9999913385645766, + "maximum": 1.000000706145498, + "runs": 5 + } + }, + { + "metric": "durable-ingest/new/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 25.03043083166518, + "minimum": 25.030303885212806, + "maximum": 25.0304635729368, + "runs": 5 + }, + "postgres": { + "median": 25.03031940360886, + "minimum": 25.030240536519056, + "maximum": 25.03035269133082, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999965156747839, + "minimum": 0.9999942402453906, + "maximum": 0.9999974691200698, + "runs": 5 + } + }, + { + "metric": "migration/elapsed_ns", + "unit": "ns", + "sqlite": null, + "postgres": { + "median": 147616197535, + "minimum": 144734628603, + "maximum": 151047832511, + "runs": 5 + }, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null, + "runs": 0 + } + }, + { + "metric": "migration/harness_validation_ns", + "unit": "ns", + "sqlite": null, + "postgres": { + "median": 54063127899, + "minimum": 53280485928, + "maximum": 56617489512, + "runs": 5 + }, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null, + "runs": 0 + } + }, + { + "metric": "migration/migration_tool_ns", + "unit": "ns", + "sqlite": null, + "postgres": { + "median": 90985078119, + "minimum": 89942496573, + "maximum": 94125867624, + "runs": 5 + }, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null, + "runs": 0 + } + }, + { + "metric": "migration/role_grants_ns", + "unit": "ns", + "sqlite": null, + "postgres": { + "median": 773319309, + "minimum": 702549749, + "maximum": 826441189, + "runs": 5 + }, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null, + "runs": 0 + } + }, + { + "metric": "mixed-http/analytics_channels/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 0.4108872023978114, + "minimum": 0.4108823317592926, + "maximum": 0.4108919547066046, + "runs": 5 + }, + "postgres": { + "median": 0.4108851580052166, + "minimum": 0.41088079265731003, + "maximum": 0.41088613215818687, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999950244432465, + "minimum": 0.9999728345878601, + "maximum": 1.000006251961873, + "runs": 5 + } + }, + { + "metric": "mixed-http/analytics_rf/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 0.6157285752179937, + "minimum": 0.6157255961925494, + "maximum": 0.6157299032173998, + "runs": 5 + }, + "postgres": { + "median": 0.6157265741676229, + "minimum": 0.6157240798681688, + "maximum": 0.6157285760324033, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.999997844533797, + "minimum": 0.9999917572105244, + "maximum": 1.0000026436362384, + "runs": 5 + } + }, + { + "metric": "mixed-http/analytics_rf_filtered/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 0.4109541207607805, + "minimum": 0.4109529557578954, + "maximum": 0.41095440282186896, + "runs": 5 + }, + "postgres": { + "median": 0.410952802471435, + "minimum": 0.41095215148316555, + "maximum": 0.41095430615484063, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.999998042903163, + "minimum": 0.9999959835932295, + "maximum": 1.0000001949578643, + "runs": 5 + } + }, + { + "metric": "mixed-http/analytics_topology/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 0.6156305737496135, + "minimum": 0.6156225331026979, + "maximum": 0.6156343787186107, + "runs": 5 + }, + "postgres": { + "median": 0.6156281904355956, + "minimum": 0.6156153153228161, + "maximum": 0.6156319924589164, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999996384245101, + "minimum": 0.9999690345496393, + "maximum": 1.0000091896131047, + "runs": 5 + } + }, + { + "metric": "mixed-http/channels/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.4326209010181128, + "minimum": 1.4326191525360814, + "maximum": 1.4326236467879043, + "runs": 5 + }, + "postgres": { + "median": 1.432624963125401, + "minimum": 1.4326165813832683, + "maximum": 1.4326264217093045, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0000018217020261, + "minimum": 0.9999950682060484, + "maximum": 1.0000042713210824, + "runs": 5 + } + }, + { + "metric": "mixed-http/messages_0/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.4326164829744195, + "minimum": 1.4326140382563404, + "maximum": 1.4326202182199135, + "runs": 5 + }, + "postgres": { + "median": 1.4326150396163264, + "minimum": 1.432613761901396, + "maximum": 1.4326181626051169, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999989925021036, + "minimum": 0.9999954933495736, + "maximum": 1.0000015156361113, + "runs": 5 + } + }, + { + "metric": "mixed-http/messages_1000/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.2290053084144184, + "minimum": 1.2290040265927031, + "maximum": 1.2290071107313902, + "runs": 5 + }, + "postgres": { + "median": 1.229002983760107, + "minimum": 1.2281871884052704, + "maximum": 1.2290078773315032, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999991514815465, + "minimum": 0.9993343234536526, + "maximum": 1.000002643299088, + "runs": 5 + } + }, + { + "metric": "mixed-http/metrics_dense_24h/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.0262033558961197, + "minimum": 1.0261974421851585, + "maximum": 1.0262036792134246, + "runs": 5 + }, + "postgres": { + "median": 1.0262024879197815, + "minimum": 1.0261996446974946, + "maximum": 1.0262030967884224, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999988391255389, + "minimum": 0.9999960778763601, + "maximum": 1.0000055102488385, + "runs": 5 + } + }, + { + "metric": "mixed-http/metrics_sparse_7d/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.0262144336331263, + "minimum": 1.0262135951837557, + "maximum": 1.0262156329348024, + "runs": 5 + }, + "postgres": { + "median": 1.0262101842337945, + "minimum": 1.0262089894205495, + "maximum": 1.0262132901317251, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999955895310654, + "minimum": 0.9999945813086569, + "maximum": 0.9999991510194463, + "runs": 5 + } + }, + { + "metric": "mixed-http/nodes_2000/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.4321358220980442, + "minimum": 1.4321151292709695, + "maximum": 1.432137778742715, + "runs": 5 + }, + "postgres": { + "median": 1.4321175676230997, + "minimum": 1.4318860417452008, + "maximum": 1.4321360214695593, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999872536705926, + "minimum": 0.9998335887043469, + "maximum": 1.000014275044328, + "runs": 5 + } + }, + { + "metric": "mixed-http/nodes_50/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.4326015608049392, + "minimum": 1.4325632998114946, + "maximum": 1.4326041214710346, + "runs": 5 + }, + "postgres": { + "median": 1.4325874801798992, + "minimum": 1.4325843262635118, + "maximum": 1.4325923358567263, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999934787765171, + "minimum": 0.9999874411458306, + "maximum": 1.0000146775029208, + "runs": 5 + } + }, + { + "metric": "mixed-http/nodes_region/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.2289781632667003, + "minimum": 1.2289738820537766, + "maximum": 1.2289804125731867, + "runs": 5 + }, + "postgres": { + "median": 1.2289646640065988, + "minimum": 1.228932099005017, + "maximum": 1.2289690282603238, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999898317227492, + "minimum": 0.99996068808935, + "maximum": 0.9999960505316479, + "runs": 5 + } + }, + { + "metric": "mixed-http/observers/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 2.051823181146627, + "minimum": 2.0518223234808373, + "maximum": 2.051835513931382, + "runs": 5 + }, + "postgres": { + "median": 2.051829810083036, + "minimum": 2.0518226444329977, + "maximum": 2.0518385051145525, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0000009907582572, + "minimum": 1.000000156422979, + "maximum": 1.0000054124763982, + "runs": 5 + } + }, + { + "metric": "mixed-http/packet_detail_memory/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.0250479314243583, + "minimum": 1.0250412312510828, + "maximum": 1.025048531895542, + "runs": 5 + }, + "postgres": { + "median": 1.0250382001440197, + "minimum": 1.0250369375445914, + "maximum": 1.025040348918518, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999905065119002, + "minimum": 0.9999889593670083, + "maximum": 0.9999974599428916, + "runs": 5 + } + }, + { + "metric": "mixed-http/packet_detail_pruned/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.1083413262638564, + "minimum": 1.1083374477990473, + "maximum": 1.108346888743566, + "runs": 5 + }, + "postgres": { + "median": 1.1083471949510062, + "minimum": 1.1083273270877296, + "maximum": 1.10835276821805, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0000002762740108, + "minimum": 0.9999908685650406, + "maximum": 1.000010323493244, + "runs": 5 + } + }, + { + "metric": "mixed-http/packet_detail_sql/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.0336749881763692, + "minimum": 1.0336731117556959, + "maximum": 1.0336777876436403, + "runs": 5 + }, + "postgres": { + "median": 1.0336763215669507, + "minimum": 1.0336737483565326, + "maximum": 1.0336784590909618, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999995856802658, + "minimum": 0.9999990149290094, + "maximum": 1.0000038292809939, + "runs": 5 + } + }, + { + "metric": "mixed-http/packets_memory/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.0248767414004638, + "minimum": 1.0248652808168723, + "maximum": 1.024883969868769, + "runs": 5 + }, + "postgres": { + "median": 1.024850728895416, + "minimum": 1.0248381718202302, + "maximum": 1.0248571049842008, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999786169107658, + "minimum": 0.9999553139185651, + "maximum": 0.9999868813621767, + "runs": 5 + } + }, + { + "metric": "mixed-http/reach/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.02621879967198, + "minimum": 1.02621541276287, + "maximum": 1.026220640613885, + "runs": 5 + }, + "postgres": { + "median": 1.0262151208327726, + "minimum": 1.02621462714909, + "maximum": 1.026217147836336, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999964151512244, + "minimum": 0.9999955978568508, + "maximum": 1.0000008391719688, + "runs": 5 + } + }, + { + "metric": "mixed-http/rx_coverage/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.026212081427731, + "minimum": 1.0262095059219531, + "maximum": 1.0262140751349913, + "runs": 5 + }, + "postgres": { + "median": 1.0262117597550047, + "minimum": 1.0262104535192633, + "maximum": 1.0262128471981375, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0000007462106717, + "minimum": 0.9999976944405704, + "maximum": 1.0000017118990405, + "runs": 5 + } + }, + { + "metric": "mixed-http/stats/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.027390984809161, + "minimum": 1.0273895609978374, + "maximum": 1.027393961682574, + "runs": 5 + }, + "postgres": { + "median": 1.027391889518365, + "minimum": 1.0273883881301913, + "maximum": 1.027394360605857, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0000013735601714, + "minimum": 0.999997451150525, + "maximum": 1.0000045528861528, + "runs": 5 + } + }, + { + "metric": "resource/aggregate_accounted_io_dbytes", + "unit": "bytes", + "sqlite": { + "median": 0, + "minimum": 0, + "maximum": 0, + "runs": 5 + }, + "postgres": { + "median": 4096, + "minimum": 0, + "maximum": 4096, + "runs": 5 + }, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null, + "runs": 0 + } + }, + { + "metric": "resource/aggregate_accounted_io_dios", + "unit": "operations", + "sqlite": { + "median": 0, + "minimum": 0, + "maximum": 0, + "runs": 5 + }, + "postgres": { + "median": 1, + "minimum": 0, + "maximum": 1, + "runs": 5 + }, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null, + "runs": 0 + } + }, + { + "metric": "resource/aggregate_accounted_io_rbytes", + "unit": "bytes", + "sqlite": { + "median": 0, + "minimum": 0, + "maximum": 0, + "runs": 5 + }, + "postgres": { + "median": 6041600, + "minimum": 5840896, + "maximum": 7208960, + "runs": 5 + }, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null, + "runs": 0 + } + }, + { + "metric": "resource/aggregate_accounted_io_rios", + "unit": "operations", + "sqlite": { + "median": 0, + "minimum": 0, + "maximum": 0, + "runs": 5 + }, + "postgres": { + "median": 658, + "minimum": 641, + "maximum": 780, + "runs": 5 + }, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null, + "runs": 0 + } + }, + { + "metric": "resource/aggregate_accounted_io_wbytes", + "unit": "bytes", + "sqlite": { + "median": 2474217472, + "minimum": 2448142336, + "maximum": 2493005824, + "runs": 5 + }, + "postgres": { + "median": 454680576, + "minimum": 395005952, + "maximum": 532303872, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.1857247306718689, + "minimum": 0.15964884108618888, + "maximum": 0.21525468085388316, + "runs": 5 + } + }, + { + "metric": "resource/aggregate_accounted_io_wios", + "unit": "operations", + "sqlite": { + "median": 218688, + "minimum": 213997, + "maximum": 223605, + "runs": 5 + }, + "postgres": { + "median": 21733, + "minimum": 17970, + "maximum": 25300, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.10155749846960471, + "minimum": 0.08217186128182616, + "maximum": 0.11618295371050698, + "runs": 5 + } + }, + { + "metric": "resource/average_cores_over_sampled_span", + "unit": "cores", + "sqlite": { + "median": 0.2421726130982453, + "minimum": 0.22948731146480547, + "maximum": 0.24844286438872823, + "runs": 5 + }, + "postgres": { + "median": 0.3190538891109188, + "minimum": 0.3099210433113059, + "maximum": 0.37064201455642537, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.3244161537813468, + "minimum": 1.279752649758031, + "maximum": 1.4918601726330816, + "runs": 5 + } + }, + { + "metric": "resource/cpu_delta_usec", + "unit": "us", + "sqlite": { + "median": 43453559, + "minimum": 41180142, + "maximum": 44576923, + "runs": 5 + }, + "postgres": { + "median": 56857596, + "minimum": 55349210, + "maximum": 66467712, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.331544725935759, + "minimum": 1.2737555052740328, + "maximum": 1.4910789602952181, + "runs": 5 + } + }, + { + "metric": "resource/observed_peak_cgroup_memory_bytes", + "unit": "bytes", + "sqlite": { + "median": 4239114240, + "minimum": 4208447488, + "maximum": 4276535296, + "runs": 5 + }, + "postgres": { + "median": 5956444160, + "minimum": 5852491776, + "maximum": 6442434560, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.4153542789791844, + "minimum": 1.3887448790657666, + "maximum": 1.5183966312342583, + "runs": 5 + } + }, + { + "metric": "resource/observed_peak_pss_bytes", + "unit": "bytes", + "sqlite": { + "median": 3064301568, + "minimum": 3032899584, + "maximum": 3067539456, + "runs": 5 + }, + "postgres": { + "median": 3404889088, + "minimum": 3355164672, + "maximum": 3482287104, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1099740156039903, + "minimum": 1.1004889879199078, + "maximum": 1.136404830505246, + "runs": 5 + } + }, + { + "metric": "resource/observed_peak_rss_bytes", + "unit": "bytes", + "sqlite": { + "median": 3067809792, + "minimum": 3036344320, + "maximum": 3071049728, + "runs": 5 + }, + "postgres": { + "median": 5300330496, + "minimum": 5206446080, + "maximum": 5474234368, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.727724616376738, + "minimum": 1.6961002257721889, + "maximum": 1.8029030278094416, + "runs": 5 + } + }, + { + "metric": "resource/observed_peak_storage_bytes", + "unit": "bytes", + "sqlite": { + "median": 1150824992, + "minimum": 1147743112, + "maximum": 1176985880, + "runs": 5 + }, + "postgres": { + "median": 2439638844, + "minimum": 2439212860, + "maximum": 2439704380, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 2.119961242551813, + "minimum": 2.072785141653526, + "maximum": 2.125653688958928, + "runs": 5 + } + }, + { + "metric": "resource/observed_peak_wal_bytes", + "unit": "bytes", + "sqlite": { + "median": 41949872, + "minimum": 30323232, + "maximum": 55591192, + "runs": 5 + }, + "postgres": { + "median": 1073741824, + "minimum": 1073741824, + "maximum": 1073741824, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 25.595830757242837, + "minimum": 19.314963133008554, + "maximum": 35.40987398704729, + "runs": 5 + } + }, + { + "metric": "retention-empty/elapsed_ns", + "unit": "ns", + "sqlite": { + "median": 267672, + "minimum": 257609, + "maximum": 307003, + "runs": 5 + }, + "postgres": { + "median": 1732086, + "minimum": 1672475, + "maximum": 1941161, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 6.470927104814848, + "minimum": 5.936102369156306, + "maximum": 7.458889213365713, + "runs": 5 + } + }, + { + "metric": "retention/elapsed_ns", + "unit": "ns", + "sqlite": { + "median": 10489658913, + "minimum": 10454347498, + "maximum": 10732519816, + "runs": 5 + }, + "postgres": { + "median": 904965331, + "minimum": 844435612, + "maximum": 962246927, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.08625904967316574, + "minimum": 0.08050172260162602, + "maximum": 0.09126157824213459, + "runs": 5 + } + }, + { + "metric": "sql-miss/channel_messages_sql/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 13.803637268524518, + "minimum": 12.727422448967895, + "maximum": 14.11584387767697, + "runs": 5 + }, + "postgres": { + "median": 8.947721906222263, + "minimum": 7.6990947866757455, + "maximum": 9.50359522922496, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.6471224599012999, + "minimum": 0.6049217598886323, + "maximum": 0.6732573207510536, + "runs": 5 + } + }, + { + "metric": "sql-miss/observer_metrics_sql/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 13.916314694145312, + "minimum": 12.833074294306549, + "maximum": 14.234787782862767, + "runs": 5 + }, + "postgres": { + "median": 9.088872506759705, + "minimum": 7.870279764029642, + "maximum": 9.656813430464279, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.6520169161197115, + "minimum": 0.6132809320305522, + "maximum": 0.6783953212207419, + "runs": 5 + } + }, + { + "metric": "startup-hot1/first_http_ns", + "unit": "ns", + "sqlite": { + "median": 9565475272, + "minimum": 9466903660, + "maximum": 10177266363, + "runs": 5 + }, + "postgres": { + "median": 11482653279, + "minimum": 10976838200, + "maximum": 11679804322, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1870159211915485, + "minimum": 1.1475476009154855, + "maximum": 1.2226419423602755, + "runs": 5 + } + }, + { + "metric": "startup-hot1/full_ready_ns", + "unit": "ns", + "sqlite": { + "median": 20519912326, + "minimum": 20126865883, + "maximum": 21753020764, + "runs": 5 + }, + "postgres": { + "median": 20145659861, + "minimum": 19918712935, + "maximum": 20651780523, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9707016588838812, + "minimum": 0.9493752958291435, + "maximum": 1.0057338078203957, + "runs": 5 + } + }, + { + "metric": "startup-hot1/ready_ns", + "unit": "ns", + "sqlite": { + "median": 9565477948, + "minimum": 9466905553, + "maximum": 10177269181, + "runs": 5 + }, + "postgres": { + "median": 11482655439, + "minimum": 10976841225, + "maximum": 11679807005, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1870159040970771, + "minimum": 1.1475475961235262, + "maximum": 1.2226420229081163, + "runs": 5 + } + }, + { + "metric": "startup/first_http_ns", + "unit": "ns", + "sqlite": { + "median": 28044152778, + "minimum": 27441447101, + "maximum": 30175141980, + "runs": 5 + }, + "postgres": { + "median": 27234077804, + "minimum": 27136667613, + "maximum": 28553473233, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9713297647748653, + "minimum": 0.9382991305301575, + "maximum": 0.9895916210658721, + "runs": 5 + } + }, + { + "metric": "startup/full_ready_ns", + "unit": "ns", + "sqlite": { + "median": 28057500751, + "minimum": 27452984839, + "maximum": 30187755720, + "runs": 5 + }, + "postgres": { + "median": 27443441037, + "minimum": 27218956850, + "maximum": 29313354989, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.978339257672393, + "minimum": 0.9402844960321558, + "maximum": 0.9988984779409157, + "runs": 5 + } + }, + { + "metric": "startup/ready_ns", + "unit": "ns", + "sqlite": { + "median": 28044154793, + "minimum": 27441450067, + "maximum": 30175144085, + "runs": 5 + }, + "postgres": { + "median": 27234080235, + "minimum": 27136669504, + "maximum": 28553475797, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9713297618216723, + "minimum": 0.9382991448170441, + "maximum": 0.9895916676699824, + "runs": 5 + } + } + ], + "runs": [ + { + "pair": 0, + "backend": "sqlite", + "metrics": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "samples": 2700, + "p50": 822717, + "p95": 1353189, + "p99": 3475992, + "maximum": 404485452 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "samples": 2700, + "p50": 642805, + "p95": 1460251883, + "p99": 6889936133, + "maximum": 8471902779 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "end_to_end", + "samples": 2700, + "p50": 1483596, + "p95": 1480236807, + "p99": 6890210177, + "maximum": 8472258330 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "samples": 1350, + "p50": 699461, + "p95": 1108514, + "p99": 2023963, + "maximum": 349841168 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "samples": 1350, + "p50": 622088, + "p95": 1420940299, + "p99": 6811915680, + "maximum": 8354270160 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "end_to_end", + "samples": 1350, + "p50": 1327799, + "p95": 1421166145, + "p99": 6831905094, + "maximum": 8354689608 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "samples": 450, + "p50": 1004443, + "p95": 1519725, + "p99": null, + "maximum": 394749837 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "samples": 450, + "p50": 703181, + "p95": 1381375643, + "p99": null, + "maximum": 8295180714 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "end_to_end", + "samples": 450, + "p50": 1814254, + "p95": 1381879868, + "p99": null, + "maximum": 8295765208 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "samples": 4500, + "p50": 1073497, + "p95": 1803359, + "p99": 4040757, + "maximum": 401354757 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "samples": 4500, + "p50": 638381, + "p95": 1283830060, + "p99": 7045907109, + "maximum": 8490684254 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "end_to_end", + "samples": 4500, + "p50": 1830858, + "p95": 1303820971, + "p99": 7046454983, + "maximum": 8510657560 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "samples": 72, + "p50": 30106300, + "p95": 34090682, + "p99": null, + "maximum": 45578951 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "samples": 72, + "p50": 355485, + "p95": 1042778, + "p99": null, + "maximum": 1099915 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "end_to_end", + "samples": 72, + "p50": 30578375, + "p95": 34202694, + "p99": null, + "maximum": 46113464 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "samples": 108, + "p50": 1748543, + "p95": 2581029, + "p99": null, + "maximum": 2987197 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "samples": 108, + "p50": 609677, + "p95": 1107672, + "p99": null, + "maximum": 1145775 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "end_to_end", + "samples": 108, + "p50": 2564913, + "p95": 3126130, + "p99": null, + "maximum": 4109964 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "samples": 72, + "p50": 1774758, + "p95": 582035908, + "p99": null, + "maximum": 746932903 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "samples": 72, + "p50": 146997, + "p95": 862300, + "p99": null, + "maximum": 1082937 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "end_to_end", + "samples": 72, + "p50": 2016137, + "p95": 582522495, + "p99": null, + "maximum": 747067759 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "samples": 108, + "p50": 30390158, + "p95": 35624242, + "p99": null, + "maximum": 52101412 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "samples": 108, + "p50": 793207, + "p95": 1086638, + "p99": null, + "maximum": 1144822 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "end_to_end", + "samples": 108, + "p50": 31061956, + "p95": 36612682, + "p99": null, + "maximum": 52751236 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "samples": 252, + "p50": 697135, + "p95": 1063634, + "p99": null, + "maximum": 17910848 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "samples": 252, + "p50": 536300, + "p95": 1076397, + "p99": null, + "maximum": 1228131 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "end_to_end", + "samples": 252, + "p50": 1262361, + "p95": 1822537, + "p99": null, + "maximum": 18870086 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "samples": 252, + "p50": 1365008, + "p95": 63539454, + "p99": null, + "maximum": 71010924 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "samples": 252, + "p50": 650462, + "p95": 1053613, + "p99": null, + "maximum": 2513445 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "end_to_end", + "samples": 252, + "p50": 2063745, + "p95": 64019267, + "p99": null, + "maximum": 71554399 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "samples": 216, + "p50": 1393196, + "p95": 86201479, + "p99": null, + "maximum": 89038098 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "samples": 216, + "p50": 726902, + "p95": 1087887, + "p99": null, + "maximum": 1178760 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "end_to_end", + "samples": 216, + "p50": 2142243, + "p95": 86906543, + "p99": null, + "maximum": 89997231 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "samples": 180, + "p50": 3395092, + "p95": 4075310, + "p99": null, + "maximum": 4510443 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "samples": 180, + "p50": 482327, + "p95": 1055173, + "p99": null, + "maximum": 1099812 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "end_to_end", + "samples": 180, + "p50": 3950121, + "p95": 4579403, + "p99": null, + "maximum": 5263838 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "samples": 180, + "p50": 1429911, + "p95": 1632132, + "p99": null, + "maximum": 2094765 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "samples": 180, + "p50": 678346, + "p95": 1053972, + "p99": null, + "maximum": 1259463 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "end_to_end", + "samples": 180, + "p50": 2160152, + "p95": 2548932, + "p99": null, + "maximum": 2843618 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "samples": 252, + "p50": 62987426, + "p95": 119444892, + "p99": null, + "maximum": 165655812 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "samples": 252, + "p50": 618306, + "p95": 1080958, + "p99": null, + "maximum": 1493506 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "end_to_end", + "samples": 252, + "p50": 63601574, + "p95": 120081659, + "p99": null, + "maximum": 165782510 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "samples": 252, + "p50": 3413817, + "p95": 7703376, + "p99": null, + "maximum": 21052099 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "samples": 252, + "p50": 229739, + "p95": 1005852, + "p99": null, + "maximum": 2471378 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "end_to_end", + "samples": 252, + "p50": 3765067, + "p95": 8125197, + "p99": null, + "maximum": 21477196 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "samples": 216, + "p50": 5637300, + "p95": 11606900, + "p99": null, + "maximum": 38950594 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "samples": 216, + "p50": 481205, + "p95": 1051484, + "p99": null, + "maximum": 1094662 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "end_to_end", + "samples": 216, + "p50": 6172381, + "p95": 12196883, + "p99": null, + "maximum": 39528431 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "samples": 360, + "p50": 2967040, + "p95": 4060831, + "p99": null, + "maximum": 19753323 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "samples": 360, + "p50": 566791, + "p95": 1055795, + "p99": null, + "maximum": 1226552 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "end_to_end", + "samples": 360, + "p50": 3605090, + "p95": 4899550, + "p99": null, + "maximum": 20292297 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "samples": 180, + "p50": 1280459, + "p95": 1505468, + "p99": null, + "maximum": 1792154 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "samples": 180, + "p50": 540534, + "p95": 1057173, + "p99": null, + "maximum": 1106582 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 1867882, + "p95": 2345033, + "p99": null, + "maximum": 2827333 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "samples": 45, + "p50": 693450, + "p95": 772831, + "p99": null, + "maximum": 803187 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "samples": 45, + "p50": 540982, + "p95": 1060010, + "p99": null, + "maximum": 1118134 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "end_to_end", + "samples": 45, + "p50": 1259735, + "p95": 1790719, + "p99": null, + "maximum": 1863197 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "samples": 135, + "p50": 1155457, + "p95": 1351718, + "p99": null, + "maximum": 1457190 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "samples": 135, + "p50": 566105, + "p95": 1032975, + "p99": null, + "maximum": 1084758 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "end_to_end", + "samples": 135, + "p50": 1695614, + "p95": 2224465, + "p99": null, + "maximum": 2347883 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "samples": 180, + "p50": 32670356, + "p95": 35263850, + "p99": null, + "maximum": 39281021 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "samples": 180, + "p50": 547346, + "p95": 1055509, + "p99": null, + "maximum": 1088314 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 33274626, + "p95": 35972527, + "p99": null, + "maximum": 40105335 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "samples": 180, + "p50": 804994, + "p95": 912227, + "p99": null, + "maximum": 1512065 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "samples": 180, + "p50": 611834, + "p95": 1056347, + "p99": null, + "maximum": 1095785 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "end_to_end", + "samples": 180, + "p50": 1386337, + "p95": 1885075, + "p99": null, + "maximum": 1996264 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "samples": 180, + "p50": 1603798, + "p95": 1830097, + "p99": null, + "maximum": 3317848 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "samples": 180, + "p50": 601193, + "p95": 1045229, + "p99": null, + "maximum": 1089815 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "end_to_end", + "samples": 180, + "p50": 2200032, + "p95": 2722840, + "p99": null, + "maximum": 4225314 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "samples": 180, + "p50": 487079, + "p95": 2302026, + "p99": null, + "maximum": 5978163 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "samples": 180, + "p50": 575428, + "p95": 1035433, + "p99": null, + "maximum": 1540466 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "end_to_end", + "samples": 180, + "p50": 1144805, + "p95": 2703840, + "p99": null, + "maximum": 6459395 + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "samples": 100, + "p50": 81735095, + "p95": 102979309, + "p99": null, + "maximum": 177324852 + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "samples": 100, + "p50": 765374, + "p95": 1466828, + "p99": null, + "maximum": 3049952 + }, + { + "scenario": "websocket", + "event_class": "new_transmission", + "timing": "receipt_after_ingest_return", + "samples": 4500, + "p50": 506551784, + "p95": 985117025, + "p99": 1330045457, + "maximum": 2216773588 + }, + { + "scenario": "handler-control", + "event_class": "200_valid_envelopes", + "timing": "service", + "samples": 200, + "p50": 875069, + "p95": 2087313, + "p99": null, + "maximum": 3516211 + } + ], + "counts": { + "durable-ingest": { + "total": 12000, + "measured": 9000, + "successful": 12000, + "errors": 0, + "drops": 0 + }, + "mixed-http": { + "total": 4800, + "measured": 3600, + "successful": 4800, + "errors": 0, + "drops": 0 + }, + "sql-miss": { + "total": 200, + "measured": 200, + "successful": 200, + "errors": 0, + "drops": 0 + } + }, + "throughput_including_drain": { + "durable-ingest/new": 25.030303885212806, + "durable-ingest/additional": 15.024883039516112, + "durable-ingest/duplicate": 7.514968153875817, + "durable-ingest/late": 2.505553877065402, + "mixed-http/nodes_50": 1.432603768572011, + "mixed-http/nodes_region": 1.2289781632667003, + "mixed-http/nodes_2000": 1.4321151292709695, + "mixed-http/channels": 1.4326196041451482, + "mixed-http/messages_0": 1.432615991280587, + "mixed-http/messages_1000": 1.2290040265927031, + "mixed-http/observers": 2.0518231676513077, + "mixed-http/metrics_dense_24h": 1.0262036695951664, + "mixed-http/metrics_sparse_7d": 1.0262144336331263, + "mixed-http/reach": 1.0262162866643942, + "mixed-http/rx_coverage": 1.0262140751349913, + "mixed-http/packets_memory": 1.0248652808168723, + "mixed-http/packet_detail_memory": 1.0250479314243583, + "mixed-http/packet_detail_sql": 1.0336749881763692, + "mixed-http/analytics_rf": 0.6157291551939691, + "mixed-http/analytics_topology": 0.6156225331026979, + "mixed-http/analytics_channels": 0.410888368563797, + "mixed-http/analytics_rf_filtered": 0.4109542260360824, + "mixed-http/stats": 1.027390984809161, + "mixed-http/packet_detail_pruned": 1.1083413262638564, + "sql-miss/channel_messages_sql": 12.727422448967895, + "sql-miss/observer_metrics_sql": 12.833074294306549 + }, + "scalars_ns": { + "startup/first_http_ns": 30175141980, + "startup/ready_ns": 30175144085, + "startup/full_ready_ns": 30187755720, + "startup-hot1/first_http_ns": 10177266363, + "startup-hot1/ready_ns": 10177269181, + "startup-hot1/full_ready_ns": 21753020764, + "retention/elapsed_ns": 10454347498, + "retention-empty/elapsed_ns": 307003 + }, + "resources": { + "sample_count": 180, + "window_start_ns": 933793567979, + "window_end_ns": 1113793567979, + "sampled_start_ns": 934111560487, + "sampled_end_ns": 1113536810271, + "sampled_span_ns": 179425249784, + "sampled_span_fraction": 0.9968069432444444, + "first_boundary_gap_ns": 317992508, + "last_boundary_gap_ns": 256757708, + "sampling_interval_ns": { + "median": 1002233643, + "minimum": 1001956569, + "maximum": 1005527344, + "runs": 179 + }, + "phases": [ + "mixed_with_retention" + ], + "cpu_delta_usec": 44576923, + "average_cores_over_sampled_span": 0.24844286438872823, + "per_device_io": { + "8:0": { + "rbytes": 0, + "wbytes": 2472902656, + "wios": 217760, + "rios": 0, + "dios": 0, + "dbytes": 0 + } + }, + "aggregate_accounted_io": { + "dbytes": 0, + "dios": 0, + "rbytes": 0, + "rios": 0, + "wbytes": 2472902656, + "wios": 217760 + }, + "observed_peaks": { + "cgroup_memory_bytes": 4214231040, + "rss_bytes": 3049676800, + "pss_bytes": 3046232064, + "storage_bytes": 1148091056, + "wal_bytes": 41949872 + }, + "memory_event_deltas": { + "oom_group_kill": 0, + "oom": 0, + "high": 0, + "max": 0, + "oom_kill": 0, + "low": 0 + }, + "minimum_free_bytes": 85368725504 + }, + "websocket": { + "full_stream_sentinels": 6000, + "full_stream_received": 6000, + "measured_sentinels": 4500, + "measured_missing": 0, + "full_stream_missing": 0, + "negative_measured_lags": 1, + "negative_full_stream_lags": 1, + "duplicate_measured_receipts": 0, + "duplicate_full_stream_receipts": 0 + }, + "retention": { + "retention": { + "elapsed_ns": 10454347498, + "transmissions_deleted": 16000, + "observations_deleted": 256000 + }, + "retention-empty": { + "elapsed_ns": 307003, + "transmissions_deleted": 0, + "observations_deleted": 0 + } + }, + "startup_counts": [ + { + "transmissions": 112000, + "observations": 1792000 + }, + { + "transmissions": 112000, + "observations": 1792000 + } + ], + "database_settings": { + "busy_timeout": "5000", + "cache_size": "-2000", + "cache_spill": "483", + "foreign_keys": "1", + "journal_mode": "wal", + "journal_size_limit": "-1", + "mmap_size": "0", + "page_size": "4096", + "sqlite_version": "3.53.4", + "synchronous": "2", + "wal_autocheckpoint": "1000" + }, + "durable_counts": { + "scheduled": 12000, + "completed": 12000, + "dropped_schedules": 0, + "write_errors": 0, + "actual_transmissions": 118000, + "actual_observations": 1802200 + } + }, + { + "pair": 0, + "backend": "postgres", + "metrics": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "samples": 2700, + "p50": 1450290, + "p95": 2185353, + "p99": 3568311, + "maximum": 89163536 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "samples": 2700, + "p50": 629399, + "p95": 1079794, + "p99": 1137461, + "maximum": 70095378 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "end_to_end", + "samples": 2700, + "p50": 2149712, + "p95": 2956499, + "p99": 4621416, + "maximum": 90067012 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "samples": 1350, + "p50": 1314432, + "p95": 1761448, + "p99": 3129539, + "maximum": 35314208 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "samples": 1350, + "p50": 629875, + "p95": 1078714, + "p99": 1139867, + "maximum": 52608569 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "end_to_end", + "samples": 1350, + "p50": 1978513, + "p95": 2628900, + "p99": 4712941, + "maximum": 59810955 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "samples": 450, + "p50": 1630812, + "p95": 2122728, + "p99": null, + "maximum": 41333058 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "samples": 450, + "p50": 643936, + "p95": 1072073, + "p99": null, + "maximum": 20794903 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "end_to_end", + "samples": 450, + "p50": 2265928, + "p95": 2952505, + "p99": null, + "maximum": 51037560 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "samples": 4500, + "p50": 1606693, + "p95": 2211119, + "p99": 2952996, + "maximum": 29983348 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "samples": 4500, + "p50": 645825, + "p95": 1084325, + "p99": 1152250, + "maximum": 31063041 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "end_to_end", + "samples": 4500, + "p50": 2310597, + "p95": 2997415, + "p99": 4424840, + "maximum": 42526825 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "samples": 72, + "p50": 30538591, + "p95": 40572855, + "p99": null, + "maximum": 55667691 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "samples": 72, + "p50": 511851, + "p95": 1064886, + "p99": null, + "maximum": 4357770 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "end_to_end", + "samples": 72, + "p50": 30712876, + "p95": 41080871, + "p99": null, + "maximum": 56405383 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "samples": 108, + "p50": 1595196, + "p95": 2590758, + "p99": null, + "maximum": 4254177 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "samples": 108, + "p50": 588067, + "p95": 1043172, + "p99": null, + "maximum": 1087950 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "end_to_end", + "samples": 108, + "p50": 2258955, + "p95": 3242347, + "p99": null, + "maximum": 4748663 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "samples": 72, + "p50": 1987458, + "p95": 621344736, + "p99": null, + "maximum": 783338035 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "samples": 72, + "p50": 207353, + "p95": 984944, + "p99": null, + "maximum": 1058814 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "end_to_end", + "samples": 72, + "p50": 2325365, + "p95": 622329680, + "p99": null, + "maximum": 783488450 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "samples": 108, + "p50": 30370710, + "p95": 39288601, + "p99": null, + "maximum": 80319487 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "samples": 108, + "p50": 587783, + "p95": 1003407, + "p99": null, + "maximum": 1099439 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "end_to_end", + "samples": 108, + "p50": 30846835, + "p95": 39603734, + "p99": null, + "maximum": 80937781 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "samples": 252, + "p50": 705143, + "p95": 1458218, + "p99": null, + "maximum": 28057663 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "samples": 252, + "p50": 579771, + "p95": 1063279, + "p99": null, + "maximum": 1096737 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "end_to_end", + "samples": 252, + "p50": 1385291, + "p95": 2155784, + "p99": null, + "maximum": 28711354 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "samples": 252, + "p50": 1477472, + "p95": 178265731, + "p99": null, + "maximum": 331713480 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "samples": 252, + "p50": 650513, + "p95": 1045280, + "p99": null, + "maximum": 2727024 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "end_to_end", + "samples": 252, + "p50": 2117237, + "p95": 178983058, + "p99": null, + "maximum": 332161785 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "samples": 216, + "p50": 1455159, + "p95": 165207339, + "p99": null, + "maximum": 283088905 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "samples": 216, + "p50": 644556, + "p95": 1050120, + "p99": null, + "maximum": 1199013 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "end_to_end", + "samples": 216, + "p50": 2199043, + "p95": 165416750, + "p99": null, + "maximum": 283773072 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "samples": 180, + "p50": 3870696, + "p95": 6617901, + "p99": null, + "maximum": 12096423 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "samples": 180, + "p50": 585372, + "p95": 1056445, + "p99": null, + "maximum": 1135696 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "end_to_end", + "samples": 180, + "p50": 4495293, + "p95": 7075678, + "p99": null, + "maximum": 12700233 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "samples": 180, + "p50": 1787384, + "p95": 2118583, + "p99": null, + "maximum": 5251559 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "samples": 180, + "p50": 515866, + "p95": 963861, + "p99": null, + "maximum": 1099149 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "end_to_end", + "samples": 180, + "p50": 2337098, + "p95": 2986641, + "p99": null, + "maximum": 5731642 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "samples": 252, + "p50": 64361304, + "p95": 103797969, + "p99": null, + "maximum": 272290669 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "samples": 252, + "p50": 475594, + "p95": 1054562, + "p99": null, + "maximum": 4322208 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "end_to_end", + "samples": 252, + "p50": 64695942, + "p95": 104746733, + "p99": null, + "maximum": 273160204 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "samples": 252, + "p50": 5322495, + "p95": 9643424, + "p99": null, + "maximum": 30001856 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "samples": 252, + "p50": 175403, + "p95": 1066486, + "p99": null, + "maximum": 1211644 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "end_to_end", + "samples": 252, + "p50": 5645859, + "p95": 9891781, + "p99": null, + "maximum": 30393030 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "samples": 216, + "p50": 7812237, + "p95": 15279315, + "p99": null, + "maximum": 38672611 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "samples": 216, + "p50": 511210, + "p95": 1065395, + "p99": null, + "maximum": 1328858 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "end_to_end", + "samples": 216, + "p50": 8484868, + "p95": 15574976, + "p99": null, + "maximum": 39127178 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "samples": 360, + "p50": 2975966, + "p95": 4617742, + "p99": null, + "maximum": 112010937 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "samples": 360, + "p50": 581177, + "p95": 1047726, + "p99": null, + "maximum": 1098343 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "end_to_end", + "samples": 360, + "p50": 3543096, + "p95": 5226672, + "p99": null, + "maximum": 112120519 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "samples": 180, + "p50": 2778915, + "p95": 3295266, + "p99": null, + "maximum": 18708475 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "samples": 180, + "p50": 588398, + "p95": 1034694, + "p99": null, + "maximum": 1087439 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 3347848, + "p95": 4101247, + "p99": null, + "maximum": 19197908 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "samples": 45, + "p50": 732517, + "p95": 1373714, + "p99": null, + "maximum": 1564420 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "samples": 45, + "p50": 513818, + "p95": 1036393, + "p99": null, + "maximum": 1114998 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "end_to_end", + "samples": 45, + "p50": 1353143, + "p95": 2102929, + "p99": null, + "maximum": 2410107 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "samples": 135, + "p50": 1625972, + "p95": 2641595, + "p99": null, + "maximum": 8957371 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "samples": 135, + "p50": 634898, + "p95": 1048145, + "p99": null, + "maximum": 1101724 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "end_to_end", + "samples": 135, + "p50": 2315509, + "p95": 3427111, + "p99": null, + "maximum": 9893578 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "samples": 180, + "p50": 36264010, + "p95": 39133225, + "p99": null, + "maximum": 140352889 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "samples": 180, + "p50": 587488, + "p95": 1055496, + "p99": null, + "maximum": 1092965 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 36850675, + "p95": 40126398, + "p99": null, + "maximum": 140618585 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "samples": 180, + "p50": 840284, + "p95": 982117, + "p99": null, + "maximum": 2006872 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "samples": 180, + "p50": 632978, + "p95": 1053007, + "p99": null, + "maximum": 1102769 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "end_to_end", + "samples": 180, + "p50": 1475761, + "p95": 1927129, + "p99": null, + "maximum": 2578942 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "samples": 180, + "p50": 1845767, + "p95": 2229272, + "p99": null, + "maximum": 3531773 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "samples": 180, + "p50": 567655, + "p95": 1013900, + "p99": null, + "maximum": 1086401 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "end_to_end", + "samples": 180, + "p50": 2420877, + "p95": 3055676, + "p99": null, + "maximum": 4215331 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "samples": 180, + "p50": 462567, + "p95": 4775971, + "p99": null, + "maximum": 21444108 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "samples": 180, + "p50": 561443, + "p95": 1027094, + "p99": null, + "maximum": 1119776 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "end_to_end", + "samples": 180, + "p50": 1142754, + "p95": 5315999, + "p99": null, + "maximum": 22463119 + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "samples": 100, + "p50": 118295581, + "p95": 183925783, + "p99": null, + "maximum": 404823519 + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "samples": 100, + "p50": 892350, + "p95": 1382099, + "p99": null, + "maximum": 4560109 + }, + { + "scenario": "websocket", + "event_class": "new_transmission", + "timing": "receipt_after_ingest_return", + "samples": 4500, + "p50": 508362757, + "p95": 954551023, + "p99": 994595773, + "maximum": 1043938984 + }, + { + "scenario": "handler-control", + "event_class": "200_valid_envelopes", + "timing": "service", + "samples": 200, + "p50": 1727646, + "p95": 2395291, + "p99": null, + "maximum": 5784460 + } + ], + "counts": { + "durable-ingest": { + "total": 12000, + "measured": 9000, + "successful": 12000, + "errors": 0, + "drops": 0 + }, + "mixed-http": { + "total": 4800, + "measured": 3600, + "successful": 4800, + "errors": 0, + "drops": 0 + }, + "sql-miss": { + "total": 200, + "measured": 200, + "successful": 200, + "errors": 0, + "drops": 0 + } + }, + "throughput_including_drain": { + "durable-ingest/new": 25.030240536519056, + "durable-ingest/additional": 15.024894453131463, + "durable-ingest/duplicate": 7.514973916479687, + "durable-ingest/late": 2.505532175372296, + "mixed-http/nodes_50": 1.432585776710199, + "mixed-http/nodes_region": 1.228968491025172, + "mixed-http/nodes_2000": 1.4321355727779226, + "mixed-http/channels": 1.432617330831885, + "mixed-http/messages_0": 1.4326181626051169, + "mixed-http/messages_1000": 1.229002983760107, + "mixed-http/observers": 2.051829810083036, + "mixed-http/metrics_dense_24h": 1.0261996446974946, + "mixed-http/metrics_sparse_7d": 1.0262120032342275, + "mixed-http/reach": 1.026217147836336, + "mixed-http/rx_coverage": 1.0262117597550047, + "mixed-http/packets_memory": 1.0248433660311196, + "mixed-http/packet_detail_memory": 1.0250382001440197, + "mixed-http/packet_detail_sql": 1.0336745599044228, + "mixed-http/analytics_rf": 0.6157240798681688, + "mixed-http/analytics_topology": 0.6156281904355956, + "mixed-http/analytics_channels": 0.41088541298749126, + "mixed-http/analytics_rf_filtered": 0.41095430615484063, + "mixed-http/stats": 1.0273883881301913, + "mixed-http/packet_detail_pruned": 1.10835276821805, + "sql-miss/channel_messages_sql": 7.6990947866757455, + "sql-miss/observer_metrics_sql": 7.870279764029642 + }, + "scalars_ns": { + "startup/first_http_ns": 28553473233, + "startup/ready_ns": 28553475797, + "startup/full_ready_ns": 29313354989, + "startup-hot1/first_http_ns": 11679804322, + "startup-hot1/ready_ns": 11679807005, + "startup-hot1/full_ready_ns": 20651780523, + "retention/elapsed_ns": 908732990, + "retention-empty/elapsed_ns": 1906958, + "migration/elapsed_ns": 144734628603, + "migration/migration_tool_ns": 89942496573, + "migration/role_grants_ns": 826441189, + "migration/harness_validation_ns": 53965692102 + }, + "resources": { + "sample_count": 175, + "window_start_ns": 1486680727718, + "window_end_ns": 1666680727718, + "sampled_start_ns": 1486978768703, + "sampled_end_ns": 1666310062484, + "sampled_span_ns": 179331293781, + "sampled_span_fraction": 0.99628496545, + "first_boundary_gap_ns": 298040985, + "last_boundary_gap_ns": 370665234, + "sampling_interval_ns": { + "median": 1029612202.5, + "minimum": 1027417903, + "maximum": 1057627376, + "runs": 174 + }, + "phases": [ + "mixed_with_retention" + ], + "cpu_delta_usec": 66467712, + "average_cores_over_sampled_span": 0.37064201455642537, + "per_device_io": { + "8:0": { + "rbytes": 5840896, + "wbytes": 532303872, + "wios": 25300, + "rios": 652, + "dios": 1, + "dbytes": 4096 + } + }, + "aggregate_accounted_io": { + "dbytes": 4096, + "dios": 1, + "rbytes": 5840896, + "rios": 652, + "wbytes": 532303872, + "wios": 25300 + }, + "observed_peaks": { + "cgroup_memory_bytes": 5852491776, + "rss_bytes": 5408137216, + "pss_bytes": 3355164672, + "storage_bytes": 2439212860, + "wal_bytes": 1073741824 + }, + "memory_event_deltas": { + "oom_group_kill": 0, + "oom": 0, + "high": 0, + "max": 0, + "oom_kill": 0, + "low": 0 + }, + "minimum_free_bytes": 79697489920 + }, + "websocket": { + "full_stream_sentinels": 6000, + "full_stream_received": 6000, + "measured_sentinels": 4500, + "measured_missing": 0, + "full_stream_missing": 0, + "negative_measured_lags": 0, + "negative_full_stream_lags": 0, + "duplicate_measured_receipts": 0, + "duplicate_full_stream_receipts": 0 + }, + "retention": { + "retention": { + "elapsed_ns": 908732990, + "transmissions_deleted": 16000, + "observations_deleted": 256000 + }, + "retention-empty": { + "elapsed_ns": 1906958, + "transmissions_deleted": 0, + "observations_deleted": 0 + } + }, + "startup_counts": [ + { + "transmissions": 112000, + "observations": 1792000 + }, + { + "transmissions": 112000, + "observations": 1792000 + } + ], + "database_settings": { + "autovacuum": "on", + "checkpoint_timeout": "5min", + "effective_cache_size": "4GB", + "fsync": "on", + "full_page_writes": "on", + "jit": "on", + "maintenance_work_mem": "64MB", + "max_connections": "32", + "max_parallel_workers_per_gather": "2", + "max_wal_size": "1GB", + "server_version": "18.6 (Ubuntu 18.6-1.pgdg24.04+2)", + "shared_buffers": "256MB", + "synchronous_commit": "on", + "work_mem": "4MB" + }, + "durable_counts": { + "scheduled": 12000, + "completed": 12000, + "dropped_schedules": 0, + "write_errors": 0, + "actual_transmissions": 118000, + "actual_observations": 1802200 + } + }, + { + "pair": 1, + "backend": "sqlite", + "metrics": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "samples": 2700, + "p50": 761548, + "p95": 1252327, + "p99": 3211164, + "maximum": 393142873 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "samples": 2700, + "p50": 632631, + "p95": 1433325433, + "p99": 6995259868, + "maximum": 8561484192 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "end_to_end", + "samples": 2700, + "p50": 1461095, + "p95": 1453320040, + "p99": 7015254002, + "maximum": 8561831134 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "samples": 1350, + "p50": 643791, + "p95": 1005604, + "p99": 1490989, + "maximum": 502093344 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "samples": 1350, + "p50": 634966, + "p95": 1393778079, + "p99": 7196012542, + "maximum": 8443222453 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "end_to_end", + "samples": 1350, + "p50": 1340017, + "p95": 1394158178, + "p99": 7248279720, + "maximum": 8443385701 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "samples": 450, + "p50": 938949, + "p95": 1463580, + "p99": null, + "maximum": 408243164 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "samples": 450, + "p50": 627724, + "p95": 1354374580, + "p99": null, + "maximum": 8383724192 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "end_to_end", + "samples": 450, + "p50": 1612892, + "p95": 1354740231, + "p99": null, + "maximum": 8384202466 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "samples": 4500, + "p50": 1002572, + "p95": 1705386, + "p99": 3760552, + "maximum": 410874562 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "samples": 4500, + "p50": 629344, + "p95": 1237271650, + "p99": 7112861352, + "maximum": 8580902537 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "end_to_end", + "samples": 4500, + "p50": 1660259, + "p95": 1257262620, + "p99": 7113311177, + "maximum": 8600879038 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "samples": 72, + "p50": 29513239, + "p95": 33608550, + "p99": null, + "maximum": 74262516 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "samples": 72, + "p50": 426083, + "p95": 918210, + "p99": null, + "maximum": 3519544 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "end_to_end", + "samples": 72, + "p50": 30108886, + "p95": 33707141, + "p99": null, + "maximum": 77782060 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "samples": 108, + "p50": 1741725, + "p95": 2558364, + "p99": null, + "maximum": 4647535 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "samples": 108, + "p50": 823685, + "p95": 1051723, + "p99": null, + "maximum": 1121182 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "end_to_end", + "samples": 108, + "p50": 2569790, + "p95": 3261937, + "p99": null, + "maximum": 5667467 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "samples": 72, + "p50": 1652674, + "p95": 783197789, + "p99": null, + "maximum": 1202712431 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "samples": 72, + "p50": 119520, + "p95": 845888, + "p99": null, + "maximum": 1814300 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "end_to_end", + "samples": 72, + "p50": 1897638, + "p95": 783293577, + "p99": null, + "maximum": 1204526731 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "samples": 108, + "p50": 29630043, + "p95": 32519796, + "p99": null, + "maximum": 83946421 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "samples": 108, + "p50": 808375, + "p95": 1026070, + "p99": null, + "maximum": 1175558 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "end_to_end", + "samples": 108, + "p50": 30277588, + "p95": 33416628, + "p99": null, + "maximum": 84900109 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "samples": 252, + "p50": 669180, + "p95": 996697, + "p99": null, + "maximum": 18942776 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "samples": 252, + "p50": 607359, + "p95": 1060203, + "p99": null, + "maximum": 1740128 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "end_to_end", + "samples": 252, + "p50": 1316364, + "p95": 1768156, + "p99": null, + "maximum": 19602485 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "samples": 252, + "p50": 1342809, + "p95": 60342156, + "p99": null, + "maximum": 73836438 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "samples": 252, + "p50": 562789, + "p95": 1015061, + "p99": null, + "maximum": 1081174 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "end_to_end", + "samples": 252, + "p50": 1969350, + "p95": 60759749, + "p99": null, + "maximum": 74602292 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "samples": 216, + "p50": 1408771, + "p95": 82877413, + "p99": null, + "maximum": 90795405 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "samples": 216, + "p50": 695104, + "p95": 1038778, + "p99": null, + "maximum": 3864083 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "end_to_end", + "samples": 216, + "p50": 2120618, + "p95": 83206815, + "p99": null, + "maximum": 90903921 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "samples": 180, + "p50": 3283949, + "p95": 4095037, + "p99": null, + "maximum": 5156794 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "samples": 180, + "p50": 566798, + "p95": 1048472, + "p99": null, + "maximum": 1090003 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "end_to_end", + "samples": 180, + "p50": 3886390, + "p95": 4909050, + "p99": null, + "maximum": 5845728 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "samples": 180, + "p50": 1389020, + "p95": 2146589, + "p99": null, + "maximum": 3002890 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "samples": 180, + "p50": 625627, + "p95": 1036853, + "p99": null, + "maximum": 1146816 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "end_to_end", + "samples": 180, + "p50": 2043880, + "p95": 2898488, + "p99": null, + "maximum": 3835006 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "samples": 252, + "p50": 61977627, + "p95": 121738785, + "p99": null, + "maximum": 268460359 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "samples": 252, + "p50": 599957, + "p95": 1033180, + "p99": null, + "maximum": 1824408 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "end_to_end", + "samples": 252, + "p50": 62581783, + "p95": 122082210, + "p99": null, + "maximum": 268728978 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "samples": 252, + "p50": 3340335, + "p95": 7375630, + "p99": null, + "maximum": 119748619 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "samples": 252, + "p50": 279939, + "p95": 1050931, + "p99": null, + "maximum": 3150568 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "end_to_end", + "samples": 252, + "p50": 3730003, + "p95": 8432182, + "p99": null, + "maximum": 120600483 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "samples": 216, + "p50": 5510226, + "p95": 12538040, + "p99": null, + "maximum": 159763466 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "samples": 216, + "p50": 525179, + "p95": 1059345, + "p99": null, + "maximum": 5019793 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "end_to_end", + "samples": 216, + "p50": 6098029, + "p95": 13621889, + "p99": null, + "maximum": 160189116 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "samples": 360, + "p50": 2814772, + "p95": 3757436, + "p99": null, + "maximum": 18362717 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "samples": 360, + "p50": 564717, + "p95": 1040904, + "p99": null, + "maximum": 1123011 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "end_to_end", + "samples": 360, + "p50": 3440043, + "p95": 4434308, + "p99": null, + "maximum": 18954324 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "samples": 180, + "p50": 1368605, + "p95": 1954966, + "p99": null, + "maximum": 6647826 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "samples": 180, + "p50": 531046, + "p95": 990454, + "p99": null, + "maximum": 1100438 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 1919097, + "p95": 2501474, + "p99": null, + "maximum": 7571663 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "samples": 45, + "p50": 652325, + "p95": 788506, + "p99": null, + "maximum": 893701 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "samples": 45, + "p50": 714878, + "p95": 1069767, + "p99": null, + "maximum": 1091766 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "end_to_end", + "samples": 45, + "p50": 1407753, + "p95": 1724933, + "p99": null, + "maximum": 1837185 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "samples": 135, + "p50": 1090014, + "p95": 1292966, + "p99": null, + "maximum": 1513014 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "samples": 135, + "p50": 570114, + "p95": 1047041, + "p99": null, + "maximum": 1123163 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "end_to_end", + "samples": 135, + "p50": 1680197, + "p95": 2151266, + "p99": null, + "maximum": 2414747 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "samples": 180, + "p50": 29796902, + "p95": 31499654, + "p99": null, + "maximum": 33813759 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "samples": 180, + "p50": 621192, + "p95": 1040615, + "p99": null, + "maximum": 1083582 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 30327771, + "p95": 32370208, + "p99": null, + "maximum": 34270426 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "samples": 180, + "p50": 792163, + "p95": 942957, + "p99": null, + "maximum": 1386715 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "samples": 180, + "p50": 590203, + "p95": 1043436, + "p99": null, + "maximum": 1106226 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "end_to_end", + "samples": 180, + "p50": 1404999, + "p95": 1864808, + "p99": null, + "maximum": 2141291 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "samples": 180, + "p50": 1555822, + "p95": 1810218, + "p99": null, + "maximum": 3264907 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "samples": 180, + "p50": 568150, + "p95": 1035079, + "p99": null, + "maximum": 1078975 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "end_to_end", + "samples": 180, + "p50": 2137228, + "p95": 2677591, + "p99": null, + "maximum": 3931181 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "samples": 180, + "p50": 460287, + "p95": 2278827, + "p99": null, + "maximum": 464324162 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "samples": 180, + "p50": 519204, + "p95": 999447, + "p99": null, + "maximum": 1129911 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "end_to_end", + "samples": 180, + "p50": 1068535, + "p95": 3020829, + "p99": null, + "maximum": 464515717 + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "samples": 100, + "p50": 66715720, + "p95": 80727265, + "p99": null, + "maximum": 85632678 + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "samples": 100, + "p50": 810965, + "p95": 1313757, + "p99": null, + "maximum": 1550608 + }, + { + "scenario": "websocket", + "event_class": "new_transmission", + "timing": "receipt_after_ingest_return", + "samples": 4500, + "p50": 534605779, + "p95": 991805005, + "p99": 1892262507, + "maximum": 2807199210 + }, + { + "scenario": "handler-control", + "event_class": "200_valid_envelopes", + "timing": "service", + "samples": 200, + "p50": 846119, + "p95": 2168709, + "p99": null, + "maximum": 4027477 + } + ], + "counts": { + "durable-ingest": { + "total": 12000, + "measured": 9000, + "successful": 12000, + "errors": 0, + "drops": 0 + }, + "mixed-http": { + "total": 4800, + "measured": 3600, + "successful": 4800, + "errors": 0, + "drops": 0 + }, + "sql-miss": { + "total": 200, + "measured": 200, + "successful": 200, + "errors": 0, + "drops": 0 + } + }, + "throughput_including_drain": { + "durable-ingest/new": 25.03043083166518, + "durable-ingest/additional": 15.024921379583459, + "durable-ingest/duplicate": 7.514979226456401, + "durable-ingest/late": 2.5055475140567047, + "mixed-http/nodes_50": 1.4325632998114946, + "mixed-http/nodes_region": 1.2289771605871025, + "mixed-http/nodes_2000": 1.4321243634160532, + "mixed-http/channels": 1.4326236467879043, + "mixed-http/messages_0": 1.4326140382563404, + "mixed-http/messages_1000": 1.2290071107313902, + "mixed-http/observers": 2.051835513931382, + "mixed-http/metrics_dense_24h": 1.0262030021972959, + "mixed-http/metrics_sparse_7d": 1.0262135951837557, + "mixed-http/reach": 1.02621541276287, + "mixed-http/rx_coverage": 1.0262132243062714, + "mixed-http/packets_memory": 1.024883969868769, + "mixed-http/packet_detail_memory": 1.0250482547261706, + "mixed-http/packet_detail_sql": 1.0336772074065332, + "mixed-http/analytics_rf": 0.6157299032173998, + "mixed-http/analytics_topology": 0.6156305737496135, + "mixed-http/analytics_channels": 0.4108823317592926, + "mixed-http/analytics_rf_filtered": 0.4109541207607805, + "mixed-http/stats": 1.0273895609978374, + "mixed-http/packet_detail_pruned": 1.108346888743566, + "sql-miss/channel_messages_sql": 14.11584387767697, + "sql-miss/observer_metrics_sql": 14.234787782862767 + }, + "scalars_ns": { + "startup/first_http_ns": 28944340758, + "startup/ready_ns": 28944343230, + "startup/full_ready_ns": 28955951187, + "startup-hot1/first_http_ns": 9673546137, + "startup-hot1/ready_ns": 9673548096, + "startup-hot1/full_ready_ns": 20825357213, + "retention/elapsed_ns": 10543833950, + "retention-empty/elapsed_ns": 267672 + }, + "resources": { + "sample_count": 180, + "window_start_ns": 2541247212756, + "window_end_ns": 2721247212756, + "sampled_start_ns": 2541428664761, + "sampled_end_ns": 2720871545286, + "sampled_span_ns": 179442880525, + "sampled_span_fraction": 0.9969048918055555, + "first_boundary_gap_ns": 181452005, + "last_boundary_gap_ns": 375667470, + "sampling_interval_ns": { + "median": 1002295337, + "minimum": 1002056947, + "maximum": 1006895570, + "runs": 179 + }, + "phases": [ + "mixed_with_retention" + ], + "cpu_delta_usec": 43603440, + "average_cores_over_sampled_span": 0.2429934242719937, + "per_device_io": { + "8:0": { + "rbytes": 0, + "wbytes": 2474217472, + "wios": 218688, + "rios": 0, + "dios": 0, + "dbytes": 0 + } + }, + "aggregate_accounted_io": { + "dbytes": 0, + "dios": 0, + "rbytes": 0, + "rios": 0, + "wbytes": 2474217472, + "wios": 218688 + }, + "observed_peaks": { + "cgroup_memory_bytes": 4242919424, + "rss_bytes": 3069657088, + "pss_bytes": 3066212352, + "storage_bytes": 1152363712, + "wal_bytes": 42040512 + }, + "memory_event_deltas": { + "oom_group_kill": 0, + "oom": 0, + "high": 0, + "max": 0, + "oom_kill": 0, + "low": 0 + }, + "minimum_free_bytes": 83181256704 + }, + "websocket": { + "full_stream_sentinels": 6000, + "full_stream_received": 6000, + "measured_sentinels": 4500, + "measured_missing": 0, + "full_stream_missing": 0, + "negative_measured_lags": 0, + "negative_full_stream_lags": 0, + "duplicate_measured_receipts": 0, + "duplicate_full_stream_receipts": 0 + }, + "retention": { + "retention": { + "elapsed_ns": 10543833950, + "transmissions_deleted": 16000, + "observations_deleted": 256000 + }, + "retention-empty": { + "elapsed_ns": 267672, + "transmissions_deleted": 0, + "observations_deleted": 0 + } + }, + "startup_counts": [ + { + "transmissions": 112000, + "observations": 1792000 + }, + { + "transmissions": 112000, + "observations": 1792000 + } + ], + "database_settings": { + "busy_timeout": "5000", + "cache_size": "-2000", + "cache_spill": "483", + "foreign_keys": "1", + "journal_mode": "wal", + "journal_size_limit": "-1", + "mmap_size": "0", + "page_size": "4096", + "sqlite_version": "3.53.4", + "synchronous": "2", + "wal_autocheckpoint": "1000" + }, + "durable_counts": { + "scheduled": 12000, + "completed": 12000, + "dropped_schedules": 0, + "write_errors": 0, + "actual_transmissions": 118000, + "actual_observations": 1802200 + } + }, + { + "pair": 1, + "backend": "postgres", + "metrics": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "samples": 2700, + "p50": 1416145, + "p95": 2062885, + "p99": 2708564, + "maximum": 34156688 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "samples": 2700, + "p50": 617638, + "p95": 1075628, + "p99": 1125685, + "maximum": 14658911 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "end_to_end", + "samples": 2700, + "p50": 2088767, + "p95": 2844395, + "p99": 4160408, + "maximum": 34621051 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "samples": 1350, + "p50": 1283204, + "p95": 1696043, + "p99": 2027802, + "maximum": 31014614 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "samples": 1350, + "p50": 598808, + "p95": 1065135, + "p99": 1117893, + "maximum": 10518589 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "end_to_end", + "samples": 1350, + "p50": 1926637, + "p95": 2525053, + "p99": 3137324, + "maximum": 31940547 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "samples": 450, + "p50": 1576205, + "p95": 2080699, + "p99": null, + "maximum": 20249905 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "samples": 450, + "p50": 639938, + "p95": 1078886, + "p99": null, + "maximum": 11970610 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "end_to_end", + "samples": 450, + "p50": 2239742, + "p95": 2903790, + "p99": null, + "maximum": 31390503 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "samples": 4500, + "p50": 1546454, + "p95": 2085505, + "p99": 2586040, + "maximum": 32540654 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "samples": 4500, + "p50": 621090, + "p95": 1076507, + "p99": 1125895, + "maximum": 11769621 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "end_to_end", + "samples": 4500, + "p50": 2213135, + "p95": 2902473, + "p99": 3415905, + "maximum": 33305535 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "samples": 72, + "p50": 30488073, + "p95": 38967436, + "p99": null, + "maximum": 84165141 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "samples": 72, + "p50": 464414, + "p95": 1062605, + "p99": null, + "maximum": 2034440 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "end_to_end", + "samples": 72, + "p50": 31021204, + "p95": 39178776, + "p99": null, + "maximum": 86199581 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "samples": 108, + "p50": 1587626, + "p95": 2588219, + "p99": null, + "maximum": 5741967 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "samples": 108, + "p50": 605844, + "p95": 1067713, + "p99": null, + "maximum": 1093579 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "end_to_end", + "samples": 108, + "p50": 2271091, + "p95": 3336811, + "p99": null, + "maximum": 5854707 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "samples": 72, + "p50": 1995707, + "p95": 576780683, + "p99": null, + "maximum": 605713157 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "samples": 72, + "p50": 159993, + "p95": 946777, + "p99": null, + "maximum": 1088773 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "end_to_end", + "samples": 72, + "p50": 2277592, + "p95": 576901965, + "p99": null, + "maximum": 606196692 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "samples": 108, + "p50": 30584877, + "p95": 47812178, + "p99": null, + "maximum": 101259493 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "samples": 108, + "p50": 525352, + "p95": 1011479, + "p99": null, + "maximum": 1152137 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "end_to_end", + "samples": 108, + "p50": 31153570, + "p95": 48289233, + "p99": null, + "maximum": 101781869 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "samples": 252, + "p50": 672386, + "p95": 931447, + "p99": null, + "maximum": 25367892 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "samples": 252, + "p50": 586968, + "p95": 1061946, + "p99": null, + "maximum": 1090670 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "end_to_end", + "samples": 252, + "p50": 1269834, + "p95": 1794376, + "p99": null, + "maximum": 26090086 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "samples": 252, + "p50": 1354168, + "p95": 107712355, + "p99": null, + "maximum": 117683826 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "samples": 252, + "p50": 570097, + "p95": 1031272, + "p99": null, + "maximum": 1130519 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "end_to_end", + "samples": 252, + "p50": 2016844, + "p95": 108292762, + "p99": null, + "maximum": 118712442 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "samples": 216, + "p50": 1411343, + "p95": 111369213, + "p99": null, + "maximum": 166303468 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "samples": 216, + "p50": 633636, + "p95": 1060963, + "p99": null, + "maximum": 1112925 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "end_to_end", + "samples": 216, + "p50": 2065070, + "p95": 111890610, + "p99": null, + "maximum": 167203405 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "samples": 180, + "p50": 3850807, + "p95": 4688150, + "p99": null, + "maximum": 5547857 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "samples": 180, + "p50": 572725, + "p95": 1010156, + "p99": null, + "maximum": 1094284 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "end_to_end", + "samples": 180, + "p50": 4400685, + "p95": 5166891, + "p99": null, + "maximum": 6125852 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "samples": 180, + "p50": 1751231, + "p95": 1961007, + "p99": null, + "maximum": 2719882 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "samples": 180, + "p50": 501337, + "p95": 982341, + "p99": null, + "maximum": 1091158 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "end_to_end", + "samples": 180, + "p50": 2239556, + "p95": 2828506, + "p99": null, + "maximum": 2969579 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "samples": 252, + "p50": 64009316, + "p95": 91576524, + "p99": null, + "maximum": 193817164 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "samples": 252, + "p50": 490237, + "p95": 1019702, + "p99": null, + "maximum": 3263174 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "end_to_end", + "samples": 252, + "p50": 64555899, + "p95": 91659010, + "p99": null, + "maximum": 194404834 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "samples": 252, + "p50": 5231837, + "p95": 9547653, + "p99": null, + "maximum": 27047776 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "samples": 252, + "p50": 195652, + "p95": 982448, + "p99": null, + "maximum": 1074390 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "end_to_end", + "samples": 252, + "p50": 5558499, + "p95": 9671301, + "p99": null, + "maximum": 27725838 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "samples": 216, + "p50": 7723789, + "p95": 15623571, + "p99": null, + "maximum": 28753952 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "samples": 216, + "p50": 474799, + "p95": 1046342, + "p99": null, + "maximum": 1231218 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "end_to_end", + "samples": 216, + "p50": 8312232, + "p95": 16186909, + "p99": null, + "maximum": 29529149 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "samples": 360, + "p50": 2931575, + "p95": 4266545, + "p99": null, + "maximum": 89232830 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "samples": 360, + "p50": 544970, + "p95": 1043002, + "p99": null, + "maximum": 1122931 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "end_to_end", + "samples": 360, + "p50": 3518928, + "p95": 4859936, + "p99": null, + "maximum": 89796059 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "samples": 180, + "p50": 2684360, + "p95": 3304445, + "p99": null, + "maximum": 88988984 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "samples": 180, + "p50": 509408, + "p95": 1028775, + "p99": null, + "maximum": 1090268 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 3166755, + "p95": 3847159, + "p99": null, + "maximum": 89376908 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "samples": 45, + "p50": 723078, + "p95": 1051024, + "p99": null, + "maximum": 1397407 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "samples": 45, + "p50": 477932, + "p95": 1008922, + "p99": null, + "maximum": 1040169 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "end_to_end", + "samples": 45, + "p50": 1217322, + "p95": 1823438, + "p99": null, + "maximum": 2065196 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "samples": 135, + "p50": 1517718, + "p95": 1936821, + "p99": null, + "maximum": 32974180 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "samples": 135, + "p50": 620332, + "p95": 1046120, + "p99": null, + "maximum": 1097616 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "end_to_end", + "samples": 135, + "p50": 2145375, + "p95": 2814057, + "p99": null, + "maximum": 33585817 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "samples": 180, + "p50": 35657776, + "p95": 39238667, + "p99": null, + "maximum": 132564208 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "samples": 180, + "p50": 582940, + "p95": 1037863, + "p99": null, + "maximum": 1120658 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 36243301, + "p95": 39798037, + "p99": null, + "maximum": 133638397 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "samples": 180, + "p50": 841431, + "p95": 946870, + "p99": null, + "maximum": 1696288 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "samples": 180, + "p50": 580461, + "p95": 1011119, + "p99": null, + "maximum": 1116521 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "end_to_end", + "samples": 180, + "p50": 1413174, + "p95": 1842015, + "p99": null, + "maximum": 2127367 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "samples": 180, + "p50": 1799312, + "p95": 1954786, + "p99": null, + "maximum": 2372872 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "samples": 180, + "p50": 583301, + "p95": 1047818, + "p99": null, + "maximum": 1087753 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "end_to_end", + "samples": 180, + "p50": 2377749, + "p95": 2880123, + "p99": null, + "maximum": 3418725 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "samples": 180, + "p50": 461068, + "p95": 3787246, + "p99": null, + "maximum": 6058634 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "samples": 180, + "p50": 588746, + "p95": 1050899, + "p99": null, + "maximum": 1115331 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "end_to_end", + "samples": 180, + "p50": 1075203, + "p95": 4551554, + "p99": null, + "maximum": 6989851 + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "samples": 100, + "p50": 102155105, + "p95": 111946799, + "p99": null, + "maximum": 167908434 + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "samples": 100, + "p50": 816310, + "p95": 1408799, + "p99": null, + "maximum": 3140359 + }, + { + "scenario": "websocket", + "event_class": "new_transmission", + "timing": "receipt_after_ingest_return", + "samples": 4500, + "p50": 518441443, + "p95": 959560523, + "p99": 998959615, + "maximum": 1224689305 + }, + { + "scenario": "handler-control", + "event_class": "200_valid_envelopes", + "timing": "service", + "samples": 200, + "p50": 1610418, + "p95": 2285630, + "p99": null, + "maximum": 5273595 + } + ], + "counts": { + "durable-ingest": { + "total": 12000, + "measured": 9000, + "successful": 12000, + "errors": 0, + "drops": 0 + }, + "mixed-http": { + "total": 4800, + "measured": 3600, + "successful": 4800, + "errors": 0, + "drops": 0 + }, + "sql-miss": { + "total": 200, + "measured": 200, + "successful": 200, + "errors": 0, + "drops": 0 + } + }, + "throughput_including_drain": { + "durable-ingest/new": 25.030307544485115, + "durable-ingest/additional": 15.02485874515304, + "durable-ingest/duplicate": 7.514925769106568, + "durable-ingest/late": 2.5055338903411295, + "mixed-http/nodes_50": 1.4325843262635118, + "mixed-http/nodes_region": 1.2289646640065988, + "mixed-http/nodes_2000": 1.4318860417452008, + "mixed-http/channels": 1.4326165813832683, + "mixed-http/messages_0": 1.4326160247622126, + "mixed-http/messages_1000": 1.2281926892005706, + "mixed-http/observers": 2.05183754680436, + "mixed-http/metrics_dense_24h": 1.0262029444000669, + "mixed-http/metrics_sparse_7d": 1.0262089894205495, + "mixed-http/reach": 1.02621462714909, + "mixed-http/rx_coverage": 1.0262108583106953, + "mixed-http/packets_memory": 1.0248381718202302, + "mixed-http/packet_detail_memory": 1.0250369375445914, + "mixed-http/packet_detail_sql": 1.0336763215669507, + "mixed-http/analytics_rf": 0.6157285760324033, + "mixed-http/analytics_topology": 0.6156274872300467, + "mixed-http/analytics_channels": 0.41088490057996513, + "mixed-http/analytics_rf_filtered": 0.410952802471435, + "mixed-http/stats": 1.0273942385855432, + "mixed-http/packet_detail_pruned": 1.1083471949510062, + "sql-miss/channel_messages_sql": 9.50359522922496, + "sql-miss/observer_metrics_sql": 9.656813430464279 + }, + "scalars_ns": { + "startup/first_http_ns": 27158449767, + "startup/ready_ns": 27158452500, + "startup/full_ready_ns": 27226831969, + "startup-hot1/first_http_ns": 11482653279, + "startup-hot1/ready_ns": 11482655439, + "startup-hot1/full_ready_ns": 20009917073, + "retention/elapsed_ns": 962246927, + "retention-empty/elapsed_ns": 1732086, + "migration/elapsed_ns": 151047832511, + "migration/migration_tool_ns": 94125867624, + "migration/role_grants_ns": 790897550, + "migration/harness_validation_ns": 56131068630 + }, + "resources": { + "sample_count": 174, + "window_start_ns": 2138786661803, + "window_end_ns": 2318786661803, + "sampled_start_ns": 2139661891871, + "sampled_end_ns": 2317868765194, + "sampled_span_ns": 178206873323, + "sampled_span_fraction": 0.9900381851277777, + "first_boundary_gap_ns": 875230068, + "last_boundary_gap_ns": 917896609, + "sampling_interval_ns": { + "median": 1028445173, + "minimum": 1026151466, + "maximum": 1075378901, + "runs": 173 + }, + "phases": [ + "mixed_with_retention" + ], + "cpu_delta_usec": 56857596, + "average_cores_over_sampled_span": 0.3190538891109188, + "per_device_io": { + "8:0": { + "rbytes": 7208960, + "wbytes": 395005952, + "wios": 17970, + "rios": 780, + "dios": 0, + "dbytes": 0 + } + }, + "aggregate_accounted_io": { + "dbytes": 0, + "dios": 0, + "rbytes": 7208960, + "rios": 780, + "wbytes": 395005952, + "wios": 17970 + }, + "observed_peaks": { + "cgroup_memory_bytes": 6442434560, + "rss_bytes": 5206446080, + "pss_bytes": 3374332928, + "storage_bytes": 2439597884, + "wal_bytes": 1073741824 + }, + "memory_event_deltas": { + "oom_group_kill": 0, + "oom": 0, + "high": 0, + "max": 444, + "oom_kill": 0, + "low": 0 + }, + "minimum_free_bytes": 79668822016 + }, + "websocket": { + "full_stream_sentinels": 6000, + "full_stream_received": 6000, + "measured_sentinels": 4500, + "measured_missing": 0, + "full_stream_missing": 0, + "negative_measured_lags": 0, + "negative_full_stream_lags": 0, + "duplicate_measured_receipts": 0, + "duplicate_full_stream_receipts": 0 + }, + "retention": { + "retention": { + "elapsed_ns": 962246927, + "transmissions_deleted": 16000, + "observations_deleted": 256000 + }, + "retention-empty": { + "elapsed_ns": 1732086, + "transmissions_deleted": 0, + "observations_deleted": 0 + } + }, + "startup_counts": [ + { + "transmissions": 112000, + "observations": 1792000 + }, + { + "transmissions": 112000, + "observations": 1792000 + } + ], + "database_settings": { + "autovacuum": "on", + "checkpoint_timeout": "5min", + "effective_cache_size": "4GB", + "fsync": "on", + "full_page_writes": "on", + "jit": "on", + "maintenance_work_mem": "64MB", + "max_connections": "32", + "max_parallel_workers_per_gather": "2", + "max_wal_size": "1GB", + "server_version": "18.6 (Ubuntu 18.6-1.pgdg24.04+2)", + "shared_buffers": "256MB", + "synchronous_commit": "on", + "work_mem": "4MB" + }, + "durable_counts": { + "scheduled": 12000, + "completed": 12000, + "dropped_schedules": 0, + "write_errors": 0, + "actual_transmissions": 118000, + "actual_observations": 1802200 + } + }, + { + "pair": 2, + "backend": "sqlite", + "metrics": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "samples": 2700, + "p50": 771876, + "p95": 1325862, + "p99": 3536701, + "maximum": 404668851 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "samples": 2700, + "p50": 622789, + "p95": 1610055353, + "p99": 7307703962, + "maximum": 8752564471 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "end_to_end", + "samples": 2700, + "p50": 1516862, + "p95": 1630047018, + "p99": 7476023526, + "maximum": 8752956768 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "samples": 1350, + "p50": 660703, + "p95": 1012552, + "p99": 1484077, + "maximum": 354040593 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "samples": 1350, + "p50": 653777, + "p95": 1570563908, + "p99": 7436413274, + "maximum": 8634694600 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "end_to_end", + "samples": 1350, + "p50": 1419332, + "p95": 1570732361, + "p99": 7436618017, + "maximum": 8634948025 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "samples": 450, + "p50": 937704, + "p95": 1361402, + "p99": null, + "maximum": 301156103 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "samples": 450, + "p50": 594151, + "p95": 1530905355, + "p99": null, + "maximum": 8576286247 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "end_to_end", + "samples": 450, + "p50": 1624934, + "p95": 1531316113, + "p99": null, + "maximum": 8576654631 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "samples": 4500, + "p50": 1006997, + "p95": 1633878, + "p99": 3649676, + "maximum": 508632253 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "samples": 4500, + "p50": 644579, + "p95": 1432959194, + "p99": 7282426629, + "maximum": 8771881541 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "end_to_end", + "samples": 4500, + "p50": 1721534, + "p95": 1452953774, + "p99": 7302409744, + "maximum": 8791856458 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "samples": 72, + "p50": 29157459, + "p95": 33765859, + "p99": null, + "maximum": 85377922 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "samples": 72, + "p50": 299402, + "p95": 897122, + "p99": null, + "maximum": 1082309 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "end_to_end", + "samples": 72, + "p50": 29579173, + "p95": 34076747, + "p99": null, + "maximum": 85391371 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "samples": 108, + "p50": 1742604, + "p95": 2659179, + "p99": null, + "maximum": 20775052 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "samples": 108, + "p50": 827007, + "p95": 1081218, + "p99": null, + "maximum": 1126340 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "end_to_end", + "samples": 108, + "p50": 2541082, + "p95": 3432011, + "p99": null, + "maximum": 21150257 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "samples": 72, + "p50": 1711493, + "p95": 719498090, + "p99": null, + "maximum": 995423055 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "samples": 72, + "p50": 116691, + "p95": 863496, + "p99": null, + "maximum": 1033810 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "end_to_end", + "samples": 72, + "p50": 1867875, + "p95": 719582554, + "p99": null, + "maximum": 995472138 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "samples": 108, + "p50": 29635707, + "p95": 31248067, + "p99": null, + "maximum": 114019302 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "samples": 108, + "p50": 745307, + "p95": 1018115, + "p99": null, + "maximum": 1094594 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "end_to_end", + "samples": 108, + "p50": 30333498, + "p95": 32160768, + "p99": null, + "maximum": 114441012 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "samples": 252, + "p50": 664665, + "p95": 929269, + "p99": null, + "maximum": 17681346 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "samples": 252, + "p50": 584467, + "p95": 1046275, + "p99": null, + "maximum": 1092821 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "end_to_end", + "samples": 252, + "p50": 1296687, + "p95": 1747663, + "p99": null, + "maximum": 18267038 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "samples": 252, + "p50": 1338129, + "p95": 58731213, + "p99": null, + "maximum": 61912178 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "samples": 252, + "p50": 544969, + "p95": 1038010, + "p99": null, + "maximum": 1171281 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "end_to_end", + "samples": 252, + "p50": 1936174, + "p95": 58820296, + "p99": null, + "maximum": 62436684 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "samples": 216, + "p50": 1343552, + "p95": 82157070, + "p99": null, + "maximum": 83581508 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "samples": 216, + "p50": 585777, + "p95": 1054293, + "p99": null, + "maximum": 1087732 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "end_to_end", + "samples": 216, + "p50": 1928511, + "p95": 82997570, + "p99": null, + "maximum": 84439905 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "samples": 180, + "p50": 3291871, + "p95": 4210769, + "p99": null, + "maximum": 5011871 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "samples": 180, + "p50": 636269, + "p95": 1057474, + "p99": null, + "maximum": 1288638 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "end_to_end", + "samples": 180, + "p50": 3906297, + "p95": 4845487, + "p99": null, + "maximum": 5454830 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "samples": 180, + "p50": 1353340, + "p95": 1606237, + "p99": null, + "maximum": 2226938 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "samples": 180, + "p50": 599844, + "p95": 1057795, + "p99": null, + "maximum": 1131036 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "end_to_end", + "samples": 180, + "p50": 1948411, + "p95": 2483813, + "p99": null, + "maximum": 3197640 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "samples": 252, + "p50": 61854758, + "p95": 119434458, + "p99": null, + "maximum": 191847633 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "samples": 252, + "p50": 549668, + "p95": 1069702, + "p99": null, + "maximum": 1314563 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "end_to_end", + "samples": 252, + "p50": 62495104, + "p95": 119552875, + "p99": null, + "maximum": 192117285 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "samples": 252, + "p50": 3292222, + "p95": 8104951, + "p99": null, + "maximum": 49946718 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "samples": 252, + "p50": 153388, + "p95": 1029170, + "p99": null, + "maximum": 10002488 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "end_to_end", + "samples": 252, + "p50": 3558430, + "p95": 8895935, + "p99": null, + "maximum": 50576330 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "samples": 216, + "p50": 5388514, + "p95": 13741682, + "p99": null, + "maximum": 42097155 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "samples": 216, + "p50": 429079, + "p95": 1039048, + "p99": null, + "maximum": 6295388 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "end_to_end", + "samples": 216, + "p50": 5803879, + "p95": 14106060, + "p99": null, + "maximum": 43155319 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "samples": 360, + "p50": 2851565, + "p95": 3986033, + "p99": null, + "maximum": 19136079 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "samples": 360, + "p50": 539879, + "p95": 1033286, + "p99": null, + "maximum": 1119289 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "end_to_end", + "samples": 360, + "p50": 3475660, + "p95": 4546494, + "p99": null, + "maximum": 19908557 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "samples": 180, + "p50": 1229184, + "p95": 1597596, + "p99": null, + "maximum": 80845588 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "samples": 180, + "p50": 560073, + "p95": 1027279, + "p99": null, + "maximum": 1195014 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 1846237, + "p95": 2389053, + "p99": null, + "maximum": 81782517 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "samples": 45, + "p50": 691132, + "p95": 837369, + "p99": null, + "maximum": 984544 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "samples": 45, + "p50": 653656, + "p95": 1021301, + "p99": null, + "maximum": 1097042 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "end_to_end", + "samples": 45, + "p50": 1430893, + "p95": 1734242, + "p99": null, + "maximum": 1747916 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "samples": 135, + "p50": 1080978, + "p95": 1321987, + "p99": null, + "maximum": 28375171 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "samples": 135, + "p50": 630278, + "p95": 1045903, + "p99": null, + "maximum": 1091769 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "end_to_end", + "samples": 135, + "p50": 1729592, + "p95": 2164018, + "p99": null, + "maximum": 28475122 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "samples": 180, + "p50": 29351432, + "p95": 31503966, + "p99": null, + "maximum": 126568253 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "samples": 180, + "p50": 658505, + "p95": 1051354, + "p99": null, + "maximum": 1101934 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 30066971, + "p95": 31874787, + "p99": null, + "maximum": 127156211 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "samples": 180, + "p50": 802021, + "p95": 913143, + "p99": null, + "maximum": 1591698 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "samples": 180, + "p50": 587119, + "p95": 1031534, + "p99": null, + "maximum": 1095857 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "end_to_end", + "samples": 180, + "p50": 1424358, + "p95": 1811044, + "p99": null, + "maximum": 2608054 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "samples": 180, + "p50": 1550419, + "p95": 1769381, + "p99": null, + "maximum": 2065768 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "samples": 180, + "p50": 606329, + "p95": 1045129, + "p99": null, + "maximum": 1105898 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "end_to_end", + "samples": 180, + "p50": 2143945, + "p95": 2695541, + "p99": null, + "maximum": 3035507 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "samples": 180, + "p50": 477282, + "p95": 2711671, + "p99": null, + "maximum": 23687849 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "samples": 180, + "p50": 615901, + "p95": 1052569, + "p99": null, + "maximum": 1130259 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "end_to_end", + "samples": 180, + "p50": 1115073, + "p95": 3130427, + "p99": null, + "maximum": 24445164 + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "samples": 100, + "p50": 76110000, + "p95": 79008480, + "p99": null, + "maximum": 167647478 + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "samples": 100, + "p50": 729945, + "p95": 1309627, + "p99": null, + "maximum": 2952632 + }, + { + "scenario": "websocket", + "event_class": "new_transmission", + "timing": "receipt_after_ingest_return", + "samples": 4500, + "p50": 506091765, + "p95": 969220451, + "p99": 1389020095, + "maximum": 1930258093 + }, + { + "scenario": "handler-control", + "event_class": "200_valid_envelopes", + "timing": "service", + "samples": 200, + "p50": 812859, + "p95": 2036319, + "p99": null, + "maximum": 4157001 + } + ], + "counts": { + "durable-ingest": { + "total": 12000, + "measured": 9000, + "successful": 12000, + "errors": 0, + "drops": 0 + }, + "mixed-http": { + "total": 4800, + "measured": 3600, + "successful": 4800, + "errors": 0, + "drops": 0 + }, + "sql-miss": { + "total": 200, + "measured": 200, + "successful": 200, + "errors": 0, + "drops": 0 + } + }, + "throughput_including_drain": { + "durable-ingest/new": 25.0304635729368, + "durable-ingest/additional": 15.024899342488451, + "durable-ingest/duplicate": 7.514958763057562, + "durable-ingest/late": 2.505541098486846, + "mixed-http/nodes_50": 1.4326041214710346, + "mixed-http/nodes_region": 1.2289738820537766, + "mixed-http/nodes_2000": 1.4321358220980442, + "mixed-http/channels": 1.4326209010181128, + "mixed-http/messages_0": 1.4326164829744195, + "mixed-http/messages_1000": 1.2290053084144184, + "mixed-http/observers": 2.051823181146627, + "mixed-http/metrics_dense_24h": 1.0261974421851585, + "mixed-http/metrics_sparse_7d": 1.0262156329348024, + "mixed-http/reach": 1.02621879967198, + "mixed-http/rx_coverage": 1.0262095059219531, + "mixed-http/packets_memory": 1.0248767414004638, + "mixed-http/packet_detail_memory": 1.0250412312510828, + "mixed-http/packet_detail_sql": 1.0336731117556959, + "mixed-http/analytics_rf": 0.615726563222311, + "mixed-http/analytics_topology": 0.6156258801281858, + "mixed-http/analytics_channels": 0.4108850379202887, + "mixed-http/analytics_rf_filtered": 0.41095440282186896, + "mixed-http/stats": 1.027393961682574, + "mixed-http/packet_detail_pruned": 1.1083374477990473, + "sql-miss/channel_messages_sql": 13.826937651935282, + "sql-miss/observer_metrics_sql": 13.939626844115455 + }, + "scalars_ns": { + "startup/first_http_ns": 27441447101, + "startup/ready_ns": 27441450067, + "startup/full_ready_ns": 27452984839, + "startup-hot1/first_http_ns": 9473558221, + "startup-hot1/ready_ns": 9473560171, + "startup-hot1/full_ready_ns": 20273227139, + "retention/elapsed_ns": 10732519816, + "retention-empty/elapsed_ns": 257609 + }, + "resources": { + "sample_count": 180, + "window_start_ns": 3005008029069, + "window_end_ns": 3185008029069, + "sampled_start_ns": 3005215282219, + "sampled_end_ns": 3184647458484, + "sampled_span_ns": 179432176265, + "sampled_span_fraction": 0.9968454236944444, + "first_boundary_gap_ns": 207253150, + "last_boundary_gap_ns": 360570585, + "sampling_interval_ns": { + "median": 1002233408, + "minimum": 1001962449, + "maximum": 1009409190, + "runs": 179 + }, + "phases": [ + "mixed_with_retention" + ], + "cpu_delta_usec": 43453559, + "average_cores_over_sampled_span": 0.2421726130982453, + "per_device_io": { + "8:0": { + "rbytes": 0, + "wbytes": 2493005824, + "wios": 223216, + "rios": 0, + "dios": 0, + "dbytes": 0 + } + }, + "aggregate_accounted_io": { + "dbytes": 0, + "dios": 0, + "rbytes": 0, + "rios": 0, + "wbytes": 2493005824, + "wios": 223216 + }, + "observed_peaks": { + "cgroup_memory_bytes": 4239114240, + "rss_bytes": 3067809792, + "pss_bytes": 3064301568, + "storage_bytes": 1147743112, + "wal_bytes": 31439752 + }, + "memory_event_deltas": { + "oom_group_kill": 0, + "oom": 0, + "high": 0, + "max": 0, + "oom_kill": 0, + "low": 0 + }, + "minimum_free_bytes": 82062835712 + }, + "websocket": { + "full_stream_sentinels": 6000, + "full_stream_received": 6000, + "measured_sentinels": 4500, + "measured_missing": 0, + "full_stream_missing": 0, + "negative_measured_lags": 0, + "negative_full_stream_lags": 0, + "duplicate_measured_receipts": 0, + "duplicate_full_stream_receipts": 0 + }, + "retention": { + "retention": { + "elapsed_ns": 10732519816, + "transmissions_deleted": 16000, + "observations_deleted": 256000 + }, + "retention-empty": { + "elapsed_ns": 257609, + "transmissions_deleted": 0, + "observations_deleted": 0 + } + }, + "startup_counts": [ + { + "transmissions": 112000, + "observations": 1792000 + }, + { + "transmissions": 112000, + "observations": 1792000 + } + ], + "database_settings": { + "busy_timeout": "5000", + "cache_size": "-2000", + "cache_spill": "483", + "foreign_keys": "1", + "journal_mode": "wal", + "journal_size_limit": "-1", + "mmap_size": "0", + "page_size": "4096", + "sqlite_version": "3.53.4", + "synchronous": "2", + "wal_autocheckpoint": "1000" + }, + "durable_counts": { + "scheduled": 12000, + "completed": 12000, + "dropped_schedules": 0, + "write_errors": 0, + "actual_transmissions": 118000, + "actual_observations": 1802200 + } + }, + { + "pair": 2, + "backend": "postgres", + "metrics": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "samples": 2700, + "p50": 1339585, + "p95": 1991272, + "p99": 3024104, + "maximum": 34604113 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "samples": 2700, + "p50": 625782, + "p95": 1076913, + "p99": 1123515, + "maximum": 15357823 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "end_to_end", + "samples": 2700, + "p50": 2024854, + "p95": 2806627, + "p99": 3954301, + "maximum": 35335495 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "samples": 1350, + "p50": 1215396, + "p95": 1629151, + "p99": 1943498, + "maximum": 22715400 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "samples": 1350, + "p50": 601103, + "p95": 1080537, + "p99": 1118114, + "maximum": 6711762 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "end_to_end", + "samples": 1350, + "p50": 1850921, + "p95": 2425550, + "p99": 2793261, + "maximum": 23609713 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "samples": 450, + "p50": 1514880, + "p95": 1993531, + "p99": null, + "maximum": 20361526 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "samples": 450, + "p50": 637320, + "p95": 1084884, + "p99": null, + "maximum": 1379217 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "end_to_end", + "samples": 450, + "p50": 2188247, + "p95": 2838691, + "p99": null, + "maximum": 20474563 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "samples": 4500, + "p50": 1480235, + "p95": 2027637, + "p99": 2578411, + "maximum": 53649424 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "samples": 4500, + "p50": 630326, + "p95": 1073768, + "p99": 1115403, + "maximum": 34441925 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "end_to_end", + "samples": 4500, + "p50": 2152369, + "p95": 2841009, + "p99": 3460706, + "maximum": 54413057 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "samples": 72, + "p50": 30441723, + "p95": 33935456, + "p99": null, + "maximum": 103993367 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "samples": 72, + "p50": 315689, + "p95": 972809, + "p99": null, + "maximum": 1065612 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "end_to_end", + "samples": 72, + "p50": 31003092, + "p95": 34930533, + "p99": null, + "maximum": 104263958 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "samples": 108, + "p50": 1576282, + "p95": 2618167, + "p99": null, + "maximum": 6076231 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "samples": 108, + "p50": 636470, + "p95": 1020589, + "p99": null, + "maximum": 1078464 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "end_to_end", + "samples": 108, + "p50": 2354243, + "p95": 3151328, + "p99": null, + "maximum": 6736885 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "samples": 72, + "p50": 1916525, + "p95": 548415278, + "p99": null, + "maximum": 913563014 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "samples": 72, + "p50": 126975, + "p95": 1011493, + "p99": null, + "maximum": 2288533 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "end_to_end", + "samples": 72, + "p50": 2163585, + "p95": 549166657, + "p99": null, + "maximum": 913676095 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "samples": 108, + "p50": 30006796, + "p95": 38553942, + "p99": null, + "maximum": 143819004 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "samples": 108, + "p50": 650104, + "p95": 1036273, + "p99": null, + "maximum": 1088570 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "end_to_end", + "samples": 108, + "p50": 30548049, + "p95": 39207753, + "p99": null, + "maximum": 144855123 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "samples": 252, + "p50": 658119, + "p95": 1186260, + "p99": null, + "maximum": 32028863 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "samples": 252, + "p50": 605369, + "p95": 1030172, + "p99": null, + "maximum": 1120934 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "end_to_end", + "samples": 252, + "p50": 1322769, + "p95": 1932422, + "p99": null, + "maximum": 32098788 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "samples": 252, + "p50": 1394098, + "p95": 111608159, + "p99": null, + "maximum": 120650104 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "samples": 252, + "p50": 551383, + "p95": 1063140, + "p99": null, + "maximum": 3921018 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "end_to_end", + "samples": 252, + "p50": 1963119, + "p95": 112150008, + "p99": null, + "maximum": 121574737 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "samples": 216, + "p50": 1390344, + "p95": 112766857, + "p99": null, + "maximum": 173045756 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "samples": 216, + "p50": 617711, + "p95": 1039658, + "p99": null, + "maximum": 1107790 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "end_to_end", + "samples": 216, + "p50": 2121777, + "p95": 113384568, + "p99": null, + "maximum": 173463605 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "samples": 180, + "p50": 3858659, + "p95": 4756509, + "p99": null, + "maximum": 11446366 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "samples": 180, + "p50": 565458, + "p95": 1071728, + "p99": null, + "maximum": 1112183 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "end_to_end", + "samples": 180, + "p50": 4450965, + "p95": 5388815, + "p99": null, + "maximum": 11912133 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "samples": 180, + "p50": 1708666, + "p95": 2035361, + "p99": null, + "maximum": 3208726 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "samples": 180, + "p50": 543962, + "p95": 1052802, + "p99": null, + "maximum": 1082431 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "end_to_end", + "samples": 180, + "p50": 2314167, + "p95": 2813734, + "p99": null, + "maximum": 3886089 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "samples": 252, + "p50": 62763292, + "p95": 72346081, + "p99": null, + "maximum": 214180114 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "samples": 252, + "p50": 557085, + "p95": 1060228, + "p99": null, + "maximum": 4850221 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "end_to_end", + "samples": 252, + "p50": 63362643, + "p95": 72775318, + "p99": null, + "maximum": 214280308 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "samples": 252, + "p50": 5067802, + "p95": 6928485, + "p99": null, + "maximum": 50973757 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "samples": 252, + "p50": 138223, + "p95": 1029463, + "p99": null, + "maximum": 2172324 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "end_to_end", + "samples": 252, + "p50": 5443298, + "p95": 7728956, + "p99": null, + "maximum": 51660788 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "samples": 216, + "p50": 7528155, + "p95": 14147839, + "p99": null, + "maximum": 95429557 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "samples": 216, + "p50": 494180, + "p95": 1024145, + "p99": null, + "maximum": 1148467 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "end_to_end", + "samples": 216, + "p50": 7987993, + "p95": 14593040, + "p99": null, + "maximum": 95893539 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "samples": 360, + "p50": 2867017, + "p95": 4250012, + "p99": null, + "maximum": 99535705 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "samples": 360, + "p50": 574897, + "p95": 1063925, + "p99": null, + "maximum": 1294911 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "end_to_end", + "samples": 360, + "p50": 3462715, + "p95": 4884393, + "p99": null, + "maximum": 99626435 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "samples": 180, + "p50": 2495053, + "p95": 3093005, + "p99": null, + "maximum": 48547903 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "samples": 180, + "p50": 567261, + "p95": 1011553, + "p99": null, + "maximum": 1082055 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 3068597, + "p95": 3763013, + "p99": null, + "maximum": 49158897 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "samples": 45, + "p50": 679433, + "p95": 1119225, + "p99": null, + "maximum": 1402905 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "samples": 45, + "p50": 619487, + "p95": 1050543, + "p99": null, + "maximum": 1451292 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "end_to_end", + "samples": 45, + "p50": 1356879, + "p95": 1892750, + "p99": null, + "maximum": 2069302 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "samples": 135, + "p50": 1545907, + "p95": 1790464, + "p99": null, + "maximum": 10191219 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "samples": 135, + "p50": 566948, + "p95": 1000333, + "p99": null, + "maximum": 1097365 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "end_to_end", + "samples": 135, + "p50": 2105552, + "p95": 2728989, + "p99": null, + "maximum": 10280084 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "samples": 180, + "p50": 34356118, + "p95": 36809693, + "p99": null, + "maximum": 108901312 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "samples": 180, + "p50": 543907, + "p95": 1011785, + "p99": null, + "maximum": 1078291 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 35045841, + "p95": 37207627, + "p99": null, + "maximum": 109261020 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "samples": 180, + "p50": 827035, + "p95": 983246, + "p99": null, + "maximum": 1504066 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "samples": 180, + "p50": 565897, + "p95": 1037205, + "p99": null, + "maximum": 1141440 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "end_to_end", + "samples": 180, + "p50": 1392669, + "p95": 1871723, + "p99": null, + "maximum": 2441450 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "samples": 180, + "p50": 1790425, + "p95": 1958771, + "p99": null, + "maximum": 2401831 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "samples": 180, + "p50": 545820, + "p95": 1037542, + "p99": null, + "maximum": 1092317 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "end_to_end", + "samples": 180, + "p50": 2331168, + "p95": 2872466, + "p99": null, + "maximum": 3060112 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "samples": 180, + "p50": 442406, + "p95": 3787331, + "p99": null, + "maximum": 27067460 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "samples": 180, + "p50": 512707, + "p95": 1013361, + "p99": null, + "maximum": 1415954 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "end_to_end", + "samples": 180, + "p50": 1049635, + "p95": 4480686, + "p99": null, + "maximum": 28073605 + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "samples": 100, + "p50": 104111127, + "p95": 141129664, + "p99": null, + "maximum": 235816687 + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "samples": 100, + "p50": 847299, + "p95": 1401232, + "p99": null, + "maximum": 3261404 + }, + { + "scenario": "websocket", + "event_class": "new_transmission", + "timing": "receipt_after_ingest_return", + "samples": 4500, + "p50": 519671849, + "p95": 961119981, + "p99": 1000351713, + "maximum": 1504357991 + }, + { + "scenario": "handler-control", + "event_class": "200_valid_envelopes", + "timing": "service", + "samples": 200, + "p50": 1576230, + "p95": 2221027, + "p99": null, + "maximum": 5061624 + } + ], + "counts": { + "durable-ingest": { + "total": 12000, + "measured": 9000, + "successful": 12000, + "errors": 0, + "drops": 0 + }, + "mixed-http": { + "total": 4800, + "measured": 3600, + "successful": 4800, + "errors": 0, + "drops": 0 + }, + "sql-miss": { + "total": 200, + "measured": 200, + "successful": 200, + "errors": 0, + "drops": 0 + } + }, + "throughput_including_drain": { + "durable-ingest/new": 25.03031940360886, + "durable-ingest/additional": 15.02486346818685, + "durable-ingest/duplicate": 7.514955154081024, + "durable-ingest/late": 2.505542867763413, + "mixed-http/nodes_50": 1.4325894320322825, + "mixed-http/nodes_region": 1.2289690282603238, + "mixed-http/nodes_2000": 1.4321175676230997, + "mixed-http/channels": 1.4326264217093045, + "mixed-http/messages_0": 1.4326150396163264, + "mixed-http/messages_1000": 1.2281871884052704, + "mixed-http/observers": 2.051824106278096, + "mixed-http/metrics_dense_24h": 1.0262030967884224, + "mixed-http/metrics_sparse_7d": 1.0262100721890361, + "mixed-http/reach": 1.0262151208327726, + "mixed-http/rx_coverage": 1.0262104535192633, + "mixed-http/packets_memory": 1.024855066066107, + "mixed-http/packet_detail_memory": 1.025038627587817, + "mixed-http/packet_detail_sql": 1.0336737483565326, + "mixed-http/analytics_rf": 0.6157265741676229, + "mixed-http/analytics_topology": 0.6156289761662707, + "mixed-http/analytics_channels": 0.41088613215818687, + "mixed-http/analytics_rf_filtered": 0.4109527522618231, + "mixed-http/stats": 1.0273913430100141, + "mixed-http/packet_detail_pruned": 1.1083273270877296, + "sql-miss/channel_messages_sql": 8.947721906222263, + "sql-miss/observer_metrics_sql": 9.088872506759705 + }, + "scalars_ns": { + "startup/first_http_ns": 27136667613, + "startup/ready_ns": 27136669504, + "startup/full_ready_ns": 27218956850, + "startup-hot1/first_http_ns": 11281796284, + "startup-hot1/ready_ns": 11281798423, + "startup-hot1/full_ready_ns": 20145659861, + "retention/elapsed_ns": 904965331, + "retention-empty/elapsed_ns": 1672475, + "migration/elapsed_ns": 148035946753, + "migration/migration_tool_ns": 90715908562, + "migration/role_grants_ns": 702549749, + "migration/harness_validation_ns": 56617489512 + }, + "resources": { + "sample_count": 174, + "window_start_ns": 3559332179160, + "window_end_ns": 3739332179160, + "sampled_start_ns": 3560207077889, + "sampled_end_ns": 3738798403630, + "sampled_span_ns": 178591325741, + "sampled_span_fraction": 0.9921740318944444, + "first_boundary_gap_ns": 874898729, + "last_boundary_gap_ns": 533775530, + "sampling_interval_ns": { + "median": 1030542767, + "minimum": 1028460520, + "maximum": 1101586886, + "runs": 173 + }, + "phases": [ + "mixed_with_retention" + ], + "cpu_delta_usec": 55349210, + "average_cores_over_sampled_span": 0.3099210433113059, + "per_device_io": { + "8:0": { + "rbytes": 7036928, + "wbytes": 450928640, + "wios": 20662, + "rios": 739, + "dios": 1, + "dbytes": 4096 + } + }, + "aggregate_accounted_io": { + "dbytes": 4096, + "dios": 1, + "rbytes": 7036928, + "rios": 739, + "wbytes": 450928640, + "wios": 20662 + }, + "observed_peaks": { + "cgroup_memory_bytes": 6021308416, + "rss_bytes": 5300330496, + "pss_bytes": 3482287104, + "storage_bytes": 2439704380, + "wal_bytes": 1073741824 + }, + "memory_event_deltas": { + "oom_group_kill": 0, + "oom": 0, + "high": 0, + "max": 0, + "oom_kill": 0, + "low": 0 + }, + "minimum_free_bytes": 77401784320 + }, + "websocket": { + "full_stream_sentinels": 6000, + "full_stream_received": 6000, + "measured_sentinels": 4500, + "measured_missing": 0, + "full_stream_missing": 0, + "negative_measured_lags": 0, + "negative_full_stream_lags": 0, + "duplicate_measured_receipts": 0, + "duplicate_full_stream_receipts": 0 + }, + "retention": { + "retention": { + "elapsed_ns": 904965331, + "transmissions_deleted": 16000, + "observations_deleted": 256000 + }, + "retention-empty": { + "elapsed_ns": 1672475, + "transmissions_deleted": 0, + "observations_deleted": 0 + } + }, + "startup_counts": [ + { + "transmissions": 112000, + "observations": 1792000 + }, + { + "transmissions": 112000, + "observations": 1792000 + } + ], + "database_settings": { + "autovacuum": "on", + "checkpoint_timeout": "5min", + "effective_cache_size": "4GB", + "fsync": "on", + "full_page_writes": "on", + "jit": "on", + "maintenance_work_mem": "64MB", + "max_connections": "32", + "max_parallel_workers_per_gather": "2", + "max_wal_size": "1GB", + "server_version": "18.6 (Ubuntu 18.6-1.pgdg24.04+2)", + "shared_buffers": "256MB", + "synchronous_commit": "on", + "work_mem": "4MB" + }, + "durable_counts": { + "scheduled": 12000, + "completed": 12000, + "dropped_schedules": 0, + "write_errors": 0, + "actual_transmissions": 118000, + "actual_observations": 1802200 + } + }, + { + "pair": 3, + "backend": "sqlite", + "metrics": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "samples": 2700, + "p50": 703363, + "p95": 1159387, + "p99": 3075125, + "maximum": 398262025 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "samples": 2700, + "p50": 639691, + "p95": 1393818642, + "p99": 7075714964, + "maximum": 8504287843 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "end_to_end", + "samples": 2700, + "p50": 1375210, + "p95": 1394052328, + "p99": 7115326735, + "maximum": 8504802966 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "samples": 1350, + "p50": 576349, + "p95": 930528, + "p99": 1678444, + "maximum": 393662958 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "samples": 1350, + "p50": 635398, + "p95": 1314878153, + "p99": 7189398088, + "maximum": 8386854812 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "end_to_end", + "samples": 1350, + "p50": 1228897, + "p95": 1315047968, + "p99": 7189695261, + "maximum": 8387055981 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "samples": 450, + "p50": 848965, + "p95": 1421064, + "p99": null, + "maximum": 300276985 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "samples": 450, + "p50": 674453, + "p95": 1275228598, + "p99": null, + "maximum": 8327635058 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "end_to_end", + "samples": 450, + "p50": 1660435, + "p95": 1275692851, + "p99": null, + "maximum": 8328070245 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "samples": 4500, + "p50": 939502, + "p95": 1525241, + "p99": 3689803, + "maximum": 405413087 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "samples": 4500, + "p50": 630112, + "p95": 1216603038, + "p99": 7034943960, + "maximum": 8308085770 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "end_to_end", + "samples": 4500, + "p50": 1671092, + "p95": 1236594667, + "p99": 7054929545, + "maximum": 8524267094 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "samples": 72, + "p50": 29127123, + "p95": 34817275, + "p99": null, + "maximum": 40932933 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "samples": 72, + "p50": 324715, + "p95": 1077937, + "p99": null, + "maximum": 1113478 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "end_to_end", + "samples": 72, + "p50": 29536575, + "p95": 35526494, + "p99": null, + "maximum": 41062085 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "samples": 108, + "p50": 1783152, + "p95": 2626019, + "p99": null, + "maximum": 3037849 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "samples": 108, + "p50": 844082, + "p95": 1060426, + "p99": null, + "maximum": 1098528 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "end_to_end", + "samples": 108, + "p50": 2608613, + "p95": 3398649, + "p99": null, + "maximum": 3696629 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "samples": 72, + "p50": 1703501, + "p95": 482259186, + "p99": null, + "maximum": 555048230 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "samples": 72, + "p50": 115940, + "p95": 934107, + "p99": null, + "maximum": 1062254 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "end_to_end", + "samples": 72, + "p50": 1862673, + "p95": 482539009, + "p99": null, + "maximum": 556062408 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "samples": 108, + "p50": 29567465, + "p95": 31944728, + "p99": null, + "maximum": 53677986 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "samples": 108, + "p50": 715768, + "p95": 977377, + "p99": null, + "maximum": 1094484 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "end_to_end", + "samples": 108, + "p50": 30175679, + "p95": 32402062, + "p99": null, + "maximum": 54251904 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "samples": 252, + "p50": 652985, + "p95": 909904, + "p99": null, + "maximum": 17755571 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "samples": 252, + "p50": 632933, + "p95": 1027657, + "p99": null, + "maximum": 1086234 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "end_to_end", + "samples": 252, + "p50": 1321596, + "p95": 1776743, + "p99": null, + "maximum": 18434606 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "samples": 252, + "p50": 1319800, + "p95": 59127634, + "p99": null, + "maximum": 62575529 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "samples": 252, + "p50": 549013, + "p95": 1040682, + "p99": null, + "maximum": 1391839 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "end_to_end", + "samples": 252, + "p50": 1937285, + "p95": 59242020, + "p99": null, + "maximum": 63089180 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "samples": 216, + "p50": 1352606, + "p95": 81992900, + "p99": null, + "maximum": 84384459 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "samples": 216, + "p50": 622575, + "p95": 1049420, + "p99": null, + "maximum": 1309872 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "end_to_end", + "samples": 216, + "p50": 2010534, + "p95": 82649912, + "p99": null, + "maximum": 85314122 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "samples": 180, + "p50": 3328298, + "p95": 4210735, + "p99": null, + "maximum": 4647029 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "samples": 180, + "p50": 568858, + "p95": 1039131, + "p99": null, + "maximum": 1116798 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "end_to_end", + "samples": 180, + "p50": 3934922, + "p95": 4921916, + "p99": null, + "maximum": 5475931 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "samples": 180, + "p50": 1369154, + "p95": 1580763, + "p99": null, + "maximum": 1721544 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "samples": 180, + "p50": 590603, + "p95": 1050599, + "p99": null, + "maximum": 1086418 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "end_to_end", + "samples": 180, + "p50": 2022376, + "p95": 2451622, + "p99": null, + "maximum": 2790548 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "samples": 252, + "p50": 61200154, + "p95": 69454249, + "p99": null, + "maximum": 154809971 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "samples": 252, + "p50": 486503, + "p95": 1012446, + "p99": null, + "maximum": 1387842 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "end_to_end", + "samples": 252, + "p50": 61842826, + "p95": 69925563, + "p99": null, + "maximum": 154920841 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "samples": 252, + "p50": 3237866, + "p95": 4911310, + "p99": null, + "maximum": 14589003 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "samples": 252, + "p50": 134430, + "p95": 986360, + "p99": null, + "maximum": 4676922 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "end_to_end", + "samples": 252, + "p50": 3536827, + "p95": 5561475, + "p99": null, + "maximum": 17673985 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "samples": 216, + "p50": 5335928, + "p95": 8665244, + "p99": null, + "maximum": 14643634 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "samples": 216, + "p50": 502242, + "p95": 1035773, + "p99": null, + "maximum": 1185371 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "end_to_end", + "samples": 216, + "p50": 5746777, + "p95": 9102632, + "p99": null, + "maximum": 15218251 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "samples": 360, + "p50": 2846359, + "p95": 3911868, + "p99": null, + "maximum": 19139868 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "samples": 360, + "p50": 549714, + "p95": 1027495, + "p99": null, + "maximum": 1118526 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "end_to_end", + "samples": 360, + "p50": 3463433, + "p95": 4667783, + "p99": null, + "maximum": 19402630 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "samples": 180, + "p50": 1212069, + "p95": 1547680, + "p99": null, + "maximum": 49576773 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "samples": 180, + "p50": 586605, + "p95": 1027984, + "p99": null, + "maximum": 1090237 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 1867204, + "p95": 2407444, + "p99": null, + "maximum": 49706620 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "samples": 45, + "p50": 662585, + "p95": 802375, + "p99": null, + "maximum": 1372648 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "samples": 45, + "p50": 480343, + "p95": 1071692, + "p99": null, + "maximum": 1079755 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "end_to_end", + "samples": 45, + "p50": 1235647, + "p95": 1727971, + "p99": null, + "maximum": 2232740 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "samples": 135, + "p50": 1065890, + "p95": 1330433, + "p99": null, + "maximum": 7472033 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "samples": 135, + "p50": 682154, + "p95": 1057630, + "p99": null, + "maximum": 1079821 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "end_to_end", + "samples": 135, + "p50": 1756956, + "p95": 2201178, + "p99": null, + "maximum": 8292324 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "samples": 180, + "p50": 29464145, + "p95": 31859822, + "p99": null, + "maximum": 84748961 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "samples": 180, + "p50": 672579, + "p95": 1059253, + "p99": null, + "maximum": 1092325 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 30151854, + "p95": 32319736, + "p99": null, + "maximum": 85578692 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "samples": 180, + "p50": 776938, + "p95": 949553, + "p99": null, + "maximum": 10340821 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "samples": 180, + "p50": 554532, + "p95": 1010964, + "p99": null, + "maximum": 1097016 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "end_to_end", + "samples": 180, + "p50": 1291446, + "p95": 1793966, + "p99": null, + "maximum": 10975249 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "samples": 180, + "p50": 1539357, + "p95": 1766429, + "p99": null, + "maximum": 7726342 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "samples": 180, + "p50": 577601, + "p95": 1053979, + "p99": null, + "maximum": 1097821 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "end_to_end", + "samples": 180, + "p50": 2166539, + "p95": 2688784, + "p99": null, + "maximum": 7942952 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "samples": 180, + "p50": 454118, + "p95": 2099687, + "p99": null, + "maximum": 2908427 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "samples": 180, + "p50": 592001, + "p95": 1032978, + "p99": null, + "maximum": 3417868 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "end_to_end", + "samples": 180, + "p50": 1081378, + "p95": 2705952, + "p99": null, + "maximum": 3960884 + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "samples": 100, + "p50": 75935594, + "p95": 94929415, + "p99": null, + "maximum": 168691464 + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "samples": 100, + "p50": 845636, + "p95": 1384812, + "p99": null, + "maximum": 5597965 + }, + { + "scenario": "websocket", + "event_class": "new_transmission", + "timing": "receipt_after_ingest_return", + "samples": 4500, + "p50": 507442696, + "p95": 986769601, + "p99": 1401338010, + "maximum": 2322782732 + }, + { + "scenario": "handler-control", + "event_class": "200_valid_envelopes", + "timing": "service", + "samples": 200, + "p50": 802914, + "p95": 1808647, + "p99": null, + "maximum": 3863637 + } + ], + "counts": { + "durable-ingest": { + "total": 12000, + "measured": 9000, + "successful": 12000, + "errors": 0, + "drops": 0 + }, + "mixed-http": { + "total": 4800, + "measured": 3600, + "successful": 4800, + "errors": 0, + "drops": 0 + }, + "sql-miss": { + "total": 200, + "measured": 200, + "successful": 200, + "errors": 0, + "drops": 0 + } + }, + "throughput_including_drain": { + "durable-ingest/new": 25.030439905523753, + "durable-ingest/additional": 15.024982037215919, + "durable-ingest/duplicate": 7.51497417095111, + "durable-ingest/late": 2.5055425541970617, + "mixed-http/nodes_50": 1.4326015608049392, + "mixed-http/nodes_region": 1.228979678131115, + "mixed-http/nodes_2000": 1.4321369562196102, + "mixed-http/channels": 1.4326191525360814, + "mixed-http/messages_0": 1.432617519920778, + "mixed-http/messages_1000": 1.2290041713302788, + "mixed-http/observers": 2.0518223234808373, + "mixed-http/metrics_dense_24h": 1.0262036792134246, + "mixed-http/metrics_sparse_7d": 1.0262147103218946, + "mixed-http/reach": 1.0262195329810109, + "mixed-http/rx_coverage": 1.026210655803778, + "mixed-http/packets_memory": 1.0248705498897606, + "mixed-http/packet_detail_memory": 1.025048531895542, + "mixed-http/packet_detail_sql": 1.0336777876436403, + "mixed-http/analytics_rf": 0.6157255961925494, + "mixed-http/analytics_topology": 0.6156322150564362, + "mixed-http/analytics_channels": 0.4108872023978114, + "mixed-http/analytics_rf_filtered": 0.4109529557578954, + "mixed-http/stats": 1.0273915374568166, + "mixed-http/packet_detail_pruned": 1.1083434128901368, + "sql-miss/channel_messages_sql": 13.740803391933806, + "sql-miss/observer_metrics_sql": 13.851959854423377 + }, + "scalars_ns": { + "startup/first_http_ns": 28037931907, + "startup/ready_ns": 28037934495, + "startup/full_ready_ns": 28051047550, + "startup-hot1/first_http_ns": 9466903660, + "startup-hot1/ready_ns": 9466905553, + "startup-hot1/full_ready_ns": 20126865883, + "retention/elapsed_ns": 10489658913, + "retention-empty/elapsed_ns": 260248 + }, + "resources": { + "sample_count": 180, + "window_start_ns": 4598997645610, + "window_end_ns": 4778997645610, + "sampled_start_ns": 4599380206390, + "sampled_end_ns": 4778824297382, + "sampled_span_ns": 179444090992, + "sampled_span_fraction": 0.9969116166222223, + "first_boundary_gap_ns": 382560780, + "last_boundary_gap_ns": 173348228, + "sampling_interval_ns": { + "median": 1002308451, + "minimum": 1002054999, + "maximum": 1006680108, + "runs": 179 + }, + "phases": [ + "mixed_with_retention" + ], + "cpu_delta_usec": 41180142, + "average_cores_over_sampled_span": 0.22948731146480547, + "per_device_io": { + "8:0": { + "rbytes": 0, + "wbytes": 2490695680, + "wios": 223605, + "rios": 0, + "dios": 0, + "dbytes": 0 + } + }, + "aggregate_accounted_io": { + "dbytes": 0, + "dios": 0, + "rbytes": 0, + "rios": 0, + "wbytes": 2490695680, + "wios": 223605 + }, + "observed_peaks": { + "cgroup_memory_bytes": 4208447488, + "rss_bytes": 3036344320, + "pss_bytes": 3032899584, + "storage_bytes": 1150824992, + "wal_bytes": 30323232 + }, + "memory_event_deltas": { + "oom_group_kill": 0, + "oom": 0, + "high": 0, + "max": 0, + "oom_kill": 0, + "low": 0 + }, + "minimum_free_bytes": 80926326784 + }, + "websocket": { + "full_stream_sentinels": 6000, + "full_stream_received": 6000, + "measured_sentinels": 4500, + "measured_missing": 0, + "full_stream_missing": 0, + "negative_measured_lags": 0, + "negative_full_stream_lags": 0, + "duplicate_measured_receipts": 0, + "duplicate_full_stream_receipts": 0 + }, + "retention": { + "retention": { + "elapsed_ns": 10489658913, + "transmissions_deleted": 16000, + "observations_deleted": 256000 + }, + "retention-empty": { + "elapsed_ns": 260248, + "transmissions_deleted": 0, + "observations_deleted": 0 + } + }, + "startup_counts": [ + { + "transmissions": 112000, + "observations": 1792000 + }, + { + "transmissions": 112000, + "observations": 1792000 + } + ], + "database_settings": { + "busy_timeout": "5000", + "cache_size": "-2000", + "cache_spill": "483", + "foreign_keys": "1", + "journal_mode": "wal", + "journal_size_limit": "-1", + "mmap_size": "0", + "page_size": "4096", + "sqlite_version": "3.53.4", + "synchronous": "2", + "wal_autocheckpoint": "1000" + }, + "durable_counts": { + "scheduled": 12000, + "completed": 12000, + "dropped_schedules": 0, + "write_errors": 0, + "actual_transmissions": 118000, + "actual_observations": 1802200 + } + }, + { + "pair": 3, + "backend": "postgres", + "metrics": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "samples": 2700, + "p50": 1333597, + "p95": 2040614, + "p99": 3115148, + "maximum": 54827570 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "samples": 2700, + "p50": 614822, + "p95": 1068853, + "p99": 1123446, + "maximum": 35918944 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "end_to_end", + "samples": 2700, + "p50": 2006288, + "p95": 2851894, + "p99": 4914909, + "maximum": 55898858 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "samples": 1350, + "p50": 1210811, + "p95": 1612256, + "p99": 1935544, + "maximum": 27071929 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "samples": 1350, + "p50": 621053, + "p95": 1071966, + "p99": 1112423, + "maximum": 18442827 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "end_to_end", + "samples": 1350, + "p50": 1853954, + "p95": 2472320, + "p99": 3075857, + "maximum": 27663711 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "samples": 450, + "p50": 1505178, + "p95": 1918817, + "p99": null, + "maximum": 22650542 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "samples": 450, + "p50": 599580, + "p95": 1068727, + "p99": null, + "maximum": 2894040 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "end_to_end", + "samples": 450, + "p50": 2120271, + "p95": 2752027, + "p99": null, + "maximum": 22714147 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "samples": 4500, + "p50": 1479998, + "p95": 2072637, + "p99": 2993042, + "maximum": 62367875 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "samples": 4500, + "p50": 620323, + "p95": 1070358, + "p99": 1120713, + "maximum": 67440507 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "end_to_end", + "samples": 4500, + "p50": 2130649, + "p95": 2831329, + "p99": 4338892, + "maximum": 87414264 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "samples": 72, + "p50": 30607540, + "p95": 36474216, + "p99": null, + "maximum": 37473417 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "samples": 72, + "p50": 344902, + "p95": 1000273, + "p99": null, + "maximum": 1118028 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "end_to_end", + "samples": 72, + "p50": 31105091, + "p95": 36573822, + "p99": null, + "maximum": 37626014 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "samples": 108, + "p50": 1666121, + "p95": 2394004, + "p99": null, + "maximum": 2821201 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "samples": 108, + "p50": 650844, + "p95": 1074081, + "p99": null, + "maximum": 1096165 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "end_to_end", + "samples": 108, + "p50": 2337254, + "p95": 3175122, + "p99": null, + "maximum": 3474309 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "samples": 72, + "p50": 1897057, + "p95": 550401549, + "p99": null, + "maximum": 1051965590 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "samples": 72, + "p50": 137712, + "p95": 1054308, + "p99": null, + "maximum": 1137318 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "end_to_end", + "samples": 72, + "p50": 2187634, + "p95": 550515514, + "p99": null, + "maximum": 1052114377 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "samples": 108, + "p50": 29975967, + "p95": 35191087, + "p99": null, + "maximum": 36501505 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "samples": 108, + "p50": 672836, + "p95": 1047385, + "p99": null, + "maximum": 1115618 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "end_to_end", + "samples": 108, + "p50": 30572256, + "p95": 35259107, + "p99": null, + "maximum": 37387954 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "samples": 252, + "p50": 666549, + "p95": 1005267, + "p99": null, + "maximum": 27516391 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "samples": 252, + "p50": 573890, + "p95": 1036760, + "p99": null, + "maximum": 1105978 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "end_to_end", + "samples": 252, + "p50": 1265774, + "p95": 1825677, + "p99": null, + "maximum": 28316390 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "samples": 252, + "p50": 1428465, + "p95": 116133908, + "p99": null, + "maximum": 508648445 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "samples": 252, + "p50": 547257, + "p95": 1048862, + "p99": null, + "maximum": 2550870 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "end_to_end", + "samples": 252, + "p50": 1989834, + "p95": 116463409, + "p99": null, + "maximum": 509348942 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "samples": 216, + "p50": 1403935, + "p95": 117877971, + "p99": null, + "maximum": 469592222 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "samples": 216, + "p50": 573203, + "p95": 1051472, + "p99": null, + "maximum": 1083901 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "end_to_end", + "samples": 216, + "p50": 2072784, + "p95": 118115976, + "p99": null, + "maximum": 470477157 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "samples": 180, + "p50": 3795591, + "p95": 4503436, + "p99": null, + "maximum": 5222700 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "samples": 180, + "p50": 512560, + "p95": 1026766, + "p99": null, + "maximum": 1093784 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "end_to_end", + "samples": 180, + "p50": 4334900, + "p95": 5237076, + "p99": null, + "maximum": 6000662 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "samples": 180, + "p50": 1714038, + "p95": 1920762, + "p99": null, + "maximum": 2133821 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "samples": 180, + "p50": 576003, + "p95": 1017079, + "p99": null, + "maximum": 1085620 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "end_to_end", + "samples": 180, + "p50": 2270758, + "p95": 2774734, + "p99": null, + "maximum": 2931590 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "samples": 252, + "p50": 62823541, + "p95": 120169199, + "p99": null, + "maximum": 172735771 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "samples": 252, + "p50": 550824, + "p95": 1039970, + "p99": null, + "maximum": 1091818 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "end_to_end", + "samples": 252, + "p50": 63354364, + "p95": 120764909, + "p99": null, + "maximum": 173489331 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "samples": 252, + "p50": 5126332, + "p95": 9456650, + "p99": null, + "maximum": 33632009 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "samples": 252, + "p50": 175965, + "p95": 1004705, + "p99": null, + "maximum": 1466254 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "end_to_end", + "samples": 252, + "p50": 5488742, + "p95": 9994621, + "p99": null, + "maximum": 34198798 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "samples": 216, + "p50": 7636628, + "p95": 15747938, + "p99": null, + "maximum": 62955102 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "samples": 216, + "p50": 554883, + "p95": 1067326, + "p99": null, + "maximum": 1173454 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "end_to_end", + "samples": 216, + "p50": 8267731, + "p95": 16300579, + "p99": null, + "maximum": 63510985 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "samples": 360, + "p50": 2990849, + "p95": 4498084, + "p99": null, + "maximum": 106986048 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "samples": 360, + "p50": 557758, + "p95": 1033918, + "p99": null, + "maximum": 1113092 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "end_to_end", + "samples": 360, + "p50": 3541517, + "p95": 5022293, + "p99": null, + "maximum": 107587643 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "samples": 180, + "p50": 2545176, + "p95": 3043911, + "p99": null, + "maximum": 9215681 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "samples": 180, + "p50": 641479, + "p95": 1031314, + "p99": null, + "maximum": 1088290 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 3135607, + "p95": 3731687, + "p99": null, + "maximum": 10192136 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "samples": 45, + "p50": 677662, + "p95": 1019459, + "p99": null, + "maximum": 1376520 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "samples": 45, + "p50": 424549, + "p95": 1041592, + "p99": null, + "maximum": 1063549 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "end_to_end", + "samples": 45, + "p50": 1237684, + "p95": 1693827, + "p99": null, + "maximum": 2061051 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "samples": 135, + "p50": 1508968, + "p95": 1829288, + "p99": null, + "maximum": 2506169 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "samples": 135, + "p50": 618291, + "p95": 1068505, + "p99": null, + "maximum": 1094991 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "end_to_end", + "samples": 135, + "p50": 2159094, + "p95": 2694851, + "p99": null, + "maximum": 3196471 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "samples": 180, + "p50": 34078962, + "p95": 37451081, + "p99": null, + "maximum": 123330698 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "samples": 180, + "p50": 580773, + "p95": 1015080, + "p99": null, + "maximum": 1099182 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 34692184, + "p95": 38324784, + "p99": null, + "maximum": 123788430 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "samples": 180, + "p50": 828113, + "p95": 960232, + "p99": null, + "maximum": 2164723 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "samples": 180, + "p50": 589940, + "p95": 1054793, + "p99": null, + "maximum": 1113526 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "end_to_end", + "samples": 180, + "p50": 1429086, + "p95": 1917114, + "p99": null, + "maximum": 3097855 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "samples": 180, + "p50": 1680802, + "p95": 1841533, + "p99": null, + "maximum": 2376853 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "samples": 180, + "p50": 593967, + "p95": 1002055, + "p99": null, + "maximum": 1066229 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "end_to_end", + "samples": 180, + "p50": 2276651, + "p95": 2744902, + "p99": null, + "maximum": 2867160 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "samples": 180, + "p50": 444647, + "p95": 3708109, + "p99": null, + "maximum": 5806389 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "samples": 180, + "p50": 563564, + "p95": 1051609, + "p99": null, + "maximum": 1100716 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "end_to_end", + "samples": 180, + "p50": 1034430, + "p95": 4322928, + "p99": null, + "maximum": 6399867 + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "samples": 100, + "p50": 108853343, + "p95": 122686767, + "p99": null, + "maximum": 218790346 + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "samples": 100, + "p50": 885560, + "p95": 1210486, + "p99": null, + "maximum": 3240313 + }, + { + "scenario": "websocket", + "event_class": "new_transmission", + "timing": "receipt_after_ingest_return", + "samples": 4500, + "p50": 511175986, + "p95": 971019472, + "p99": 1569019521, + "maximum": 2088790429 + }, + { + "scenario": "handler-control", + "event_class": "200_valid_envelopes", + "timing": "service", + "samples": 200, + "p50": 1545732, + "p95": 2112537, + "p99": null, + "maximum": 5151475 + } + ], + "counts": { + "durable-ingest": { + "total": 12000, + "measured": 9000, + "successful": 12000, + "errors": 0, + "drops": 0 + }, + "mixed-http": { + "total": 4800, + "measured": 3600, + "successful": 4800, + "errors": 0, + "drops": 0 + }, + "sql-miss": { + "total": 200, + "measured": 200, + "successful": 200, + "errors": 0, + "drops": 0 + } + }, + "throughput_including_drain": { + "durable-ingest/new": 25.03035269133082, + "durable-ingest/additional": 15.024877578444668, + "durable-ingest/duplicate": 7.514971838959406, + "durable-ingest/late": 2.5055393442095704, + "mixed-http/nodes_50": 1.4325923358567263, + "mixed-http/nodes_region": 1.2289593607145823, + "mixed-http/nodes_2000": 1.4321360214695593, + "mixed-http/channels": 1.4326252717124708, + "mixed-http/messages_0": 1.4326142854382222, + "mixed-http/messages_1000": 1.2290074199558843, + "mixed-http/observers": 2.0518226444329977, + "mixed-http/metrics_dense_24h": 1.0262024879197815, + "mixed-http/metrics_sparse_7d": 1.0262101842337945, + "mixed-http/reach": 1.0262150154157241, + "mixed-http/rx_coverage": 1.0262124125728151, + "mixed-http/packets_memory": 1.0248571049842008, + "mixed-http/packet_detail_memory": 1.025040348918518, + "mixed-http/packet_detail_sql": 1.033676769397638, + "mixed-http/analytics_rf": 0.6157272239470484, + "mixed-http/analytics_topology": 0.6156319924589164, + "mixed-http/analytics_channels": 0.4108851580052166, + "mixed-http/analytics_rf_filtered": 0.41095215148316555, + "mixed-http/stats": 1.027394360605857, + "mixed-http/packet_detail_pruned": 1.108335836479854, + "sql-miss/channel_messages_sql": 8.804470889468686, + "sql-miss/observer_metrics_sql": 8.911988890842238 + }, + "scalars_ns": { + "startup/first_http_ns": 27234077804, + "startup/ready_ns": 27234080235, + "startup/full_ready_ns": 27443441037, + "startup-hot1/first_http_ns": 11574633479, + "startup-hot1/ready_ns": 11574636556, + "startup-hot1/full_ready_ns": 20242269464, + "retention/elapsed_ns": 844435612, + "retention-empty/elapsed_ns": 1941161, + "migration/elapsed_ns": 147616197535, + "migration/migration_tool_ns": 92797633416, + "migration/role_grants_ns": 755437541, + "migration/harness_validation_ns": 54063127899 + }, + "resources": { + "sample_count": 175, + "window_start_ns": 4201532482936, + "window_end_ns": 4381532482936, + "sampled_start_ns": 4201641716811, + "sampled_end_ns": 4380795532257, + "sampled_span_ns": 179153815446, + "sampled_span_fraction": 0.9952989747, + "first_boundary_gap_ns": 109233875, + "last_boundary_gap_ns": 736950679, + "sampling_interval_ns": { + "median": 1028134695.0, + "minimum": 1026505967, + "maximum": 1086977692, + "runs": 174 + }, + "phases": [ + "mixed_with_retention" + ], + "cpu_delta_usec": 58134110, + "average_cores_over_sampled_span": 0.3244927262937507, + "per_device_io": { + "8:0": { + "rbytes": 6041600, + "wbytes": 511025152, + "wios": 24726, + "rios": 658, + "dios": 1, + "dbytes": 4096 + } + }, + "aggregate_accounted_io": { + "dbytes": 4096, + "dios": 1, + "rbytes": 6041600, + "rios": 658, + "wbytes": 511025152, + "wios": 24726 + }, + "observed_peaks": { + "cgroup_memory_bytes": 5956444160, + "rss_bytes": 5474234368, + "pss_bytes": 3439024128, + "storage_bytes": 2439704380, + "wal_bytes": 1073741824 + }, + "memory_event_deltas": { + "oom_group_kill": 0, + "oom": 0, + "high": 0, + "max": 0, + "oom_kill": 0, + "low": 0 + }, + "minimum_free_bytes": 77379325952 + }, + "websocket": { + "full_stream_sentinels": 6000, + "full_stream_received": 6000, + "measured_sentinels": 4500, + "measured_missing": 0, + "full_stream_missing": 0, + "negative_measured_lags": 0, + "negative_full_stream_lags": 0, + "duplicate_measured_receipts": 0, + "duplicate_full_stream_receipts": 0 + }, + "retention": { + "retention": { + "elapsed_ns": 844435612, + "transmissions_deleted": 16000, + "observations_deleted": 256000 + }, + "retention-empty": { + "elapsed_ns": 1941161, + "transmissions_deleted": 0, + "observations_deleted": 0 + } + }, + "startup_counts": [ + { + "transmissions": 112000, + "observations": 1792000 + }, + { + "transmissions": 112000, + "observations": 1792000 + } + ], + "database_settings": { + "autovacuum": "on", + "checkpoint_timeout": "5min", + "effective_cache_size": "4GB", + "fsync": "on", + "full_page_writes": "on", + "jit": "on", + "maintenance_work_mem": "64MB", + "max_connections": "32", + "max_parallel_workers_per_gather": "2", + "max_wal_size": "1GB", + "server_version": "18.6 (Ubuntu 18.6-1.pgdg24.04+2)", + "shared_buffers": "256MB", + "synchronous_commit": "on", + "work_mem": "4MB" + }, + "durable_counts": { + "scheduled": 12000, + "completed": 12000, + "dropped_schedules": 0, + "write_errors": 0, + "actual_transmissions": 118000, + "actual_observations": 1802200 + } + }, + { + "pair": 4, + "backend": "sqlite", + "metrics": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "samples": 2700, + "p50": 728915, + "p95": 1267221, + "p99": 3273407, + "maximum": 404500289 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "samples": 2700, + "p50": 644889, + "p95": 1418025850, + "p99": 7282994145, + "maximum": 8517142686 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "end_to_end", + "samples": 2700, + "p50": 1416808, + "p95": 1418631405, + "p99": 7283266492, + "maximum": 8517639363 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "samples": 1350, + "p50": 620009, + "p95": 995318, + "p99": 1602756, + "maximum": 395165063 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "samples": 1350, + "p50": 623236, + "p95": 1339572698, + "p99": 7204174062, + "maximum": 8399497142 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "end_to_end", + "samples": 1350, + "p50": 1281234, + "p95": 1339908672, + "p99": 7204380610, + "maximum": 8399746894 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "samples": 450, + "p50": 887300, + "p95": 1471023, + "p99": null, + "maximum": 222213114 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "samples": 450, + "p50": 634351, + "p95": 1300115746, + "p99": null, + "maximum": 8342461834 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "end_to_end", + "samples": 450, + "p50": 1534555, + "p95": 1300493011, + "p99": null, + "maximum": 8343004986 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "samples": 4500, + "p50": 969291, + "p95": 1592996, + "p99": 3642435, + "maximum": 412921309 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "samples": 4500, + "p50": 630744, + "p95": 1496191553, + "p99": 7252420371, + "maximum": 8633174274 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "end_to_end", + "samples": 4500, + "p50": 1602181, + "p95": 1516178442, + "p99": 7291978626, + "maximum": 8653147615 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "samples": 72, + "p50": 29620151, + "p95": 32457613, + "p99": null, + "maximum": 46103961 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "samples": 72, + "p50": 393849, + "p95": 1032801, + "p99": null, + "maximum": 1079159 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "end_to_end", + "samples": 72, + "p50": 30164830, + "p95": 33489189, + "p99": null, + "maximum": 46166644 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "samples": 108, + "p50": 1777736, + "p95": 2472415, + "p99": null, + "maximum": 2736034 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "samples": 108, + "p50": 755837, + "p95": 1068985, + "p99": null, + "maximum": 1164132 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "end_to_end", + "samples": 108, + "p50": 2553349, + "p95": 3277188, + "p99": null, + "maximum": 3666353 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "samples": 72, + "p50": 1748556, + "p95": 542635483, + "p99": null, + "maximum": 608843451 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "samples": 72, + "p50": 124344, + "p95": 808610, + "p99": null, + "maximum": 871738 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "end_to_end", + "samples": 72, + "p50": 1951621, + "p95": 542753626, + "p99": null, + "maximum": 609406443 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "samples": 108, + "p50": 29795987, + "p95": 36502620, + "p99": null, + "maximum": 47544772 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "samples": 108, + "p50": 726680, + "p95": 1022097, + "p99": null, + "maximum": 1118298 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "end_to_end", + "samples": 108, + "p50": 30388414, + "p95": 37017366, + "p99": null, + "maximum": 48434122 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "samples": 252, + "p50": 671267, + "p95": 874879, + "p99": null, + "maximum": 18478667 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "samples": 252, + "p50": 587628, + "p95": 1043880, + "p99": null, + "maximum": 1204661 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "end_to_end", + "samples": 252, + "p50": 1295682, + "p95": 1765052, + "p99": null, + "maximum": 19471171 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "samples": 252, + "p50": 1338143, + "p95": 59905684, + "p99": null, + "maximum": 63326323 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "samples": 252, + "p50": 497273, + "p95": 1034135, + "p99": null, + "maximum": 2187451 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "end_to_end", + "samples": 252, + "p50": 1951684, + "p95": 60085376, + "p99": null, + "maximum": 63735691 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "samples": 216, + "p50": 1377416, + "p95": 82786779, + "p99": null, + "maximum": 163758481 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "samples": 216, + "p50": 591989, + "p95": 1066348, + "p99": null, + "maximum": 2735926 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "end_to_end", + "samples": 216, + "p50": 2011405, + "p95": 83512558, + "p99": null, + "maximum": 164037809 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "samples": 180, + "p50": 3327173, + "p95": 4125651, + "p99": null, + "maximum": 4815691 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "samples": 180, + "p50": 538129, + "p95": 1041495, + "p99": null, + "maximum": 1105546 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "end_to_end", + "samples": 180, + "p50": 3844893, + "p95": 4801167, + "p99": null, + "maximum": 5551578 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "samples": 180, + "p50": 1369118, + "p95": 1554201, + "p99": null, + "maximum": 1863207 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "samples": 180, + "p50": 670805, + "p95": 1063729, + "p99": null, + "maximum": 1283637 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "end_to_end", + "samples": 180, + "p50": 2047631, + "p95": 2502633, + "p99": null, + "maximum": 3086668 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "samples": 252, + "p50": 61738200, + "p95": 82184023, + "p99": null, + "maximum": 206354101 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "samples": 252, + "p50": 602775, + "p95": 986390, + "p99": null, + "maximum": 1154851 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "end_to_end", + "samples": 252, + "p50": 62362964, + "p95": 82787187, + "p99": null, + "maximum": 206742595 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "samples": 252, + "p50": 3331942, + "p95": 6110372, + "p99": null, + "maximum": 26488964 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "samples": 252, + "p50": 237058, + "p95": 1035627, + "p99": null, + "maximum": 2223423 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "end_to_end", + "samples": 252, + "p50": 3661213, + "p95": 7137790, + "p99": null, + "maximum": 26504533 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "samples": 216, + "p50": 5476232, + "p95": 10094717, + "p99": null, + "maximum": 13984493 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "samples": 216, + "p50": 456769, + "p95": 1065376, + "p99": null, + "maximum": 1768703 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "end_to_end", + "samples": 216, + "p50": 5921280, + "p95": 10618082, + "p99": null, + "maximum": 14324285 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "samples": 360, + "p50": 2845242, + "p95": 3812894, + "p99": null, + "maximum": 18423110 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "samples": 360, + "p50": 577949, + "p95": 1018988, + "p99": null, + "maximum": 1109517 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "end_to_end", + "samples": 360, + "p50": 3489156, + "p95": 4572379, + "p99": null, + "maximum": 19150617 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "samples": 180, + "p50": 1263712, + "p95": 1585619, + "p99": null, + "maximum": 63300275 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "samples": 180, + "p50": 582744, + "p95": 1037678, + "p99": null, + "maximum": 1090601 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 1864229, + "p95": 2432617, + "p99": null, + "maximum": 64337953 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "samples": 45, + "p50": 680745, + "p95": 792570, + "p99": null, + "maximum": 841925 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "samples": 45, + "p50": 544909, + "p95": 1062243, + "p99": null, + "maximum": 1089944 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "end_to_end", + "samples": 45, + "p50": 1275193, + "p95": 1752626, + "p99": null, + "maximum": 1889782 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "samples": 135, + "p50": 1097317, + "p95": 1337115, + "p99": null, + "maximum": 19596451 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "samples": 135, + "p50": 556303, + "p95": 1045532, + "p99": null, + "maximum": 1082599 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "end_to_end", + "samples": 135, + "p50": 1726965, + "p95": 2196763, + "p99": null, + "maximum": 20129292 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "samples": 180, + "p50": 29910908, + "p95": 31885043, + "p99": null, + "maximum": 95091321 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "samples": 180, + "p50": 576248, + "p95": 1060265, + "p99": null, + "maximum": 1127982 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 30462193, + "p95": 32599106, + "p99": null, + "maximum": 95419001 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "samples": 180, + "p50": 807890, + "p95": 931987, + "p99": null, + "maximum": 1739692 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "samples": 180, + "p50": 555119, + "p95": 973458, + "p99": null, + "maximum": 1055659 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "end_to_end", + "samples": 180, + "p50": 1393898, + "p95": 1797060, + "p99": null, + "maximum": 2598981 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "samples": 180, + "p50": 1556988, + "p95": 1764734, + "p99": null, + "maximum": 1906389 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "samples": 180, + "p50": 590583, + "p95": 1004059, + "p99": null, + "maximum": 1077759 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "end_to_end", + "samples": 180, + "p50": 2163966, + "p95": 2623178, + "p99": null, + "maximum": 2779949 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "samples": 180, + "p50": 469071, + "p95": 2251394, + "p99": null, + "maximum": 3179848 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "samples": 180, + "p50": 585749, + "p95": 1054275, + "p99": null, + "maximum": 1114595 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "end_to_end", + "samples": 180, + "p50": 1110411, + "p95": 2746312, + "p99": null, + "maximum": 3531975 + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "samples": 100, + "p50": 76441452, + "p95": 85136438, + "p99": null, + "maximum": 162459050 + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "samples": 100, + "p50": 760640, + "p95": 1304741, + "p99": null, + "maximum": 1969281 + }, + { + "scenario": "websocket", + "event_class": "new_transmission", + "timing": "receipt_after_ingest_return", + "samples": 4500, + "p50": 507054442, + "p95": 985455250, + "p99": 1345331140, + "maximum": 1979785508 + }, + { + "scenario": "handler-control", + "event_class": "200_valid_envelopes", + "timing": "service", + "samples": 200, + "p50": 808459, + "p95": 1798700, + "p99": null, + "maximum": 3612669 + } + ], + "counts": { + "durable-ingest": { + "total": 12000, + "measured": 9000, + "successful": 12000, + "errors": 0, + "drops": 0 + }, + "mixed-http": { + "total": 4800, + "measured": 3600, + "successful": 4800, + "errors": 0, + "drops": 0 + }, + "sql-miss": { + "total": 200, + "measured": 200, + "successful": 200, + "errors": 0, + "drops": 0 + } + }, + "throughput_including_drain": { + "durable-ingest/new": 25.030416454931984, + "durable-ingest/additional": 15.024764325650635, + "durable-ingest/duplicate": 7.514981423623983, + "durable-ingest/late": 2.505538498823182, + "mixed-http/nodes_50": 1.4325968224639394, + "mixed-http/nodes_region": 1.2289804125731867, + "mixed-http/nodes_2000": 1.432137778742715, + "mixed-http/channels": 1.4326223533143574, + "mixed-http/messages_0": 1.4326202182199135, + "mixed-http/messages_1000": 1.2290053969508077, + "mixed-http/observers": 2.0518273996471788, + "mixed-http/metrics_dense_24h": 1.0262033558961197, + "mixed-http/metrics_sparse_7d": 1.0262141613675921, + "mixed-http/reach": 1.026220640613885, + "mixed-http/rx_coverage": 1.026212081427731, + "mixed-http/packets_memory": 1.0248771569337634, + "mixed-http/packet_detail_memory": 1.0250475432172843, + "mixed-http/packet_detail_sql": 1.033674500860842, + "mixed-http/analytics_rf": 0.6157285752179937, + "mixed-http/analytics_topology": 0.6156343787186107, + "mixed-http/analytics_channels": 0.4108919547066046, + "mixed-http/analytics_rf_filtered": 0.410954063699347, + "mixed-http/stats": 1.0273904783357235, + "mixed-http/packet_detail_pruned": 1.1083410622900038, + "sql-miss/channel_messages_sql": 13.803637268524518, + "sql-miss/observer_metrics_sql": 13.916314694145312 + }, + "scalars_ns": { + "startup/first_http_ns": 28044152778, + "startup/ready_ns": 28044154793, + "startup/full_ready_ns": 28057500751, + "startup-hot1/first_http_ns": 9565475272, + "startup-hot1/ready_ns": 9565477948, + "startup-hot1/full_ready_ns": 20519912326, + "retention/elapsed_ns": 10457033035, + "retention-empty/elapsed_ns": 288837 + }, + "resources": { + "sample_count": 179, + "window_start_ns": 5063636324030, + "window_end_ns": 5243636324030, + "sampled_start_ns": 5064469308804, + "sampled_end_ns": 5242929761439, + "sampled_span_ns": 178460452635, + "sampled_span_fraction": 0.9914469590833334, + "first_boundary_gap_ns": 832984774, + "last_boundary_gap_ns": 706562591, + "sampling_interval_ns": { + "median": 1002316097.0, + "minimum": 1002079160, + "maximum": 1025885458, + "runs": 178 + }, + "phases": [ + "mixed_with_retention" + ], + "cpu_delta_usec": 42554895, + "average_cores_over_sampled_span": 0.23845560387004228, + "per_device_io": { + "8:0": { + "rbytes": 0, + "wbytes": 2448142336, + "wios": 213997, + "rios": 0, + "dios": 0, + "dbytes": 0 + } + }, + "aggregate_accounted_io": { + "dbytes": 0, + "dios": 0, + "rbytes": 0, + "rios": 0, + "wbytes": 2448142336, + "wios": 213997 + }, + "observed_peaks": { + "cgroup_memory_bytes": 4276535296, + "rss_bytes": 3071049728, + "pss_bytes": 3067539456, + "storage_bytes": 1176985880, + "wal_bytes": 55591192 + }, + "memory_event_deltas": { + "oom_group_kill": 0, + "oom": 0, + "high": 0, + "max": 0, + "oom_kill": 0, + "low": 0 + }, + "minimum_free_bytes": 79768514560 + }, + "websocket": { + "full_stream_sentinels": 6000, + "full_stream_received": 6000, + "measured_sentinels": 4500, + "measured_missing": 0, + "full_stream_missing": 0, + "negative_measured_lags": 0, + "negative_full_stream_lags": 0, + "duplicate_measured_receipts": 0, + "duplicate_full_stream_receipts": 0 + }, + "retention": { + "retention": { + "elapsed_ns": 10457033035, + "transmissions_deleted": 16000, + "observations_deleted": 256000 + }, + "retention-empty": { + "elapsed_ns": 288837, + "transmissions_deleted": 0, + "observations_deleted": 0 + } + }, + "startup_counts": [ + { + "transmissions": 112000, + "observations": 1792000 + }, + { + "transmissions": 112000, + "observations": 1792000 + } + ], + "database_settings": { + "busy_timeout": "5000", + "cache_size": "-2000", + "cache_spill": "483", + "foreign_keys": "1", + "journal_mode": "wal", + "journal_size_limit": "-1", + "mmap_size": "0", + "page_size": "4096", + "sqlite_version": "3.53.4", + "synchronous": "2", + "wal_autocheckpoint": "1000" + }, + "durable_counts": { + "scheduled": 12000, + "completed": 12000, + "dropped_schedules": 0, + "write_errors": 0, + "actual_transmissions": 118000, + "actual_observations": 1802200 + } + }, + { + "pair": 4, + "backend": "postgres", + "metrics": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "samples": 2700, + "p50": 1366552, + "p95": 1987759, + "p99": 2628122, + "maximum": 28874054 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "samples": 2700, + "p50": 620986, + "p95": 1071175, + "p99": 1117584, + "maximum": 10417899 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "end_to_end", + "samples": 2700, + "p50": 2049150, + "p95": 2787618, + "p99": 3517210, + "maximum": 29757860 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "samples": 1350, + "p50": 1251706, + "p95": 1673785, + "p99": 2074184, + "maximum": 36632634 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "samples": 1350, + "p50": 605116, + "p95": 1073323, + "p99": 1127280, + "maximum": 17534335 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "end_to_end", + "samples": 1350, + "p50": 1897170, + "p95": 2489061, + "p99": 3044891, + "maximum": 37508066 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "samples": 450, + "p50": 1545967, + "p95": 2070698, + "p99": null, + "maximum": 43365466 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "samples": 450, + "p50": 606007, + "p95": 1058335, + "p99": null, + "maximum": 2821214 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "end_to_end", + "samples": 450, + "p50": 2170802, + "p95": 2917657, + "p99": null, + "maximum": 44144928 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "samples": 4500, + "p50": 1514924, + "p95": 2084132, + "p99": 2785475, + "maximum": 45491880 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "samples": 4500, + "p50": 619166, + "p95": 1074208, + "p99": 1126835, + "maximum": 39732785 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "end_to_end", + "samples": 4500, + "p50": 2180463, + "p95": 2857915, + "p99": 3861511, + "maximum": 59704423 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "samples": 72, + "p50": 30081857, + "p95": 33855412, + "p99": null, + "maximum": 40165343 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "samples": 72, + "p50": 356897, + "p95": 1039591, + "p99": null, + "maximum": 1088732 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "end_to_end", + "samples": 72, + "p50": 30719712, + "p95": 34500193, + "p99": null, + "maximum": 40682489 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "samples": 108, + "p50": 1597846, + "p95": 2525169, + "p99": null, + "maximum": 2780584 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "samples": 108, + "p50": 571153, + "p95": 1057577, + "p99": null, + "maximum": 1114904 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "end_to_end", + "samples": 108, + "p50": 2373943, + "p95": 3014884, + "p99": null, + "maximum": 3706896 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "samples": 72, + "p50": 1965424, + "p95": 576027263, + "p99": null, + "maximum": 582735311 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "samples": 72, + "p50": 128416, + "p95": 961977, + "p99": null, + "maximum": 1072865 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "end_to_end", + "samples": 72, + "p50": 2224535, + "p95": 576142647, + "p99": null, + "maximum": 582849745 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "samples": 108, + "p50": 30081038, + "p95": 32516371, + "p99": null, + "maximum": 51233172 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "samples": 108, + "p50": 738119, + "p95": 1074273, + "p99": null, + "maximum": 1093714 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "end_to_end", + "samples": 108, + "p50": 30593146, + "p95": 33590024, + "p99": null, + "maximum": 51892530 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "samples": 252, + "p50": 676735, + "p95": 1058122, + "p99": null, + "maximum": 31827470 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "samples": 252, + "p50": 637022, + "p95": 1057938, + "p99": null, + "maximum": 2517357 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "end_to_end", + "samples": 252, + "p50": 1307488, + "p95": 1852827, + "p99": null, + "maximum": 32072090 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "samples": 252, + "p50": 1410775, + "p95": 112703993, + "p99": null, + "maximum": 239097510 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "samples": 252, + "p50": 599614, + "p95": 1056078, + "p99": null, + "maximum": 2262967 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "end_to_end", + "samples": 252, + "p50": 2062937, + "p95": 113275690, + "p99": null, + "maximum": 241360477 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "samples": 216, + "p50": 1427051, + "p95": 114125189, + "p99": null, + "maximum": 126343021 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "samples": 216, + "p50": 661902, + "p95": 1074946, + "p99": null, + "maximum": 2534750 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "end_to_end", + "samples": 216, + "p50": 2089331, + "p95": 114909699, + "p99": null, + "maximum": 127313528 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "samples": 180, + "p50": 3805277, + "p95": 4574740, + "p99": null, + "maximum": 5148741 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "samples": 180, + "p50": 566823, + "p95": 1038893, + "p99": null, + "maximum": 1101740 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "end_to_end", + "samples": 180, + "p50": 4404624, + "p95": 5151929, + "p99": null, + "maximum": 5721499 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "samples": 180, + "p50": 1735658, + "p95": 1951446, + "p99": null, + "maximum": 2326790 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "samples": 180, + "p50": 536064, + "p95": 1051906, + "p99": null, + "maximum": 1097096 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "end_to_end", + "samples": 180, + "p50": 2283101, + "p95": 2819113, + "p99": null, + "maximum": 3007227 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "samples": 252, + "p50": 63765017, + "p95": 126902611, + "p99": null, + "maximum": 198101869 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "samples": 252, + "p50": 510944, + "p95": 1045438, + "p99": null, + "maximum": 3818149 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "end_to_end", + "samples": 252, + "p50": 64354893, + "p95": 126961061, + "p99": null, + "maximum": 200588513 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "samples": 252, + "p50": 5230810, + "p95": 9377179, + "p99": null, + "maximum": 46910004 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "samples": 252, + "p50": 142630, + "p95": 1016385, + "p99": null, + "maximum": 1203553 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "end_to_end", + "samples": 252, + "p50": 5525959, + "p95": 9995458, + "p99": null, + "maximum": 48113557 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "samples": 216, + "p50": 7742004, + "p95": 14597220, + "p99": null, + "maximum": 36576562 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "samples": 216, + "p50": 492863, + "p95": 1033667, + "p99": null, + "maximum": 1208978 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "end_to_end", + "samples": 216, + "p50": 8262204, + "p95": 15520041, + "p99": null, + "maximum": 36984503 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "samples": 360, + "p50": 2922301, + "p95": 4180163, + "p99": null, + "maximum": 110194847 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "samples": 360, + "p50": 522037, + "p95": 1038465, + "p99": null, + "maximum": 1151491 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "end_to_end", + "samples": 360, + "p50": 3494786, + "p95": 4819054, + "p99": null, + "maximum": 110899965 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "samples": 180, + "p50": 2581827, + "p95": 3057684, + "p99": null, + "maximum": 3275525 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "samples": 180, + "p50": 571591, + "p95": 1039489, + "p99": null, + "maximum": 1100363 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 3145195, + "p95": 3747685, + "p99": null, + "maximum": 4019907 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "samples": 45, + "p50": 715544, + "p95": 869329, + "p99": null, + "maximum": 1292736 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "samples": 45, + "p50": 500297, + "p95": 1035042, + "p99": null, + "maximum": 1057308 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "end_to_end", + "samples": 45, + "p50": 1254152, + "p95": 1798169, + "p99": null, + "maximum": 1943326 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "samples": 135, + "p50": 1523597, + "p95": 1933827, + "p99": null, + "maximum": 2400599 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "samples": 135, + "p50": 621935, + "p95": 1044552, + "p99": null, + "maximum": 1091641 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "end_to_end", + "samples": 135, + "p50": 2128550, + "p95": 2661574, + "p99": null, + "maximum": 3410829 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "samples": 180, + "p50": 34739808, + "p95": 37123162, + "p99": null, + "maximum": 51840432 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "samples": 180, + "p50": 508948, + "p95": 1012698, + "p99": null, + "maximum": 1098975 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 35339795, + "p95": 37685779, + "p99": null, + "maximum": 52223254 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "samples": 180, + "p50": 818668, + "p95": 958522, + "p99": null, + "maximum": 1628938 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "samples": 180, + "p50": 592804, + "p95": 1066532, + "p99": null, + "maximum": 1118779 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "end_to_end", + "samples": 180, + "p50": 1408413, + "p95": 1877009, + "p99": null, + "maximum": 2679391 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "samples": 180, + "p50": 1800943, + "p95": 2011174, + "p99": null, + "maximum": 2364247 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "samples": 180, + "p50": 527788, + "p95": 1037729, + "p99": null, + "maximum": 1131074 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "end_to_end", + "samples": 180, + "p50": 2315039, + "p95": 2870395, + "p99": null, + "maximum": 3411645 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "samples": 180, + "p50": 460190, + "p95": 3719797, + "p99": null, + "maximum": 5582829 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "samples": 180, + "p50": 523900, + "p95": 1031617, + "p99": null, + "maximum": 1367605 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "end_to_end", + "samples": 180, + "p50": 1100963, + "p95": 4162587, + "p99": null, + "maximum": 5986869 + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "samples": 100, + "p50": 106304598, + "p95": 115012422, + "p99": null, + "maximum": 175465000 + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "samples": 100, + "p50": 814669, + "p95": 1322049, + "p99": null, + "maximum": 3269535 + }, + { + "scenario": "websocket", + "event_class": "new_transmission", + "timing": "receipt_after_ingest_return", + "samples": 4500, + "p50": 505540967, + "p95": 946833676, + "p99": 986662688, + "maximum": 1156836646 + }, + { + "scenario": "handler-control", + "event_class": "200_valid_envelopes", + "timing": "service", + "samples": 200, + "p50": 1574486, + "p95": 2274351, + "p99": null, + "maximum": 5713231 + } + ], + "counts": { + "durable-ingest": { + "total": 12000, + "measured": 9000, + "successful": 12000, + "errors": 0, + "drops": 0 + }, + "mixed-http": { + "total": 4800, + "measured": 3600, + "successful": 4800, + "errors": 0, + "drops": 0 + }, + "sql-miss": { + "total": 200, + "measured": 200, + "successful": 200, + "errors": 0, + "drops": 0 + } + }, + "throughput_including_drain": { + "durable-ingest/new": 25.030346347344757, + "durable-ingest/additional": 15.024829572423004, + "durable-ingest/duplicate": 7.514953461769194, + "durable-ingest/late": 2.5055331317161813, + "mixed-http/nodes_50": 1.4325874801798992, + "mixed-http/nodes_region": 1.228932099005017, + "mixed-http/nodes_2000": 1.4320777478872302, + "mixed-http/channels": 1.432624963125401, + "mixed-http/messages_0": 1.432613761901396, + "mixed-http/messages_1000": 1.2290078773315032, + "mixed-http/observers": 2.0518385051145525, + "mixed-http/metrics_dense_24h": 1.0262020202712916, + "mixed-http/metrics_sparse_7d": 1.0262132901317251, + "mixed-http/reach": 1.0262167020325899, + "mixed-http/rx_coverage": 1.0262128471981375, + "mixed-http/packets_memory": 1.024850728895416, + "mixed-http/packet_detail_memory": 1.0250378075488775, + "mixed-http/packet_detail_sql": 1.0336784590909618, + "mixed-http/analytics_rf": 0.6157263576485035, + "mixed-http/analytics_topology": 0.6156153153228161, + "mixed-http/analytics_channels": 0.41088079265731003, + "mixed-http/analytics_rf_filtered": 0.41095336441836106, + "mixed-http/stats": 1.027391889518365, + "mixed-http/packet_detail_pruned": 1.1083481604732834, + "sql-miss/channel_messages_sql": 9.02603258052588, + "sql-miss/observer_metrics_sql": 9.17066540165261 + }, + "scalars_ns": { + "startup/first_http_ns": 27752258609, + "startup/ready_ns": 27752261910, + "startup/full_ready_ns": 28026594795, + "startup-hot1/first_http_ns": 10976838200, + "startup-hot1/ready_ns": 10976841225, + "startup-hot1/full_ready_ns": 19918712935, + "retention/elapsed_ns": 902013732, + "retention-empty/elapsed_ns": 1714566, + "migration/elapsed_ns": 145038882300, + "migration/migration_tool_ns": 90985078119, + "migration/role_grants_ns": 773319309, + "migration/harness_validation_ns": 53280485928 + }, + "resources": { + "sample_count": 175, + "window_start_ns": 5615428545133, + "window_end_ns": 5795428545133, + "sampled_start_ns": 5615723348117, + "sampled_end_ns": 5795144350961, + "sampled_span_ns": 179421002844, + "sampled_span_fraction": 0.9967833491333333, + "first_boundary_gap_ns": 294802984, + "last_boundary_gap_ns": 284194172, + "sampling_interval_ns": { + "median": 1029175286.0, + "minimum": 1026984875, + "maximum": 1118218742, + "runs": 174 + }, + "phases": [ + "mixed_with_retention" + ], + "cpu_delta_usec": 56663746, + "average_cores_over_sampled_span": 0.3158144537251698, + "per_device_io": { + "8:0": { + "rbytes": 6021120, + "wbytes": 454680576, + "wios": 21733, + "rios": 641, + "dios": 1, + "dbytes": 4096 + } + }, + "aggregate_accounted_io": { + "dbytes": 4096, + "dios": 1, + "rbytes": 6021120, + "rios": 641, + "wbytes": 454680576, + "wios": 21733 + }, + "observed_peaks": { + "cgroup_memory_bytes": 5954797568, + "rss_bytes": 5289295872, + "pss_bytes": 3404889088, + "storage_bytes": 2439638844, + "wal_bytes": 1073741824 + }, + "memory_event_deltas": { + "oom_group_kill": 0, + "oom": 0, + "high": 0, + "max": 0, + "oom_kill": 0, + "low": 0 + }, + "minimum_free_bytes": 79554400256 + }, + "websocket": { + "full_stream_sentinels": 6000, + "full_stream_received": 6000, + "measured_sentinels": 4500, + "measured_missing": 0, + "full_stream_missing": 0, + "negative_measured_lags": 0, + "negative_full_stream_lags": 0, + "duplicate_measured_receipts": 0, + "duplicate_full_stream_receipts": 0 + }, + "retention": { + "retention": { + "elapsed_ns": 902013732, + "transmissions_deleted": 16000, + "observations_deleted": 256000 + }, + "retention-empty": { + "elapsed_ns": 1714566, + "transmissions_deleted": 0, + "observations_deleted": 0 + } + }, + "startup_counts": [ + { + "transmissions": 112000, + "observations": 1792000 + }, + { + "transmissions": 112000, + "observations": 1792000 + } + ], + "database_settings": { + "autovacuum": "on", + "checkpoint_timeout": "5min", + "effective_cache_size": "4GB", + "fsync": "on", + "full_page_writes": "on", + "jit": "on", + "maintenance_work_mem": "64MB", + "max_connections": "32", + "max_parallel_workers_per_gather": "2", + "max_wal_size": "1GB", + "server_version": "18.6 (Ubuntu 18.6-1.pgdg24.04+2)", + "shared_buffers": "256MB", + "synchronous_commit": "on", + "work_mem": "4MB" + }, + "durable_counts": { + "scheduled": 12000, + "completed": 12000, + "dropped_schedules": 0, + "write_errors": 0, + "actual_transmissions": 118000, + "actual_observations": 1802200 + } + } + ], + "limits": [ + "Five paired runs are the independent repetitions; request records are never pooled across runs.", + "Absolute entries are medians of per-run quantiles, with observed min/max, not confidence intervals.", + "p99 is omitted unless every contributing run has at least 1000 successful samples.", + "Ingest service excludes replay queue wait; HTTP service includes response reading and validation. Queue and schedule-to-complete distributions are separate.", + "Resource deltas use the first and last samples strictly inside the measured window; no boundary interpolation. Coverage and sampling gaps are reported.", + "Memory/RSS/PSS/storage/WAL are observed sampled peaks, not proof against shorter spikes. Storage includes selected database files and WAL.", + "I/O is per-device cgroup accounting. The aggregate can count layered devices twice and is not physical storage bytes.", + "WebSocket lag is signed receipt minus ingest return; negative lags remain. Nonpositive baseline denominators have no ordinary speed ratio.", + "Accounts are disabled by the reviewed harness. The OS cache is warm; startup uses a new process and approximately 100ms controller probes.", + "No capacity, OS-cold, 65-minute soak or engine-wide causal claim follows from this report.", + "Validation uses public digests and runtime evidence; it does not reopen private databases or payload files." + ], + "publication": { + "run_url": "https://github.com/Kpa-clawbot/CoreScope/actions/runs/37914893623", + "raw_artifact_url": "https://github.com/Kpa-clawbot/CoreScope/actions/runs/37914893623/artifacts/11613951167", + "analysis_reference_note": "analysis_reference is the analyzer source-review anchor; provenance.candidate_sha is the measured application revision.", + "qualification_history": { + "run_url": "https://github.com/Kpa-clawbot/CoreScope/actions/runs/37897747657", + "candidate_sha": "7b2c19a1eaf0050113c1ce8ae73cc69b6e541893", + "status": "failed; comparison-ineligible", + "ingest_rate": 100, + "http_rate": 20, + "scheduled": 24000, + "completed": 23918, + "dropped": 82, + "write_errors": 0, + "drop_schedule_range_seconds": [ + 205.92, + 206.76 + ], + "retention_scheduled_seconds": 195, + "retention_elapsed_ns": 11744610088, + "queue_limit": 1024, + "interpretation": "Maintenance-related queue saturation; retained independently, never pooled with the supplemental common 50/s experiment." + }, + "resource_interpretation": [ + "The PostgreSQL pair-1 measured window has 444 memory.events.max increments; all 174 samples are at least 99% of the 6-GiB limit and closest sampled headroom is 16 KiB.", + "No final OOM, OOM-kill or OOM-group-kill events occurred; every PostgreSQL full leg approached the memory cap during import/startup.", + "PostgreSQL retained 1 GiB of WAL segments from import; this is not WAL generated during timed ingestion.", + "Minimum observed free filesystem space is runner evidence, not an operator minimum or migration disk-capacity guarantee." + ] + } +} diff --git a/docs/performance/postgresql-ce023e0-details.md b/docs/performance/postgresql-ce023e0-details.md new file mode 100644 index 000000000..7ccd788ec --- /dev/null +++ b/docs/performance/postgresql-ce023e0-details.md @@ -0,0 +1,273 @@ +# PostgreSQL ce023: complete paired results + +[Summary](../postgresql-performance.md) · [Complete JSON](postgresql-ce023e0-details.json) · [Run](https://github.com/Kpa-clawbot/CoreScope/actions/runs/37914893623) · [Raw public artifact](https://github.com/Kpa-clawbot/CoreScope/actions/runs/37914893623/artifacts/11613951167) + +All generated tables below are preserved from the validated final analysis. +The artifact-relative source directory is `corescope-bench-37914893623-1/public`. +The JSON's `analysis_reference` identifies the analyzer's source-review anchor +(`c315d7d`); its `provenance.candidate_sha` identifies the measured candidate +(`ce023e0`). The review checked 189 input hashes; the raw public bundle contains +234 files. No host-local paths or private payload databases are published here. + +The [intermediate c315 run](https://github.com/Kpa-clawbot/CoreScope/actions/runs/37901327267) +was diagnostic evidence for the RX generic-plan correction. It ran on another +Actions runner; no quantified cross-run speedup is claimed. All paired ratios +below compare ce023 with the unchanged upstream on the final run's same host. + +Candidate: ce023e0d623116fe66987761d06016590f98fb11 + +Completed eligible B; five pairs. Values are median [min, max] across runs. Ratios are paired PostgreSQL/SQLite; no request populations are pooled. + +## Per-run latency quantiles + +| Scenario / class / timing | SQLite p50 ms | PostgreSQL p50 ms | SQLite p95 ms | PostgreSQL p95 ms | Paired p50 ratio (n) | Paired p95 ratio (n) | +|---|---:|---:|---:|---:|---:|---:| +| durable-ingest / additional / end_to_end | 1.461 [1.375, 1.517] | 2.049 [2.006, 2.15] | 1453 [1394, 1630] | 2.844 [2.788, 2.956] | 1.446 [1.335, 1.459] (n=5) | 0.001965 [0.001722, 0.002046] (n=5) | +| durable-ingest / additional / queue | 0.6397 [0.6228, 0.6449] | 0.621 [0.6148, 0.6294] | 1433 [1394, 1610] | 1.076 [1.069, 1.08] | 0.9763 [0.9611, 1.005] (n=5) | 0.0007504 [0.0006689, 0.0007669] (n=5) | +| durable-ingest / additional / service | 0.7615 [0.7034, 0.8227] | 1.367 [1.334, 1.45] | 1.267 [1.159, 1.353] | 2.041 [1.988, 2.185] | 1.86 [1.735, 1.896] (n=5) | 1.615 [1.502, 1.76] (n=5) | +| durable-ingest / duplicate / end_to_end | 1.328 [1.229, 1.419] | 1.897 [1.851, 1.979] | 1394 [1315, 1571] | 2.489 [2.426, 2.629] | 1.481 [1.304, 1.509] (n=5) | 0.00185 [0.001544, 0.00188] (n=5) | +| durable-ingest / duplicate / queue | 0.635 [0.6221, 0.6538] | 0.6051 [0.5988, 0.6299] | 1394 [1315, 1571] | 1.073 [1.065, 1.081] | 0.9709 [0.9194, 1.013] (n=5) | 0.0007642 [0.000688, 0.0008153] (n=5) | +| durable-ingest / duplicate / service | 0.6438 [0.5763, 0.6995] | 1.252 [1.211, 1.314] | 1.006 [0.9305, 1.109] | 1.674 [1.612, 1.761] | 1.993 [1.84, 2.101] (n=5) | 1.682 [1.589, 1.733] (n=5) | +| durable-ingest / late / end_to_end | 1.625 [1.535, 1.814] | 2.188 [2.12, 2.266] | 1355 [1276, 1531] | 2.904 [2.752, 2.953] | 1.347 [1.249, 1.415] (n=5) | 0.002143 [0.001854, 0.002244] (n=5) | +| durable-ingest / late / queue | 0.6344 [0.5942, 0.7032] | 0.6373 [0.5996, 0.6439] | 1354 [1275, 1531] | 1.072 [1.058, 1.085] | 0.9553 [0.889, 1.073] (n=5) | 0.0007966 [0.0007087, 0.0008381] (n=5) | +| durable-ingest / late / service | 0.9377 [0.849, 1.004] | 1.546 [1.505, 1.631] | 1.464 [1.361, 1.52] | 2.071 [1.919, 2.123] | 1.679 [1.616, 1.773] (n=5) | 1.408 [1.35, 1.464] (n=5) | +| durable-ingest / new / end_to_end | 1.671 [1.602, 1.831] | 2.18 [2.131, 2.311] | 1304 [1237, 1516] | 2.858 [2.831, 2.997] | 1.275 [1.25, 1.361] (n=5) | 0.00229 [0.001885, 0.002309] (n=5) | +| durable-ingest / new / queue | 0.6307 [0.6293, 0.6446] | 0.6211 [0.6192, 0.6458] | 1284 [1217, 1496] | 1.074 [1.07, 1.084] | 0.9845 [0.9779, 1.012] (n=5) | 0.0008446 [0.000718, 0.0008798] (n=5) | +| durable-ingest / new / service | 1.003 [0.9395, 1.073] | 1.515 [1.48, 1.607] | 1.634 [1.525, 1.803] | 2.084 [2.028, 2.211] | 1.542 [1.47, 1.575] (n=5) | 1.241 [1.223, 1.359] (n=5) | +| handler-control / 200_valid_envelopes / service | 0.8129 [0.8029, 0.8751] | 1.576 [1.546, 1.728] | 2.036 [1.799, 2.169] | 2.274 [2.113, 2.395] | 1.939 [1.903, 1.974] (n=5) | 1.148 [1.054, 1.264] (n=5) | +| mixed-http / analytics_channels / end_to_end | 30.11 [29.54, 30.58] | 31 [30.71, 31.11] | 34.08 [33.49, 35.53] | 36.57 [34.5, 41.08] | 1.03 [1.004, 1.053] (n=5) | 1.03 [1.025, 1.201] (n=5) | +| mixed-http / analytics_channels / queue | 0.3555 [0.2994, 0.4261] | 0.3569 [0.3157, 0.5119] | 1.033 [0.8971, 1.078] | 1.04 [0.9728, 1.065] | 1.062 [0.9062, 1.44] (n=5) | 1.021 [0.928, 1.157] (n=5) | +| mixed-http / analytics_channels / service | 29.51 [29.13, 30.11] | 30.49 [30.08, 30.61] | 33.77 [32.46, 34.82] | 36.47 [33.86, 40.57] | 1.033 [1.014, 1.051] (n=5) | 1.048 [1.005, 1.19] (n=5) | +| mixed-http / analytics_rf / end_to_end | 2.565 [2.541, 2.609] | 2.337 [2.259, 2.374] | 3.277 [3.126, 3.432] | 3.175 [3.015, 3.337] | 0.896 [0.8807, 0.9297] (n=5) | 0.9342 [0.9182, 1.037] (n=5) | +| mixed-http / analytics_rf / queue | 0.8237 [0.6097, 0.8441] | 0.6058 [0.5712, 0.6508] | 1.069 [1.052, 1.108] | 1.058 [1.021, 1.074] | 0.7696 [0.7355, 0.9646] (n=5) | 0.9893 [0.9418, 1.015] (n=5) | +| mixed-http / analytics_rf / service | 1.749 [1.742, 1.783] | 1.595 [1.576, 1.666] | 2.581 [2.472, 2.659] | 2.588 [2.394, 2.618] | 0.9115 [0.8988, 0.9344] (n=5) | 1.004 [0.9116, 1.021] (n=5) | +| mixed-http / analytics_rf_filtered / end_to_end | 1.898 [1.863, 2.016] | 2.225 [2.164, 2.325] | 582.5 [482.5, 783.3] | 576.1 [549.2, 622.3] | 1.158 [1.14, 1.2] (n=5) | 1.062 [0.7365, 1.141] (n=5) | +| mixed-http / analytics_rf_filtered / queue | 0.1195 [0.1159, 0.147] | 0.1377 [0.127, 0.2074] | 0.8623 [0.8086, 0.9341] | 0.9849 [0.9468, 1.054] | 1.188 [1.033, 1.411] (n=5) | 1.142 [1.119, 1.19] (n=5) | +| mixed-http / analytics_rf_filtered / service | 1.711 [1.653, 1.775] | 1.965 [1.897, 1.996] | 582 [482.3, 783.2] | 576 [548.4, 621.3] | 1.12 [1.114, 1.208] (n=5) | 1.062 [0.7364, 1.141] (n=5) | +| mixed-http / analytics_topology / end_to_end | 30.33 [30.18, 31.06] | 30.59 [30.55, 31.15] | 33.42 [32.16, 37.02] | 39.21 [33.59, 48.29] | 1.007 [0.9931, 1.029] (n=5) | 1.088 [0.9074, 1.445] (n=5) | +| mixed-http / analytics_topology / queue | 0.7453 [0.7158, 0.8084] | 0.6501 [0.5254, 0.7381] | 1.022 [0.9774, 1.087] | 1.036 [1.003, 1.074] | 0.8723 [0.6499, 1.016] (n=5) | 1.018 [0.9234, 1.072] (n=5) | +| mixed-http / analytics_topology / service | 29.64 [29.57, 30.39] | 30.08 [29.98, 30.58] | 32.52 [31.25, 36.5] | 38.55 [32.52, 47.81] | 1.013 [0.9994, 1.032] (n=5) | 1.103 [0.8908, 1.47] (n=5) | +| mixed-http / channels / end_to_end | 1.297 [1.262, 1.322] | 1.307 [1.266, 1.385] | 1.768 [1.748, 1.823] | 1.853 [1.794, 2.156] | 1.009 [0.9578, 1.097] (n=5) | 1.05 [1.015, 1.183] (n=5) | +| mixed-http / channels / queue | 0.5876 [0.5363, 0.6329] | 0.587 [0.5739, 0.637] | 1.046 [1.028, 1.076] | 1.058 [1.03, 1.063] | 1.036 [0.9067, 1.084] (n=5) | 1.002 [0.9846, 1.013] (n=5) | +| mixed-http / channels / service | 0.6692 [0.653, 0.6971] | 0.6724 [0.6581, 0.7051] | 0.9293 [0.8749, 1.064] | 1.058 [0.9314, 1.458] | 1.008 [0.9902, 1.021] (n=5) | 1.209 [0.9345, 1.371] (n=5) | +| mixed-http / messages_0 / end_to_end | 1.952 [1.936, 2.064] | 2.017 [1.963, 2.117] | 60.09 [58.82, 64.02] | 113.3 [108.3, 179] | 1.026 [1.014, 1.057] (n=5) | 1.907 [1.782, 2.796] (n=5) | +| mixed-http / messages_0 / queue | 0.549 [0.4973, 0.6505] | 0.5701 [0.5473, 0.6505] | 1.038 [1.015, 1.054] | 1.049 [1.031, 1.063] | 1.012 [0.9968, 1.206] (n=5) | 1.016 [0.9921, 1.024] (n=5) | +| mixed-http / messages_0 / service | 1.338 [1.32, 1.365] | 1.411 [1.354, 1.477] | 59.91 [58.73, 63.54] | 112.7 [107.7, 178.3] | 1.054 [1.008, 1.082] (n=5) | 1.9 [1.785, 2.806] (n=5) | +| mixed-http / messages_1000 / end_to_end | 2.011 [1.929, 2.142] | 2.089 [2.065, 2.199] | 83.21 [82.65, 86.91] | 114.9 [111.9, 165.4] | 1.031 [0.9738, 1.1] (n=5) | 1.376 [1.345, 1.903] (n=5) | +| mixed-http / messages_1000 / queue | 0.6226 [0.5858, 0.7269] | 0.6336 [0.5732, 0.6619] | 1.054 [1.039, 1.088] | 1.051 [1.04, 1.075] | 0.9207 [0.8867, 1.118] (n=5) | 1.002 [0.9653, 1.021] (n=5) | +| mixed-http / messages_1000 / service | 1.377 [1.344, 1.409] | 1.411 [1.39, 1.455] | 82.79 [81.99, 86.2] | 114.1 [111.4, 165.2] | 1.036 [1.002, 1.044] (n=5) | 1.379 [1.344, 1.917] (n=5) | +| mixed-http / metrics_dense_24h / end_to_end | 3.906 [3.845, 3.95] | 4.405 [4.335, 4.495] | 4.845 [4.579, 4.922] | 5.237 [5.152, 7.076] | 1.138 [1.102, 1.146] (n=5) | 1.073 [1.053, 1.545] (n=5) | +| mixed-http / metrics_dense_24h / queue | 0.5668 [0.4823, 0.6363] | 0.5668 [0.5126, 0.5854] | 1.048 [1.039, 1.057] | 1.039 [1.01, 1.072] | 1.01 [0.8887, 1.214] (n=5) | 0.9975 [0.9635, 1.013] (n=5) | +| mixed-http / metrics_dense_24h / service | 3.327 [3.284, 3.395] | 3.851 [3.796, 3.871] | 4.126 [4.075, 4.211] | 4.688 [4.503, 6.618] | 1.144 [1.14, 1.173] (n=5) | 1.13 [1.07, 1.624] (n=5) | +| mixed-http / metrics_sparse_7d / end_to_end | 2.044 [1.948, 2.16] | 2.283 [2.24, 2.337] | 2.503 [2.452, 2.898] | 2.819 [2.775, 2.987] | 1.115 [1.082, 1.188] (n=5) | 1.132 [0.9759, 1.172] (n=5) | +| mixed-http / metrics_sparse_7d / queue | 0.6256 [0.5906, 0.6783] | 0.5361 [0.5013, 0.576] | 1.054 [1.037, 1.064] | 1.017 [0.9639, 1.053] | 0.8013 [0.7605, 0.9753] (n=5) | 0.9681 [0.9145, 0.9953] (n=5) | +| mixed-http / metrics_sparse_7d / service | 1.369 [1.353, 1.43] | 1.736 [1.709, 1.787] | 1.606 [1.554, 2.147] | 1.961 [1.921, 2.119] | 1.261 [1.25, 1.268] (n=5) | 1.256 [0.9135, 1.298] (n=5) | +| mixed-http / nodes_2000 / end_to_end | 62.5 [61.84, 63.6] | 64.35 [63.35, 64.7] | 119.6 [69.93, 122.1] | 104.7 [72.78, 127] | 1.024 [1.014, 1.032] (n=5) | 0.8723 [0.6087, 1.727] (n=5) | +| mixed-http / nodes_2000 / queue | 0.6 [0.4865, 0.6183] | 0.5109 [0.4756, 0.5571] | 1.033 [0.9864, 1.081] | 1.045 [1.02, 1.06] | 0.8477 [0.7692, 1.132] (n=5) | 0.9911 [0.9756, 1.06] (n=5) | +| mixed-http / nodes_2000 / service | 61.85 [61.2, 62.99] | 63.77 [62.76, 64.36] | 119.4 [69.45, 121.7] | 103.8 [72.35, 126.9] | 1.027 [1.015, 1.033] (n=5) | 0.869 [0.6057, 1.73] (n=5) | +| mixed-http / nodes_50 / end_to_end | 3.661 [3.537, 3.765] | 5.526 [5.443, 5.646] | 8.125 [5.561, 8.896] | 9.892 [7.729, 9.995] | 1.509 [1.49, 1.552] (n=5) | 1.217 [0.8688, 1.797] (n=5) | +| mixed-http / nodes_50 / queue | 0.2297 [0.1344, 0.2799] | 0.1754 [0.1382, 0.1957] | 1.029 [0.9864, 1.051] | 1.016 [0.9824, 1.066] | 0.7635 [0.6017, 1.309] (n=5) | 1 [0.9348, 1.06] (n=5) | +| mixed-http / nodes_50 / service | 3.332 [3.238, 3.414] | 5.231 [5.068, 5.322] | 7.376 [4.911, 8.105] | 9.457 [6.928, 9.643] | 1.566 [1.539, 1.583] (n=5) | 1.294 [0.8548, 1.925] (n=5) | +| mixed-http / nodes_region / end_to_end | 5.921 [5.747, 6.172] | 8.268 [7.988, 8.485] | 12.2 [9.103, 14.11] | 15.57 [14.59, 16.3] | 1.376 [1.363, 1.439] (n=5) | 1.277 [1.035, 1.791] (n=5) | +| mixed-http / nodes_region / queue | 0.4812 [0.4291, 0.5252] | 0.4942 [0.4748, 0.5549] | 1.051 [1.036, 1.065] | 1.046 [1.024, 1.067] | 1.079 [0.9041, 1.152] (n=5) | 0.9877 [0.9702, 1.03] (n=5) | +| mixed-http / nodes_region / service | 5.476 [5.336, 5.637] | 7.724 [7.528, 7.812] | 11.61 [8.665, 13.74] | 15.28 [14.15, 15.75] | 1.402 [1.386, 1.431] (n=5) | 1.316 [1.03, 1.817] (n=5) | +| mixed-http / observers / end_to_end | 3.476 [3.44, 3.605] | 3.519 [3.463, 3.543] | 4.572 [4.434, 4.9] | 4.884 [4.819, 5.227] | 1.002 [0.9828, 1.023] (n=5) | 1.074 [1.054, 1.096] (n=5) | +| mixed-http / observers / queue | 0.5647 [0.5399, 0.5779] | 0.5578 [0.522, 0.5812] | 1.033 [1.019, 1.056] | 1.043 [1.034, 1.064] | 1.015 [0.9033, 1.065] (n=5) | 1.006 [0.9924, 1.03] (n=5) | +| mixed-http / observers / service | 2.846 [2.815, 2.967] | 2.932 [2.867, 2.991] | 3.912 [3.757, 4.061] | 4.267 [4.18, 4.618] | 1.027 [1.003, 1.051] (n=5) | 1.135 [1.066, 1.15] (n=5) | +| mixed-http / packet_detail_memory / end_to_end | 1.867 [1.846, 1.919] | 3.145 [3.069, 3.348] | 2.407 [2.345, 2.501] | 3.763 [3.732, 4.101] | 1.679 [1.65, 1.792] (n=5) | 1.55 [1.538, 1.749] (n=5) | +| mixed-http / packet_detail_memory / queue | 0.5601 [0.531, 0.5866] | 0.5716 [0.5094, 0.6415] | 1.028 [0.9905, 1.057] | 1.031 [1.012, 1.039] | 1.013 [0.9593, 1.094] (n=5) | 1.002 [0.9787, 1.039] (n=5) | +| mixed-http / packet_detail_memory / service | 1.264 [1.212, 1.369] | 2.582 [2.495, 2.779] | 1.586 [1.505, 1.955] | 3.093 [3.044, 3.304] | 2.043 [1.961, 2.17] (n=5) | 1.936 [1.69, 2.189] (n=5) | +| mixed-http / packet_detail_pruned / end_to_end | 1.275 [1.236, 1.431] | 1.254 [1.217, 1.357] | 1.734 [1.725, 1.791] | 1.823 [1.694, 2.103] | 0.9835 [0.8647, 1.074] (n=5) | 1.057 [0.9802, 1.174] (n=5) | +| mixed-http / packet_detail_pruned / queue | 0.5449 [0.4803, 0.7149] | 0.5003 [0.4245, 0.6195] | 1.062 [1.021, 1.072] | 1.036 [1.009, 1.051] | 0.9181 [0.6686, 0.9498] (n=5) | 0.9744 [0.9431, 1.029] (n=5) | +| mixed-http / packet_detail_pruned / service | 0.6807 [0.6523, 0.6935] | 0.7155 [0.6777, 0.7325] | 0.7926 [0.7728, 0.8374] | 1.051 [0.8693, 1.374] | 1.051 [0.9831, 1.108] (n=5) | 1.333 [1.097, 1.778] (n=5) | +| mixed-http / packet_detail_sql / end_to_end | 1.727 [1.68, 1.757] | 2.145 [2.106, 2.316] | 2.197 [2.151, 2.224] | 2.729 [2.662, 3.427] | 1.233 [1.217, 1.366] (n=5) | 1.261 [1.212, 1.541] (n=5) | +| mixed-http / packet_detail_sql / queue | 0.5701 [0.5563, 0.6822] | 0.6203 [0.5669, 0.6349] | 1.046 [1.033, 1.058] | 1.046 [1, 1.069] | 1.088 [0.8995, 1.122] (n=5) | 0.9991 [0.9564, 1.015] (n=5) | +| mixed-http / packet_detail_sql / service | 1.09 [1.066, 1.155] | 1.524 [1.509, 1.626] | 1.33 [1.293, 1.352] | 1.934 [1.79, 2.642] | 1.407 [1.388, 1.43] (n=5) | 1.446 [1.354, 1.954] (n=5) | +| mixed-http / packets_memory / end_to_end | 30.33 [30.07, 33.27] | 35.34 [34.69, 36.85] | 32.37 [31.87, 35.97] | 38.32 [37.21, 40.13] | 1.16 [1.107, 1.195] (n=5) | 1.167 [1.115, 1.229] (n=5) | +| mixed-http / packets_memory / queue | 0.6212 [0.5473, 0.6726] | 0.5808 [0.5089, 0.5875] | 1.056 [1.041, 1.06] | 1.015 [1.012, 1.055] | 0.8832 [0.826, 1.073] (n=5) | 0.9624 [0.9551, 1] (n=5) | +| mixed-http / packets_memory / service | 29.8 [29.35, 32.67] | 34.74 [34.08, 36.26] | 31.86 [31.5, 35.26] | 37.45 [36.81, 39.24] | 1.161 [1.11, 1.197] (n=5) | 1.168 [1.11, 1.246] (n=5) | +| mixed-http / reach / end_to_end | 1.394 [1.291, 1.424] | 1.413 [1.393, 1.476] | 1.811 [1.794, 1.885] | 1.877 [1.842, 1.927] | 1.01 [0.9778, 1.107] (n=5) | 1.034 [0.9878, 1.069] (n=5) | +| mixed-http / reach / queue | 0.5871 [0.5545, 0.6118] | 0.5899 [0.5659, 0.633] | 1.032 [0.9735, 1.056] | 1.053 [1.011, 1.067] | 1.035 [0.9639, 1.068] (n=5) | 1.005 [0.969, 1.096] (n=5) | +| mixed-http / reach / service | 0.802 [0.7769, 0.8079] | 0.8281 [0.8187, 0.8414] | 0.932 [0.9122, 0.9496] | 0.9602 [0.9469, 0.9832] | 1.044 [1.013, 1.066] (n=5) | 1.028 [1.004, 1.077] (n=5) | +| mixed-http / rx_coverage / end_to_end | 2.164 [2.137, 2.2] | 2.331 [2.277, 2.421] | 2.689 [2.623, 2.723] | 2.872 [2.745, 3.056] | 1.087 [1.051, 1.113] (n=5) | 1.076 [1.021, 1.122] (n=5) | +| mixed-http / rx_coverage / queue | 0.5906 [0.5682, 0.6063] | 0.5677 [0.5278, 0.594] | 1.045 [1.004, 1.054] | 1.038 [1.002, 1.048] | 0.9442 [0.8937, 1.028] (n=5) | 0.9927 [0.9507, 1.034] (n=5) | +| mixed-http / rx_coverage / service | 1.556 [1.539, 1.604] | 1.799 [1.681, 1.846] | 1.769 [1.765, 1.83] | 1.959 [1.842, 2.229] | 1.155 [1.092, 1.157] (n=5) | 1.107 [1.043, 1.218] (n=5) | +| mixed-http / stats / end_to_end | 1.11 [1.069, 1.145] | 1.075 [1.034, 1.143] | 2.746 [2.704, 3.13] | 4.481 [4.163, 5.316] | 0.9915 [0.9413, 1.006] (n=5) | 1.516 [1.431, 1.966] (n=5) | +| mixed-http / stats / queue | 0.5857 [0.5192, 0.6159] | 0.5614 [0.5127, 0.5887] | 1.035 [0.9994, 1.054] | 1.032 [1.013, 1.052] | 0.952 [0.8325, 1.134] (n=5) | 0.9919 [0.9628, 1.051] (n=5) | +| mixed-http / stats / service | 0.4691 [0.4541, 0.4871] | 0.4602 [0.4424, 0.4626] | 2.279 [2.1, 2.712] | 3.787 [3.708, 4.776] | 0.9791 [0.9269, 1.002] (n=5) | 1.662 [1.397, 2.075] (n=5) | +| sql-miss / channel_messages_sql / service | 76.11 [66.72, 81.74] | 106.3 [102.2, 118.3] | 85.14 [79.01, 103] | 122.7 [111.9, 183.9] | 1.433 [1.368, 1.531] (n=5) | 1.387 [1.292, 1.786] (n=5) | +| sql-miss / observer_metrics_sql / service | 0.7654 [0.7299, 0.8456] | 0.8473 [0.8147, 0.8923] | 1.314 [1.305, 1.467] | 1.382 [1.21, 1.409] | 1.071 [1.007, 1.166] (n=5) | 1.013 [0.8741, 1.072] (n=5) | +| websocket / new_transmission / receipt_after_ingest_return | 507.1 [506.1, 534.6] | 511.2 [505.5, 519.7] | 985.5 [969.2, 991.8] | 959.6 [946.8, 971] | 1.004 [0.9698, 1.027] (n=5) | 0.969 [0.9608, 0.9916] (n=5) | + +## Eligible p99 + +| Scenario / class / timing | SQLite ms | PostgreSQL ms | Paired ratio (n) | +|---|---:|---:|---:| +| durable-ingest / additional / end_to_end | 7115 [6890, 7476] | 4.16 [3.517, 4.915] | 0.0005931 [0.0004829, 0.0006907] (n=5) | +| durable-ingest / additional / queue | 7076 [6890, 7308] | 1.124 [1.118, 1.137] | 0.0001588 [0.0001535, 0.0001651] (n=5) | +| durable-ingest / additional / service | 3.273 [3.075, 3.537] | 3.024 [2.628, 3.568] | 0.8551 [0.8029, 1.027] (n=5) | +| durable-ingest / duplicate / end_to_end | 7204 [6832, 7437] | 3.076 [2.793, 4.713] | 0.0004278 [0.0003756, 0.0006898] (n=5) | +| durable-ingest / duplicate / queue | 7196 [6812, 7436] | 1.118 [1.112, 1.14] | 0.0001553 [0.0001504, 0.0001673] (n=5) | +| durable-ingest / duplicate / service | 1.603 [1.484, 2.024] | 2.028 [1.936, 3.13] | 1.31 [1.153, 1.546] (n=5) | +| durable-ingest / new / end_to_end | 7113 [7046, 7302] | 3.862 [3.416, 4.425] | 0.0005296 [0.0004739, 0.000628] (n=5) | +| durable-ingest / new / queue | 7113 [7035, 7282] | 1.126 [1.115, 1.152] | 0.0001583 [0.0001532, 0.0001635] (n=5) | +| durable-ingest / new / service | 3.69 [3.642, 4.041] | 2.785 [2.578, 2.993] | 0.7308 [0.6877, 0.8112] (n=5) | +| websocket / new_transmission / receipt_after_ingest_return | 1389 [1330, 1892] | 999 [986.7, 1569] | 0.7334 [0.5279, 1.12] (n=5) | + +## Durations, throughput and sampled resources + +| Metric | Unit | SQLite | PostgreSQL | Paired ratio (n) | +|---|---|---:|---:|---:| +| durable-ingest/additional/completed_per_second_including_drain | per_second | 15.02 [15.02, 15.02] | 15.02 [15.02, 15.02] | 1 [1, 1] (n=5) | +| durable-ingest/duplicate/completed_per_second_including_drain | per_second | 7.515 [7.515, 7.515] | 7.515 [7.515, 7.515] | 1 [1, 1] (n=5) | +| durable-ingest/late/completed_per_second_including_drain | per_second | 2.506 [2.506, 2.506] | 2.506 [2.506, 2.506] | 1 [1, 1] (n=5) | +| durable-ingest/new/completed_per_second_including_drain | per_second | 25.03 [25.03, 25.03] | 25.03 [25.03, 25.03] | 1 [1, 1] (n=5) | +| migration/elapsed_ns | s | — | 147.6 [144.7, 151] | — (n=0) | +| migration/harness_validation_ns | s | — | 54.06 [53.28, 56.62] | — (n=0) | +| migration/migration_tool_ns | s | — | 90.99 [89.94, 94.13] | — (n=0) | +| migration/role_grants_ns | s | — | 0.7733 [0.7025, 0.8264] | — (n=0) | +| mixed-http/analytics_channels/completed_per_second_including_drain | per_second | 0.4109 [0.4109, 0.4109] | 0.4109 [0.4109, 0.4109] | 1 [1, 1] (n=5) | +| mixed-http/analytics_rf/completed_per_second_including_drain | per_second | 0.6157 [0.6157, 0.6157] | 0.6157 [0.6157, 0.6157] | 1 [1, 1] (n=5) | +| mixed-http/analytics_rf_filtered/completed_per_second_including_drain | per_second | 0.411 [0.411, 0.411] | 0.411 [0.411, 0.411] | 1 [1, 1] (n=5) | +| mixed-http/analytics_topology/completed_per_second_including_drain | per_second | 0.6156 [0.6156, 0.6156] | 0.6156 [0.6156, 0.6156] | 1 [1, 1] (n=5) | +| mixed-http/channels/completed_per_second_including_drain | per_second | 1.433 [1.433, 1.433] | 1.433 [1.433, 1.433] | 1 [1, 1] (n=5) | +| mixed-http/messages_0/completed_per_second_including_drain | per_second | 1.433 [1.433, 1.433] | 1.433 [1.433, 1.433] | 1 [1, 1] (n=5) | +| mixed-http/messages_1000/completed_per_second_including_drain | per_second | 1.229 [1.229, 1.229] | 1.229 [1.228, 1.229] | 1 [0.9993, 1] (n=5) | +| mixed-http/metrics_dense_24h/completed_per_second_including_drain | per_second | 1.026 [1.026, 1.026] | 1.026 [1.026, 1.026] | 1 [1, 1] (n=5) | +| mixed-http/metrics_sparse_7d/completed_per_second_including_drain | per_second | 1.026 [1.026, 1.026] | 1.026 [1.026, 1.026] | 1 [1, 1] (n=5) | +| mixed-http/nodes_2000/completed_per_second_including_drain | per_second | 1.432 [1.432, 1.432] | 1.432 [1.432, 1.432] | 1 [0.9998, 1] (n=5) | +| mixed-http/nodes_50/completed_per_second_including_drain | per_second | 1.433 [1.433, 1.433] | 1.433 [1.433, 1.433] | 1 [1, 1] (n=5) | +| mixed-http/nodes_region/completed_per_second_including_drain | per_second | 1.229 [1.229, 1.229] | 1.229 [1.229, 1.229] | 1 [1, 1] (n=5) | +| mixed-http/observers/completed_per_second_including_drain | per_second | 2.052 [2.052, 2.052] | 2.052 [2.052, 2.052] | 1 [1, 1] (n=5) | +| mixed-http/packet_detail_memory/completed_per_second_including_drain | per_second | 1.025 [1.025, 1.025] | 1.025 [1.025, 1.025] | 1 [1, 1] (n=5) | +| mixed-http/packet_detail_pruned/completed_per_second_including_drain | per_second | 1.108 [1.108, 1.108] | 1.108 [1.108, 1.108] | 1 [1, 1] (n=5) | +| mixed-http/packet_detail_sql/completed_per_second_including_drain | per_second | 1.034 [1.034, 1.034] | 1.034 [1.034, 1.034] | 1 [1, 1] (n=5) | +| mixed-http/packets_memory/completed_per_second_including_drain | per_second | 1.025 [1.025, 1.025] | 1.025 [1.025, 1.025] | 1 [1, 1] (n=5) | +| mixed-http/reach/completed_per_second_including_drain | per_second | 1.026 [1.026, 1.026] | 1.026 [1.026, 1.026] | 1 [1, 1] (n=5) | +| mixed-http/rx_coverage/completed_per_second_including_drain | per_second | 1.026 [1.026, 1.026] | 1.026 [1.026, 1.026] | 1 [1, 1] (n=5) | +| mixed-http/stats/completed_per_second_including_drain | per_second | 1.027 [1.027, 1.027] | 1.027 [1.027, 1.027] | 1 [1, 1] (n=5) | +| resource/aggregate_accounted_io_dbytes | MiB | 0 [0, 0] | 0.003906 [0, 0.003906] | — (n=0) | +| resource/aggregate_accounted_io_dios | operations | 0 [0, 0] | 1 [0, 1] | — (n=0) | +| resource/aggregate_accounted_io_rbytes | MiB | 0 [0, 0] | 5.762 [5.57, 6.875] | — (n=0) | +| resource/aggregate_accounted_io_rios | operations | 0 [0, 0] | 658 [641, 780] | — (n=0) | +| resource/aggregate_accounted_io_wbytes | MiB | 2360 [2335, 2378] | 433.6 [376.7, 507.6] | 0.1857 [0.1596, 0.2153] (n=5) | +| resource/aggregate_accounted_io_wios | operations | 2.187e+05 [2.14e+05, 2.236e+05] | 2.173e+04 [1.797e+04, 2.53e+04] | 0.1016 [0.08217, 0.1162] (n=5) | +| resource/average_cores_over_sampled_span | cores | 0.2422 [0.2295, 0.2484] | 0.3191 [0.3099, 0.3706] | 1.324 [1.28, 1.492] (n=5) | +| resource/cpu_delta_usec | CPU s | 43.45 [41.18, 44.58] | 56.86 [55.35, 66.47] | 1.332 [1.274, 1.491] (n=5) | +| resource/observed_peak_cgroup_memory_bytes | MiB | 4043 [4013, 4078] | 5681 [5581, 6144] | 1.415 [1.389, 1.518] (n=5) | +| resource/observed_peak_pss_bytes | MiB | 2922 [2892, 2925] | 3247 [3200, 3321] | 1.11 [1.1, 1.136] (n=5) | +| resource/observed_peak_rss_bytes | MiB | 2926 [2896, 2929] | 5055 [4965, 5221] | 1.728 [1.696, 1.803] (n=5) | +| resource/observed_peak_storage_bytes | MiB | 1098 [1095, 1122] | 2327 [2326, 2327] | 2.12 [2.073, 2.126] (n=5) | +| resource/observed_peak_wal_bytes | MiB | 40.01 [28.92, 53.02] | 1024 [1024, 1024] | 25.6 [19.31, 35.41] (n=5) | +| retention-empty/elapsed_ns | s | 0.0002677 [0.0002576, 0.000307] | 0.001732 [0.001672, 0.001941] | 6.471 [5.936, 7.459] (n=5) | +| retention/elapsed_ns | s | 10.49 [10.45, 10.73] | 0.905 [0.8444, 0.9622] | 0.08626 [0.0805, 0.09126] (n=5) | +| sql-miss/channel_messages_sql/completed_per_second_including_drain | per_second | 13.8 [12.73, 14.12] | 8.948 [7.699, 9.504] | 0.6471 [0.6049, 0.6733] (n=5) | +| sql-miss/observer_metrics_sql/completed_per_second_including_drain | per_second | 13.92 [12.83, 14.23] | 9.089 [7.87, 9.657] | 0.652 [0.6133, 0.6784] (n=5) | +| startup-hot1/first_http_ns | s | 9.565 [9.467, 10.18] | 11.48 [10.98, 11.68] | 1.187 [1.148, 1.223] (n=5) | +| startup-hot1/full_ready_ns | s | 20.52 [20.13, 21.75] | 20.15 [19.92, 20.65] | 0.9707 [0.9494, 1.006] (n=5) | +| startup-hot1/ready_ns | s | 9.565 [9.467, 10.18] | 11.48 [10.98, 11.68] | 1.187 [1.148, 1.223] (n=5) | +| startup/first_http_ns | s | 28.04 [27.44, 30.18] | 27.23 [27.14, 28.55] | 0.9713 [0.9383, 0.9896] (n=5) | +| startup/full_ready_ns | s | 28.06 [27.45, 30.19] | 27.44 [27.22, 29.31] | 0.9783 [0.9403, 0.9989] (n=5) | +| startup/ready_ns | s | 28.04 [27.44, 30.18] | 27.23 [27.14, 28.55] | 0.9713 [0.9383, 0.9896] (n=5) | + +## Measured-window resource coverage + +| Pair / backend | Samples | Sampled span / window s | Boundary gaps first / last s | Max interval s | +|---|---:|---:|---:|---:| +| 0 / sqlite | 180 | 179.425 / 180 | 0.318 / 0.257 | 1.006 | +| 0 / postgres | 175 | 179.331 / 180 | 0.298 / 0.371 | 1.058 | +| 1 / sqlite | 180 | 179.443 / 180 | 0.181 / 0.376 | 1.007 | +| 1 / postgres | 174 | 178.207 / 180 | 0.875 / 0.918 | 1.075 | +| 2 / sqlite | 180 | 179.432 / 180 | 0.207 / 0.361 | 1.009 | +| 2 / postgres | 174 | 178.591 / 180 | 0.875 / 0.534 | 1.102 | +| 3 / sqlite | 180 | 179.444 / 180 | 0.383 / 0.173 | 1.007 | +| 3 / postgres | 175 | 179.154 / 180 | 0.109 / 0.737 | 1.087 | +| 4 / sqlite | 179 | 178.460 / 180 | 0.833 / 0.707 | 1.026 | +| 4 / postgres | 175 | 179.421 / 180 | 0.295 / 0.284 | 1.118 | + +## Acceptance counts + +| Pair / backend | Scheduled / completed ingest | HTTP successes total / measured | Measured WS sentinels / missing / negative | Errors / drops | +|---|---:|---:|---:|---:| +| 0 / sqlite | 12000 / 12000 | 4800 / 3600 | 4500 / 0 / 1 | 0 / 0 | +| 0 / postgres | 12000 / 12000 | 4800 / 3600 | 4500 / 0 / 0 | 0 / 0 | +| 1 / sqlite | 12000 / 12000 | 4800 / 3600 | 4500 / 0 / 0 | 0 / 0 | +| 1 / postgres | 12000 / 12000 | 4800 / 3600 | 4500 / 0 / 0 | 0 / 0 | +| 2 / sqlite | 12000 / 12000 | 4800 / 3600 | 4500 / 0 / 0 | 0 / 0 | +| 2 / postgres | 12000 / 12000 | 4800 / 3600 | 4500 / 0 / 0 | 0 / 0 | +| 3 / sqlite | 12000 / 12000 | 4800 / 3600 | 4500 / 0 / 0 | 0 / 0 | +| 3 / postgres | 12000 / 12000 | 4800 / 3600 | 4500 / 0 / 0 | 0 / 0 | +| 4 / sqlite | 12000 / 12000 | 4800 / 3600 | 4500 / 0 / 0 | 0 / 0 | +| 4 / postgres | 12000 / 12000 | 4800 / 3600 | 4500 / 0 / 0 | 0 / 0 | + +Per-run JSON retains per-device I/O deltas, exact sample bounds, counts, retention deletions, settings, provenance and input hashes. + +## Provenance + +- baseline_sha: 9dbc287579a237ffa744dd0c91fa7227d09763ac +- candidate_sha: ce023e0d623116fe66987761d06016590f98fb11 +- harness_sha256: 207f4317b3dcf0395ca8f76f3a8d4398e2520d2aff429a3381b502becf856f8a +- go_version: go version go1.27.2 linux/amd64 +- postgres_version: postgres (PostgreSQL) 18.6 (Ubuntu 18.6-1.pgdg24.04+2) +- compiler: cc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0 +- Workload: {"http_concurrency": 64, "http_rate": 20, "ingest_rate": 50, "ingestor_memory_mib": 512, "measured_seconds": 180, "packet_store_mib": 2048, "queue_limit": 1024, "retention_hours": 168, "server_memory_mib": 3072, "sql_reader_pool": 4, "sql_writer_pool": 1, "warmup_seconds": 60} +- Resource budget: {"allowed_cpus": [0, 1, 2, 3], "cpu_limit": 3.0, "enforced": true, "inactive_charge_reset_bytes": 5099520, "memory_limit_bytes": 6442450944, "memory_peak_reset": true, "resource_counters": ["cgroup.procs", "memory.current", "memory.events", "cpu.stat", "io.stat"]} + +## Limits + +- Five paired runs are the independent repetitions; request records are never pooled across runs. +- Absolute entries are medians of per-run quantiles, with observed min/max, not confidence intervals. +- p99 is omitted unless every contributing run has at least 1000 successful samples. +- Ingest service excludes replay queue wait; HTTP service includes response reading and validation. Queue and schedule-to-complete distributions are separate. +- Resource deltas use the first and last samples strictly inside the measured window; no boundary interpolation. Coverage and sampling gaps are reported. +- Memory/RSS/PSS/storage/WAL are observed sampled peaks, not proof against shorter spikes. Storage includes selected database files and WAL. +- I/O is per-device cgroup accounting. The aggregate can count layered devices twice and is not physical storage bytes. +- WebSocket lag is signed receipt minus ingest return; negative lags remain. Nonpositive baseline denominators have no ordinary speed ratio. +- Accounts are disabled by the reviewed harness. The OS cache is warm; startup uses a new process and approximately 100ms controller probes. +- No capacity, OS-cold, 65-minute soak or engine-wide causal claim follows from this report. +- Validation uses public digests and runtime evidence; it does not reopen private databases or payload files. + +## Resource interpretation + +The measured PostgreSQL cgroup-memory peaks for pairs 0–4 were 5581.371, +6143.984, 5742.367, 5680.508 and 5678.938 MiB. Their measured-window +`memory.events.max` deltas were 0, 444, 0, 0 and 0. All 174 samples in pair 1 +were at least 99% of the 6-GiB limit; the nearest sampled headroom was 16 KiB. +SQLite's measured-window memory-event deltas were zero. All final OOM, OOM-kill +and OOM-group-kill counters were zero. Every PostgreSQL full leg approached +6144 MiB during import/startup, so the measured-window table must not be read as +a complete-run headroom or capacity guarantee. + +Native `memory.peak` resets were checked for each cgroup transition; SQLite full +legs measured about 4231–4340 MiB after PostgreSQL legs near 6144 MiB, without +inheriting their prior peaks. PostgreSQL WAL was 1024 MiB in every measured +window, at its start and end: retained segments from import, not measured WAL +generation. Storage includes WAL. Cgroup per-device I/O accounting is not a +physical-disk-write estimate. Observed free filesystem minima (72.065 GiB in +measured windows; 70.817 GiB across full legs) describe this runner only, not a +required operator minimum. + +## Preserved failed qualification + +The [100-event/s run 37897747657](https://github.com/Kpa-clawbot/CoreScope/actions/runs/37897747657) +at `7b2c19a1eaf0050113c1ce8ae73cc69b6e541893` failed its SQLite leg. It completed +23,918 of 24,000 events and dropped 82, all during measured time, with zero +write errors and correct durable effects for completed work. The drop schedule +window was 205.92–206.76 seconds. Retention was scheduled at 195 seconds and took +11.744610088 seconds, deleting 16,000 transmissions and 256,000 observations. +Maximum ingestion queue wait was 11.054 seconds; source and raw timing review +identified maintenance-related queue saturation. The bounded queue held 1024 +events: 10.24 seconds of offered arrivals at 100/s versus 20.48 seconds at 50/s. +This failed artifact supplies no cross-engine conclusion and is not pooled +with the final separately labeled common 50/s experiment. + +## Production and harness hashes + +The application source hash was equal before and after each build/measurement. +Documentation-only publication does not change which application was measured. + +| Material | SHA-256 | +|---|---| +| Candidate production source | `74f4bdc613e1311f65dbbb27d216c186d499d82c05fbb7c9337e239a07b5ae5c` | +| Baseline production source | `7b786814eec6adc6c322a331727d9a4c2bd51a63456b6bb233f2a3327b39381c` | +| Benchmark harness | `207f4317b3dcf0395ca8f76f3a8d4398e2520d2aff429a3381b502becf856f8a` | diff --git a/docs/postgresql-migration-plan.md b/docs/postgresql-migration-plan.md index afc7f281f..4ef123de5 100644 --- a/docs/postgresql-migration-plan.md +++ b/docs/postgresql-migration-plan.md @@ -1,6 +1,6 @@ # PostgreSQL conversion plan -Status: approved for implementation; application conversion, migration and benchmark work in progress. +Status: implemented and validated at `ce023e0d623116fe66987761d06016590f98fb11`. The [operator guide](postgresql-upgrade.md) covers cutover, interruption recovery and rollback. The [performance report](postgresql-performance.md) records the completed five-pair Linux comparison, including regressions and resource limits. Maintainer acceptance of the replacement remains a separate decision. ## Goal and baseline diff --git a/docs/postgresql-performance.md b/docs/postgresql-performance.md index 0a2247168..8f5afbb40 100644 --- a/docs/postgresql-performance.md +++ b/docs/postgresql-performance.md @@ -1,51 +1,111 @@ # PostgreSQL conversion performance -**Status: not measured.** No same-host, source-pinned paired comparison has been -executed for the final candidate. A PostgreSQL speedup, regression, capacity -increase or equivalent resource cost has not been established. - -The comparison baseline is unmodified upstream commit -`9dbc287579a237ffa744dd0c91fa7227d09763ac`. The final candidate must be identified by -its full commit SHA. The [benchmark harness](../scripts/postgres-benchmark/README.md) -archives both revisions and adds test-only overlays. Production source hashes -before and after measurement prove that a later documentation-only commit did -not change the application being measured. - -The primary corpus is synthetic B: 128,000 transmissions and 2,048,000 -observations, plus the documented nodes, observers, metrics and feature tables. -Five alternating paired runs use one Linux host, Go 1.27.2, PostgreSQL 18.6, -durable settings, equal cache policy, and one shared 3-CPU/6-GiB application plus -database envelope. The actual corpus size, configuration, binaries, settings and -resource enforcement must be present in the result manifest. - -| Acceptance evidence | Current result | -| --- | --- | -| Durable ingestion by event class and full-handler control | Not measured | -| First/full startup, including hot startup and loaded counts | Not measured | -| SQL misses separately from cached/mixed HTTP requests | Not measured | -| Concurrent ingest/API and WebSocket visibility | Not measured | -| Bounded retention under ingestion and empty maintenance pass | Not measured | -| Application + PostgreSQL CPU, cgroup memory, PSS/RSS, I/O, storage and WAL | Not measured | -| Offline migration duration, source immutability, parity and disk headroom | Not measured | -| Five paired repetitions, absolute p50/p95 and adequately sampled p99 | Not measured | -| Actual query plans and raw artifacts | Not measured | -| OS-cold, long soak, L and multi-GiB qualification | Not measured; optional capacity/extended runs | - -The generator and harness controls have local correctness checks. These checks -are preparation evidence, not a database performance comparison. Existing unit -timing assertions likewise do not establish matched upstream/candidate results. - -When the Linux run completes, replace this placeholder with links to the -sanitized manifest and raw artifacts; report absolute measurements, paired -variation, errors, gains and regressions together. Keep p99 unavailable for -classes with fewer than 1,000 successful samples. Preserve a failed or overloaded -run when selecting a lower common offered rate for a separately labeled run. - -The conversion changes more than the driver: packet writes become atomic and -serialized through commit; observer identity becomes explicit; the native SQL -and indexes differ; repeated neighbor-edge writes are aggregated before bounded -upserts; PostgreSQL owns vacuum and checkpoint maintenance. Results therefore -describe this conversion as shipped, not an isolated engine substitution. - -No production data or endpoint is a benchmark input. Publish only the harness's -`public/` bundle, and retain the private disposable state for diagnosing failures. +The PostgreSQL conversion reduced retention-related ingestion delays in this +test. Median writes and several reads were slower, and CPU, memory and storage +costs increased. These results describe the implemented conversion under one +synthetic workload. + +Five alternating pairs completed in [run 37914893623](https://github.com/Kpa-clawbot/CoreScope/actions/runs/37914893623), comparing +[candidate `ce023e0`](https://github.com/Kpa-clawbot/CoreScope/commit/ce023e0d623116fe66987761d06016590f98fb11) +with unmodified upstream +[`9dbc287`](https://github.com/Kpa-clawbot/CoreScope/commit/9dbc287579a237ffa744dd0c91fa7227d09763ac). +The [raw public artifact](https://github.com/Kpa-clawbot/CoreScope/actions/runs/37914893623/artifacts/11613951167), +[complete tables](performance/postgresql-ce023e0-details.md) and +[machine-readable results](performance/postgresql-ce023e0-details.json) retain +every measured class, regression, range, input hash and limitation. + +Both engines ran serially on the same Linux x86-64 runner, with Go 1.27.2, +PostgreSQL 18.6, durable settings and a common budget of **3 CPUs and 6 GiB** for +application and database. Corpus B +contained 128,000 transmissions, 2,048,000 observations, 2,000 nodes and 128 +observers across eight days. Accounts were disabled; OS caches were warm. +Application startup used a new process; +channel SQL tests explicitly cleared the application cache. + +The supplemental workload offered **50 ingest events/s and 20 HTTP requests/s**, +with 60 seconds of warmup and 180 measured seconds per engine. Every leg completed +12,000 scheduled ingest events and 4,800 expected HTTP responses with zero errors +or dropped schedules. Both startup modes loaded all 112,000 transmissions and +1,792,000 observations in the seven-day retained window, with zero evictions. +All 18-table migration and post-replay digests matched, source files stayed +unchanged, and retention deleted the expected rows. + +Values below are **median [minimum, maximum] across five runs**. The ratio column +is the median and range of the five paired PostgreSQL/SQLite ratios; it is not +the quotient of the displayed medians. Ranges are observed variation, not +confidence intervals. Latencies are milliseconds. Ingest service excludes queue +wait in the replay driver; schedule→return includes it. HTTP timing includes scheduling delay, response +reading and validation. + +| Latency metric (ms) | SQLite | PostgreSQL | Paired PG/SQLite | +|---|---:|---:|---:| +| New ingest: schedule→return p95 | 1304 [1237, 1516] | 2.858 [2.831, 2.997] | 0.00229 [0.001885, 0.002309] | +| New ingest: service p50 | 1.003 [0.9395, 1.073] | 1.515 [1.48, 1.607] | 1.542 [1.47, 1.575] | +| Channel messages: SQL miss p50 | 76.11 [66.72, 81.74] | 106.3 [102.2, 118.3] | 1.433 [1.368, 1.531] | +| Channel messages: HTTP first-page p95 | 60.09 [58.82, 64.02] | 113.3 [108.3, 179] | 1.907 [1.782, 2.796] | +| In-memory packet list: HTTP p50 | 30.33 [30.07, 33.27] | 35.34 [34.69, 36.85] | 1.16 [1.107, 1.195] | +| RX coverage: HTTP p50 | 2.164 [2.137, 2.2] | 2.331 [2.277, 2.421] | 1.087 [1.051, 1.113] | +| RX coverage: HTTP p95 | 2.689 [2.623, 2.723] | 2.872 [2.745, 3.056] | 1.076 [1.021, 1.122] | +| RF analytics: HTTP p50 | 2.565 [2.541, 2.609] | 2.337 [2.259, 2.374] | 0.896 [0.8807, 0.9297] | +| WebSocket receipt after ingest return: p95 | 985.5 [969.2, 991.8] | 959.6 [946.8, 971] | 0.969 [0.9608, 0.9916] | + +Ingestion queue tails improved while median service costs rose across event +classes. Channel SQL misses, HTTP message-page tails and the in-memory packet +list regressed. RF analytics improved; RX coverage remained slightly slower. +WebSocket median lag was +similar (507.1 versus 511.2 ms); its p95 improved modestly. One negative SQLite +receipt-minus-return sample is retained, reflecting visibility before the final +ingest return. The details show p99 only with at least 1,000 samples per run. + +| Duration or resource metric | SQLite | PostgreSQL | Paired PG/SQLite | +|---|---:|---:|---:| +| Retention pass (s) | 10.49 [10.45, 10.73] | 0.905 [0.8444, 0.9622] | 0.08626 [0.0805, 0.09126] | +| Full startup, full readiness (s) | 28.06 [27.45, 30.19] | 27.44 [27.22, 29.31] | 0.9783 [0.9403, 0.9989] | +| Hot startup, initial readiness (s) | 9.565 [9.467, 10.18] | 11.48 [10.98, 11.68] | 1.187 [1.148, 1.223] | +| Hot startup, full background fill (s) | 20.52 [20.13, 21.75] | 20.15 [19.92, 20.65] | 0.9707 [0.9494, 1.006] | +| Measured-window CPU time (CPU s) | 43.45 [41.18, 44.58] | 56.86 [55.35, 66.47] | 1.332 [1.274, 1.491] | +| Sampled cgroup memory peak (MiB) | 4043 [4013, 4078] | 5681 [5581, 6144] | 1.415 [1.389, 1.518] | +| Sampled process PSS peak (MiB) | 2922 [2892, 2925] | 3247 [3200, 3321] | 1.11 [1.1, 1.136] | +| Sampled database + WAL footprint (MiB) | 1098 [1095, 1122] | 2327 [2326, 2327] | 2.12 [2.073, 2.126] | +| Accounted device writes (MiB) | 2360 [2335, 2378] | 433.6 [376.7, 507.6] | 0.1857 [0.1596, 0.2153] | + +Retention removed 16,000 transmissions and 256,000 observations. The empty pass +was slower on PostgreSQL, but small in absolute terms: 0.268 versus 1.732 ms. +Full startup and hot background completion were close; initial hot readiness +was slower on PostgreSQL. Both sustained the offered rate. + +Paired CPU time increased about **33%**, PSS about **11%**, and sampled database +plus WAL footprint about **2.12×**. Resource deltas cover 178.2–179.4 sampled +seconds of each 180-second window; sampled peaks can miss spikes. Cgroup memory +includes file cache and charges retained from preparation/import, so it is not +an application-heap estimate. PostgreSQL's run in pair 1 had all 174 measured samples +at or above 99% of the memory limit, reached within **16 KiB** of it, and recorded +444 `memory.events.max` increments. No OOM or process/group OOM kills occurred. +Every PostgreSQL full leg approached the cap during import/startup. This supplies +no spare-capacity assurance. + +The lower write counter is **cgroup device accounting**, not physical disk-write +volume; layered devices can be counted more than once. PostgreSQL retained +1,024 MiB of WAL segments from import throughout the measured windows. That is +part of the stored footprint, not WAL generated by timed ingestion. PSS accounts +for shared pages; summed RSS is reported separately in the details. + +The synthetic telemetry migration tool took **90.99 s [89.94, 94.13]**. Including +role grants and independent harness validation, the phase took **147.6 s +[144.7, 151.0]**. These timings exclude a real operator's shutdown, account +migration, backups, restore checks and rollback work; they are not an outage +estimate. Follow the [upgrade runbook](postgresql-upgrade.md) and rehearse on +copies of the deployment's own data. + +The earlier [100-event/s qualification](https://github.com/Kpa-clawbot/CoreScope/actions/runs/37897747657) +remains failed: SQLite dropped 82 of 24,000 schedules during an 11.745-second +retention burst. The bounded 1,024-event queue was unchanged for this separately +labeled 50-event/s comparison. Failed and successful experiments are not pooled. + +Production hashes matched before and after measurement; the full candidate, +baseline and harness hashes are in the details. Atomic serialized writes, native +SQL/indexes and maintenance behavior differ between implementations, so these +are conversion results rather than isolated engine effects. No L corpus, +OS-cold run, long soak, account workload or capacity qualification was performed. +Use the [harness instructions](../scripts/postgres-benchmark/README.md) to reproduce +the defined experiment; deployment hardware and workload can change the result. diff --git a/docs/postgresql-upgrade.md b/docs/postgresql-upgrade.md index f31f6dd2e..a82c2495a 100644 --- a/docs/postgresql-upgrade.md +++ b/docs/postgresql-upgrade.md @@ -4,6 +4,14 @@ CoreScope uses PostgreSQL for telemetry and optional accounts. The tested databa Existing SQLite instances need a planned outage. Stop **both** the ingestor and server: the account janitor, sessions, proposals and notification evaluator also write data. MQTT messages arriving during this window may be lost unless the broker and publishers provide a separately verified replay mechanism. +For an existing instance, the upgrade path is: + +1. [Stop the old deployment and save its recovery bundle](#preflight-and-recovery). +2. [Preserve configuration and select the correct source directories](#point-the-new-deployment-at-the-right-files). +3. [Run the offline import](#import-with-the-application-stopped). On interruption, use the [resume procedure](#interruption-and-resume). +4. Read the [rollback boundary](#acceptance-and-rollback), then [verify the sources, grant permissions and start](#check-the-retained-source-then-grant-and-start). +5. Check live ingestion and account access, then [verify native backup recovery](#native-backups-and-restores). + ## Databases and credentials The supplied deployment creates `corescope_telemetry` and `corescope_accounts` on one PostgreSQL service. They are separate logical databases, with `C` text collation. Runtime URLs must name one host and database; use the default `public` schema for native backups. From 7edb4739b269266519e15edf33a1b1b0a619eee0 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 12:48:06 -0400 Subject: [PATCH 16/67] feat(storage): add explicit backend resolution and native SQLite schema support --- internal/dbconfig/dbconfig.go | 13 +- internal/dbconfig/storage.go | 172 ++++++++ internal/dbconfig/storage_test.go | 107 +++++ internal/dbschema/backend_test.go | 26 ++ internal/dbschema/collation_test.go | 3 +- internal/dbschema/dbschema.go | 5 +- internal/dbschema/go.mod | 3 + .../dbschema/parameter_conformance_test.go | 62 +++ .../dbschema/postgres_integration_test.go | 12 +- internal/dbschema/sqlite.go | 381 +++++++++++++++++ internal/dbschema/sqlite_test.go | 394 ++++++++++++++++++ 11 files changed, 1162 insertions(+), 16 deletions(-) create mode 100644 internal/dbconfig/storage.go create mode 100644 internal/dbconfig/storage_test.go create mode 100644 internal/dbschema/backend_test.go create mode 100644 internal/dbschema/parameter_conformance_test.go create mode 100644 internal/dbschema/sqlite.go create mode 100644 internal/dbschema/sqlite_test.go diff --git a/internal/dbconfig/dbconfig.go b/internal/dbconfig/dbconfig.go index ccb0b0555..8072a202d 100644 --- a/internal/dbconfig/dbconfig.go +++ b/internal/dbconfig/dbconfig.go @@ -1,13 +1,14 @@ // Package dbconfig provides the shared DBConfig struct used by both the server -// and ingestor binaries for startup loading and legacy maintenance settings. +// and ingestor binaries for storage selection, startup and maintenance settings. package dbconfig -// DBConfig controls startup loading. The three former SQLite controls remain -// decodable for old config files; PostgreSQL maintenance does not use them. +// DBConfig controls storage and startup loading. SQLite maintenance options +// remain active for SQLite and are ignored by PostgreSQL. type DBConfig struct { - VacuumOnStartup bool `json:"vacuumOnStartup"` // Deprecated: PostgreSQL autovacuum owns maintenance. - IncrementalVacuumPages int `json:"incrementalVacuumPages"` // Deprecated: retained for legacy configuration decoding. - AnalysisLimit int `json:"analysisLimit"` // Deprecated: PostgreSQL owns planner statistics. + Backend Backend `json:"backend,omitempty"` + VacuumOnStartup bool `json:"vacuumOnStartup"` + IncrementalVacuumPages int `json:"incrementalVacuumPages"` + AnalysisLimit int `json:"analysisLimit"` // Load controls chunked startup loading (#1009). Load *LoadConfig `json:"load,omitempty"` diff --git a/internal/dbconfig/storage.go b/internal/dbconfig/storage.go new file mode 100644 index 000000000..c99043113 --- /dev/null +++ b/internal/dbconfig/storage.go @@ -0,0 +1,172 @@ +package dbconfig + +import ( + "errors" + "net/url" + "path/filepath" + "runtime" + "strconv" + "strings" +) + +type Backend string + +const ( + SQLite Backend = "sqlite" + Postgres Backend = "postgres" +) + +// Parameter emits a numbered positional placeholder. SQLite drivers differ in +// how they bind dollar names; question-mark numbers work positionally in both. +// Construct queries once when preparing; this is not a SQL-text rewriting API. +// Invalid backend/position values are programming errors. +func (b Backend) Parameter(position int) string { + if position < 1 { + panic("database parameter position must be positive") + } + switch b { + case SQLite: + return "?" + strconv.Itoa(position) + case Postgres: + return "$" + strconv.Itoa(position) + default: + panic("database parameter requires a validated backend") + } +} + +// StorageInputs contains raw configured values, before defaults are inserted. +// The caller supplies installation facts; resolution does no filesystem, +// environment or database reads. ExistingBackend is the recorded choice, +// and FreshInstall must not be inferred from a failed database connection. +type StorageInputs struct { + Backend, EnvBackend, ExistingBackend Backend + FreshInstall bool + BaseDir, StateDir string + DBPath, UsersDBPath string + DatabaseURL, ReaderDatabaseURL, WriterDatabaseURL string + UsersDatabaseURL, ApprovedChannelsDatabaseURL string +} + +// Storage contains only the selected engine's targets. Role URLs can be empty +// when that service/optional feature is not configured; the caller opening a +// connection must require its role's URL. Never log this value: URLs may carry +// credentials. Runtime openers retain their engine-specific validation and +// read-only/role checks, including actual telemetry/account identity checks. +type Storage struct { + Backend Backend + StateDir string + DBPath, UsersDBPath string + ReaderDatabaseURL, WriterDatabaseURL string + UsersDatabaseURL, ApprovedChannelsDatabaseURL string +} + +func ResolveStorage(in StorageInputs) (Storage, error) { + var out Storage + for _, backend := range []Backend{in.Backend, in.EnvBackend, in.ExistingBackend} { + if backend != "" && backend != SQLite && backend != Postgres { + return out, errors.New("unsupported database backend; choose sqlite or postgres") + } + } + selected := in.Backend + if in.EnvBackend != "" { + if selected != "" && selected != in.EnvBackend { + return out, errors.New("database backend conflicts between configuration and environment") + } + selected = in.EnvBackend + } + if in.ExistingBackend != "" { + if selected != "" && selected != in.ExistingBackend { + return out, errors.New("changing an existing database backend requires verified offline conversion") + } + selected = in.ExistingBackend + } + if selected == "" { + hasSQLite := in.DBPath != "" || in.UsersDBPath != "" + hasPostgres := in.DatabaseURL != "" || in.ReaderDatabaseURL != "" || in.WriterDatabaseURL != "" || in.UsersDatabaseURL != "" || in.ApprovedChannelsDatabaseURL != "" + switch { + case hasSQLite && hasPostgres: + return out, errors.New("ambiguous legacy database settings; record the existing backend explicitly") + case hasPostgres: + selected = Postgres + case hasSQLite || in.FreshInstall: + selected = SQLite + default: + return out, errors.New("existing database backend is unknown; supply the recorded choice or unambiguous legacy settings") + } + } + if !filepath.IsAbs(in.BaseDir) { + return out, errors.New("storage base directory must be absolute") + } + out.Backend = selected + stateDir := in.StateDir + if selected == SQLite { + path := in.DBPath + if path == "" { + path = filepath.Join("data", "meshcore.db") + } + var err error + if out.DBPath, err = storagePath(in.BaseDir, path); err != nil { + return Storage{}, err + } + path = in.UsersDBPath + if path == "" { + path = filepath.Join(filepath.Dir(out.DBPath), "users.db") + } + if out.UsersDBPath, err = storagePath(in.BaseDir, path); err != nil { + return Storage{}, err + } + same := out.DBPath == out.UsersDBPath + if runtime.GOOS == "windows" { + same = strings.EqualFold(out.DBPath, out.UsersDBPath) + } + if same { + return Storage{}, errors.New("telemetry and accounts require separate database paths") + } + if stateDir == "" { + stateDir = filepath.Dir(out.DBPath) + } + } else { + out.ReaderDatabaseURL = in.ReaderDatabaseURL + if out.ReaderDatabaseURL == "" { + out.ReaderDatabaseURL = in.DatabaseURL + } + out.WriterDatabaseURL = in.WriterDatabaseURL + if out.WriterDatabaseURL == "" { + out.WriterDatabaseURL = in.DatabaseURL + } + if out.ReaderDatabaseURL == "" && out.WriterDatabaseURL == "" { + return Storage{}, errors.New("PostgreSQL is selected but its telemetry connection settings are missing") + } + out.UsersDatabaseURL, out.ApprovedChannelsDatabaseURL = in.UsersDatabaseURL, in.ApprovedChannelsDatabaseURL + for _, value := range []string{out.ReaderDatabaseURL, out.WriterDatabaseURL, out.UsersDatabaseURL, out.ApprovedChannelsDatabaseURL} { + if value == "" { + continue + } + u, err := url.Parse(value) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") || u.Host == "" || strings.Trim(u.Path, "/") == "" { + return Storage{}, errors.New("PostgreSQL targets require explicit PostgreSQL database URLs") + } + } + if stateDir == "" { + stateDir = "data" + } + } + var err error + if out.StateDir, err = storagePath(in.BaseDir, stateDir); err != nil { + return Storage{}, err + } + return out, nil +} + +func storagePath(base, path string) (string, error) { + if strings.Contains(path, "://") || strings.HasPrefix(path, "postgres:") || strings.HasPrefix(path, "postgresql:") { + return "", errors.New("a database URL cannot be used as a storage path") + } + if filepath.IsAbs(path) { + return filepath.Clean(path), nil + } + if filepath.VolumeName(path) != "" { + return "", errors.New("drive-relative storage paths are ambiguous; use an absolute path") + } + return filepath.Join(base, path), nil +} diff --git a/internal/dbconfig/storage_test.go b/internal/dbconfig/storage_test.go new file mode 100644 index 000000000..b322585b4 --- /dev/null +++ b/internal/dbconfig/storage_test.go @@ -0,0 +1,107 @@ +package dbconfig + +import ( + "path/filepath" + "strings" + "testing" +) + +func TestStorageSelection(t *testing.T) { + base := t.TempDir() + pg := "postgres://reader:private-value@localhost/telemetry" + cases := []struct { + name string + in StorageInputs + want Backend + err bool + }{ + {"fresh", StorageInputs{FreshInstall: true}, SQLite, false}, + {"unknown existing", StorageInputs{}, "", true}, + {"legacy sqlite", StorageInputs{DBPath: "old/mesh.db"}, SQLite, false}, + {"legacy postgres", StorageInputs{DatabaseURL: pg}, Postgres, false}, + {"legacy conflict", StorageInputs{DBPath: "old.db", DatabaseURL: pg}, "", true}, + {"recorded postgres missing url", StorageInputs{ExistingBackend: Postgres, FreshInstall: true}, "", true}, + {"recorded postgres", StorageInputs{ExistingBackend: Postgres, ReaderDatabaseURL: pg}, Postgres, false}, + {"ordinary backend change", StorageInputs{Backend: SQLite, ExistingBackend: Postgres}, "", true}, + {"selector conflict", StorageInputs{Backend: SQLite, EnvBackend: Postgres, DatabaseURL: pg}, "", true}, + {"unknown selector", StorageInputs{Backend: "mysql", FreshInstall: true}, "", true}, + {"explicit sqlite ignores inactive pg", StorageInputs{Backend: SQLite, DatabaseURL: "private-invalid-pg"}, SQLite, false}, + {"explicit postgres ignores old path", StorageInputs{Backend: Postgres, DBPath: "old.db", DatabaseURL: pg}, Postgres, false}, + {"postgres path is not url", StorageInputs{Backend: Postgres, DatabaseURL: "old.db"}, "", true}, + {"sqlite url is not path", StorageInputs{Backend: SQLite, DBPath: pg}, "", true}, + {"accounts alone cannot configure telemetry", StorageInputs{UsersDatabaseURL: pg}, "", true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + c.in.BaseDir = base + got, err := ResolveStorage(c.in) + if (err != nil) != c.err { + t.Fatalf("error = %v, want error %v", err, c.err) + } + if err != nil { + if strings.Contains(err.Error(), "private-") { + t.Fatal("error leaked connection value") + } + return + } + if got.Backend != c.want { + t.Fatalf("backend=%q, want %q", got.Backend, c.want) + } + }) + } +} + +func TestStorageTargets(t *testing.T) { + base := t.TempDir() + got, err := ResolveStorage(StorageInputs{BaseDir: base, Backend: SQLite, DBPath: "legacy/mesh.db"}) + if err != nil { + t.Fatal(err) + } + if got.DBPath != filepath.Join(base, "legacy", "mesh.db") || got.UsersDBPath != filepath.Join(base, "legacy", "users.db") || got.StateDir != filepath.Join(base, "legacy") { + t.Fatalf("SQLite paths: %+v", got) + } + if _, err := ResolveStorage(StorageInputs{BaseDir: base, Backend: SQLite, DBPath: "same.db", UsersDBPath: "same.db"}); err == nil { + t.Fatal("accepted same telemetry/account target") + } + if _, err := ResolveStorage(StorageInputs{BaseDir: "relative", Backend: SQLite}); err == nil { + t.Fatal("accepted non-absolute base instead of caller-resolved location") + } + common := "postgres://common@localhost/telemetry" + reader := "postgres://reader@localhost/telemetry" + got, err = ResolveStorage(StorageInputs{BaseDir: base, Backend: Postgres, DatabaseURL: common, ReaderDatabaseURL: reader, StateDir: "state"}) + if err != nil { + t.Fatal(err) + } + if got.ReaderDatabaseURL != reader || got.WriterDatabaseURL != common || got.StateDir != filepath.Join(base, "state") || got.DBPath != "" || got.UsersDBPath != "" { + t.Fatal("PostgreSQL roles/state were not resolved independently of file paths") + } + got, err = ResolveStorage(StorageInputs{BaseDir: base, FreshInstall: true}) + if err != nil { + t.Fatal(err) + } + if got.DBPath != filepath.Join(base, "data", "meshcore.db") || got.UsersDBPath != filepath.Join(base, "data", "users.db") { + t.Fatal("fresh SQLite defaults changed") + } +} + +func TestBackendParameters(t *testing.T) { + for _, backend := range []Backend{SQLite, Postgres} { + prefix := "$" + if backend == SQLite { + prefix = "?" + } + if got := backend.Parameter(12); got != prefix+"12" { + t.Fatalf("parameter=%q", got) + } + } + for _, call := range []func(){func() { SQLite.Parameter(0) }, func() { Backend("unknown").Parameter(1) }} { + func() { + defer func() { + if recover() == nil { + t.Error("invalid parameter construction did not fail") + } + }() + call() + }() + } +} diff --git a/internal/dbschema/backend_test.go b/internal/dbschema/backend_test.go new file mode 100644 index 000000000..f05d72737 --- /dev/null +++ b/internal/dbschema/backend_test.go @@ -0,0 +1,26 @@ +package dbschema + +import ( + "os" + "testing" + + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +// PostgreSQL-only assertions belong to the explicitly selected PG matrix. +// A missing URL in that matrix is an error, never a skipped integration test. +func postgresSchema(t testing.TB) string { + t.Helper() + switch os.Getenv("CORESCOPE_TEST_BACKEND") { + case "", "sqlite": + t.Skip("PostgreSQL-specific test; select CORESCOPE_TEST_BACKEND=postgres") + case "postgres": + if os.Getenv("CORESCOPE_TEST_POSTGRES_URL") == "" { + t.Fatal("PostgreSQL test matrix requires CORESCOPE_TEST_POSTGRES_URL") + } + return pgtest.NewSchema(t) + default: + t.Fatal("CORESCOPE_TEST_BACKEND must be sqlite or postgres") + } + return "" +} diff --git a/internal/dbschema/collation_test.go b/internal/dbschema/collation_test.go index 29faacf6a..23239ee55 100644 --- a/internal/dbschema/collation_test.go +++ b/internal/dbschema/collation_test.go @@ -4,11 +4,10 @@ import ( "testing" "github.com/meshcore-analyzer/pgutil" - "github.com/meshcore-analyzer/pgutil/pgtest" ) func TestTelemetryTextUsesBinaryCollation(t *testing.T) { - db, err := pgutil.Open(pgtest.NewSchema(t), false) + db, err := pgutil.Open(postgresSchema(t), false) if err != nil { t.Fatal(err) } diff --git a/internal/dbschema/dbschema.go b/internal/dbschema/dbschema.go index 46cccfdd6..3cece8c42 100644 --- a/internal/dbschema/dbschema.go +++ b/internal/dbschema/dbschema.go @@ -1,5 +1,6 @@ -// Package dbschema owns the canonical PostgreSQL telemetry schema. Only the -// offline migration command uses Apply; both runtime processes assert readiness. +// Package dbschema owns telemetry schema entrypoints for SQLite and PostgreSQL. +// SQLite's writer applies migrations; PostgreSQL uses offline bootstrap. Readers +// only assert readiness. Apply and AssertReady retain PostgreSQL compatibility. package dbschema import ( diff --git a/internal/dbschema/go.mod b/internal/dbschema/go.mod index 5996cad5c..da81f1bb0 100644 --- a/internal/dbschema/go.mod +++ b/internal/dbschema/go.mod @@ -4,6 +4,7 @@ go 1.25.0 require ( github.com/mattn/go-sqlite3 v1.14.52 + github.com/meshcore-analyzer/dbconfig v0.0.0 github.com/meshcore-analyzer/pgutil v0.0.0 ) @@ -18,3 +19,5 @@ require ( ) replace github.com/meshcore-analyzer/pgutil => ../pgutil + +replace github.com/meshcore-analyzer/dbconfig => ../dbconfig diff --git a/internal/dbschema/parameter_conformance_test.go b/internal/dbschema/parameter_conformance_test.go new file mode 100644 index 000000000..4848deda4 --- /dev/null +++ b/internal/dbschema/parameter_conformance_test.go @@ -0,0 +1,62 @@ +package dbschema + +import ( + "database/sql" + "fmt" + "testing" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" +) + +func testNumberedParameters(t *testing.T, db *sql.DB, backend dbconfig.Backend) { + t.Helper() + query := fmt.Sprintf("SELECT CAST(%s AS BIGINT),CAST(%s AS BIGINT),CAST(%s AS BIGINT)", backend.Parameter(2), backend.Parameter(1), backend.Parameter(2)) + for _, second := range []any{int64(22), nil} { + var a, b, c sql.NullInt64 + if err := db.QueryRow(query, int64(11), second).Scan(&a, &b, &c); err != nil { + t.Fatal(err) + } + if !b.Valid || b.Int64 != 11 || a != c { + t.Fatalf("reordered/repeated parameters: %v,%v,%v", a, b, c) + } + if second == nil { + if a.Valid { + t.Fatal("NULL changed") + } + } else if !a.Valid || a.Int64 != 22 { + t.Fatal("second argument bound by appearance, not position") + } + } +} + +func TestSQLiteNumberedParameterConformance(t *testing.T) { + db, _ := sqliteTestDB(t) + testNumberedParameters(t, db, dbconfig.SQLite) +} + +func TestPostgresNumberedParameterConformance(t *testing.T) { + db, err := pgutil.Open(postgresSchema(t), false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + testNumberedParameters(t, db, dbconfig.Postgres) +} + +func TestPostgresNamedEntrypointsKeepImportGate(t *testing.T) { + db, err := pgutil.Open(postgresSchema(t), false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := ApplyForImport(db, nil); err != nil { + t.Fatal(err) + } + if err := AssertPostgresReady(db); err == nil { + t.Fatal("named reader entrypoint bypassed incomplete import") + } + if err := ApplyPostgres(db, nil); err == nil { + t.Fatal("named bootstrap entrypoint bypassed incomplete import") + } +} diff --git a/internal/dbschema/postgres_integration_test.go b/internal/dbschema/postgres_integration_test.go index 179841084..76416512d 100644 --- a/internal/dbschema/postgres_integration_test.go +++ b/internal/dbschema/postgres_integration_test.go @@ -9,7 +9,7 @@ import ( ) func TestPostgresImportInitializationNeverReady(t *testing.T) { - db, err := pgutil.Open(pgtest.NewSchema(t), false) + db, err := pgutil.Open(postgresSchema(t), false) if err != nil { t.Fatal(err) } @@ -26,7 +26,7 @@ func TestPostgresImportInitializationNeverReady(t *testing.T) { } func TestPostgresConcurrentInitialization(t *testing.T) { - db, err := pgutil.Open(pgtest.NewSchema(t), false) + db, err := pgutil.Open(postgresSchema(t), false) if err != nil { t.Fatal(err) } @@ -48,7 +48,7 @@ func TestPostgresConcurrentInitialization(t *testing.T) { } func TestPostgresIdentityRawValuesAndNullDedup(t *testing.T) { - db, err := pgutil.Open(pgtest.NewSchema(t), false) + db, err := pgutil.Open(postgresSchema(t), false) if err != nil { t.Fatal(err) } @@ -97,7 +97,7 @@ func TestPostgresIdentityRawValuesAndNullDedup(t *testing.T) { } func TestPostgresWriterRejectsOwnerAndReader(t *testing.T) { - dsn := pgtest.NewSchema(t) + dsn := postgresSchema(t) owner, err := pgutil.Open(dsn, false) if err != nil { t.Fatal(err) @@ -128,7 +128,7 @@ func TestPostgresWriterRejectsOwnerAndReader(t *testing.T) { } func TestPostgresInitializationRejectsUnversionedData(t *testing.T) { - db, err := pgutil.Open(pgtest.NewSchema(t), false) + db, err := pgutil.Open(postgresSchema(t), false) if err != nil { t.Fatal(err) } @@ -149,7 +149,7 @@ func TestPostgresInitializationRejectsUnversionedData(t *testing.T) { } func TestPostgresSchemaFreshRepeatedAndReadiness(t *testing.T) { - dsn := pgtest.NewSchema(t) + dsn := postgresSchema(t) db, err := pgutil.Open(dsn, false) if err != nil { t.Fatal(err) diff --git a/internal/dbschema/sqlite.go b/internal/dbschema/sqlite.go new file mode 100644 index 000000000..704dd981d --- /dev/null +++ b/internal/dbschema/sqlite.go @@ -0,0 +1,381 @@ +package dbschema + +import ( + "context" + "database/sql" + "errors" + "fmt" + "regexp" + "strings" + + "github.com/meshcore-analyzer/dbschema/legacy" +) + +// The existing logical migration table records this SQLite physical-layout +// change. No new data table or PostgreSQL manifest column is introduced. +const SQLiteObserverIdentityMigration = "observers_identity_autoincrement_v1" + +// Existing entrypoints remain PostgreSQL-compatible while callers migrate to +// explicit engine selection. PostgreSQL grants/readiness checks are unchanged. +func ApplyPostgres(db *sql.DB, logf Logger) error { return Apply(db, logf) } +func AssertPostgresReady(db *sql.DB) error { return AssertReady(db) } + +// ApplySQLite is a writer-only startup/upgrade entrypoint. Existing v3 files +// are checked before historical migrations can perform cleanup. Older v2 or +// unknown layouts require an explicit supported offline upgrade first. +func ApplySQLite(db *sql.DB, logf Logger) error { + migrated, cols, err := sqliteObserverLayout(db) + if err != nil { + return err + } + if err := sqliteIdentityVersion(db); err != nil { + return err + } + var ledger, applied int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='table' AND name='_migrations'`).Scan(&ledger); err != nil { + return err + } + if ledger != 0 { + if err := db.QueryRow(`SELECT count(*) FROM _migrations WHERE name=?`, SQLiteObserverIdentityMigration).Scan(&applied); err != nil { + return err + } + } + if applied != 0 && !migrated { + return errors.New("SQLite observer identity marker does not match its physical schema") + } + needsIdentity := applied == 0 + var tables int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'`).Scan(&tables); err != nil { + return err + } + if tables != 0 && needsIdentity { + if len(cols) == 0 { + return errors.New("existing SQLite file is not a supported telemetry database") + } + if err := legacy.CheckLegacySource(db); err != nil { + return err + } + if err := sqliteForeignKeysValid(db); err != nil { + return err + } + } + if err := legacy.ApplyBase(db); err != nil { + return err + } + if err := legacy.Apply(db, legacy.Logger(logf)); err != nil { + return err + } + // The identity migration's corpus/FK scans run once. Ordinary restarts + // retain existing schema maintenance without rescanning all observations. + if !needsIdentity { + return AssertSQLiteReady(db) + } + return migrateSQLiteObserverIdentity(db) +} + +// AssertSQLiteReady only reads schema metadata. It is safe on a mode=ro handle; +// it never applies migrations or marks a partially initialized target ready. +func AssertSQLiteReady(db *sql.DB) error { + if err := legacy.AssertReady(db); err != nil { + return err + } + if err := sqliteIdentityVersion(db); err != nil { + return err + } + migrated, _, err := sqliteObserverLayout(db) + if err != nil { + return err + } + var count int + if err := db.QueryRow(`SELECT count(*) FROM _migrations WHERE name=?`, SQLiteObserverIdentityMigration).Scan(&count); err != nil { + return err + } + if !migrated || count != 1 { + return errors.New("SQLite observer identity schema is not migrated; run the writer upgrade") + } + return nil +} + +type sqliteColumn struct { + name, kind string + notNull, primaryKey, hidden int + defaultSQL sql.NullString +} + +var unsupportedObserverConstraint = regexp.MustCompile(`(?i)\b(CHECK|COLLATE|REFERENCES|CONSTRAINT|GENERATED|WITHOUT|STRICT)\b|\bON\s+CONFLICT\b`) + +func sqliteQuote(value string) string { return `"` + strings.ReplaceAll(value, `"`, `""`) + `"` } + +// Rebuilding unknown constraints can silently change semantics. Accept the +// ordinary supported layout; refuse extensions we cannot reproduce before any +// schema writes. Explicit indexes and triggers are preserved by the rebuild. +func sqliteObserverLayout(db Querier) (bool, []sqliteColumn, error) { + var ddl string + err := db.QueryRow(`SELECT sql FROM sqlite_master WHERE type='table' AND name='observers'`).Scan(&ddl) + if errors.Is(err, sql.ErrNoRows) { + return false, nil, nil + } + if err != nil { + return false, nil, err + } + if unsupportedObserverConstraint.MatchString(ddl) { + return false, nil, errors.New("unsupported SQLite observer constraints; source was not changed") + } + rows, err := db.Query(`PRAGMA table_xinfo(observers)`) + if err != nil { + return false, nil, err + } + var cols []sqliteColumn + for rows.Next() { + var cid int + var c sqliteColumn + if err = rows.Scan(&cid, &c.name, &c.kind, &c.notNull, &c.defaultSQL, &c.primaryKey, &c.hidden); err != nil { + rows.Close() + return false, nil, err + } + known := false + for _, table := range Tables { + if table.Name == "observers" { + for _, col := range table.Columns { + if col.Name == c.name { + known = true + } + } + } + } + c.kind = strings.ToUpper(c.kind) + if !known || c.hidden != 0 || (c.kind != "TEXT" && c.kind != "INTEGER" && c.kind != "REAL") { + rows.Close() + return false, nil, errors.New("unsupported SQLite observer column layout; source was not changed") + } + cols = append(cols, c) + } + if err = rows.Err(); err != nil { + rows.Close() + return false, nil, err + } + rows.Close() + var id, rowid *sqliteColumn + for i := range cols { + if cols[i].name == "id" { + id = &cols[i] + } + if cols[i].name == "rowid" { + rowid = &cols[i] + } + } + if id == nil || id.kind != "TEXT" { + return false, nil, errors.New("unsupported SQLite observer text identity") + } + if rowid == nil { + if id.primaryKey != 1 || strings.Contains(strings.ToUpper(ddl), "UNIQUE") { + return false, nil, errors.New("unsupported SQLite observer primary key") + } + for _, c := range cols { + if c.name != "id" && c.primaryKey != 0 { + return false, nil, errors.New("unsupported SQLite composite observer primary key") + } + } + return false, cols, nil + } + if rowid.kind != "INTEGER" || rowid.primaryKey != 1 || id.primaryKey != 0 || !strings.Contains(strings.ToUpper(ddl), "AUTOINCREMENT") { + return false, nil, errors.New("unsupported SQLite observer rowid identity") + } + // A text identity remains a BINARY unique key (including SQLite's existing + // multiple-NULL behavior), so foreign keys naming observers(id) still work. + idx, err := db.Query(`SELECT name FROM pragma_index_list('observers') WHERE "unique"=1 AND partial=0`) + if err != nil { + return false, nil, err + } + var names []string + for idx.Next() { + var name string + if err = idx.Scan(&name); err != nil { + idx.Close() + return false, nil, err + } + names = append(names, name) + } + if err = idx.Err(); err != nil { + idx.Close() + return false, nil, err + } + idx.Close() + for _, name := range names { + var total, matching int + err = db.QueryRow(`SELECT count(*),coalesce(sum(name='id' AND coll='BINARY'),0) FROM pragma_index_xinfo(?) WHERE key=1`, name).Scan(&total, &matching) + if err != nil { + return false, nil, err + } + if total == 1 && matching == 1 { + return true, cols, nil + } + } + return false, nil, errors.New("SQLite observer text identity is not a BINARY unique key") +} + +func sqliteIdentityVersion(db Querier) error { + var exists int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='table' AND name='_migrations'`).Scan(&exists); err != nil { + return err + } + if exists == 0 { + return nil + } + const prefix = "observers_identity_autoincrement_v" + rows, err := db.Query(`SELECT name FROM _migrations WHERE substr(name,1,?)=?`, len(prefix), prefix) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return err + } + if name != SQLiteObserverIdentityMigration { + return errors.New("unsupported newer SQLite observer identity migration") + } + } + return rows.Err() +} + +func migrateSQLiteObserverIdentity(db *sql.DB) (err error) { + ctx := context.Background() + conn, err := db.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + var foreignKeys, legacyAlter int + if err = conn.QueryRowContext(ctx, `PRAGMA foreign_keys`).Scan(&foreignKeys); err != nil { + return err + } + if err = conn.QueryRowContext(ctx, `PRAGMA legacy_alter_table`).Scan(&legacyAlter); err != nil { + return err + } + defer func() { + for _, q := range []string{fmt.Sprintf("PRAGMA legacy_alter_table=%d", legacyAlter), fmt.Sprintf("PRAGMA foreign_keys=%d", foreignKeys)} { + if _, restoreErr := conn.ExecContext(ctx, q); err == nil && restoreErr != nil { + err = restoreErr + } + } + }() + // Use the same reserved connection throughout. Disabling FK actions avoids + // cascades while replacing the parent; references are checked before commit. + if _, err = conn.ExecContext(ctx, `PRAGMA foreign_keys=OFF`); err != nil { + return err + } + if _, err = conn.ExecContext(ctx, `PRAGMA legacy_alter_table=ON`); err != nil { + return err + } + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + migrated, cols, err := sqliteObserverLayout(tx) + if err != nil { + return err + } + if err = sqliteForeignKeysValid(tx); err != nil { + return err + } + if !migrated { + var objects []string + rows, e := tx.Query(`SELECT sql FROM sqlite_master WHERE tbl_name='observers' AND type IN ('index','trigger') AND sql IS NOT NULL ORDER BY type,name`) + if e != nil { + return e + } + for rows.Next() { + var ddl string + if e = rows.Scan(&ddl); e != nil { + rows.Close() + return e + } + objects = append(objects, ddl) + } + if e = rows.Err(); e != nil { + rows.Close() + return e + } + rows.Close() + fields := []string{`rowid INTEGER PRIMARY KEY AUTOINCREMENT`} + names := []string{`rowid`} + for _, c := range cols { + field := sqliteQuote(c.name) + " " + c.kind + if c.name == "id" { + field += " UNIQUE" + } + if c.notNull != 0 { + field += " NOT NULL" + } + if c.defaultSQL.Valid { + field += " DEFAULT " + c.defaultSQL.String + } + fields = append(fields, field) + names = append(names, sqliteQuote(c.name)) + } + const replacement = "_corescope_observers_identity_v1" + if _, err = tx.Exec(`CREATE TABLE ` + replacement + ` (` + strings.Join(fields, ",") + `)`); err != nil { + return err + } + columns := strings.Join(names, ",") + if _, err = tx.Exec(`INSERT INTO ` + replacement + ` (` + columns + `) SELECT ` + columns + ` FROM observers`); err != nil { + return err + } + if _, err = tx.Exec(`DROP TABLE observers`); err != nil { + return err + } + if _, err = tx.Exec(`ALTER TABLE ` + replacement + ` RENAME TO observers`); err != nil { + return err + } + for _, ddl := range objects { + if _, err = tx.Exec(ddl); err != nil { + return err + } + } + } + // Include retained orphan links and the sequence itself, so deleted IDs can + // never be reassigned to a new observer. Other table sequences and cursors + // are untouched by this migration; reverse import must seed them separately. + var high int64 + if err = tx.QueryRow(`SELECT max(value) FROM ( + SELECT coalesce(max(rowid),0) AS value FROM observers UNION ALL + SELECT coalesce(max(observer_idx),0) FROM observations UNION ALL + SELECT coalesce(max(seq),0) FROM sqlite_sequence WHERE name='observers')`).Scan(&high); err != nil { + return err + } + res, err := tx.Exec(`UPDATE sqlite_sequence SET seq=? WHERE name='observers'`, high) + if err != nil { + return err + } + n, err := res.RowsAffected() + if err != nil { + return err + } + if n == 0 { + if _, err = tx.Exec(`INSERT INTO sqlite_sequence(name,seq) VALUES('observers',?)`, high); err != nil { + return err + } + } + if err = sqliteForeignKeysValid(tx); err != nil { + return err + } + if _, err = tx.Exec(`INSERT OR IGNORE INTO _migrations(name) VALUES(?)`, SQLiteObserverIdentityMigration); err != nil { + return err + } + return tx.Commit() +} + +func sqliteForeignKeysValid(db Querier) error { + rows, err := db.Query(`PRAGMA foreign_key_check`) + if err != nil { + return err + } + defer rows.Close() + if rows.Next() { + return errors.New("SQLite foreign-key violations prevent observer identity migration") + } + return rows.Err() +} diff --git a/internal/dbschema/sqlite_test.go b/internal/dbschema/sqlite_test.go new file mode 100644 index 000000000..b38bc17c8 --- /dev/null +++ b/internal/dbschema/sqlite_test.go @@ -0,0 +1,394 @@ +package dbschema + +import ( + "context" + "database/sql" + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbschema/legacy" +) + +func sqliteTestDB(t *testing.T) (*sql.DB, string) { + t.Helper() + path := filepath.Join(t.TempDir(), "telemetry.db") + db, err := sql.Open("sqlite3", legacy.WriterDSN(path)) + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + t.Cleanup(func() { db.Close() }) + return db, path +} + +func sqliteExec(t *testing.T, db *sql.DB, query string, args ...any) { + t.Helper() + if _, err := db.Exec(query, args...); err != nil { + t.Fatal(err) + } +} + +func TestSQLiteFreshAndReadOnlyReadiness(t *testing.T) { + db, path := sqliteTestDB(t) + if err := AssertSQLiteReady(db); err == nil { + t.Fatal("empty schema ready") + } + if err := ApplySQLite(db, nil); err != nil { + t.Fatal(err) + } + if err := ApplySQLite(db, nil); err != nil { + t.Fatal(err) + } + if err := AssertSQLiteReady(db); err != nil { + t.Fatal(err) + } + var marker int + if err := db.QueryRow(`SELECT count(*) FROM _migrations WHERE name=?`, SQLiteObserverIdentityMigration).Scan(&marker); err != nil || marker != 1 { + t.Fatalf("marker=%d err=%v", marker, err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + ro, err := sql.Open("sqlite3", "file:"+filepath.ToSlash(path)+"?mode=ro") + if err != nil { + t.Fatal(err) + } + defer ro.Close() + ro.SetMaxOpenConns(1) + if err := AssertSQLiteReady(ro); err != nil { + t.Fatal(err) + } + if _, err := ro.Exec(`INSERT INTO observers(id) VALUES('reader-write')`); err == nil { + t.Fatal("reader wrote telemetry") + } + if err := ApplySQLite(ro, nil); err == nil { + t.Fatal("reader ran writer schema entrypoint") + } +} + +func TestSQLiteObserverIdentityPreservesOldRowsAndReferences(t *testing.T) { + db, _ := sqliteTestDB(t) + if err := legacy.ApplyBase(db); err != nil { + t.Fatal(err) + } + if err := legacy.Apply(db, nil); err != nil { + t.Fatal(err) + } + sqliteExec(t, db, `INSERT INTO observers(rowid,id,name) VALUES(0,'zero','zero'),(7,'Node','upper'),(8,'node','lower'),(20,NULL,'null-one'),(30,NULL,'null-two')`) + sqliteExec(t, db, `INSERT INTO transmissions(id,raw_hex,hash,first_seen) VALUES(1,'aa','hash','2026-01-01'),(77,'bb','deleted','2026-01-01')`) + sqliteExec(t, db, `DELETE FROM transmissions WHERE id=77`) + sqliteExec(t, db, `INSERT INTO observations(id,transmission_id,observer_idx,timestamp) VALUES(1,1,7,1),(2,1,99,1),(3,1,NULL,1),(88,1,8,2)`) + sqliteExec(t, db, `DELETE FROM observations WHERE id=88`) + sqliteExec(t, db, `INSERT INTO advert_evidence_backfill(id,tx_cursor,obs_cursor) VALUES(1,700,900)`) + sqliteExec(t, db, `CREATE TABLE observer_child(observer_id TEXT REFERENCES observers(id) ON DELETE CASCADE)`) + sqliteExec(t, db, `INSERT INTO observer_child VALUES('Node')`) + sqliteExec(t, db, `CREATE INDEX custom_observer_name ON observers(name)`) + sqliteExec(t, db, `CREATE TABLE observer_audit(value TEXT)`) + sqliteExec(t, db, `CREATE TRIGGER observer_name_changed AFTER UPDATE OF name ON observers BEGIN INSERT INTO observer_audit VALUES(new.name); END`) + sqliteExec(t, db, `CREATE VIEW observer_view AS SELECT rowid,id,name FROM observers`) + before := sqliteObserverRows(t, db) + if err := ApplySQLite(db, nil); err != nil { + t.Fatal(err) + } + if got := sqliteObserverRows(t, db); got != before { + t.Fatalf("rows changed: before=%s after=%s", before, got) + } + for query, want := range map[string]int64{ + `SELECT count(*) FROM observers`: 5, + `SELECT count(*) FROM observer_child`: 1, + `SELECT count(*) FROM observer_view`: 5, + `SELECT count(*) FROM observer_audit`: 0, + `SELECT count(*) FROM sqlite_master WHERE name='custom_observer_name'`: 1, + `SELECT count(*) FROM observations WHERE observer_idx=99`: 1, + `SELECT count(*) FROM observations WHERE observer_idx IS NULL`: 1, + `SELECT seq FROM sqlite_sequence WHERE name='observers'`: 99, + `SELECT seq FROM sqlite_sequence WHERE name='transmissions'`: 77, + `SELECT seq FROM sqlite_sequence WHERE name='observations'`: 88, + `SELECT tx_cursor FROM advert_evidence_backfill WHERE id=1`: 700, + `SELECT obs_cursor FROM advert_evidence_backfill WHERE id=1`: 900, + `PRAGMA foreign_keys`: 1, + `PRAGMA legacy_alter_table`: 0, + } { + var got int64 + if err := db.QueryRow(query).Scan(&got); err != nil || got != want { + t.Fatalf("%s: %d want %d err=%v", query, got, want, err) + } + } + if _, err := db.Exec(`INSERT INTO observers(id) VALUES('Node')`); err == nil { + t.Fatal("lost text-id uniqueness") + } + if _, err := db.Exec(`INSERT INTO observer_child VALUES('NODE')`); err == nil { + t.Fatal("lost binary foreign-key matching") + } + sqliteExec(t, db, `UPDATE observers SET name='updated' WHERE id='Node'`) + var value string + if err := db.QueryRow(`SELECT value FROM observer_audit`).Scan(&value); err != nil || value != "updated" { + t.Fatalf("trigger: %q %v", value, err) + } + sqliteExec(t, db, `DELETE FROM observers WHERE id='Node'`) + var count int + if err := db.QueryRow(`SELECT count(*) FROM observer_child`).Scan(&count); err != nil || count != 0 { + t.Fatal("foreign-key cascade changed") + } + for _, want := range []int64{100, 101} { + res, err := db.Exec(`INSERT INTO observers(id) VALUES('new')`) + if err != nil { + t.Fatal(err) + } + got, err := res.LastInsertId() + if err != nil || got != want { + t.Fatalf("next id=%d want=%d err=%v", got, want, err) + } + sqliteExec(t, db, `DELETE FROM observers WHERE id='new'`) + if err := ApplySQLite(db, nil); err != nil { + t.Fatal(err) + } + } +} + +func sqliteObserverRows(t *testing.T, db *sql.DB) string { + t.Helper() + rows, err := db.Query(`SELECT rowid,id,name FROM observers ORDER BY rowid`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var out strings.Builder + for rows.Next() { + var id int64 + var key, name sql.NullString + if err := rows.Scan(&id, &key, &name); err != nil { + t.Fatal(err) + } + fmt.Fprintf(&out, "%d:%#v:%#v\n", id, key, name) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return out.String() +} + +func TestSQLiteUnknownObserverLayoutRefusesBeforeMutation(t *testing.T) { + for _, tc := range []struct{ name, old, replacement string }{ + {"collation", "id TEXT PRIMARY KEY", "id TEXT COLLATE NOCASE PRIMARY KEY"}, + {"check", "name TEXT", "name TEXT CHECK(name<>'')"}, + {"shadow rowid", "id TEXT PRIMARY KEY", "rowid TEXT,id TEXT PRIMARY KEY"}, + {"conflict policy", "id TEXT PRIMARY KEY", "id TEXT PRIMARY KEY ON CONFLICT IGNORE"}, + {"composite key", "id TEXT PRIMARY KEY", "id TEXT"}, + } { + t.Run(tc.name, func(t *testing.T) { + db, _ := sqliteTestDB(t) + if err := legacy.ApplyBase(db); err != nil { + t.Fatal(err) + } + if err := legacy.Apply(db, nil); err != nil { + t.Fatal(err) + } + var ddl string + if err := db.QueryRow(`SELECT sql FROM sqlite_master WHERE name='observers' AND type='table'`).Scan(&ddl); err != nil { + t.Fatal(err) + } + if !strings.Contains(ddl, tc.old) { + t.Fatal("fixture does not contain expected declaration") + } + ddl = strings.Replace(ddl, tc.old, tc.replacement, 1) + if tc.name == "composite key" { + ddl = strings.TrimSuffix(strings.TrimSpace(ddl), ")") + ",PRIMARY KEY(id,name))" + } + sqliteExec(t, db, `DROP TABLE observers`) + sqliteExec(t, db, ddl) + sqliteExec(t, db, `INSERT INTO observers(id,name) VALUES('kept','value')`) + var before, after int + if err := db.QueryRow(`PRAGMA schema_version`).Scan(&before); err != nil { + t.Fatal(err) + } + if err := ApplySQLite(db, nil); err == nil { + t.Fatal("unsupported schema accepted") + } + if err := db.QueryRow(`PRAGMA schema_version`).Scan(&after); err != nil { + t.Fatal(err) + } + if before != after { + t.Fatal("refusal mutated schema") + } + var name string + if err := db.QueryRow(`SELECT name FROM observers`).Scan(&name); err != nil || name != "value" { + t.Fatal("refusal changed row") + } + }) + } +} + +func TestSQLiteObserverMigrationFailureRollsBackAndRestoresConnection(t *testing.T) { + db, _ := sqliteTestDB(t) + if err := legacy.ApplyBase(db); err != nil { + t.Fatal(err) + } + if err := legacy.Apply(db, nil); err != nil { + t.Fatal(err) + } + sqliteExec(t, db, `INSERT INTO observers(rowid,id,name) VALUES(12,'preserve','value')`) + // A conflicting pre-existing object is never overwritten or repurposed. + sqliteExec(t, db, `CREATE TABLE _corescope_observers_identity_v1(value TEXT)`) + sqliteExec(t, db, `INSERT INTO _corescope_observers_identity_v1 VALUES('owned elsewhere')`) + before := sqliteObserverRows(t, db) + if err := ApplySQLite(db, nil); err == nil { + t.Fatal("conflicting replacement table accepted") + } + if got := sqliteObserverRows(t, db); got != before { + t.Fatal("failed rebuild changed observers") + } + var got string + if err := db.QueryRow(`SELECT value FROM _corescope_observers_identity_v1`).Scan(&got); err != nil || got != "owned elsewhere" { + t.Fatal("failed rebuild modified existing object") + } + for query, want := range map[string]int{ + `SELECT count(*) FROM _migrations WHERE name='observers_identity_autoincrement_v1'`: 0, + `PRAGMA foreign_keys`: 1, + `PRAGMA legacy_alter_table`: 0, + } { + var got int + if err := db.QueryRow(query).Scan(&got); err != nil || got != want { + t.Fatalf("%s: %d want %d err=%v", query, got, want, err) + } + } +} + +func TestSQLiteFutureIdentityVersionRefusesBeforeMutation(t *testing.T) { + db, _ := sqliteTestDB(t) + if err := legacy.ApplyBase(db); err != nil { + t.Fatal(err) + } + if err := legacy.Apply(db, nil); err != nil { + t.Fatal(err) + } + sqliteExec(t, db, `INSERT INTO _migrations(name) VALUES('observers_identity_autoincrement_v2')`) + var before, after int + if err := db.QueryRow(`PRAGMA schema_version`).Scan(&before); err != nil { + t.Fatal(err) + } + if err := ApplySQLite(db, nil); err == nil { + t.Fatal("future version accepted") + } + if err := db.QueryRow(`PRAGMA schema_version`).Scan(&after); err != nil { + t.Fatal(err) + } + if before != after { + t.Fatal("refusal modified schema") + } +} + +func sqliteAuthorizer(t *testing.T, db *sql.DB, callback func(int, string, string, string) int) { + t.Helper() + conn, err := db.Conn(context.Background()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if err := conn.Raw(func(driver any) error { driver.(*sqlite3.SQLiteConn).RegisterAuthorizer(callback); return nil }); err != nil { + t.Fatal(err) + } +} + +func TestSQLiteIdentityRollbackAfterDroppingOriginal(t *testing.T) { + db, _ := sqliteTestDB(t) + if err := legacy.ApplyBase(db); err != nil { + t.Fatal(err) + } + if err := legacy.Apply(db, nil); err != nil { + t.Fatal(err) + } + sqliteExec(t, db, `INSERT INTO observers(rowid,id,name) VALUES(12,'preserve','value')`) + sqliteExec(t, db, `CREATE TABLE observer_child(observer_id TEXT REFERENCES observers(id) ON DELETE CASCADE)`) + sqliteExec(t, db, `INSERT INTO observer_child VALUES('preserve')`) + before := sqliteObserverRows(t, db) + dropped, denied := false, false + sqliteAuthorizer(t, db, func(op int, a, b, c string) int { + if op == sqlite3.SQLITE_DROP_TABLE && a == "observers" { + dropped = true + } + if op == sqlite3.SQLITE_ALTER_TABLE { + denied = true + return sqlite3.SQLITE_DENY + } + return sqlite3.SQLITE_OK + }) + err := ApplySQLite(db, nil) + sqliteAuthorizer(t, db, nil) + if err == nil || !dropped || !denied { + t.Fatalf("failure did not occur after drop: err=%v dropped=%v denied=%v", err, dropped, denied) + } + if got := sqliteObserverRows(t, db); got != before { + t.Fatal("rollback changed original rows") + } + for query, want := range map[string]int{ + `SELECT count(*) FROM observer_child`: 1, + `SELECT count(*) FROM sqlite_master WHERE name='_corescope_observers_identity_v1'`: 0, + `SELECT count(*) FROM _migrations WHERE name='observers_identity_autoincrement_v1'`: 0, + `PRAGMA foreign_keys`: 1, + `PRAGMA legacy_alter_table`: 0, + } { + var got int + if err := db.QueryRow(query).Scan(&got); err != nil || got != want { + t.Fatalf("%s: got=%d err=%v", query, got, err) + } + } +} + +func TestSQLiteRepeatedApplyDoesNotRepeatCorpusForeignKeyScan(t *testing.T) { + db, _ := sqliteTestDB(t) + if err := ApplySQLite(db, nil); err != nil { + t.Fatal(err) + } + sqliteAuthorizer(t, db, func(op int, a, b, c string) int { + if op == sqlite3.SQLITE_PRAGMA && a == "foreign_key_check" { + return sqlite3.SQLITE_DENY + } + return sqlite3.SQLITE_OK + }) + defer sqliteAuthorizer(t, db, nil) + if err := ApplySQLite(db, nil); err != nil { + t.Fatal(err) + } + if err := AssertSQLiteReady(db); err != nil { + t.Fatal(err) + } +} + +func TestSQLitePartialUniqueObserverKeyIsNotReady(t *testing.T) { + db, _ := sqliteTestDB(t) + if err := ApplySQLite(db, nil); err != nil { + t.Fatal(err) + } + var ddl string + if err := db.QueryRow(`SELECT sql FROM sqlite_master WHERE name='observers' AND type='table'`).Scan(&ddl); err != nil { + t.Fatal(err) + } + const fullKey = `"id" TEXT UNIQUE` + if !strings.Contains(ddl, fullKey) { + t.Fatal("fixture lacks full unique text key") + } + ddl = strings.Replace(ddl, fullKey, `"id" TEXT`, 1) + sqliteExec(t, db, `DROP TABLE observers`) + sqliteExec(t, db, ddl) + sqliteExec(t, db, `CREATE UNIQUE INDEX observers_partial_identity ON observers(id) WHERE id IS NOT NULL`) + if err := AssertSQLiteReady(db); err == nil { + t.Fatal("partial unique text key accepted as ready") + } + var before, after int + if err := db.QueryRow(`PRAGMA schema_version`).Scan(&before); err != nil { + t.Fatal(err) + } + if err := ApplySQLite(db, nil); err == nil { + t.Fatal("partial unique text key accepted for upgrade") + } + if err := db.QueryRow(`PRAGMA schema_version`).Scan(&after); err != nil { + t.Fatal(err) + } + if before != after { + t.Fatal("refusal mutated partial-key schema") + } +} From 2dd25d5a0ffe877b0b2afbf80d160f2539948ca0 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 16:03:30 -0400 Subject: [PATCH 17/67] feat(storage): keep SQLite default with verified optional PostgreSQL --- .env.example | 29 +- .env.postgres.example | 21 + .github/workflows/deploy.yml | 173 +++- .github/workflows/postgres-benchmark.yml | 2 +- DEPLOY.md | 23 +- Dockerfile | 34 +- Makefile | 23 +- README.md | 37 +- cmd/decrypt/README.md | 23 +- cmd/decrypt/cli_test.go | 80 ++ cmd/decrypt/go.mod | 4 + cmd/decrypt/go.sum | 2 + cmd/decrypt/main.go | 66 +- cmd/decrypt/main_test.go | 31 +- cmd/decrypt/postgres_test.go | 33 +- cmd/decrypt/storage.go | 173 ++++ cmd/decrypt/storage_test.go | 306 +++++++ .../advert_evidence_migration_race_test.go | 16 +- cmd/ingestor/advert_route_evidence.go | 10 +- cmd/ingestor/advert_route_evidence_test.go | 28 +- cmd/ingestor/approved_channels.go | 115 ++- cmd/ingestor/approved_channels_import_test.go | 31 + cmd/ingestor/approved_channels_test.go | 124 ++- cmd/ingestor/async_migration.go | 53 +- cmd/ingestor/async_migration_test.go | 21 +- cmd/ingestor/client_reception.go | 12 +- cmd/ingestor/client_reception_test.go | 48 +- cmd/ingestor/client_regions_test.go | 10 +- cmd/ingestor/client_rf_sample.go | 2 +- cmd/ingestor/config.go | 4 +- cmd/ingestor/config_test.go | 4 +- cmd/ingestor/coverage_boost_test.go | 8 +- cmd/ingestor/db.go | 336 ++++--- cmd/ingestor/db_test.go | 48 +- cmd/ingestor/go.mod | 1 + cmd/ingestor/go.sum | 2 + cmd/ingestor/main.go | 114 ++- cmd/ingestor/maintenance.go | 24 +- cmd/ingestor/multibyte_persist.go | 10 +- .../multibyte_persist_helpers_test.go | 2 +- cmd/ingestor/neighbor_builder.go | 12 +- cmd/ingestor/neighbor_builder_delta_test.go | 34 + cmd/ingestor/postgres_config_test.go | 7 +- cmd/ingestor/postgres_test.go | 10 + cmd/ingestor/pragma_parity_test.go | 10 +- cmd/ingestor/prune_chunked_test.go | 12 +- cmd/ingestor/prune_geofilter.go | 2 +- cmd/ingestor/refresh_planner_stats_test.go | 228 +++++ cmd/ingestor/region_keys.go | 2 +- cmd/ingestor/retention_yield_test.go | 103 +++ cmd/ingestor/scope_match_tally.go | 2 +- cmd/ingestor/sig_validate_ingest_test.go | 4 +- cmd/ingestor/sqlite_maintenance.go | 141 +++ cmd/ingestor/storage_adoption.go | 147 ++++ cmd/ingestor/storage_adoption_test.go | 125 +++ cmd/ingestor/storage_backend.go | 72 ++ cmd/ingestor/storage_backend_test.go | 100 +++ cmd/ingestor/storage_config.go | 120 +++ cmd/ingestor/storage_config_test.go | 88 ++ cmd/ingestor/storage_runtime_open_test.go | 121 +++ cmd/migrate/account_setup.go | 186 ++++ cmd/migrate/account_setup_test.go | 244 ++++++ cmd/migrate/backend_test.go | 41 + cmd/migrate/cli_test.go | 9 +- cmd/migrate/datetime_test.go | 7 +- cmd/migrate/destination_empty_test.go | 4 +- cmd/migrate/generated_columns_test.go | 5 +- cmd/migrate/go.mod | 18 + cmd/migrate/go.sum | 41 + cmd/migrate/grants.go | 123 +++ cmd/migrate/identity_layout_test.go | 107 +++ cmd/migrate/import.go | 41 +- cmd/migrate/import_safety_test.go | 27 +- cmd/migrate/import_test.go | 9 +- cmd/migrate/layout.go | 30 +- cmd/migrate/main.go | 19 +- cmd/migrate/main_test.go | 15 +- cmd/migrate/operator_paths_test.go | 6 +- cmd/migrate/postgres_layout.go | 125 +++ cmd/migrate/postgres_layout.json | 575 ++++++++++++ cmd/migrate/postgres_layout_test.go | 83 ++ cmd/migrate/resume_finalized_test.go | 9 +- cmd/migrate/reverse.go | 817 ++++++++++++++++++ cmd/migrate/reverse_test.go | 190 ++++ cmd/migrate/source.go | 141 +-- cmd/migrate/storage_command.go | 475 ++++++++++ cmd/migrate/storage_command_test.go | 149 ++++ cmd/migrate/switch.go | 425 +++++++++ cmd/migrate/switch_test.go | 454 ++++++++++ cmd/migrate/target_identity_test.go | 3 +- cmd/server/account_test.go | 2 +- cmd/server/advert_route_evidence.go | 6 +- cmd/server/advert_route_evidence_test.go | 16 +- cmd/server/advert_stats.go | 11 +- cmd/server/advert_stats_test.go | 6 +- cmd/server/auth_fixture_test.go | 36 +- cmd/server/auth_flow_test.go | 2 +- cmd/server/auth_login_audit_test.go | 2 +- cmd/server/auth_mail_test.go | 2 +- cmd/server/auth_service.go | 24 +- cmd/server/backup.go | 19 +- cmd/server/backup_test.go | 8 +- cmd/server/bounded_load_test.go | 42 +- cmd/server/cache_refresh.go | 50 ++ cmd/server/cache_swr_test.go | 240 +++++ cmd/server/chunked_load.go | 7 +- cmd/server/chunked_load_id_zero_test.go | 20 +- cmd/server/chunked_load_oldest_test.go | 20 +- cmd/server/config.go | 9 +- cmd/server/config_test.go | 46 +- cmd/server/coverage_test.go | 46 +- cmd/server/db.go | 359 ++++---- cmd/server/db_test.go | 274 +++++- cmd/server/go.mod | 12 + cmd/server/go.sum | 19 + cmd/server/hash_migrate_test.go | 15 +- cmd/server/hot_startup_test.go | 6 +- cmd/server/issue1809_bg_load_race_test.go | 20 +- cmd/server/issue810_repro_test.go | 2 +- cmd/server/load_graph_invariant_test.go | 20 +- cmd/server/loadchunk_ambiguous_prefix_test.go | 23 +- .../loadchunk_pathjson_fallback_test.go | 23 +- .../loadchunk_resolved_path_1558_test.go | 23 +- cmd/server/main.go | 39 +- cmd/server/multibyte_capability_test.go | 12 +- cmd/server/neighbor_api_test.go | 6 +- cmd/server/neighbor_recomputer_test.go | 6 +- cmd/server/node_battery.go | 4 +- cmd/server/node_blacklist_test.go | 14 +- cmd/server/node_reach.go | 29 +- cmd/server/node_reach_bench_test.go | 10 +- cmd/server/node_reach_endpoint_test.go | 19 +- cmd/server/node_reach_test.go | 79 +- cmd/server/node_resolve.go | 2 +- cmd/server/notify_service_test.go | 9 +- cmd/server/notify_source.go | 2 +- cmd/server/observer_blacklist_test.go | 8 +- cmd/server/openapi.go | 1 + cmd/server/parity_test.go | 6 + cmd/server/paths_anchor_bias_test.go | 4 +- cmd/server/paths_observations_race_test.go | 55 ++ cmd/server/perf_io.go | 19 + cmd/server/postgres_fixture_test.go | 76 +- cmd/server/postgres_stats.go | 45 +- cmd/server/postgres_stats_test.go | 1 + cmd/server/postgres_test.go | 22 +- cmd/server/postgres_value_parity_test.go | 3 +- cmd/server/preparable_schema_test.go | 22 +- cmd/server/proposal_handlers_test.go | 3 +- cmd/server/readonly_invariant_test.go | 4 +- cmd/server/resolved_index.go | 100 ++- cmd/server/resolved_index_test.go | 10 +- cmd/server/resolved_path_unlocked_test.go | 133 +++ cmd/server/rf_noise.go | 4 +- cmd/server/rf_noise_test.go | 10 +- cmd/server/routes.go | 46 +- cmd/server/routes_test.go | 8 +- cmd/server/rx_coverage.go | 11 +- cmd/server/rx_coverage_endpoint_test.go | 6 +- cmd/server/rx_coverage_postgres_plan_test.go | 5 +- cmd/server/rx_dashboard.go | 18 +- cmd/server/scope_audit.go | 16 +- cmd/server/scope_audit_test.go | 52 +- cmd/server/scope_verify.go | 4 +- cmd/server/state_dir.go | 8 +- cmd/server/storage_backend.go | 117 +++ cmd/server/storage_backend_test.go | 73 ++ cmd/server/storage_config.go | 112 +++ cmd/server/storage_config_test.go | 155 ++++ cmd/server/storage_diagnostics_test.go | 44 + cmd/server/store.go | 566 ++++++++---- cmd/server/store_lock_convoy_test.go | 46 + cmd/server/store_lock_sql_invariant_test.go | 313 +++++++ cmd/server/store_memory_diagnostics_test.go | 19 +- cmd/server/store_obs_memory_test.go | 43 + cmd/server/topology_dedup_test.go | 48 +- cmd/server/types.go | 4 + cmd/server/user_mgmt_config.go | 9 +- cmd/server/user_mgmt_config_test.go | 5 +- cmd/server/user_mgmt_off_test.go | 8 +- cmd/server/users_backup.go | 19 +- cmd/server/users_backup_test.go | 43 +- config.example.json | 13 +- docker-compose.example.postgres.yml | 28 + docker-compose.example.yml | 26 +- docker-compose.no-mosquitto.yml | 23 +- docker-compose.postgres.yml | 28 + docker-compose.staging.no-mosquitto.yml | 23 +- docker-compose.staging.postgres.yml | 28 + docker-compose.staging.yml | 23 +- docker-compose.yml | 23 +- docker/entrypoint-go.sh | 3 + docker/postgres-bootstrap.sh | 43 +- docker/postgres.compose.yml | 10 +- docker/storage.sh | 87 ++ docker/supervisord-go-no-caddy.conf | 2 +- .../supervisord-go-no-mosquitto-no-caddy.conf | 2 +- docker/supervisord-go-no-mosquitto.conf | 2 +- docker/supervisord-go.conf | 2 +- docs/backend-switching.md | 105 +++ docs/deployment.md | 26 +- docs/postgresql-performance.md | 5 + docs/postgresql-upgrade.md | 62 +- docs/storage.md | 97 +++ docs/user-guide/accounts.md | 17 +- internal/dbconfig/selection.go | 331 +++++++ internal/dbconfig/selection_file.go | 500 +++++++++++ internal/dbconfig/selection_test.go | 464 ++++++++++ internal/dbconfig/selection_unix.go | 33 + internal/dbconfig/selection_windows.go | 102 +++ internal/dbconfig/sqlite_import.go | 23 + internal/dbconfig/sqlite_uri.go | 29 + internal/dbconfig/sqlite_uri_test.go | 66 ++ internal/dbconfig/storage.go | 8 +- internal/dbconfig/storage_config.go | 40 + internal/dbconfig/storage_config_test.go | 32 + internal/dbschema/legacy/writer_dsn.go | 23 +- internal/dbschema/legacy/writer_dsn_test.go | 59 +- internal/dbschema/sqlite.go | 24 + internal/dbschema/sqlite_indexes.go | 16 + internal/dbschema/sqlite_test.go | 42 +- internal/sqliteutil/go.mod | 10 + internal/sqliteutil/go.sum | 2 + internal/sqliteutil/snapshot.go | 138 +++ internal/sqliteutil/snapshot_test.go | 124 +++ internal/users/audit.go | 24 +- internal/users/audit_list_test.go | 11 +- internal/users/go.mod | 12 + internal/users/go.sum | 41 + internal/users/helpers_test.go | 88 +- internal/users/legacy_schema.go | 4 +- internal/users/mail.go | 18 +- internal/users/notify.go | 44 +- internal/users/notify_test.go | 4 +- internal/users/onetime.go | 28 +- internal/users/postgres_test.go | 43 +- internal/users/proposals.go | 51 +- internal/users/proposals_test.go | 11 +- internal/users/schema.go | 7 +- internal/users/sessions.go | 18 +- internal/users/settings.go | 14 +- internal/users/settings_test.go | 4 +- internal/users/snapshot.go | 44 +- internal/users/snapshot_test.go | 47 +- internal/users/sqlite_backend_test.go | 124 +++ internal/users/sqlite_bootstrap_test.go | 51 ++ internal/users/sqlite_identity_test.go | 384 ++++++++ internal/users/sqlite_import_test.go | 32 + internal/users/sqlite_open.go | 134 +++ internal/users/sqlite_runtime_open_test.go | 216 +++++ internal/users/sqlite_schema.go | 410 +++++++++ internal/users/stats.go | 33 +- internal/users/store.go | 74 +- internal/users/store_test.go | 37 +- internal/users/users.go | 56 +- manage.sh | 602 +++++++++++-- public/perf.js | 43 +- scripts/check-dockerfile-internal-pkgs.sh | 4 +- scripts/postgres-benchmark/README.md | 7 +- scripts/postgres-benchmark/run.py | 12 +- scripts/postgres-benchmark/test_harness.py | 20 + scripts/test-postgres-compose.sh | 106 ++- scripts/test-storage-compose.js | 93 ++ test-all.sh | 2 + tests/e2e/test-postgres-backup-e2e.js | 12 +- tests/unit/test-issue-1956-release-routing.js | 40 +- tests/unit/test-perf-disk-io-1120.js | 22 +- tests/unit/test-postgres-ops.js | 104 ++- tests/unit/test-storage-entrypoint.js | 134 +++ tests/unit/test-storage-manage.js | 103 +++ 270 files changed, 15906 insertions(+), 1972 deletions(-) create mode 100644 .env.postgres.example create mode 100644 cmd/decrypt/cli_test.go create mode 100644 cmd/decrypt/storage.go create mode 100644 cmd/decrypt/storage_test.go create mode 100644 cmd/ingestor/approved_channels_import_test.go create mode 100644 cmd/ingestor/retention_yield_test.go create mode 100644 cmd/ingestor/sqlite_maintenance.go create mode 100644 cmd/ingestor/storage_adoption.go create mode 100644 cmd/ingestor/storage_adoption_test.go create mode 100644 cmd/ingestor/storage_backend.go create mode 100644 cmd/ingestor/storage_backend_test.go create mode 100644 cmd/ingestor/storage_config.go create mode 100644 cmd/ingestor/storage_config_test.go create mode 100644 cmd/ingestor/storage_runtime_open_test.go create mode 100644 cmd/migrate/account_setup.go create mode 100644 cmd/migrate/account_setup_test.go create mode 100644 cmd/migrate/backend_test.go create mode 100644 cmd/migrate/grants.go create mode 100644 cmd/migrate/identity_layout_test.go create mode 100644 cmd/migrate/postgres_layout.go create mode 100644 cmd/migrate/postgres_layout.json create mode 100644 cmd/migrate/postgres_layout_test.go create mode 100644 cmd/migrate/reverse.go create mode 100644 cmd/migrate/reverse_test.go create mode 100644 cmd/migrate/storage_command.go create mode 100644 cmd/migrate/storage_command_test.go create mode 100644 cmd/migrate/switch.go create mode 100644 cmd/migrate/switch_test.go create mode 100644 cmd/server/cache_refresh.go create mode 100644 cmd/server/cache_swr_test.go create mode 100644 cmd/server/paths_observations_race_test.go create mode 100644 cmd/server/resolved_path_unlocked_test.go create mode 100644 cmd/server/storage_backend.go create mode 100644 cmd/server/storage_backend_test.go create mode 100644 cmd/server/storage_config.go create mode 100644 cmd/server/storage_config_test.go create mode 100644 cmd/server/storage_diagnostics_test.go create mode 100644 cmd/server/store_lock_convoy_test.go create mode 100644 cmd/server/store_lock_sql_invariant_test.go create mode 100644 cmd/server/store_obs_memory_test.go create mode 100644 docker-compose.example.postgres.yml create mode 100644 docker-compose.postgres.yml create mode 100644 docker-compose.staging.postgres.yml create mode 100755 docker/storage.sh create mode 100644 docs/backend-switching.md create mode 100644 docs/storage.md create mode 100644 internal/dbconfig/selection.go create mode 100644 internal/dbconfig/selection_file.go create mode 100644 internal/dbconfig/selection_test.go create mode 100644 internal/dbconfig/selection_unix.go create mode 100644 internal/dbconfig/selection_windows.go create mode 100644 internal/dbconfig/sqlite_import.go create mode 100644 internal/dbconfig/sqlite_uri.go create mode 100644 internal/dbconfig/sqlite_uri_test.go create mode 100644 internal/dbconfig/storage_config.go create mode 100644 internal/dbconfig/storage_config_test.go create mode 100644 internal/dbschema/sqlite_indexes.go create mode 100644 internal/sqliteutil/go.mod create mode 100644 internal/sqliteutil/go.sum create mode 100644 internal/sqliteutil/snapshot.go create mode 100644 internal/sqliteutil/snapshot_test.go create mode 100644 internal/users/sqlite_backend_test.go create mode 100644 internal/users/sqlite_bootstrap_test.go create mode 100644 internal/users/sqlite_identity_test.go create mode 100644 internal/users/sqlite_import_test.go create mode 100644 internal/users/sqlite_open.go create mode 100644 internal/users/sqlite_runtime_open_test.go create mode 100644 internal/users/sqlite_schema.go create mode 100644 scripts/test-storage-compose.js create mode 100644 tests/unit/test-storage-entrypoint.js create mode 100644 tests/unit/test-storage-manage.js diff --git a/.env.example b/.env.example index 50c485f89..860d868fe 100644 --- a/.env.example +++ b/.env.example @@ -8,26 +8,17 @@ # Each environment keeps config + data together in one directory: # ~/meshcore-data/config.json, Caddyfile, theme.json, queues and backups # ~/meshcore-staging-data/config.json, Caddyfile, queues and backups -# PostgreSQL has a separate persistent directory; do not copy it as a backup. +# With PostgreSQL selected, its separate persistent directory is not a logical backup. -# --- PostgreSQL --- -# Generate a DIFFERENT value for each blank field: openssl rand -hex 32 -# Keep .env private. Hexadecimal passwords can be safely embedded in the -# generated container URLs; the initializer enforces at least 32 hex digits. -# These roles are created only when PostgreSQL's data directory is empty. -# Editing these values afterward does not change existing database passwords. -POSTGRES_ADMIN_PASSWORD= -CORESCOPE_OWNER_PASSWORD= -CORESCOPE_READER_PASSWORD= -CORESCOPE_WRITER_PASSWORD= -CORESCOPE_ACCOUNTS_PASSWORD= -CORESCOPE_CHANNELS_PASSWORD= -PROD_POSTGRES_DATA_DIR=~/corescope-postgres -STAGING_POSTGRES_DATA_DIR=~/corescope-staging-postgres -# docker-compose.example.yml uses this local default instead: -POSTGRES_DATA_DIR=./postgres-data -# Existing instances: stop both processes and follow docs/postgresql-upgrade.md -# before starting the PostgreSQL image. Never mount two servers on one directory. +# --- Storage choice --- +# SQLite is the default and needs no database service or PostgreSQL credentials. +# setup offers PostgreSQL explicitly. Once installed, storage-selection.json in +# the shared state directory is authoritative; ordinary updates keep it. +# To change engines, use the verified offline storage switch in docs/storage.md. +# Optional PostgreSQL credentials/directories: merge .env.postgres.example. +# Do not replace an existing .env or regenerate existing database passwords. +# Optional container state subdirectory, shared by setup/server/ingestor: +# CORESCOPE_STATE_DIR=/app/data # --- Example deployment (docker-compose.example.yml only) --- # For upgrades, point DATA_DIR at the existing state directory BEFORE bootstrap. diff --git a/.env.postgres.example b/.env.postgres.example new file mode 100644 index 000000000..d0bbe08a2 --- /dev/null +++ b/.env.postgres.example @@ -0,0 +1,21 @@ +# OPTIONAL PostgreSQL settings — merge into your existing private .env. +# Base Compose files need none of these. Add the matching .postgres.yml override. +# Use a different persistent directory for each production/staging instance. +# Generate a DIFFERENT value for each blank field: openssl rand -hex 32 +# Keep .env private. Hexadecimal passwords can be safely embedded in the +# generated container URLs; the initializer enforces at least 32 hex digits. +# These roles are created only when PostgreSQL's data directory is empty. +# Editing these values afterward does not change existing database passwords. +POSTGRES_ADMIN_PASSWORD= +CORESCOPE_OWNER_PASSWORD= +CORESCOPE_READER_PASSWORD= +CORESCOPE_WRITER_PASSWORD= +CORESCOPE_ACCOUNTS_PASSWORD= +CORESCOPE_CHANNELS_PASSWORD= +PROD_POSTGRES_DATA_DIR=~/corescope-postgres +STAGING_POSTGRES_DATA_DIR=~/corescope-staging-postgres +# docker-compose.example.yml uses this local default instead: +POSTGRES_DATA_DIR=./postgres-data +# Existing installations keep their recorded backend on update. Changing engines +# requires the offline storage switch in docs/storage.md. Never mount two database +# servers on one persistent directory. diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3f22852d8..46e5f454f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -142,14 +142,18 @@ jobs: # The server has -race in its own step already; this closes the same gap for # the ingestor, which carries an atomic.Pointer snapshot (the region key # set) whose safety is an argument until something checks it. - race-test: - name: "🏁 Race detector (ingestor)" + race-backend: + name: "Race detector (ingestor, ${{ matrix.backend }})" + strategy: + fail-fast: false + matrix: + backend: [sqlite, postgres] runs-on: ubuntu-24.04 needs: [changes] if: needs.changes.outputs.ingestor == 'true' services: &postgres_test_services postgres: - image: postgres:18.6-alpine3.24 + image: ${{ matrix.backend == 'postgres' && 'postgres:18.6-alpine3.24' || '' }} env: POSTGRES_PASSWORD: "ci-${{ github.run_id }}-${{ github.run_attempt }}" POSTGRES_INITDB_ARGS: "--encoding=UTF8 --locale=C" @@ -158,7 +162,9 @@ jobs: --health-cmd "pg_isready -h 127.0.0.1 -U postgres" --health-interval 5s --health-timeout 5s --health-retries 20 env: - CORESCOPE_TEST_POSTGRES_URL: "postgresql://postgres:ci-${{ github.run_id }}-${{ github.run_attempt }}@127.0.0.1:5432/postgres?sslmode=disable" + CORESCOPE_TEST_BACKEND: ${{ matrix.backend }} + CORESCOPE_DB_BACKEND: ${{ matrix.backend }} + CORESCOPE_TEST_POSTGRES_URL: ${{ matrix.backend == 'postgres' && format('postgresql://postgres:ci-{0}-{1}@127.0.0.1:5432/postgres?sslmode=disable', github.run_id, github.run_attempt) || '' }} PGHOST: 127.0.0.1 PGPORT: 5432 PGUSER: postgres @@ -169,6 +175,7 @@ jobs: uses: actions/checkout@v5 - name: Install pinned PostgreSQL 18.6 clients + if: matrix.backend == 'postgres' run: &install_postgres_clients bash scripts/install-postgres-ci.sh client - name: Set up Go 1.27.2 @@ -187,17 +194,45 @@ jobs: cd cmd/ingestor go test -timeout 30m -race ./... + race-test: + name: "🏁 Race detector (ingestor)" + runs-on: ubuntu-24.04 + needs: [race-backend, changes] + if: ${{ !cancelled() && needs.changes.outputs.ingestor == 'true' }} + steps: + - name: Require both native race matrices + run: | + result="${{ needs['race-backend'].result }}" + [ "$result" = success ] || { echo "::error::Race matrices did not all pass ($result)"; exit 1; } + + go-test: + name: "✅ Go Build & Test" + runs-on: ubuntu-24.04 + needs: [go-backend, changes] + if: ${{ !cancelled() && needs.changes.outputs.code == 'true' }} + steps: + - name: Require both native storage matrices + run: | + result="${{ needs['go-backend'].result }}" + [ "$result" = success ] || { echo "::error::Storage matrices did not all pass ($result)"; exit 1; } + # ─────────────────────────────────────────────────────────────── # 1. Go Build & Test # ─────────────────────────────────────────────────────────────── - go-test: - name: "✅ Go Build & Test" + go-backend: + name: "Go Build & Test (${{ matrix.backend }})" + strategy: + fail-fast: false + matrix: + backend: [sqlite, postgres] runs-on: ubuntu-24.04 needs: [changes] if: needs.changes.outputs.code == 'true' services: *postgres_test_services env: - CORESCOPE_TEST_POSTGRES_URL: "postgresql://postgres:ci-${{ github.run_id }}-${{ github.run_attempt }}@127.0.0.1:5432/postgres?sslmode=disable" + CORESCOPE_TEST_BACKEND: ${{ matrix.backend }} + CORESCOPE_DB_BACKEND: ${{ matrix.backend }} + CORESCOPE_TEST_POSTGRES_URL: ${{ matrix.backend == 'postgres' && format('postgresql://postgres:ci-{0}-{1}@127.0.0.1:5432/postgres?sslmode=disable', github.run_id, github.run_attempt) || '' }} PGHOST: 127.0.0.1 PGPORT: 5432 PGUSER: postgres @@ -213,6 +248,7 @@ jobs: run: rm -rf ~/go/pkg/mod 2>/dev/null || true - name: Install pinned PostgreSQL 18.6 clients + if: matrix.backend == 'postgres' run: *install_postgres_clients @@ -249,7 +285,7 @@ jobs: run: | set -e -o pipefail cd cmd/server - CGO_ENABLED=0 go build . + CGO_ENABLED=1 go build . # -race gates PR #1208's atomic.Pointer migration: the race-detector # is what makes path_inspect_atomic_race_test.go actually assert. go test -timeout 20m -race -coverprofile=server-coverage.out ./... 2>&1 | tee server-test.log @@ -262,7 +298,7 @@ jobs: run: | set -e -o pipefail cd cmd/ingestor - CGO_ENABLED=0 go build . + CGO_ENABLED=1 go build . go test -timeout 20m -coverprofile=ingestor-coverage.out ./... 2>&1 | tee ingestor-test.log echo "--- Go Ingestor Coverage ---" go tool cover -func=ingestor-coverage.out | tail -1 @@ -274,7 +310,7 @@ jobs: go test ./... echo "--- Channel library tests passed ---" cd ../../cmd/decrypt - CGO_ENABLED=0 go build -ldflags="-s -w" -o corescope-decrypt . + CGO_ENABLED=1 go build -ldflags="-s -w" -o corescope-decrypt . go test ./... echo "--- Decrypt CLI tests passed ---" @@ -285,20 +321,32 @@ jobs: cd ../mailer && go test -race ./... echo "--- users + mailer tests passed ---" + - name: Test shared selection and native SQLite snapshot modules + if: matrix.backend == 'sqlite' + run: | + set -euo pipefail + cd internal/dbconfig && go test -race ./... + cd ../sqliteutil && go test -race ./... + - name: Test migrate CLI + dbschema run: | set -e -o pipefail # Neither module had CI test execution before the mattn/go-sqlite3 # migration; both open the database, and dbschema owns the migrations. - cd internal/pgutil - go test ./... - cd ../dbschema + if [ "$CORESCOPE_TEST_BACKEND" = postgres ]; then + (cd internal/pgutil && go test ./...) + fi + cd internal/dbschema go test ./... cd ../../cmd/migrate CGO_ENABLED=1 go build -o /dev/null . go test ./... echo "--- migrate + dbschema tests passed ---" + - name: Validate default and optional Compose models + if: matrix.backend == 'sqlite' + run: node scripts/test-storage-compose.js + - name: Verify Dockerfile COPY invariants (issue #1316) run: bash scripts/check-dockerfile-internal-pkgs.sh @@ -355,7 +403,7 @@ jobs: echo "✅ All .proto files are syntactically valid" - name: Generate Go coverage badges - if: success() + if: success() && matrix.backend == 'sqlite' run: | mkdir -p .badges @@ -384,7 +432,7 @@ jobs: echo "| Ingestor | ${INGESTOR_COV}% |" >> $GITHUB_STEP_SUMMARY - name: Upload Go coverage badges - if: success() + if: success() && matrix.backend == 'sqlite' uses: actions/upload-artifact@v6 with: name: go-badges @@ -405,14 +453,17 @@ jobs: # uses nothing go-test produces. build-and-publish still needs both, so # nothing reaches GHCR unless every test job passed. e2e-shard: - name: "🎭 Playwright E2E shard ${{ matrix.shard }}/3" + name: "🎭 Playwright E2E (${{ matrix.backend }}) shard ${{ matrix.shard }}/3" needs: [changes] runs-on: ubuntu-24.04 strategy: matrix: shard: [1, 2, 3] + backend: [sqlite, postgres] env: - CORESCOPE_TEST_POSTGRES_URL: "postgresql://postgres:ci-${{ github.run_id }}-${{ github.run_attempt }}@127.0.0.1:5432/postgres?sslmode=disable" + CORESCOPE_TEST_BACKEND: ${{ matrix.backend }} + CORESCOPE_DB_BACKEND: ${{ matrix.backend }} + CORESCOPE_TEST_POSTGRES_URL: ${{ matrix.backend == 'postgres' && format('postgresql://postgres:ci-{0}-{1}@127.0.0.1:5432/postgres?sslmode=disable', github.run_id, github.run_attempt) || '' }} PGHOST: 127.0.0.1 PGPORT: 5432 PGUSER: postgres @@ -440,6 +491,7 @@ jobs: run: rm -rf ~/go/pkg/mod 2>/dev/null || true - name: Install pinned PostgreSQL 18.6 clients + if: matrix.backend == 'postgres' run: *install_postgres_clients @@ -452,7 +504,7 @@ jobs: - name: Build Go server run: | cd cmd/server - CGO_ENABLED=0 go build -o ../../corescope-server . + CGO_ENABLED=1 go build -o ../../corescope-server . echo "Go server built successfully" # Only test-user-management-e2e.js uses the e2etest build and its server, @@ -462,7 +514,7 @@ jobs: if: matrix.shard == 3 run: | cd cmd/server - CGO_ENABLED=0 go build -tags e2etest -o ../../corescope-server-e2e . + CGO_ENABLED=1 go build -tags e2etest -o ../../corescope-server-e2e . - name: Build Go migrate tool run: | @@ -572,6 +624,7 @@ jobs: go run ../../scripts/migrate-fixture-hashes.go "$E2E_SQLITE" - name: Import fixture and grant restricted runtime roles + if: matrix.backend == 'postgres' run: | set -euo pipefail for key in CORESCOPE_OWNER_PASSWORD CORESCOPE_READER_PASSWORD CORESCOPE_WRITER_PASSWORD CORESCOPE_ACCOUNTS_PASSWORD CORESCOPE_CHANNELS_PASSWORD; do @@ -588,18 +641,36 @@ jobs: PGUSER=corescope_owner PGPASSWORD="$CORESCOPE_OWNER_PASSWORD" PGDATABASE="$database" \ psql -X -v ON_ERROR_STOP=1 -f docker/postgres-grants.sql done - echo "CORESCOPE_READER_DATABASE_URL=postgresql://corescope_reader:$CORESCOPE_READER_PASSWORD@127.0.0.1:5432/corescope_telemetry?sslmode=disable" >> "$GITHUB_ENV" - echo "CORESCOPE_USERS_DATABASE_URL=postgresql://corescope_accounts:$CORESCOPE_ACCOUNTS_PASSWORD@127.0.0.1:5432/corescope_accounts?sslmode=disable" >> "$GITHUB_ENV" + export CORESCOPE_DATABASE_URL="$owner_url" CORESCOPE_USERS_OWNER_DATABASE_URL="$owner_accounts" + export CORESCOPE_READER_DATABASE_URL="postgresql://corescope_reader:$CORESCOPE_READER_PASSWORD@127.0.0.1:5432/corescope_telemetry?sslmode=disable&search_path=public" + export CORESCOPE_WRITER_DATABASE_URL="postgresql://corescope_writer:$CORESCOPE_WRITER_PASSWORD@127.0.0.1:5432/corescope_telemetry?sslmode=disable&search_path=public" + export CORESCOPE_USERS_DATABASE_URL="postgresql://corescope_accounts:$CORESCOPE_ACCOUNTS_PASSWORD@127.0.0.1:5432/corescope_accounts?sslmode=disable&search_path=public" + export CORESCOPE_APPROVED_CHANNELS_DATABASE_URL="postgresql://corescope_channels:$CORESCOPE_CHANNELS_PASSWORD@127.0.0.1:5432/corescope_accounts?sslmode=disable&search_path=public" + for key in CORESCOPE_DATABASE_URL CORESCOPE_USERS_OWNER_DATABASE_URL CORESCOPE_READER_DATABASE_URL CORESCOPE_WRITER_DATABASE_URL CORESCOPE_USERS_DATABASE_URL CORESCOPE_APPROVED_CHANNELS_DATABASE_URL; do + printf '%s=%s\n' "$key" "${!key}" >> "$GITHUB_ENV" + done mkdir -p "$RUNNER_TEMP/corescope-state" + ./corescope-migrate -storage-action=adopt -backend=postgres -offline \ + -selection-file "$RUNNER_TEMP/corescope-state/storage-selection.json" # Retain the masked owner password for the later restore step only. umask 077 printf '%s' "$CORESCOPE_OWNER_PASSWORD" > "$RUNNER_TEMP/corescope-e2e-owner-password" + - name: Adopt prepared SQLite fixture without a PostgreSQL service + if: matrix.backend == 'sqlite' + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/corescope-state" + ./corescope-migrate -storage-action=setup -backend=sqlite -offline \ + -selection-file "$RUNNER_TEMP/corescope-state/storage-selection.json" \ + -sqlite-path "$E2E_SQLITE" -users-sqlite-path "$RUNNER_TEMP/corescope-state/users.db" + printf 'DB_PATH=%s\n' "$E2E_SQLITE" >> "$GITHUB_ENV" + - name: Start Go server with fixture DB run: | fuser -k 13581/tcp 2>/dev/null || true sleep 1 - env -u PGPASSWORD -u PGUSER -u CORESCOPE_TEST_POSTGRES_URL ./corescope-server -port 13581 -state-dir "$RUNNER_TEMP/corescope-state" -public public-instrumented & + env -u PGPASSWORD -u PGUSER -u CORESCOPE_TEST_POSTGRES_URL -u CORESCOPE_DATABASE_URL -u CORESCOPE_USERS_OWNER_DATABASE_URL ./corescope-server -port 13581 -state-dir "$RUNNER_TEMP/corescope-state" -public public-instrumented & echo $! > .server.pid for i in $(seq 1 30); do if curl -sf http://localhost:13581/api/healthz > /dev/null 2>&1; then @@ -633,7 +704,14 @@ jobs: } } JSON - env -u PGPASSWORD -u PGUSER -u CORESCOPE_TEST_POSTGRES_URL ./corescope-server-e2e -config-dir /tmp/cs-um -port 13582 -state-dir /tmp/cs-um -public public-instrumented & + if [ "$CORESCOPE_TEST_BACKEND" = sqlite ]; then + ./corescope-migrate -storage-action=setup -backend=sqlite -offline -config-dir /tmp/cs-um \ + -selection-file /tmp/cs-um/storage-selection.json -sqlite-path "$E2E_SQLITE" -users-sqlite-path /tmp/cs-um/users.db + else + ./corescope-migrate -storage-action=adopt -backend=postgres -offline -config-dir /tmp/cs-um \ + -selection-file /tmp/cs-um/storage-selection.json + fi + env -u PGPASSWORD -u PGUSER -u CORESCOPE_TEST_POSTGRES_URL -u CORESCOPE_DATABASE_URL -u CORESCOPE_USERS_OWNER_DATABASE_URL ./corescope-server-e2e -config-dir /tmp/cs-um -port 13582 -state-dir /tmp/cs-um -public public-instrumented & echo $! > .server-um.pid for i in $(seq 1 30); do if curl -sf http://localhost:13582/api/healthz > /dev/null 2>&1; then echo "UM server ready after ${i}s"; break; fi @@ -811,6 +889,7 @@ jobs: done ! kill -0 "$pid" 2>/dev/null || { echo 'Fixture server did not stop' >&2; exit 1; } done + if [ "$CORESCOPE_TEST_BACKEND" = postgres ]; then # Fresh databases, same deployment roles, no overwrite or role widening. psql -X -v ON_ERROR_STOP=1 -d postgres <<'SQL' CREATE DATABASE corescope_restore_telemetry OWNER corescope_owner TEMPLATE template0 ENCODING 'UTF8' LOCALE 'C'; @@ -836,6 +915,10 @@ jobs: for key, database in ( ("CORESCOPE_READER_DATABASE_URL", "corescope_restore_telemetry"), ("CORESCOPE_USERS_DATABASE_URL", "corescope_restore_accounts"), + ("CORESCOPE_WRITER_DATABASE_URL", "corescope_restore_telemetry"), + ("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", "corescope_restore_accounts"), + ("CORESCOPE_DATABASE_URL", "corescope_restore_telemetry"), + ("CORESCOPE_USERS_OWNER_DATABASE_URL", "corescope_restore_accounts"), ): raw = os.environ[key] try: @@ -849,10 +932,33 @@ jobs: ) while IFS= read -r assignment; do export "$assignment"; done <<< "$restore_urls" unset restore_urls assignment + # End restored database URLs. + else + mkdir -m 700 "$RUNNER_TEMP/corescope-restored-sqlite" + for kind in telemetry accounts; do + cp "$E2E_BACKUP_DIR/$kind.db" "$RUNNER_TEMP/corescope-restored-sqlite/$kind.db" + test "$(sqlite3 -readonly "$RUNNER_TEMP/corescope-restored-sqlite/$kind.db" 'PRAGMA quick_check')" = ok + done + export DB_PATH="$RUNNER_TEMP/corescope-restored-sqlite/telemetry.db" + fi + # A restore uses new verified selection records; changing only role URLs + # must never silently replace the recorded old targets. + restored_state="$RUNNER_TEMP/corescope-restored-state" + restored_accounts_state="$RUNNER_TEMP/corescope-restored-accounts-state" + mkdir -m 700 "$restored_state" "$restored_accounts_state" + for selection in "$restored_state/storage-selection.json" "$restored_accounts_state/storage-selection.json"; do + if [ "$CORESCOPE_TEST_BACKEND" = sqlite ]; then + ./corescope-migrate -storage-action=adopt -backend=sqlite -offline \ + -selection-file "$selection" -sqlite-path "$DB_PATH" \ + -users-sqlite-path "$RUNNER_TEMP/corescope-restored-sqlite/accounts.db" + else + ./corescope-migrate -storage-action=adopt -backend=postgres -offline -selection-file "$selection" + fi + done # Restart restored servers. - env -u PGPASSWORD -u PGUSER -u CORESCOPE_TEST_POSTGRES_URL ./corescope-server -port 13581 -state-dir "$RUNNER_TEMP/corescope-state" -public public-instrumented & + env -u PGPASSWORD -u PGUSER -u CORESCOPE_TEST_POSTGRES_URL -u CORESCOPE_DATABASE_URL -u CORESCOPE_USERS_OWNER_DATABASE_URL ./corescope-server -port 13581 -state-dir "$restored_state" -public public-instrumented & echo $! > .server.pid - env -u PGPASSWORD -u PGUSER -u CORESCOPE_TEST_POSTGRES_URL ./corescope-server-e2e -config-dir /tmp/cs-um -port 13582 -state-dir /tmp/cs-um -public public-instrumented & + env -u PGPASSWORD -u PGUSER -u CORESCOPE_TEST_POSTGRES_URL -u CORESCOPE_DATABASE_URL -u CORESCOPE_USERS_OWNER_DATABASE_URL ./corescope-server-e2e -config-dir /tmp/cs-um -port 13582 -state-dir "$restored_accounts_state" -public public-instrumented & echo $! > .server-um.pid for port in 13581 13582; do for attempt in $(seq 1 30); do @@ -926,7 +1032,7 @@ jobs: if: success() uses: actions/upload-artifact@v6 with: - name: e2e-shard-${{ matrix.shard }} + name: e2e-shard-${{ matrix.backend }}-${{ matrix.shard }} path: | e2e-output.txt .badges/ @@ -966,7 +1072,7 @@ jobs: run: | set -e ls shards - [ "$(ls -d shards/e2e-shard-* | wc -l)" -eq 3 ] || { echo "::error::expected results from 3 shards"; exit 1; } + [ "$(ls -d shards/e2e-shard-* | wc -l)" -eq 6 ] || { echo "::error::expected results from 3 shards for each backend"; exit 1; } cat shards/e2e-shard-*/e2e-output.txt > e2e-output.txt mkdir -p .badges cp shards/e2e-shard-*/.badges/*.json .badges/ 2>/dev/null || true @@ -999,7 +1105,7 @@ jobs: # ─────────────────────────────────────────────────────────────── # 3a. Two-arch image build — beside the tests, never publishes # ─────────────────────────────────────────────────────────────── - # The offline SQLite importer uses cgo, so the container build needs a zig + # Native SQLite support uses cgo, so the container build needs a zig # cross-toolchain and static musl linking. Builds both architectures and # actually runs the arm64 binaries under QEMU, on every event: for a PR it is # the only image check (the GHCR push below is push/tag-only, so without it @@ -1113,7 +1219,9 @@ jobs: - name: Exercise packaged PostgreSQL Compose startup timeout-minutes: 10 - run: timeout --kill-after=150s 7m bash scripts/test-postgres-compose.sh "corescope-pg-smoke:$GITHUB_SHA" + run: | + timeout --kill-after=150s 10m bash scripts/test-postgres-compose.sh "corescope-pg-smoke:$GITHUB_SHA" sqlite + timeout --kill-after=150s 10m bash scripts/test-postgres-compose.sh "corescope-pg-smoke:$GITHUB_SHA" postgres # ─────────────────────────────────────────────────────────────── # 3b. Publish Docker Image — only after every test job passed @@ -1239,7 +1347,7 @@ jobs: cmd/migrate/go.sum internal/**/go.sum - # The offline importer alone needs cgo and a cross C compiler. + # Both release tools support native SQLite and need a cross C compiler. - name: Set up Zig uses: mlugg/setup-zig@v2 with: @@ -1252,8 +1360,9 @@ jobs: case "$arch" in amd64) target=x86_64-linux-musl ;; arm64) target=aarch64-linux-musl ;; esac ( cd cmd/decrypt - CGO_ENABLED=0 GOOS=linux GOARCH="$arch" go build -trimpath -tags netgo,osusergo \ - -ldflags="-s -w -X main.version=$GITHUB_REF_NAME" -o "../../corescope-decrypt-linux-$arch" . + CGO_ENABLED=1 GOOS=linux GOARCH="$arch" CC="zig cc -target $target" \ + go build -trimpath -tags netgo,osusergo,sqlite_omit_load_extension \ + -ldflags="-s -w -extldflags '-static -Wl,-s' -X main.version=$GITHUB_REF_NAME" -o "../../corescope-decrypt-linux-$arch" . ) ( cd cmd/migrate diff --git a/.github/workflows/postgres-benchmark.yml b/.github/workflows/postgres-benchmark.yml index 8a4280742..1ccb47eb5 100644 --- a/.github/workflows/postgres-benchmark.yml +++ b/.github/workflows/postgres-benchmark.yml @@ -64,7 +64,7 @@ jobs: .github/workflows/deploy.yml .github/workflows/postgres-benchmark.yml \ scripts/postgres-benchmark scripts/install-postgres-ci.sh \ || { echo 'Workflow or benchmark harness differs from candidate; update the candidate before rerunning' >&2; exit 1; } - git cat-file -e 9dbc287579a237ffa744dd0c91fa7227d09763ac^{commit} + git cat-file -e 3e21b179ecccae005aa5f6f45bc095353c94b343^{commit} - name: Test benchmark controls run: python3 -m unittest discover -s scripts/postgres-benchmark -p 'test_*.py' diff --git a/DEPLOY.md b/DEPLOY.md index 6fc60b8b7..4a71895eb 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -6,7 +6,7 @@ Pre-built images are published to GHCR for `linux/amd64` and `linux/arm64` (Rasp ### Complete Compose checkout -PostgreSQL 18.6 runs as a separate service. Existing SQLite installations must complete the [offline upgrade](docs/postgresql-upgrade.md) before starting this version. +SQLite is the default and runs inside the application container. PostgreSQL18.6 is optional. Existing installations keep their recorded backend; changing it is a verified [offline storage switch](docs/storage.md), not an image update. Set `CORESCOPE_REF` to the reviewed application revision, then retain the complete checkout: the Compose variants require `docker/postgres.compose.yml` and its initialization scripts. @@ -16,13 +16,20 @@ cd CoreScope git checkout --detach "$CORESCOPE_REF" test -f .env || cp .env.example .env chmod 600 .env -# New PostgreSQL storage only: fill missing password fields with distinct -# openssl rand -hex 32 values. Preserve an existing .env and merge missing keys. +# SQLite needs no database credentials. Preserve an existing .env and DATA_DIR. # Set CORESCOPE_IMAGE in .env to the matching reviewed image tag or digest. docker compose -f docker-compose.example.yml up -d ``` -The PostgreSQL initializer creates separate telemetry/account databases and roles. Bootstrap installs schemas and grants; the application starts only when bootstrap succeeds. Open `http://localhost` and verify `/api/healthz` plus real packet ingestion. The default HTTP port is 80; adjust `HTTP_PORT` when using the example variant. +Packaged setup validates an existing SQLite installation or initializes a provably fresh one before starting the application. Open `http://localhost` and verify `/api/healthz` plus real packet ingestion. The default HTTP port is 80; adjust `HTTP_PORT` when using the example variant. + +For a **new PostgreSQL installation**, merge `.env.postgres.example` into the private `.env`, generate distinct passwords only for the first empty PostgreSQL directory, and run: + +```sh +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml up -d +``` + +The optional overlay adds PostgreSQL and owner bootstrap. Runtime services receive only restricted role credentials. Adding/removing an overlay does not convert existing data; use [storage switching](docs/storage.md). Keep the same overlay on PostgreSQL updates. ## Image Tags @@ -54,9 +61,9 @@ docker compose -f docker-compose.example.yml up -d ## Data -PostgreSQL persists telemetry and accounts in its own host directory (`POSTGRES_DATA_DIR`, or the production/staging equivalent). `/app/data` holds config, theme, queues, statistics and account backup files. Keep these locations separate and preserve the private `.env` with your recovery material. +SQLite stores telemetry and accounts as separate files under the data mount. When selected, PostgreSQL persists them in its own host directory (`POSTGRES_DATA_DIR`, or the production/staging equivalent). `/app/data` holds config, theme, queues, statistics and account backup files. Keep these locations separate and preserve the private `.env` with your recovery material. -Use native `pg_dump` archives or the authenticated backup endpoints. `./manage.sh backup ` and `restore` operate the production Compose variant managed by that script; they do not select `docker-compose.example.yml`. The [backup and restore instructions](docs/postgresql-upgrade.md#native-backups-and-restores) include the example variant. Do not copy live PostgreSQL storage or a legacy SQLite file as a current backup. +Use native SQLite snapshots or PostgreSQL `pg_dump` archives, selected automatically by the recorded backend, or the authenticated backup endpoints. `./manage.sh backup ` and `restore` operate the production Compose variant managed by that script; they do not select `docker-compose.example.yml`. The [backup and restore instructions](docs/postgresql-upgrade.md#native-backups-and-restores) include the example variant. Do not copy live PostgreSQL storage or a SQLite main file while ignoring its WAL. Preserve the private state archive and configuration with the native backup pair. ## TLS @@ -131,7 +138,7 @@ If you're currently deploying with `manage.sh` (git clone + local build), you ha ### Option A: Keep using manage.sh -Complete the PostgreSQL migration and configure the private database credentials before using `manage.sh start` or `update`. Its backup, restore, staging-copy and status commands use PostgreSQL clients from the selected database container. Restores require empty destinations; existing staging data is retained. +`manage.sh setup` offers SQLite (default) or PostgreSQL. Start/update use the recorded selection, and storage changes require `./manage.sh storage switch sqlite|postgres` with writers stopped. Backups/restores use the selected engine's native tools; existing staging data is retained, and only telemetry is cloned into a fresh staging installation. See [storage operations](docs/storage.md). ```bash ./manage.sh update # latest release @@ -149,7 +156,7 @@ Pre-built images skip the build step entirely — faster updates, no Go toolchai ./manage.sh stop ``` -2. Preserve `PROD_DATA_DIR`, the separate PostgreSQL directory, credentials and the native backup pair. For a SQLite instance, perform the offline upgrade first. +2. Preserve `PROD_DATA_DIR`, the separate PostgreSQL directory, credentials and the native backup pair. Keep the selected backend and its recovery files; use an explicit offline switch only when changing engines. 3. Keep the complete pinned checkout and choose the matching image in `.env`: ```bash diff --git a/Dockerfile b/Dockerfile index 90fc02e82..58661a72c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,9 +2,9 @@ # Build stage always runs natively on the builder's arch ($BUILDPLATFORM) and # cross-compiles to $TARGETOS/$TARGETARCH. No QEMU for compilation. # -# The PostgreSQL runtime binaries use pure Go. The offline SQLite importer -# still needs cgo and a cross C compiler. `zig cc` targets musl for that -# binary, keeping all four release binaries independent of the runtime libc. +# All four binaries support native SQLite and PostgreSQL. SQLite requires cgo; +# the upstream Zig cross-compiler targets musl so both release architectures +# remain fully static and independent of the runtime image libc. # # BUILDPLATFORM is auto-set by buildx; default to linux/amd64 so plain # `docker build` (without buildx) doesn't fail on an empty platform string. @@ -18,8 +18,8 @@ ARG BUILD_TIME=unknown ARG TARGETOS ARG TARGETARCH -# Keep these in step with the Makefile. The importer's netgo/osusergo tags -# retain Go DNS/user lookup; sqlite_omit_load_extension permits static linking. +# Keep these in step with the Makefile. netgo/osusergo retain Go DNS/user +# lookup; sqlite_omit_load_extension permits static linking. ENV GO_BUILD_TAGS=netgo,osusergo,sqlite_omit_load_extension \ ZIG_GLOBAL_CACHE_DIR=/tmp/zig-cache @@ -39,7 +39,7 @@ RUN apk add --no-cache curl xz && \ ln -s /opt/zig/zig /usr/local/bin/zig && rm /tmp/zig.tar.xz && \ zig version -# zigcc selects the importer's C target from the requested image architecture. +# zigcc selects the native SQLite C target from the requested image architecture. RUN printf '%s\n' '#!/bin/sh' \ 'case "$TARGETARCH" in' \ ' amd64) t=x86_64-linux-musl ;;' \ @@ -47,7 +47,7 @@ RUN printf '%s\n' '#!/bin/sh' \ ' *) echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 1 ;;' \ 'esac' \ 'exec zig cc -target "$t" "$@"' > /usr/local/bin/zigcc && chmod +x /usr/local/bin/zigcc -ENV CC=zigcc CGO_ENABLED=0 +ENV CC=zigcc CGO_ENABLED=1 # Build server WORKDIR /build/server @@ -65,6 +65,7 @@ COPY internal/users/ ../../internal/users/ COPY internal/channel/ ../../internal/channel/ COPY internal/mailer/ ../../internal/mailer/ COPY internal/pgutil/ ../../internal/pgutil/ +COPY internal/sqliteutil/ ../../internal/sqliteutil/ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/tmp/zig-cache \ @@ -75,7 +76,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/tmp/zig-cache \ GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ go build -trimpath -tags ${GO_BUILD_TAGS} \ - -ldflags "-s -w -X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" \ + -ldflags "-s -w -extldflags '-static -Wl,-s' -X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" \ -o /corescope-server . # Build ingestor @@ -101,7 +102,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/tmp/zig-cache \ GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ go build -trimpath -tags ${GO_BUILD_TAGS} \ - -ldflags "-s -w" \ + -ldflags "-s -w -extldflags '-static -Wl,-s'" \ -o /corescope-ingestor . # Build decrypt CLI @@ -110,6 +111,7 @@ COPY cmd/decrypt/go.mod cmd/decrypt/go.sum ./ COPY internal/channel/ ../../internal/channel/ COPY internal/dbschema/ ../../internal/dbschema/ COPY internal/pgutil/ ../../internal/pgutil/ +COPY internal/dbconfig/ ../../internal/dbconfig/ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/tmp/zig-cache \ @@ -120,16 +122,18 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/tmp/zig-cache \ GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ go build -trimpath -tags ${GO_BUILD_TAGS} \ - -ldflags "-s -w -X main.version=${APP_VERSION}" \ + -ldflags "-s -w -extldflags '-static -Wl,-s' -X main.version=${APP_VERSION}" \ -o /corescope-decrypt . -# Offline bootstrap/import CLI retains SQLite support for existing instances. -ENV CGO_ENABLED=1 +# Offline bootstrap and verified conversion CLI for both storage engines. WORKDIR /build/migrate COPY cmd/migrate/go.mod cmd/migrate/go.sum ./ +COPY internal/packetpath/ ../../internal/packetpath/ COPY internal/dbschema/ ../../internal/dbschema/ COPY internal/users/ ../../internal/users/ COPY internal/pgutil/ ../../internal/pgutil/ +COPY internal/sqliteutil/ ../../internal/sqliteutil/ +COPY internal/dbconfig/ ../../internal/dbconfig/ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ go mod download @@ -145,7 +149,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ # The client major matches PostgreSQL 18 for native backup/restore support. FROM alpine:3.24 -RUN apk add --no-cache mosquitto mosquitto-clients supervisor caddy wget postgresql18-client +RUN apk add --no-cache mosquitto mosquitto-clients supervisor caddy wget postgresql18-client sqlite jq WORKDIR /app @@ -174,8 +178,8 @@ RUN mkdir -p /app/data /var/lib/mosquitto /data/caddy && \ # Entrypoint COPY docker/entrypoint-go.sh /entrypoint.sh -COPY docker/postgres-bootstrap.sh docker/postgres-grants.sql /app/ -RUN chmod +x /entrypoint.sh +COPY docker/storage.sh docker/postgres-bootstrap.sh docker/postgres-grants.sql /app/ +RUN chmod +x /entrypoint.sh /app/storage.sh /app/postgres-bootstrap.sh EXPOSE 80 443 1883 diff --git a/Makefile b/Makefile index 878d1025b..51037f30f 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ # corescope build entry point. # # Each Go module resolves internal dependencies through local replace directives. -# Runtime binaries use pure Go. Only the offline SQLite importer needs cgo; -# its Linux cross-build uses zig cc with musl for a fully static binary. +# SQLite and PostgreSQL are supported by the same binaries. Native SQLite +# needs cgo; Linux cross-builds use zig cc with musl for fully static binaries. # # Quick reference: # make build # all four binaries for the host @@ -21,12 +21,11 @@ GOENV_GOARCH := $(shell "$(GO)" env GOARCH) GOOS ?= $(GOENV_GOOS) GOARCH ?= $(GOENV_GOARCH) -CGO_server := 0 -CGO_ingestor := 0 -CGO_decrypt := 0 +CGO_server := 1 +CGO_ingestor := 1 +CGO_decrypt := 1 CGO_migrate := 1 -GO_BUILD_TAGS ?= netgo,osusergo -TAGS_migrate := ,sqlite_omit_load_extension +GO_BUILD_TAGS ?= netgo,osusergo,sqlite_omit_load_extension GO_BUILD_FLAGS ?= -trimpath GO_LDFLAGS_OPTIMS ?= -s -w @@ -66,7 +65,7 @@ build: $(addprefix build-,$(CMDS)) build-%: @mkdir -p $(DIST) cd cmd/$* && CGO_ENABLED=$(CGO_$*) GOOS=$(GOOS) GOARCH=$(GOARCH) "$(GO)" build \ - -tags $(GO_BUILD_TAGS)$(TAGS_$*) $(GO_BUILD_FLAGS) \ + -tags $(GO_BUILD_TAGS) $(GO_BUILD_FLAGS) \ -ldflags "$(GO_LDFLAGS_OPTIMS) $(LDFLAGS_$*)" \ -o ../../$(DIST)/corescope-$* . @@ -81,10 +80,10 @@ define CROSSBUILD_RULE .PHONY: crossbuild-$(2)-$(3)-$(4) crossbuild-$(2)-$(3)-$(4): @mkdir -p $$(DIST) - $(if $(filter migrate,$(2)),@command -v zig >/dev/null || { echo "zig not found: needed to cross-compile the offline importer. See https://ziglang.org/download/"; exit 1; },@:) - cd cmd/$(2) && CGO_ENABLED=$$(CGO_$(2)) GOOS=$(3) GOARCH=$(4) $(if $(filter migrate,$(2)),CC="zig cc -target $(1)",) \ - "$$(GO)" build -tags $$(GO_BUILD_TAGS)$$(TAGS_$(2)) $$(GO_BUILD_FLAGS) \ - -ldflags '$$(GO_LDFLAGS_OPTIMS) $(if $(filter migrate,$(2)),-extldflags "-static -Wl$(comma)-s",) $$(LDFLAGS_$(2))' \ + @command -v zig >/dev/null || { echo "zig not found: needed to cross-compile native SQLite. See https://ziglang.org/download/"; exit 1; } + cd cmd/$(2) && CGO_ENABLED=$$(CGO_$(2)) GOOS=$(3) GOARCH=$(4) CC="zig cc -target $(1)" \ + "$$(GO)" build -tags $$(GO_BUILD_TAGS) $$(GO_BUILD_FLAGS) \ + -ldflags '$$(GO_LDFLAGS_OPTIMS) -extldflags "-static -Wl$(comma)-s" $$(LDFLAGS_$(2))' \ -o ../../$$(DIST)/corescope-$(2)-$(3)-$(4) . endef diff --git a/README.md b/README.md index 151684b81..f222e9810 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,13 @@ [![Frontend Coverage](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/Kpa-clawbot/CoreScope/master/.badges/frontend-coverage.json)](https://github.com/Kpa-clawbot/CoreScope/actions/workflows/deploy.yml) [![Deploy](https://github.com/Kpa-clawbot/CoreScope/actions/workflows/deploy.yml/badge.svg)](https://github.com/Kpa-clawbot/CoreScope/actions/workflows/deploy.yml) -> MeshCore packet analysis with a Go backend, PostgreSQL persistence, real-time WebSocket updates and channel decryption. +> MeshCore packet analysis with a Go backend, SQLite persistence by default, with optional PostgreSQL, real-time WebSocket updates and channel decryption. Self-hosted, open-source MeshCore packet analyzer. Collects MeshCore packets via MQTT, decodes them in real time, and presents a full web UI with live packet feed, interactive maps, channel chat, packet tracing, and per-node analytics. ## ⚡ Performance -The Go backend combines a packet cache with PostgreSQL persistence and SQL-backed analytics. The ingestor owns telemetry writes; the server uses a restricted telemetry reader and a separate optional account writer. Existing SQLite instances require the [offline PostgreSQL upgrade](docs/postgresql-upgrade.md). +The Go backend combines a packet cache with SQLite or PostgreSQL storage and SQL-backed analytics. SQLite is the default. The ingestor owns telemetry writes; the server reads telemetry and uses a separate optional account store. Ordinary updates keep the recorded backend. Use the [offline storage workflow](docs/storage.md) to change engines without discarding existing data. Budget for the application and PostgreSQL together, including retained data, indexes, WAL and the packet cache. Historical SQLite measurements in [PERFORMANCE.md](PERFORMANCE.md) are baseline context, not PostgreSQL results. Use the [paired benchmark workflow](.github/workflows/postgres-benchmark.yml) for an exact candidate comparison. @@ -66,26 +66,26 @@ Full experience on your phone — proper touch controls, iOS safe area support, ## Quick Start -### Pre-built image with PostgreSQL +### Pre-built image — SQLite by default -Retain the complete repository at the same reviewed revision as the image. The Compose variants require the common PostgreSQL service and initialization scripts. +Retain the complete repository at the same reviewed revision as the image. The optional PostgreSQL overrides use the matching shared initialization scripts. ```bash git clone https://github.com/Kpa-clawbot/CoreScope.git cd CoreScope git checkout --detach "$CORESCOPE_REF" -cp .env.example .env +test -f .env || cp .env.example .env chmod 600 .env -# Fill each PostgreSQL password field with a different openssl rand -hex 32 value. +# No PostgreSQL service or credentials are needed for the default SQLite install. # Set CORESCOPE_IMAGE to the matching reviewed image tag or digest. docker compose -f docker-compose.example.yml up -d ``` -Open `http://localhost`. Bootstrap creates separate telemetry and account databases and grants restricted runtime roles before the application starts. Existing SQLite data requires the [offline upgrade](docs/postgresql-upgrade.md) first. See [DEPLOY.md](DEPLOY.md) for ports, MQTT, HTTPS and native backups. +Open `http://localhost` and verify `/api/healthz` plus actual ingestion. Packaged setup validates existing SQLite data or initializes a provably fresh installation before the application starts. Keep the existing `DATA_DIR` on updates. For a new PostgreSQL install, merge `.env.postgres.example` into the private `.env` and add `-f docker-compose.example.postgres.yml`; existing installations use the verified [storage switch](docs/storage.md). See [DEPLOY.md](DEPLOY.md) for ports, MQTT, HTTPS and backups. ### Build from source -Use the same complete checkout and private `.env`, then run `./manage.sh setup`. The wizard configures, builds and starts the source deployment. PostgreSQL credentials must already be filled in. +Use the same complete checkout and private `.env`, then run `./manage.sh setup`. The wizard offers SQLite (default) or PostgreSQL, configures, builds and starts the deployment. It keeps an existing recorded backend; PostgreSQL credentials are generated only for a new empty database directory, preserving existing private settings. ```bash ./manage.sh status @@ -129,7 +129,9 @@ Copy `config.example.json` to `config.json` and edit: | `mqttSources` | External MQTT broker connections (optional) | | `channelKeys` | Channel decryption keys (hex). Hashtag channels auto-derived via SHA256 | | `defaultRegion` | Default IATA region code for the UI | -| `databaseURL` | PostgreSQL telemetry URL; prefer the per-process environment credentials below | +| `db.backend` | Initial choice: `sqlite` (default) or `postgres`; the installation record wins after setup | +| `dbPath` | SQLite telemetry path; `DB_PATH` is its environment override | +| `databaseURL` | Optional PostgreSQL telemetry URL; prefer the per-process role credentials below | | `stateDir` | Filesystem directory for queues, statistics and account backups (default: `data`) | ### Environment Variables @@ -137,6 +139,7 @@ Copy `config.example.json` to `config.json` and edit: | Variable | Description | |----------|-------------| | `PORT` | Override config port | +| `CORESCOPE_DB_BACKEND` | Bootstrap choice only; use an offline switch to change an installed backend | | `CORESCOPE_READER_DATABASE_URL` | Server telemetry reader URL | | `CORESCOPE_WRITER_DATABASE_URL` | Ingestor telemetry writer URL | | `CORESCOPE_USERS_DATABASE_URL` | Optional account writer URL in a separate database | @@ -148,14 +151,14 @@ Copy `config.example.json` to `config.json` and edit: ```mermaid flowchart LR MQTT[MQTT observers] --> Ingestor[Go ingestor] - Ingestor -->|telemetry writer| Telemetry[(PostgreSQL telemetry)] + Ingestor -->|telemetry writer| Telemetry[(SQLite or PostgreSQL telemetry)] Telemetry -->|read-only role| Server[Go API server] Server <-->|account writer| Accounts[(Separate account database)] Accounts -->|approved-channel view| Ingestor Server <-->|REST and WebSocket| Browser ``` -**Two-process model:** The ingestor handles MQTT, decoding, telemetry writes and retention. The HTTP server loads its packet cache and serves REST/WebSocket requests using a read-only telemetry role. Accounts use a separate PostgreSQL database and writer role. Supervisord runs the application processes with Caddy and optional Mosquitto; PostgreSQL runs as a separate service. Bootstrap/import credentials are not runtime credentials. +**Two-process model:** The ingestor handles MQTT, decoding, telemetry writes and retention. The HTTP server loads its packet cache and serves REST/WebSocket requests using a read-only telemetry role. Accounts use a separate SQLite file or PostgreSQL database. Supervisord runs the application processes with Caddy and optional Mosquitto; PostgreSQL runs as a separate service only when selected. Bootstrap/conversion credentials are not runtime credentials. ## MQTT Setup @@ -175,14 +178,14 @@ corescope/ │ │ ├── main.go # Entry point │ │ ├── routes.go # 40+ API endpoint handlers │ │ ├── store.go # In-memory packet store (5 indexes) -│ │ ├── db.go # Read-only PostgreSQL access +│ │ ├── db.go # Read-only native storage access │ │ ├── decoder.go # MeshCore packet decoder │ │ ├── websocket.go # WebSocket broadcast │ │ └── *_test.go # Server tests │ └── ingestor/ # Go MQTT ingestor │ ├── main.go # MQTT subscription + packet processing │ ├── decoder.go # Packet decoder (shared logic) -│ ├── db.go # PostgreSQL writer and retention path +│ ├── db.go # Native writer and retention path │ └── *_test.go # Ingestor tests ├── proto/ # Protobuf API definitions ├── public/ # Vanilla JS frontend (no build step) @@ -227,10 +230,12 @@ that runs on Alpine or scratch. The container build does the same thing — see ### Test Suite -Go suites exercise real PostgreSQL, role boundaries, concurrency, migration and native backup/restore. The standalone frontend list remains `test-all.sh`; Playwright covers the imported fixture and accounts on/off. CI pins Go 1.27.2 and PostgreSQL 18.6. +Default Go suites exercise native SQLite without a PostgreSQL service. The explicit PostgreSQL matrix covers role boundaries, concurrency, migration and native backup/restore. The standalone frontend list remains `test-all.sh`; Playwright covers the imported fixture and accounts on/off. CI pins Go 1.27.2 and PostgreSQL 18.6. ```bash -# Point this at an isolated PostgreSQL 18 administrator database. +# Default SQLite tests need no service. For the separate PostgreSQL matrix: +export CORESCOPE_TEST_BACKEND=postgres +# Point this at an isolated PostgreSQL18 administrator database. # Keep the URL in private environment configuration; pg_dump/pg_restore 18 must be on PATH. # CORESCOPE_TEST_POSTGRES_URL is required; database tests do not silently skip. # Go backend tests @@ -255,7 +260,7 @@ node tools/generate-packets.js --api --count 200 ### Migrating from Node.js -Historical Node.js/SQLite deployments must first reach a supported SQLite layout using the last SQLite release, then follow the [offline PostgreSQL upgrade](docs/postgresql-upgrade.md). Preserve the original data, configuration and keys; a container/image update alone does not perform this conversion. +Historical Node.js/SQLite deployments must first reach a supported SQLite layout using the prior version's explicit upgrade procedure. They can keep SQLite. Use the [offline storage switch](docs/storage.md) only when changing engines. Preserve original data, configuration and keys; a container/image update does not perform a backend conversion. ## Contributing diff --git a/cmd/decrypt/README.md b/cmd/decrypt/README.md index d6bf5ee78..68d8b47c3 100644 --- a/cmd/decrypt/README.md +++ b/cmd/decrypt/README.md @@ -1,6 +1,6 @@ # corescope-decrypt -Standalone CLI tool to decrypt and export MeshCore hashtag channel messages from a CoreScope SQLite database. +Standalone CLI tool to decrypt and export MeshCore hashtag channel messages from CoreScope storage. SQLite is the default; PostgreSQL is optional. Both are opened read-only. ## Why @@ -24,7 +24,7 @@ docker exec corescope-prod /app/corescope-decrypt --channel "#wardriving" --db / ### From GitHub release -Download the static binary from the [Releases](https://github.com/Kpa-clawbot/CoreScope/releases) page: +Download the binary for your platform from the [Releases](https://github.com/Kpa-clawbot/CoreScope/releases) page: ```bash # Linux amd64 @@ -40,16 +40,28 @@ cd cmd/decrypt go build -ldflags="-s -w" -o corescope-decrypt . # cgo: the SQLite driver needs it ``` -The binary is statically linked — no dependencies, runs on any Linux. +Building SQLite support requires a C compiler and CGO enabled. Use the packaged image or a release binary matching your platform. ## Usage ``` -corescope-decrypt --channel NAME --db PATH [--format FORMAT] [--output FILE] +corescope-decrypt --channel NAME [--db PATH] [--format FORMAT] [--output FILE] ``` Run `corescope-decrypt --help` for full flag documentation. +### Storage selection + +The tool reads storage fields from `config.json`, then `data/config.json` if the first file is absent. `--config FILE` selects an explicit file. An unreadable or malformed selected config is an error. Relative paths use the working directory. + +For an installed instance, `storage-selection.json` is authoritative: stale `--db` paths or bootstrap backend settings do not override a completed backend switch. The tool finds that record using `--state-dir`, `CORESCOPE_STATE_DIR`, or configured `stateDir`; otherwise it uses the legacy SQLite database directory (default `data`). Keep the installation's stable state directory when exporting after a switch. A shared lease blocks a switch until the export exits. + +Without a recorded installation, `--db PATH` reads an existing SQLite file; the default is `data/meshcore.db`. It never creates, adopts, or migrates a database. Incomplete conversions, missing files, and incompatible schemas fail. Offline legacy files remain readable when they contain the columns needed for export. + +For PostgreSQL, supply a restricted reader URL through `CORESCOPE_READER_DATABASE_URL`, or the generic `CORESCOPE_DATABASE_URL`. `--database-url` is available but environment settings keep credentials out of command history. PostgreSQL role and schema-readiness checks remain enforced; a failed connection never falls back to SQLite. + +Before a selection has been recorded, `db.backend`, `CORESCOPE_DB_BACKEND`, or `--backend sqlite|postgres` makes the choice explicit; conflicting config/environment selectors fail. Otherwise unambiguous native settings determine the engine. CLI paths/URLs override their environment/config values; environment values override config. `DB_PATH` is the legacy SQLite path variable. Mixed unrecorded file and URL settings require an explicit backend choice. + ### JSON output (default) Machine-readable, includes all metadata (observers, path hops, raw hex): @@ -123,6 +135,9 @@ See the firmware source at `firmware/src/helpers/BaseChatMesh.cpp` for the canon cd cmd/decrypt go test ./... +# Explicit PostgreSQL matrix: requires a disposable administrator URL. +CORESCOPE_TEST_BACKEND=postgres go test ./... + # Manual test with the real fixture: go run . --channel "#wardriving" --db ../../test-fixtures/e2e-fixture.db --format irc ``` diff --git a/cmd/decrypt/cli_test.go b/cmd/decrypt/cli_test.go new file mode 100644 index 000000000..0d13ef233 --- /dev/null +++ b/cmd/decrypt/cli_test.go @@ -0,0 +1,80 @@ +package main + +import ( + "bytes" + "encoding/json" + "flag" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// Run the real flag/storage/query/render path without building another binary. +func TestDecryptCLIProcess(t *testing.T) { + if os.Getenv("CORESCOPE_DECRYPT_TEST_CHILD") != "1" { + return + } + for i, arg := range os.Args { + if arg == "--" { + os.Args = append([]string{"corescope-decrypt"}, os.Args[i+1:]...) + break + } + } + flag.CommandLine = flag.NewFlagSet("corescope-decrypt", flag.ExitOnError) + main() + os.Exit(0) +} + +func runDecryptCLI(t *testing.T, base string, args ...string) ([]byte, string, error) { + t.Helper() + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + cmd := exec.Command(executable, append([]string{"-test.run=^TestDecryptCLIProcess$", "--"}, args...)...) + cmd.Dir = base + for _, value := range os.Environ() { + key, _, _ := strings.Cut(value, "=") + switch key { + case "DB_PATH", "CORESCOPE_DB_BACKEND", "CORESCOPE_DATABASE_URL", "CORESCOPE_READER_DATABASE_URL", "CORESCOPE_STATE_DIR", "CORESCOPE_DECRYPT_TEST_CHILD": + continue + } + cmd.Env = append(cmd.Env, value) + } + cmd.Env = append(cmd.Env, "CORESCOPE_DECRYPT_TEST_CHILD=1") + var out, diagnostic bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &diagnostic + err = cmd.Run() + return out.Bytes(), diagnostic.String(), err +} + +func TestDecryptNativeCLI(t *testing.T) { + base := t.TempDir() + data := filepath.Join(base, "data") + if err := os.Mkdir(data, 0700); err != nil { + t.Fatal(err) + } + path := filepath.Join(data, "meshcore.db") + db := sqliteExportFixture(t, path, true) + for _, args := range [][]string{ + {"--channel", "#example"}, + {"--channel", "#example", "--db", path}, + } { + out, diagnostic, err := runDecryptCLI(t, base, args...) + if err != nil || !json.Valid(out) || !strings.Contains(diagnostic, "Scanned 0 GRP_TXT packets") { + t.Fatalf("native CLI export failed: %v; %s", err, diagnostic) + } + } + db.Close() + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if _, _, err := runDecryptCLI(t, base, "--channel", "#example"); err == nil { + t.Fatal("default CLI accepted a missing database") + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatal("default CLI created a missing database") + } +} diff --git a/cmd/decrypt/go.mod b/cmd/decrypt/go.mod index 954b4763f..fc8c6bb40 100644 --- a/cmd/decrypt/go.mod +++ b/cmd/decrypt/go.mod @@ -13,6 +13,8 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgx/v5 v5.7.6 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/mattn/go-sqlite3 v1.14.52 + github.com/meshcore-analyzer/dbconfig v0.0.0 golang.org/x/crypto v0.37.0 // indirect golang.org/x/sync v0.13.0 // indirect golang.org/x/text v0.24.0 // indirect @@ -23,3 +25,5 @@ replace github.com/meshcore-analyzer/channel => ../../internal/channel replace github.com/meshcore-analyzer/dbschema => ../../internal/dbschema replace github.com/meshcore-analyzer/pgutil => ../../internal/pgutil + +replace github.com/meshcore-analyzer/dbconfig => ../../internal/dbconfig diff --git a/cmd/decrypt/go.sum b/cmd/decrypt/go.sum index 7600ae8d6..c83dde8cc 100644 --- a/cmd/decrypt/go.sum +++ b/cmd/decrypt/go.sum @@ -9,6 +9,8 @@ github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/cmd/decrypt/main.go b/cmd/decrypt/main.go index 4b8395469..48583de9d 100644 --- a/cmd/decrypt/main.go +++ b/cmd/decrypt/main.go @@ -1,4 +1,4 @@ -// corescope-decrypt decrypts and exports hashtag channel messages from PostgreSQL. +// corescope-decrypt decrypts and exports hashtag channel messages from CoreScope storage. // // Usage: // @@ -19,8 +19,7 @@ import ( "time" "github.com/meshcore-analyzer/channel" - "github.com/meshcore-analyzer/dbschema" - "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/dbconfig" ) // Version info (set via ldflags). @@ -48,12 +47,11 @@ type Observer struct { func main() { channelName := flag.String("channel", "", "Channel name (e.g. \"#wardriving\")") - defaultURL := os.Getenv("CORESCOPE_READER_DATABASE_URL") - if defaultURL == "" { - defaultURL = os.Getenv("CORESCOPE_DATABASE_URL") - } - databaseURL := flag.String("database-url", defaultURL, "PostgreSQL reader URL (prefer CORESCOPE_READER_DATABASE_URL)") - legacyDB := flag.String("db", "", "Removed: import SQLite with corescope-migrate before exporting") + databaseURL := flag.String("database-url", "", "PostgreSQL reader URL (prefer CORESCOPE_READER_DATABASE_URL)") + dbPath := flag.String("db", "", "Existing SQLite database path (default: data/meshcore.db)") + backend := flag.String("backend", "", "Bootstrap backend: sqlite or postgres (installed selection wins)") + stateDir := flag.String("state-dir", "", "Installation state directory (CORESCOPE_STATE_DIR)") + configPath := flag.String("config", "", "Storage config file (default: config.json, then data/config.json)") format := flag.String("format", "json", "Output format: json, html, irc (or log)") output := flag.String("output", "", "Output file (default: stdout)") showVersion := flag.Bool("version", false, "Print version and exit") @@ -67,6 +65,10 @@ USAGE FLAGS --channel NAME Channel name to decrypt (e.g. "#wardriving", "wardriving") The "#" prefix is added automatically if missing. + --db PATH Existing SQLite database (default: data/meshcore.db). + --backend NAME Bootstrap choice: sqlite or postgres. + --state-dir DIR Find the installed storage selection in this directory. + --config FILE Read storage settings from this config file. --database-url PostgreSQL reader URL. Prefer CORESCOPE_READER_DATABASE_URL or CORESCOPE_DATABASE_URL to keep credentials out of argv. --format FORMAT Output format (default: json): @@ -77,6 +79,9 @@ FLAGS --output FILE Write output to FILE instead of stdout. --version Print version and exit. + The installed selection takes precedence over bootstrap paths and backend flags. + Export holds a shared selection lease and never creates or migrates a database. + EXAMPLES # Export #wardriving messages as JSON corescope-decrypt --channel "#wardriving" @@ -114,10 +119,7 @@ LIMITATIONS os.Exit(0) } - if *legacyDB != "" { - log.Fatal("SQLite runtime access has been removed; use corescope-migrate -from-sqlite during the documented offline upgrade") - } - if *channelName == "" || *databaseURL == "" { + if *channelName == "" { flag.Usage() os.Exit(1) } @@ -131,14 +133,21 @@ LIMITATIONS key := channel.DeriveKey(ch) chHash := channel.ChannelHash(key) - db, err := openExportDB(*databaseURL) + raw, err := exportStorageInputs(".", exportOptions{Backend: dbconfig.Backend(*backend), DBPath: *dbPath, DatabaseURL: *databaseURL, StateDir: *stateDir, ConfigPath: *configPath}, os.Getenv) + if err != nil { + log.Fatalf("Storage configuration: %v", err) + } + db, lease, err := openSelectedExport(raw) if err != nil { log.Fatalf("Failed to open database: %v", err) } + if lease != nil { + defer lease.Close() + } defer db.Close() // Query all GRP_TXT packets - rows, err := db.Query(`SELECT id, hash, raw_hex, first_seen FROM transmissions WHERE payload_type = 5`) + rows, err := db.Query(exportTransmissionsSQL) if err != nil { log.Fatalf("Query failed: %v", err) } @@ -286,26 +295,9 @@ func extractGRPPayload(rawHex string) ([]byte, error) { return buf[offset:], nil } -func openExportDB(databaseURL string) (*sql.DB, error) { - db, err := pgutil.Open(databaseURL, true) - if err != nil { - return nil, err - } - db.SetMaxOpenConns(4) - if err := pgutil.AssertReadOnly(db); err != nil { - db.Close() - return nil, err - } - if err := dbschema.AssertReady(db); err != nil { - db.Close() - return nil, err - } - return db, nil -} - func getPathFromDB(db *sql.DB, txID int) []string { var decodedJSON sql.NullString - err := db.QueryRow(`SELECT decoded_json FROM transmissions WHERE id = $1`, txID).Scan(&decodedJSON) + err := db.QueryRow(exportPathSQL, txID).Scan(&decodedJSON) if err != nil || !decodedJSON.Valid { return nil } @@ -322,13 +314,7 @@ func getPathFromDB(db *sql.DB, txID int) []string { } func getObservers(db *sql.DB, txID int) []Observer { - rows, err := db.Query(` - SELECT o.name, obs.snr, obs.rssi, obs.timestamp - FROM observations obs - LEFT JOIN observers o ON o.rowid = obs.observer_idx - WHERE obs.transmission_id = $1 - ORDER BY obs.timestamp, obs.id - `, txID) + rows, err := db.Query(exportObserversSQL, txID) if err != nil { return nil } diff --git a/cmd/decrypt/main_test.go b/cmd/decrypt/main_test.go index 76588a458..5d173a80f 100644 --- a/cmd/decrypt/main_test.go +++ b/cmd/decrypt/main_test.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "encoding/json" "os" + "path/filepath" "strings" "testing" @@ -113,17 +114,27 @@ func TestJSONOutputParseable(t *testing.T) { } } -// Integration test against fixture DB (skipped if DB not found) +// Exercise the actual CLI against the checked-in, captured packet fixture. func TestFixtureDecrypt(t *testing.T) { - dbPath := "../../test-fixtures/e2e-fixture.db" - if _, err := os.Stat(dbPath); os.IsNotExist(err) { - t.Skip("fixture DB not found") + dbPath, err := filepath.Abs("../../test-fixtures/e2e-fixture.db") + if err != nil { + t.Fatal(err) } - - // We know the fixture has #wardriving messages with channelHash 0x81 - key := channel.DeriveKey("#wardriving") - ch := channel.ChannelHash(key) - if ch != 0x81 { - t.Fatalf("unexpected channel hash: %02X", ch) + if _, err := os.Stat(dbPath); err != nil { + t.Fatal("checked-in fixture database is required") + } + base := t.TempDir() + out, diagnostic, err := runDecryptCLI(t, base, "--channel", "#wardriving", "--db", dbPath, "--state-dir", base) + if err != nil { + t.Fatalf("fixture export failed: %v; %s", err, diagnostic) + } + var messages []ChannelMessage + if err := json.Unmarshal(out, &messages); err != nil || len(messages) == 0 { + t.Fatalf("fixture export produced no valid messages: %v", err) + } + for _, message := range messages { + if message.Channel != "#wardriving" || message.RawHex == "" || message.Timestamp == "" { + t.Fatal("fixture export lost message fields") + } } } diff --git a/cmd/decrypt/postgres_test.go b/cmd/decrypt/postgres_test.go index 1e835ebb3..9441d1e70 100644 --- a/cmd/decrypt/postgres_test.go +++ b/cmd/decrypt/postgres_test.go @@ -1,14 +1,23 @@ package main import ( + "os" + "path/filepath" "testing" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/dbschema" "github.com/meshcore-analyzer/pgutil" "github.com/meshcore-analyzer/pgutil/pgtest" ) func TestPostgresExportReadsPathAndObservers(t *testing.T) { + if backend := os.Getenv("CORESCOPE_TEST_BACKEND"); backend != "postgres" { + if backend != "" && backend != "sqlite" { + t.Fatal("CORESCOPE_TEST_BACKEND must be sqlite or postgres") + } + t.Skip("PostgreSQL matrix is not selected") + } dsn := pgtest.NewSchema(t) writer, err := pgutil.Open(dsn, false) if err != nil { @@ -32,12 +41,24 @@ func TestPostgresExportReadsPathAndObservers(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { reader.Close() }) - path := getPathFromDB(reader, 1) - if len(path) != 2 || path[0] != "ab" || path[1] != "cd" { - t.Fatalf("PostgreSQL path query lost data: %v", path) + assertExportMetadata(t, reader) + if _, err := reader.Exec(`DELETE FROM observations`); err == nil { + t.Fatal("PostgreSQL export reader permitted a write") + } + base := t.TempDir() + readerURL := pgtest.ReadOnly(t, dsn) + recordExportStorage(t, dbconfig.Storage{Backend: dbconfig.Postgres, StateDir: base, ReaderDatabaseURL: readerURL}) + selected, lease, err := openSelectedExport(dbconfig.StorageInputs{BaseDir: base, DBPath: filepath.Join(base, "stale.db"), Backend: dbconfig.SQLite, ReaderDatabaseURL: readerURL}) + if err != nil { + t.Fatal(err) + } + assertExportMetadata(t, selected) + selected.Close() + lease.Close() + if _, err := writer.Exec(`UPDATE corescope_schema SET ready=false`); err != nil { + t.Fatal(err) } - observers := getObservers(reader, 1) - if len(observers) != 1 || observers[0].Name != "Example observer" || observers[0].SNR != 1.5 { - t.Fatalf("PostgreSQL observer query lost data: %+v", observers) + if db, lease, err := openSelectedExport(dbconfig.StorageInputs{BaseDir: base, StateDir: base, ReaderDatabaseURL: readerURL}); err == nil || db != nil || lease != nil { + t.Fatal("export accepted an incomplete PostgreSQL import") } } diff --git a/cmd/decrypt/storage.go b/cmd/decrypt/storage.go new file mode 100644 index 000000000..51f4762ad --- /dev/null +++ b/cmd/decrypt/storage.go @@ -0,0 +1,173 @@ +package main + +import ( + "database/sql" + "errors" + "io" + "net/url" + "os" + "path/filepath" + "strings" + + _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" +) + +const exportTransmissionsSQL = `SELECT id, hash, raw_hex, first_seen FROM transmissions WHERE payload_type = 5` +const exportPathSQL = `SELECT decoded_json FROM transmissions WHERE id = $1` +const exportObserversSQL = ` + SELECT o.name, obs.snr, obs.rssi, obs.timestamp + FROM observations obs + LEFT JOIN observers o ON o.rowid = obs.observer_idx + WHERE obs.transmission_id = $1 + ORDER BY obs.timestamp, obs.id` + +type exportOptions struct { + Backend dbconfig.Backend + DBPath, DatabaseURL, StateDir, ConfigPath string +} + +// Use the installation's storage fields without depending on the server's full +// configuration. Only absent search candidates fall through to defaults. +func exportStorageInputs(base string, flags exportOptions, getenv func(string) string) (dbconfig.StorageInputs, error) { + base, err := filepath.Abs(base) + if err != nil { + return dbconfig.StorageInputs{}, err + } + paths := []string{filepath.Join(base, "config.json"), filepath.Join(base, "data", "config.json")} + if flags.ConfigPath != "" { + paths = []string{flags.ConfigPath} + } + var raw dbconfig.StorageInputs + for _, path := range paths { + raw, err = dbconfig.ReadStorageConfig(path) + if err == nil { + break + } + if flags.ConfigPath != "" || !errors.Is(err, os.ErrNotExist) { + return raw, err + } + } + raw.BaseDir = base + raw.EnvBackend = dbconfig.Backend(getenv("CORESCOPE_DB_BACKEND")) + for key, target := range map[string]*string{ + "DB_PATH": &raw.DBPath, "CORESCOPE_STATE_DIR": &raw.StateDir, + "CORESCOPE_DATABASE_URL": &raw.DatabaseURL, "CORESCOPE_READER_DATABASE_URL": &raw.ReaderDatabaseURL, + } { + if value := strings.TrimSpace(getenv(key)); value != "" { + *target = value + } + } + if flags.Backend != "" { + raw.Backend = flags.Backend + } + if flags.DBPath != "" { + raw.DBPath = flags.DBPath + } + if flags.DatabaseURL != "" { + raw.DatabaseURL, raw.ReaderDatabaseURL = flags.DatabaseURL, flags.DatabaseURL + } + if flags.StateDir != "" { + raw.StateDir = flags.StateDir + } + return raw, nil +} + +func openSelectedExport(raw dbconfig.StorageInputs) (*sql.DB, io.Closer, error) { + // The legacy SQLite directory remains the discovery anchor after a switch. + state := raw.StateDir + if state == "" { + path := raw.DBPath + if path == "" { + path = filepath.Join("data", "meshcore.db") + } + if strings.Contains(path, "://") || strings.HasPrefix(path, "postgres:") || strings.HasPrefix(path, "postgresql:") { + return nil, nil, errors.New("--db requires a SQLite filesystem path") + } + state = filepath.Dir(path) + } + if strings.Contains(state, "://") || strings.ContainsRune(state, 0) || (filepath.VolumeName(state) != "" && !filepath.IsAbs(state)) { + return nil, nil, errors.New("invalid storage state directory") + } + if !filepath.IsAbs(state) { + state = filepath.Join(raw.BaseDir, state) + } + selected, lease, err := dbconfig.OpenSelection(dbconfig.SelectionPath(filepath.Clean(state))) + if err == nil { + raw, err = selected.ApplyTo(raw) + } else if errors.Is(err, dbconfig.ErrSelectionMissing) { + // An offline/legacy export can read an existing source without adopting + // it. This default is never permission to create or migrate a database. + err = nil + if raw.Backend == "" && raw.EnvBackend == "" && raw.DBPath == "" && raw.UsersDBPath == "" && raw.DatabaseURL == "" && raw.ReaderDatabaseURL == "" && raw.WriterDatabaseURL == "" && raw.UsersDatabaseURL == "" && raw.ApprovedChannelsDatabaseURL == "" { + raw.DBPath = filepath.Join("data", "meshcore.db") + } + } + var db *sql.DB + if err == nil { + var storage dbconfig.Storage + storage, err = dbconfig.ResolveStorage(raw) + if err == nil { + target := storage.DBPath + if storage.Backend == dbconfig.Postgres { + target = storage.ReaderDatabaseURL + } + db, err = openExportDB(target) + } + } + if err != nil && lease != nil { + lease.Close() + lease = nil + } + return db, lease, err +} + +func openExportDB(target string) (*sql.DB, error) { + postgres := strings.HasPrefix(target, "postgres://") || strings.HasPrefix(target, "postgresql://") + var db *sql.DB + var err error + if postgres { + db, err = pgutil.Open(target, true) + } else { + if target == "" || strings.Contains(target, "://") || strings.HasPrefix(target, "postgres:") || strings.HasPrefix(target, "postgresql:") { + return nil, errors.New("export requires a SQLite filesystem path or PostgreSQL reader URL") + } + var dsn string + dsn, err = dbconfig.SQLiteURI(target, url.Values{"mode": {"ro"}, "_query_only": {"on"}, "_busy_timeout": {"5000"}}) + if err == nil { + db, err = sql.Open("sqlite3", dsn) + } + } + if err != nil { + return nil, err + } + db.SetMaxOpenConns(4) + db.SetMaxIdleConns(4) + if postgres { + err = pgutil.AssertReadOnly(db) + if err == nil { + err = dbschema.AssertPostgresReady(db) + } + } else { + err = dbconfig.AssertSQLiteImportComplete(db) + // Export needs only its existing read columns, including on older + // offline snapshots. Do not run migrations or require writer indexes. + for _, query := range []string{exportTransmissionsSQL, exportPathSQL, exportObserversSQL} { + if err != nil { + break + } + var stmt *sql.Stmt + stmt, err = db.Prepare(query) + if err == nil { + err = stmt.Close() + } + } + } + if err != nil { + db.Close() + return nil, err + } + return db, nil +} diff --git a/cmd/decrypt/storage_test.go b/cmd/decrypt/storage_test.go new file mode 100644 index 000000000..2b342eda3 --- /dev/null +++ b/cmd/decrypt/storage_test.go @@ -0,0 +1,306 @@ +package main + +import ( + "bytes" + "database/sql" + "errors" + "net" + "os" + "path/filepath" + "strings" + "testing" + + _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" +) + +func sqliteExportFixture(t *testing.T, path string, current bool) *sql.DB { + t.Helper() + dsn, err := legacy.WriterDSN(path) + if err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + t.Cleanup(func() { db.Close() }) + if current { + err = dbschema.ApplySQLite(db, t.Logf) + } else if err = legacy.ApplyBase(db); err == nil { + err = legacy.Apply(db, t.Logf) + } + if err != nil { + t.Fatal(err) + } + for _, query := range []string{ + `INSERT INTO transmissions(id,raw_hex,hash,first_seen,decoded_json) VALUES(1,'00','export-fixture','2026-01-01','{"path":{"hops":["ab","cd"]}}')`, + `INSERT INTO observers(rowid,id,name) VALUES(10,'observer','Example observer')`, + `INSERT INTO observations(transmission_id,observer_idx,snr,rssi,timestamp,path_json) VALUES(1,10,1.5,-90,1767225600,'["ab"]')`, + } { + if _, err := db.Exec(query); err != nil { + t.Fatal(err) + } + } + return db +} + +func assertExportMetadata(t *testing.T, db *sql.DB) { + t.Helper() + path := getPathFromDB(db, 1) + if len(path) != 2 || path[0] != "ab" || path[1] != "cd" { + t.Fatalf("path query lost data: %v", path) + } + observers := getObservers(db, 1) + if len(observers) != 1 || observers[0].Name != "Example observer" || observers[0].SNR != 1.5 || observers[0].RSSI != -90 || observers[0].Timestamp != "2026-01-01T00:00:00Z" { + t.Fatalf("observer query lost data: %+v", observers) + } +} + +func TestSQLiteExportReadsPathAndObservers(t *testing.T) { + for _, current := range []bool{false, true} { + name := "legacy" + if current { + name = "current" + } + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "telemetry #percent%.db") + sqliteExportFixture(t, path, current) + db, err := openExportDB(path) + if err != nil { + t.Fatalf("native SQLite export should open its existing source: %v", err) + } + defer db.Close() + assertExportMetadata(t, db) + if _, err := db.Exec(`DELETE FROM observations`); err == nil { + t.Fatal("export reader permitted a write") + } + var queryOnly int + if err := db.QueryRow(`PRAGMA query_only`).Scan(&queryOnly); err != nil || queryOnly != 1 { + t.Fatalf("query_only=%d err=%v", queryOnly, err) + } + }) + } +} + +func TestSQLiteExportRejectsMissingEmptyWrongAndPendingSources(t *testing.T) { + for _, kind := range []string{"missing", "missing-directory", "empty", "header-only", "wrong-schema", "pending"} { + t.Run(kind, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "telemetry.db") + switch kind { + case "missing-directory": + path = filepath.Join(filepath.Dir(path), "absent", "telemetry.db") + case "empty": + if err := os.WriteFile(path, nil, 0600); err != nil { + t.Fatal(err) + } + case "header-only", "wrong-schema": + dsn, err := dbconfig.SQLiteURI(path, nil) + if err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal(err) + } + query := `PRAGMA user_version=123` + if kind == "wrong-schema" { + query = `CREATE TABLE keep(value TEXT); INSERT INTO keep VALUES('preserve')` + } + if _, err := db.Exec(query); err != nil { + t.Fatal(err) + } + db.Close() + case "pending": + db := sqliteExportFixture(t, path, true) + if _, err := db.Exec(`CREATE TABLE corescope_reverse_progress(value TEXT)`); err != nil { + t.Fatal(err) + } + db.Close() + } + before, beforeErr := os.ReadFile(path) + db, err := openExportDB(path) + if err == nil { + db.Close() + t.Fatal("invalid source was accepted") + } + if kind == "pending" && !errors.Is(err, dbconfig.ErrSQLiteImportIncomplete) { + t.Fatalf("pending import guard lost: %v", err) + } + after, afterErr := os.ReadFile(path) + if errors.Is(beforeErr, os.ErrNotExist) { + if !errors.Is(afterErr, os.ErrNotExist) { + t.Fatal("export created a replacement database") + } + } else if beforeErr != nil || afterErr != nil || !bytes.Equal(before, after) { + t.Fatal("export modified its rejected source") + } + }) + } +} + +func recordExportStorage(t *testing.T, storage dbconfig.Storage) dbconfig.Selection { + t.Helper() + selected, err := dbconfig.NewSelection(storage) + if err != nil { + t.Fatal(err) + } + selected, err = dbconfig.AdoptSelection(dbconfig.SelectionPath(storage.StateDir), selected) + if err != nil { + t.Fatal(err) + } + return selected +} + +func TestExportHonorsRecordedSQLiteAndHoldsLease(t *testing.T) { + base := t.TempDir() + path := filepath.Join(base, "installed.db") + sqliteExportFixture(t, path, true) + selected := recordExportStorage(t, dbconfig.Storage{Backend: dbconfig.SQLite, StateDir: base, DBPath: path}) + db, lease, err := openSelectedExport(dbconfig.StorageInputs{BaseDir: base, StateDir: base, Backend: dbconfig.Postgres, DBPath: "stale.db"}) + if err != nil { + t.Fatal(err) + } + defer db.Close() + defer lease.Close() + assertExportMetadata(t, db) + update, err := dbconfig.BeginSelectionUpdate(dbconfig.SelectionPath(base), selected.Generation) + if err == nil { + update.Abort() + update.Close() + t.Fatal("export did not hold the selection lease") + } + if !errors.Is(err, dbconfig.ErrSelectionBusy) { + t.Fatalf("unexpected lease error: %v", err) + } + db.Close() + lease.Close() + update, err = dbconfig.BeginSelectionUpdate(dbconfig.SelectionPath(base), selected.Generation) + if err != nil { + t.Fatalf("export did not release the lease: %v", err) + } + defer update.Close() + if err := update.Abort(); err != nil { + t.Fatal(err) + } +} + +func TestRecordedSQLiteExportDoesNotFallBackOrCreate(t *testing.T) { + base := t.TempDir() + path := filepath.Join(base, "lost.db") + recordExportStorage(t, dbconfig.Storage{Backend: dbconfig.SQLite, StateDir: base, DBPath: path}) + stale := filepath.Join(base, "stale.db") + sqliteExportFixture(t, stale, true) + db, lease, err := openSelectedExport(dbconfig.StorageInputs{BaseDir: base, StateDir: base, DBPath: stale}) + if err == nil || db != nil || lease != nil { + t.Fatalf("lost recorded source did not fail closed: db=%t lease=%t err=%v", db != nil, lease != nil, err) + } + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatal("export created the missing selected source") + } +} + +func TestExportSelectionErrorsCannotFallBack(t *testing.T) { + for _, kind := range []string{"corrupt", "switching"} { + t.Run(kind, func(t *testing.T) { + base := t.TempDir() + path := filepath.Join(base, "legacy.db") + sqliteExportFixture(t, path, true) + metadata := dbconfig.SelectionPath(base) + if kind == "switching" { + metadata += ".switch.json" + } + if err := os.WriteFile(metadata, []byte(`{}`), 0600); err != nil { + t.Fatal(err) + } + if db, lease, err := openSelectedExport(dbconfig.StorageInputs{BaseDir: base, DBPath: path}); err == nil || db != nil || lease != nil { + t.Fatal("invalid selection fell back to legacy SQLite") + } + }) + } +} + +func TestRecordedPostgresExportNeverUsesStaleSQLite(t *testing.T) { + base := t.TempDir() + stale := filepath.Join(base, "meshcore.db") + w := sqliteExportFixture(t, stale, true) + w.Close() + before, err := os.ReadFile(stale) + if err != nil { + t.Fatal(err) + } + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + endpoint := listener.Addr().String() + listener.Close() + dsn := "postgres://reader:never-print-this@" + endpoint + "/unavailable?sslmode=disable&connect_timeout=1" + recordExportStorage(t, dbconfig.Storage{Backend: dbconfig.Postgres, StateDir: base, ReaderDatabaseURL: dsn}) + for _, reader := range []string{"", dsn} { + db, lease, err := openSelectedExport(dbconfig.StorageInputs{BaseDir: base, DBPath: stale, Backend: dbconfig.SQLite, ReaderDatabaseURL: reader}) + if err == nil || db != nil || lease != nil { + t.Fatal("missing PostgreSQL selection fell back to SQLite") + } + if strings.Contains(err.Error(), "never-print-this") { + t.Fatal("connection failure exposed its password") + } + } + after, err := os.ReadFile(stale) + if err != nil || !bytes.Equal(before, after) { + t.Fatal("stale SQLite source changed") + } +} + +func TestExportConfigResolution(t *testing.T) { + base := t.TempDir() + data := filepath.Join(base, "data") + if err := os.Mkdir(data, 0700); err != nil { + t.Fatal(err) + } + path := filepath.Join(data, "meshcore.db") + sqliteExportFixture(t, path, true) + noEnv := func(string) string { return "" } + raw, err := exportStorageInputs(base, exportOptions{}, noEnv) + if err != nil { + t.Fatal(err) + } + db, lease, err := openSelectedExport(raw) + if err != nil { + t.Fatalf("default SQLite export: %v", err) + } + if lease != nil { + lease.Close() + t.Fatal("unrecorded export must not adopt a selection") + } + assertExportMetadata(t, db) + db.Close() + if _, err := os.Stat(dbconfig.SelectionPath(data)); !errors.Is(err, os.ErrNotExist) { + t.Fatal("export persisted a selection") + } + if err := os.WriteFile(filepath.Join(data, "config.json"), []byte(`{"dbPath":"data/meshcore.db"}`), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(base, "config.json"), []byte(`{"db":`), 0600); err != nil { + t.Fatal(err) + } + if _, err := exportStorageInputs(base, exportOptions{}, noEnv); err == nil { + t.Fatal("malformed primary config fell back to a valid secondary config") + } + if _, err := exportStorageInputs(base, exportOptions{ConfigPath: filepath.Join(base, "missing.json")}, noEnv); err == nil { + t.Fatal("missing explicit config was ignored") + } + getenv := func(key string) string { + return map[string]string{"DB_PATH": "env.db", "CORESCOPE_DATABASE_URL": "postgres://reader@localhost/example", "CORESCOPE_READER_DATABASE_URL": "postgres://reader@localhost/reader", "CORESCOPE_STATE_DIR": "env-state"}[key] + } + raw, err = exportStorageInputs(base, exportOptions{ConfigPath: filepath.Join(data, "config.json"), DBPath: "cli.db", DatabaseURL: "postgres://reader@localhost/cli", StateDir: "cli-state"}, getenv) + if err != nil || raw.DBPath != "cli.db" || raw.DatabaseURL != "postgres://reader@localhost/cli" || raw.ReaderDatabaseURL != raw.DatabaseURL || raw.StateDir != "cli-state" { + t.Fatal("explicit flag precedence was lost") + } + if _, err := dbconfig.ResolveStorage(raw); err == nil { + t.Fatal("ambiguous unrecorded file and URL inputs were accepted") + } +} diff --git a/cmd/ingestor/advert_evidence_migration_race_test.go b/cmd/ingestor/advert_evidence_migration_race_test.go index 93ae8b91b..2eb6f37b1 100644 --- a/cmd/ingestor/advert_evidence_migration_race_test.go +++ b/cmd/ingestor/advert_evidence_migration_race_test.go @@ -25,7 +25,7 @@ func seedUnbackfilledAdvert(t *testing.T, canonical, observed string) (*Store, s if _, err := s.db.Exec(`INSERT INTO observations(id,transmission_id,observer_idx,raw_hex,path_json,timestamp) VALUES(1,1,(SELECT rowid FROM observers WHERE id='fixture-observer'),$1,'[]',1)`, observed); err != nil { t.Fatal(err) } - if _, err := testAdmin(t, s).Exec(`SELECT setval('observations_id_seq',1,true)`); err != nil { + if _, err := testAdmin(t, s).Exec(testNativeSQL(`UPDATE sqlite_sequence SET seq=1 WHERE name='observations'`, `SELECT setval('observations_id_seq',1,true)`)); err != nil { t.Fatal(err) } @@ -39,7 +39,7 @@ func TestAdvertRouteEvidenceLegacyProtectionErrorsDoNotDropIncomingRaw(t *testin defer s.Close() stmt := `DROP TABLE advert_evidence_backfill` if failure == "write" { - stmt = `CREATE OR REPLACE FUNCTION fail_old_evidence_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.bit=2 THEN RAISE EXCEPTION 'fixture old evidence failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER fail_old_evidence BEFORE INSERT ON advert_route_evidence FOR EACH ROW EXECUTE FUNCTION fail_old_evidence_fn()` + stmt = testNativeSQL(`CREATE TRIGGER fail_old_evidence BEFORE INSERT ON advert_route_evidence WHEN NEW.bit=2 BEGIN SELECT RAISE(ABORT,'fixture old evidence failure'); END`, `CREATE OR REPLACE FUNCTION fail_old_evidence_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.bit=2 THEN RAISE EXCEPTION 'fixture old evidence failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER fail_old_evidence BEFORE INSERT ON advert_route_evidence FOR EACH ROW EXECUTE FUNCTION fail_old_evidence_fn()`) } if _, err := testAdmin(t, s).Exec(stmt); err != nil { t.Fatal(err) @@ -81,10 +81,12 @@ func TestAdvertRouteEvidenceLegacyCoalescedPathAndIndex(t *testing.T) { if mask != 3 { t.Fatalf("coalesced path lookup lost old evidence: mask=%d", mask) } - if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { - t.Fatal(err) + if testBackendValue() == "postgres" { + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } } - rows, err := s.db.Query(`EXPLAIN (COSTS OFF) `+legacyAdvertObservationSQL, 1, 1, "") + rows, err := s.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN `, `EXPLAIN (COSTS OFF) `)+legacyAdvertObservationSQL, 1, 1, "") if err != nil { t.Fatal(err) } @@ -92,7 +94,7 @@ func TestAdvertRouteEvidenceLegacyCoalescedPathAndIndex(t *testing.T) { indexed := false for rows.Next() { var detail string - if err := rows.Scan(&detail); err != nil { + if err := scanTestPlan(rows, &detail); err != nil { t.Fatal(err) } if strings.Contains(detail, "idx_observations_dedup") { @@ -169,7 +171,7 @@ func BenchmarkAdvertEvidenceLegacyPreservation(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { if err := func() error { - tx, e := beginWrite(s.db) + tx, e := beginWrite(s.db, s.Backend()) if e != nil { return e } diff --git a/cmd/ingestor/advert_route_evidence.go b/cmd/ingestor/advert_route_evidence.go index 8ffcc2b3c..16eb6438c 100644 --- a/cmd/ingestor/advert_route_evidence.go +++ b/cmd/ingestor/advert_route_evidence.go @@ -48,8 +48,8 @@ func (s *Store) backfillAdvertEvidence(ctx context.Context, db *sql.DB) error { return err } for _, source := range []struct{ cursor, table, query string }{ - {"tx_cursor", "transmissions", `SELECT id,id,COALESCE(raw_hex,''),payload_type FROM transmissions WHERE id>$1 AND id<=$2 ORDER BY id LIMIT 500`}, - {"obs_cursor", "observations", `SELECT o.id,o.transmission_id,COALESCE(o.raw_hex,''),t.payload_type FROM observations o JOIN transmissions t ON t.id=o.transmission_id WHERE o.id>$1 AND o.id<=$2 ORDER BY o.id LIMIT 500`}, + {"tx_cursor", "transmissions", `SELECT id,id,COALESCE(raw_hex,''),payload_type FROM transmissions WHERE id>` + s.parameter(1) + ` AND id<=` + s.parameter(2) + ` ORDER BY id LIMIT 500`}, + {"obs_cursor", "observations", `SELECT o.id,o.transmission_id,COALESCE(o.raw_hex,''),t.payload_type FROM observations o JOIN transmissions t ON t.id=o.transmission_id WHERE o.id>` + s.parameter(1) + ` AND o.id<=` + s.parameter(2) + ` ORDER BY o.id LIMIT 500`}, } { // A finite horizon prevents live traffic from extending this scan. var upper int64 @@ -101,17 +101,17 @@ func (s *Store) backfillAdvertEvidence(ctx context.Context, db *sql.DB) error { err = func() error { writerMu.Lock() defer writerMu.Unlock() - tx, err := beginWriteContext(ctx, db) + tx, err := beginWriteContext(ctx, db, s.Backend()) if err != nil { return err } defer tx.Rollback() for _, item := range batch { - if _, err := tx.ExecContext(ctx, insertAdvertEvidenceSQL+` AND EXISTS(SELECT 1 FROM transmissions WHERE id=$5)`, item.txID, item.bit, item.txID, item.bit, item.txID); err != nil { + if _, err := tx.ExecContext(ctx, insertAdvertEvidenceSQL+` AND EXISTS(SELECT 1 FROM transmissions WHERE id=`+s.parameter(5)+`)`, item.txID, item.bit, item.txID, item.bit, item.txID); err != nil { return err } } - if _, err := tx.ExecContext(ctx, `UPDATE advert_evidence_backfill SET `+source.cursor+`=$1 WHERE id=1`, lastID); err != nil { + if _, err := tx.ExecContext(ctx, `UPDATE advert_evidence_backfill SET `+source.cursor+`=`+s.parameter(1)+` WHERE id=1`, lastID); err != nil { return err } return tx.Commit() diff --git a/cmd/ingestor/advert_route_evidence_test.go b/cmd/ingestor/advert_route_evidence_test.go index d1a5cf80e..10494fa8a 100644 --- a/cmd/ingestor/advert_route_evidence_test.go +++ b/cmd/ingestor/advert_route_evidence_test.go @@ -65,7 +65,7 @@ func TestAdvertRouteEvidenceSurvivesObservationUpsert(t *testing.T) { if canonical != sequence.raws[0] { t.Fatalf("canonical raw changed: %s", canonical) } - if err := s.db.QueryRow(`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='advert_route_evidence'`).Scan(&count); err != nil { + if err := s.db.QueryRow(testNativeSQL(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='advert_route_evidence'`, `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='advert_route_evidence'`)).Scan(&count); err != nil { t.Fatal(err) } if count != 1 { @@ -95,7 +95,7 @@ func TestAdvertRouteEvidenceSurvivesObservationUpsert(t *testing.T) { t.Fatal("duplicate traffic appended route evidence") } var sequenceID int64 - if err := s.db.QueryRow(`SELECT last_value FROM advert_route_evidence_id_seq`).Scan(&sequenceID); err != nil { + if err := s.db.QueryRow(testNativeSQL(`SELECT seq FROM sqlite_sequence WHERE name='advert_route_evidence'`, `SELECT last_value FROM advert_route_evidence_id_seq`)).Scan(&sequenceID); err != nil { t.Fatal(err) } if sequenceID != maxID { @@ -154,7 +154,7 @@ func TestAdvertRouteEvidenceBackfillResumeAndLiveUnion(t *testing.T) { } // Abort after one committed batch; the failed batch must not move its // persisted cursor, so a retry can recover every remaining frame. - if _, err := testAdmin(t, s).Exec(`CREATE OR REPLACE FUNCTION fail_evidence_batch_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.tx_id=501 THEN RAISE EXCEPTION 'fixture failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER fail_evidence_batch BEFORE INSERT ON advert_route_evidence FOR EACH ROW EXECUTE FUNCTION fail_evidence_batch_fn()`); err != nil { + if _, err := testAdmin(t, s).Exec(testNativeSQL(`CREATE TRIGGER fail_evidence_batch BEFORE INSERT ON advert_route_evidence WHEN NEW.tx_id=501 BEGIN SELECT RAISE(ABORT,'fixture failure'); END`, `CREATE OR REPLACE FUNCTION fail_evidence_batch_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.tx_id=501 THEN RAISE EXCEPTION 'fixture failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER fail_evidence_batch BEFORE INSERT ON advert_route_evidence FOR EACH ROW EXECUTE FUNCTION fail_evidence_batch_fn()`)); err != nil { t.Fatal(err) } if err := s.backfillAdvertEvidence(context.Background(), s.db); err == nil { @@ -167,7 +167,7 @@ func TestAdvertRouteEvidenceBackfillResumeAndLiveUnion(t *testing.T) { if cursor != 500 { t.Fatalf("cursor=%d after failure, want committed batch boundary 500", cursor) } - if _, err := testAdmin(t, s).Exec(`DROP TRIGGER fail_evidence_batch ON advert_route_evidence`); err != nil { + if _, err := testAdmin(t, s).Exec(testNativeSQL(`DROP TRIGGER fail_evidence_batch`, `DROP TRIGGER fail_evidence_batch ON advert_route_evidence`)); err != nil { t.Fatal(err) } done := make(chan error, 1) @@ -188,13 +188,13 @@ func TestAdvertRouteEvidenceBackfillResumeAndLiveUnion(t *testing.T) { t.Fatalf("got %d evidence rows, want two per history transmission", count) } var before, after int64 - if err := s.db.QueryRow(`SELECT last_value FROM advert_route_evidence_id_seq`).Scan(&before); err != nil { + if err := s.db.QueryRow(testNativeSQL(`SELECT seq FROM sqlite_sequence WHERE name='advert_route_evidence'`, `SELECT last_value FROM advert_route_evidence_id_seq`)).Scan(&before); err != nil { t.Fatal(err) } if err := s.backfillAdvertEvidence(context.Background(), s.db); err != nil { t.Fatal(err) } - if err := s.db.QueryRow(`SELECT last_value FROM advert_route_evidence_id_seq`).Scan(&after); err != nil { + if err := s.db.QueryRow(testNativeSQL(`SELECT seq FROM sqlite_sequence WHERE name='advert_route_evidence'`, `SELECT last_value FROM advert_route_evidence_id_seq`)).Scan(&after); err != nil { t.Fatal(err) } if before != after { @@ -227,12 +227,12 @@ func TestAdvertRouteEvidenceFailureKeepsCoreIngestion(t *testing.T) { if _, err := s.db.Exec(`DELETE FROM advert_route_evidence; UPDATE observations SET raw_hex='1200aa',resolved_path=NULL`); err != nil { t.Fatal(err) } - stmt := `CREATE OR REPLACE FUNCTION fail_evidence_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.bit=1 THEN RAISE EXCEPTION 'fixture evidence failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER fail_evidence BEFORE INSERT ON advert_route_evidence FOR EACH ROW EXECUTE FUNCTION fail_evidence_fn()` + stmt := testNativeSQL(`CREATE TRIGGER fail_evidence BEFORE INSERT ON advert_route_evidence WHEN NEW.bit=1 BEGIN SELECT RAISE(ABORT,'fixture evidence failure'); END`, `CREATE OR REPLACE FUNCTION fail_evidence_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.bit=1 THEN RAISE EXCEPTION 'fixture evidence failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER fail_evidence BEFORE INSERT ON advert_route_evidence FOR EACH ROW EXECUTE FUNCTION fail_evidence_fn()`) if failure == "legacy-read" { stmt = `DROP TABLE advert_evidence_backfill` } if failure == "legacy-write" { - stmt = `CREATE OR REPLACE FUNCTION fail_evidence_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.bit=2 THEN RAISE EXCEPTION 'fixture evidence failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER fail_evidence BEFORE INSERT ON advert_route_evidence FOR EACH ROW EXECUTE FUNCTION fail_evidence_fn()` + stmt = testNativeSQL(`CREATE TRIGGER fail_evidence BEFORE INSERT ON advert_route_evidence WHEN NEW.bit=2 BEGIN SELECT RAISE(ABORT,'fixture evidence failure'); END`, `CREATE OR REPLACE FUNCTION fail_evidence_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.bit=2 THEN RAISE EXCEPTION 'fixture evidence failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER fail_evidence BEFORE INSERT ON advert_route_evidence FOR EACH ROW EXECUTE FUNCTION fail_evidence_fn()`) } if _, err := testAdmin(t, s).Exec(stmt); err != nil { t.Fatal(err) @@ -293,21 +293,23 @@ func TestAdvertRouteEvidenceConstraintsAndFeedIndex(t *testing.T) { t.Errorf("constraint accepted: %s", stmt) } } - s.db.Exec(`SET enable_seqscan=off`) - rows, err := s.db.Query(`EXPLAIN (COSTS OFF) SELECT id,tx_id,bit FROM advert_route_evidence WHERE id>1 ORDER BY id LIMIT 500`) + if s.Backend() == "postgres" { + s.db.Exec(`SET enable_seqscan=off`) + } + rows, err := s.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN SELECT id,tx_id,bit FROM advert_route_evidence WHERE id>1 ORDER BY id LIMIT 500`, `EXPLAIN (COSTS OFF) SELECT id,tx_id,bit FROM advert_route_evidence WHERE id>1 ORDER BY id LIMIT 500`)) if err != nil { t.Fatal(err) } var plan string for rows.Next() { var line string - if err := rows.Scan(&line); err != nil { + if err := scanTestPlan(rows, &line); err != nil { t.Fatal(err) } plan += line } rows.Close() - if !strings.Contains(plan, "advert_route_evidence_pkey") { + if !strings.Contains(plan, testNativeSQL("INTEGER PRIMARY KEY", "advert_route_evidence_pkey")) { t.Fatalf("feed must seek by primary key: %s", plan) } if _, err := s.db.Exec(`DELETE FROM observations; DELETE FROM transmissions`); err != nil { @@ -357,7 +359,7 @@ func TestAdvertRouteEvidenceRejectsUnprovenFrames(t *testing.T) { } } var count int - if err := s.db.QueryRow(`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='advert_route_evidence'`).Scan(&count); err != nil { + if err := s.db.QueryRow(testNativeSQL(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='advert_route_evidence'`, `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='advert_route_evidence'`)).Scan(&count); err != nil { t.Fatal(err) } if count != 1 { diff --git a/cmd/ingestor/approved_channels.go b/cmd/ingestor/approved_channels.go index d4da85083..fe88f6fcd 100644 --- a/cmd/ingestor/approved_channels.go +++ b/cmd/ingestor/approved_channels.go @@ -5,10 +5,14 @@ import ( "database/sql" "errors" "log" + "net/url" + "strings" "sync/atomic" "time" + _ "github.com/mattn/go-sqlite3" "github.com/meshcore-analyzer/channel" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/pgutil" ) @@ -23,7 +27,9 @@ const approvedChannelsRefresh = time.Minute // (ingestorApprovedQuery), which checks it against ApprovedSubjects. const approvedChannelsQuery = `SELECT subject FROM approved_channels WHERE kind = 'hashtag_channel' ORDER BY decided_at, id LIMIT $1` -var errUsersDBMissing = errors.New("approved channel database URL is not configured") +const approvedChannelsSQLiteQuery = `SELECT subject FROM proposals WHERE kind = 'hashtag_channel' AND status = 'approved' ORDER BY decided_at, id LIMIT ?1` + +var errUsersDBMissing = errors.New("approved channel account storage is not configured") // channelKeySet hands each message the current channel key map. Snapshots // are never mutated after they are stored, so decoders read them without a @@ -31,6 +37,7 @@ var errUsersDBMissing = errors.New("approved channel database URL is not configu type channelKeySet struct { configured map[string]string path string + backend dbconfig.Backend max int cur atomic.Pointer[map[string]string] @@ -41,8 +48,24 @@ type channelKeySet struct { // newChannelKeySet starts with the configured keys (the very map // loadChannelKeys built, so the feature off changes nothing). -func newChannelKeySet(configured map[string]string, usersDBPath string, max int) *channelKeySet { - s := &channelKeySet{configured: configured, path: usersDBPath, max: max} +// newChannelKeySet retains compatibility for an unambiguous native target. +func newChannelKeySet(configured map[string]string, target string, max int) *channelKeySet { + storage := dbconfig.Storage{Backend: dbconfig.SQLite, UsersDBPath: target} + if strings.HasPrefix(strings.ToLower(target), "postgres:") || strings.HasPrefix(strings.ToLower(target), "postgresql:") { + storage.Backend = dbconfig.Postgres + storage.ApprovedChannelsDatabaseURL = target + } + return newChannelKeySetStorage(configured, storage, max) +} + +// newChannelKeySetStorage consumes the same resolved storage choice as startup. +// Inactive addresses never cause fallback to a different account store. +func newChannelKeySetStorage(configured map[string]string, storage dbconfig.Storage, max int) *channelKeySet { + target := storage.UsersDBPath + if storage.Backend == dbconfig.Postgres { + target = storage.ApprovedChannelsDatabaseURL + } + s := &channelKeySet{configured: configured, path: target, backend: storage.Backend, max: max} s.cur.Store(&configured) return s } @@ -75,48 +98,80 @@ func (s *channelKeySet) refresh() { } func (s *channelKeySet) readApproved() ([]string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if s.backend != dbconfig.SQLite && s.backend != dbconfig.Postgres { + return nil, errors.New("approved channel storage backend must be selected") + } if s.db == nil { if s.path == "" { return nil, errUsersDBMissing } - db, err := pgutil.Open(s.path, true) - if err != nil { - return nil, err - } - if err = pgutil.AssertReadOnly(db); err != nil { - db.Close() - return nil, err + var db *sql.DB + var err error + if s.backend == dbconfig.SQLite { + uri, e := dbconfig.SQLiteURI(s.path, url.Values{"mode": {"ro"}, "_busy_timeout": {"5000"}}) + if e != nil { + return nil, e + } + db, err = sql.Open("sqlite3", uri) + } else { + db, err = pgutil.Open(s.path, true) + if err == nil { + err = pgutil.AssertReadOnly(db) + } + if err == nil { + var credentialAccess bool + err = db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=current_schema() AND c.relkind IN ('r','p','m','f') AND c.relname <> 'corescope_schema' AND has_table_privilege(c.oid,'SELECT'))`).Scan(&credentialAccess) + if err == nil && credentialAccess { + err = errors.New("approved-channel reader has account-table access") + } + } } - var credentialAccess bool - if err = db.QueryRow(`SELECT EXISTS(SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=current_schema() AND c.relkind IN ('r','p','m','f') AND c.relname <> 'corescope_schema' AND has_table_privilege(c.oid,'SELECT'))`).Scan(&credentialAccess); err != nil { - db.Close() + if err != nil { + if db != nil { + db.Close() + } return nil, err } - if credentialAccess { - db.Close() - return nil, errors.New("approved-channel reader has account-table access") - } db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) s.db = db } - // Read the gate and projection in one snapshot. The readiness marker is the - // only base table this role can read; an import must not expose partial keys. - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - tx, err := s.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true}) + if s.backend == dbconfig.SQLite { + if err := dbconfig.AssertSQLiteImportComplete(s.db); err != nil { + return nil, err + } + } + // Check schema and rows within one native read-only snapshot. PostgreSQL's + // projection role can read only the readiness marker and approved view. + opts := &sql.TxOptions{ReadOnly: true} + query := approvedChannelsSQLiteQuery + if s.backend == dbconfig.Postgres { + opts.Isolation = sql.LevelRepeatableRead + query = approvedChannelsQuery + } + tx, err := s.db.BeginTx(ctx, opts) if err != nil { return nil, err } defer tx.Rollback() var version int - var ready bool - if err := tx.QueryRowContext(ctx, `SELECT version,ready FROM corescope_schema WHERE kind='accounts'`).Scan(&version, &ready); err != nil { - return nil, errors.New("approved-channel account schema is not initialized") - } - if version != 1 || !ready { - return nil, errors.New("approved-channel account schema is incomplete or unsupported") + if s.backend == dbconfig.Postgres { + var ready bool + if err := tx.QueryRowContext(ctx, `SELECT version,ready FROM corescope_schema WHERE kind='accounts'`).Scan(&version, &ready); err != nil { + return nil, errors.New("approved-channel account schema is not initialized") + } + if version != 1 || !ready { + return nil, errors.New("approved-channel account schema is incomplete or unsupported") + } + } else { + var count int + if err := tx.QueryRowContext(ctx, `SELECT count(*),COALESCE(MAX(version),0) FROM schema_version`).Scan(&count, &version); err != nil || count != 1 || version < 4 || version > 6 { + return nil, errors.New("approved-channel SQLite account schema is incomplete or unsupported") + } } - rows, err := tx.QueryContext(ctx, approvedChannelsQuery, s.max) + rows, err := tx.QueryContext(ctx, query, s.max) if err != nil { return nil, err } @@ -174,5 +229,5 @@ func (s *channelKeySet) Close() { // Connection strings contain credentials and must never appear in logs. func logApprovedChannelsSource(databaseURL string) { - log.Print("[proposals] reading approved channels with restricted PostgreSQL reader") + log.Print("[proposals] reading approved channels from the selected account store") } diff --git a/cmd/ingestor/approved_channels_import_test.go b/cmd/ingestor/approved_channels_import_test.go new file mode 100644 index 000000000..718168ba1 --- /dev/null +++ b/cmd/ingestor/approved_channels_import_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "errors" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbconfig" +) + +func TestApprovedChannelsPendingSQLiteImportKeepsLastGoodKeys(t *testing.T) { + t.Setenv("CORESCOPE_TEST_BACKEND", "sqlite") + path := filepath.Join(t.TempDir(), "accounts.db") + writeUsersDB(t, path, testProposal{"#complete", "approved", 1}) + keys := newApprovedTestKeySet(t, configuredKeys(), path, 128) + defer keys.Close() + keys.refresh() + want := keys.Snapshot()["#complete"] + if want == "" { + t.Fatal("complete initial approval was not loaded") + } + execUsersDB(t, path, `CREATE TABLE corescope_reverse_progress (table_name TEXT)`) + execUsersDB(t, path, `UPDATE proposals SET subject='#partial'`) + if _, err := keys.readApproved(); !errors.Is(err, dbconfig.ErrSQLiteImportIncomplete) { + t.Fatalf("partial import was accepted: %v", err) + } + keys.refresh() + if keys.Snapshot()["#complete"] != want || keys.Snapshot()["#partial"] != "" { + t.Fatal("partial import replaced the last verified approval set") + } +} diff --git a/cmd/ingestor/approved_channels_test.go b/cmd/ingestor/approved_channels_test.go index 9843552a3..978e58683 100644 --- a/cmd/ingestor/approved_channels_test.go +++ b/cmd/ingestor/approved_channels_test.go @@ -3,8 +3,10 @@ package main import ( "bytes" "crypto/sha256" + "database/sql" "encoding/hex" "fmt" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/pgutil" "github.com/meshcore-analyzer/pgutil/pgtest" "log" @@ -38,17 +40,26 @@ type testProposal struct { func execUsersDB(t *testing.T, path string, stmts ...string) { t.Helper() - db, err := pgutil.Open(approvedFixture(t, path).owner, false) + db, err := openApprovedFixture(t, approvedFixture(t, path).owner) if err != nil { t.Fatal(err) } defer db.Close() for _, s := range stmts { + if approvedTestBackend(t) == dbconfig.SQLite && s == testProposalsDDL { + s = strings.Replace(s, "BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY", "INTEGER PRIMARY KEY", 1) + } if _, err := db.Exec(s); err != nil { t.Fatalf("%s: %v", s, err) } } if stmts[0] == testProposalsDDL { + if approvedTestBackend(t) == dbconfig.SQLite { + if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS schema_version(version INTEGER NOT NULL); DELETE FROM schema_version; INSERT INTO schema_version VALUES(5)`); err != nil { + t.Fatal(err) + } + return + } f := approvedFixture(t, path) for _, stmt := range []string{ `CREATE TABLE IF NOT EXISTS corescope_schema(version INTEGER NOT NULL, kind TEXT PRIMARY KEY, ready BOOLEAN NOT NULL)`, @@ -79,7 +90,7 @@ func configuredKeys() map[string]string { } func TestApprovedChannelsQueryMatchesProjection(t *testing.T) { - path := "projection" + path := filepath.Join(t.TempDir(), "projection.db") writeUsersDB(t, path, testProposal{"#fixture", "approved", 1}) reader := newApprovedTestKeySet(t, configuredKeys(), path, 128) defer reader.Close() @@ -89,7 +100,10 @@ func TestApprovedChannelsQueryMatchesProjection(t *testing.T) { } func TestApprovedChannelsRejectsIncompleteAccountImport(t *testing.T) { - path := "account-import-gate" + if approvedTestBackend(t) != dbconfig.Postgres { + t.Skip("PostgreSQL import readiness marker; SQLite schema completeness is tested separately") + } + path := filepath.Join(t.TempDir(), "account-import-gate.db") writeUsersDB(t, path, testProposal{"#fixture", "approved", 1}) reader := newApprovedTestKeySet(t, configuredKeys(), path, 128) defer reader.Close() @@ -208,7 +222,11 @@ func TestChannelKeySetReadErrorKeepsLastGoodSet(t *testing.T) { s := newApprovedTestKeySet(t, configuredKeys(), path, 128) defer s.Close() s.refresh() - execUsersDB(t, path, `DROP VIEW approved_channels`) + statement := `DROP VIEW approved_channels` + if approvedTestBackend(t) == dbconfig.SQLite { + statement = `DROP TABLE proposals` + } + execUsersDB(t, path, statement) s.refresh() if _, ok := s.Snapshot()["#mycity"]; !ok { t.Fatal("a failing read removed an approved key") @@ -248,14 +266,16 @@ func TestChannelKeySetIsReadOnly(t *testing.T) { defer s.Close() s.refresh() _, err := s.db.Exec(`INSERT INTO proposals (kind, subject, status, created_at) VALUES ('hashtag_channel', '#x', 'approved', 1)`) - if err == nil || !strings.Contains(strings.ToLower(err.Error()), "read-only") { + if err == nil || !strings.Contains(strings.ToLower(err.Error()), "read") { t.Fatalf("write through the ingestor's users.db handle: err = %v; want a read-only refusal", err) } - if _, err := s.db.Exec(`SET default_transaction_read_only=off`); err != nil { - t.Fatal(err) - } - if _, err := s.db.Query(`SELECT * FROM proposals`); err == nil { - t.Fatal("projection reader can read proposals table") + if approvedTestBackend(t) == dbconfig.Postgres { + if _, err := s.db.Exec(`SET default_transaction_read_only=off`); err != nil { + t.Fatal(err) + } + if _, err := s.db.Query(`SELECT * FROM proposals`); err == nil { + t.Fatal("projection reader can read proposals table") + } } } @@ -264,7 +284,7 @@ func TestChannelKeySetIsReadOnly(t *testing.T) { func TestChannelKeySetReadsWALWithOpenWriter(t *testing.T) { path := filepath.Join(t.TempDir(), "users.db") writeUsersDB(t, path, testProposal{"#mycity", "approved", 1}) - w, err := pgutil.Open(approvedFixture(t, path).owner, false) + w, err := openApprovedFixture(t, approvedFixture(t, path).owner) if err != nil { t.Fatal(err) } @@ -423,6 +443,9 @@ var approvedFixtures sync.Map func approvedFixture(t *testing.T, key string) approvedFixtureDB { t.Helper() + if approvedTestBackend(t) == dbconfig.SQLite { + return approvedFixtureDB{owner: key, reader: key} + } key = t.Name() + "|" + key if v, ok := approvedFixtures.Load(key); ok { return v.(approvedFixtureDB) @@ -454,3 +477,82 @@ func approvedFixture(t *testing.T, key string) approvedFixtureDB { func newApprovedTestKeySet(t *testing.T, configured map[string]string, key string, max int) *channelKeySet { return newChannelKeySet(configured, approvedFixture(t, key).reader, max) } + +func approvedTestBackend(t *testing.T) dbconfig.Backend { + t.Helper() + switch os.Getenv("CORESCOPE_TEST_BACKEND") { + case "", "sqlite": + return dbconfig.SQLite + case "postgres": + return dbconfig.Postgres + default: + t.Fatal("invalid CORESCOPE_TEST_BACKEND") + return "" + } +} +func openApprovedFixture(t *testing.T, path string) (*sql.DB, error) { + t.Helper() + if approvedTestBackend(t) == dbconfig.Postgres { + return pgutil.Open(path, false) + } + uri, err := dbconfig.SQLiteURI(path, url.Values{"_journal_mode": {"WAL"}, "_busy_timeout": {"5000"}}) + if err != nil { + return nil, err + } + return sql.Open("sqlite3", uri) +} + +func TestApprovedChannelsStorageChoiceDoesNotFallback(t *testing.T) { + path := filepath.Join(t.TempDir(), "accounts.db") + // A selected PostgreSQL target cannot fall back even when a local file exists. + if approvedTestBackend(t) == dbconfig.SQLite { + writeUsersDB(t, path, testProposal{"#local", "approved", 1}) + } + s := newChannelKeySetStorage(configuredKeys(), dbconfig.Storage{Backend: dbconfig.Postgres, UsersDBPath: path}, 128) + defer s.Close() + if _, err := s.readApproved(); err != errUsersDBMissing { + t.Fatalf("missing PostgreSQL target error=%v", err) + } + s.refresh() + if len(s.Snapshot()) != 2 { + t.Fatal("invalid backend selection added account keys") + } +} + +func TestApprovedChannelsSQLiteVersionsAndLiteralPath(t *testing.T) { + if approvedTestBackend(t) != dbconfig.SQLite { + t.Skip("native SQLite contract") + } + path := filepath.Join(t.TempDir(), "account # % ü.db") + writeUsersDB(t, path, testProposal{"#local", "approved", 1}) + s := newChannelKeySetStorage(configuredKeys(), dbconfig.Storage{Backend: dbconfig.SQLite, UsersDBPath: path, ApprovedChannelsDatabaseURL: "postgres://unused.invalid/accounts"}, 128) + defer s.Close() + for _, version := range []int{4, 5, 6} { + execUsersDB(t, path, fmt.Sprintf(`UPDATE schema_version SET version=%d`, version)) + names, err := s.readApproved() + if err != nil || len(names) != 1 || names[0] != "#local" { + t.Fatalf("version%d names=%v,%v", version, names, err) + } + } + s.refresh() + for _, version := range []int{3, 7} { + execUsersDB(t, path, fmt.Sprintf(`UPDATE schema_version SET version=%d`, version)) + if _, err := s.readApproved(); err == nil { + t.Fatalf("unsupported version%d accepted", version) + } + s.refresh() + if s.Snapshot()["#local"] == "" { + t.Fatal("unsupported schema removed last good key") + } + } + if s.db.Stats().MaxOpenConnections != 1 { + t.Fatal("account reader pool is not bounded") + } + // mode=ro remains authoritative even when a caller disables query_only. + if _, err := s.db.Exec(`PRAGMA query_only=OFF`); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`DELETE FROM proposals`); err == nil { + t.Fatal("SQLite account reader became writable") + } +} diff --git a/cmd/ingestor/async_migration.go b/cmd/ingestor/async_migration.go index 431f12a8b..e631af65e 100644 --- a/cmd/ingestor/async_migration.go +++ b/cmd/ingestor/async_migration.go @@ -1,5 +1,5 @@ // Async data backfills resume through durable bookkeeping. Schema DDL belongs -// exclusively to the offline migration command; runtime callbacks must not +// to the SQLite writer or PostgreSQL owner; PostgreSQL callbacks must not // create or alter tables/indexes. Long backfills use bounded transactions and // yield between batches so live ingestion can continue. @@ -9,19 +9,21 @@ import ( "context" "database/sql" "fmt" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" "log" ) -// ensureAsyncMigrationsTable checks bootstrap-owned bookkeeping without DDL. -func ensureAsyncMigrationsTable(db *sql.DB) error { - var present bool - if err := db.QueryRow(`SELECT to_regclass('_async_migrations') IS NOT NULL`).Scan(&present); err != nil { - return err +// SQLite owns its ledger; PostgreSQL checks the bootstrap-owned ledger without DDL. +func ensureAsyncMigrationsTable(db *sql.DB, backend dbconfig.Backend) error { + if backend == dbconfig.SQLite { + return dbschema.EnsureSQLiteAsyncMigrations(db) } - if !present { - return fmt.Errorf("missing migration bookkeeping; run the offline migration command") + rows, err := db.Query(`SELECT name FROM _async_migrations LIMIT 0`) + if err != nil { + return err } - return nil + return rows.Close() } // RunAsyncMigration registers `name` as a pending async migration and @@ -38,27 +40,30 @@ func ensureAsyncMigrationsTable(db *sql.DB) error { // - The caller's WaitGroup tracks the goroutine so tests/shutdown can // wait via Store.WaitForAsyncMigrations(). func (s *Store) RunAsyncMigration(ctx context.Context, name string, fn func(context.Context, *sql.DB) error) error { - if err := ensureAsyncMigrationsTable(s.db); err != nil { + if err := ensureAsyncMigrationsTable(s.db, s.Backend()); err != nil { return fmt.Errorf("ensure _async_migrations: %w", err) } var existing string - row := s.db.QueryRow(`SELECT status FROM _async_migrations WHERE name = $1`, name) + row := s.db.QueryRow(`SELECT status FROM _async_migrations WHERE name = `+s.parameter(1), name) switch err := row.Scan(&existing); err { case nil: if existing == "done" { return nil // already complete, nothing to do } // pending_async or failed → reset and retry. - if _, err := s.db.Exec(` + if _, err := s.db.Exec(s.nativeSQL(` + UPDATE _async_migrations + SET status = 'pending_async', started_at = datetime('now'), ended_at = NULL, error = NULL + WHERE name = ?1`, ` UPDATE _async_migrations SET status = 'pending_async', started_at = to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD HH24:MI:SS'), ended_at = NULL, error = NULL - WHERE name = $1`, name); err != nil { + WHERE name = `+s.parameter(1)), name); err != nil { return fmt.Errorf("reset async migration %q: %w", name, err) } case sql.ErrNoRows: if _, err := s.db.Exec(` - INSERT INTO _async_migrations (name, status) VALUES ($1, 'pending_async')`, + INSERT INTO _async_migrations (name, status) VALUES (`+s.parameter(1)+`, 'pending_async')`, name); err != nil { return fmt.Errorf("register async migration %q: %w", name, err) } @@ -76,19 +81,25 @@ func (s *Store) RunAsyncMigration(ctx context.Context, name string, fn func(cont log.Printf("[async-migration] %q panic recovered: %v", name, r) } if runErr != nil { - if _, err := s.db.Exec(` + if _, err := s.db.Exec(s.nativeSQL(` + UPDATE _async_migrations + SET status = 'failed', ended_at = datetime('now'), error = ?1 + WHERE name = ?2`, ` UPDATE _async_migrations - SET status = 'failed', ended_at = to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD HH24:MI:SS'), error = $1 - WHERE name = $2`, runErr.Error(), name); err != nil { + SET status = 'failed', ended_at = to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD HH24:MI:SS'), error = `+s.parameter(1)+` + WHERE name = `+s.parameter(2)), runErr.Error(), name); err != nil { log.Printf("[async-migration] failed to record failure for %q: %v", name, err) } log.Printf("[async-migration] %q FAILED: %v", name, runErr) return } - if _, err := s.db.Exec(` + if _, err := s.db.Exec(s.nativeSQL(` + UPDATE _async_migrations + SET status = 'done', ended_at = datetime('now'), error = NULL + WHERE name = ?1`, ` UPDATE _async_migrations SET status = 'done', ended_at = to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD HH24:MI:SS'), error = NULL - WHERE name = $1`, name); err != nil { + WHERE name = `+s.parameter(1)), name); err != nil { log.Printf("[async-migration] failed to mark %q done: %v", name, err) return } @@ -105,11 +116,11 @@ func (s *Store) RunAsyncMigration(ctx context.Context, name string, fn func(cont // (one of "pending_async", "done", "failed") or sql.ErrNoRows if no such // migration has been registered. func (s *Store) AsyncMigrationStatus(name string) (string, error) { - if err := ensureAsyncMigrationsTable(s.db); err != nil { + if err := ensureAsyncMigrationsTable(s.db, s.Backend()); err != nil { return "", err } var status string - err := s.db.QueryRow(`SELECT status FROM _async_migrations WHERE name = $1`, name).Scan(&status) + err := s.db.QueryRow(`SELECT status FROM _async_migrations WHERE name = `+s.parameter(1), name).Scan(&status) return status, err } diff --git a/cmd/ingestor/async_migration_test.go b/cmd/ingestor/async_migration_test.go index d7abd672c..4e31d412d 100644 --- a/cmd/ingestor/async_migration_test.go +++ b/cmd/ingestor/async_migration_test.go @@ -157,7 +157,7 @@ func TestRunAsyncMigration_RestartSafetyFailedIsRetried(t *testing.T) { s := newTestStore(t) const name = "test_restart_failed_v1" - if err := ensureAsyncMigrationsTable(s.db); err != nil { + if err := ensureAsyncMigrationsTable(s.db, s.Backend()); err != nil { t.Fatalf("ensure table: %v", err) } if _, err := s.db.Exec(`INSERT INTO _async_migrations (name, status, error) VALUES ($1, 'failed', 'simulated prior crash')`, name); err != nil { @@ -197,7 +197,7 @@ func TestRunAsyncMigration_RestartSafetyPendingIsRetried(t *testing.T) { s := newTestStore(t) const name = "test_restart_pending_v1" - if err := ensureAsyncMigrationsTable(s.db); err != nil { + if err := ensureAsyncMigrationsTable(s.db, s.Backend()); err != nil { t.Fatalf("ensure table: %v", err) } if _, err := s.db.Exec(`INSERT INTO _async_migrations (name, status) VALUES ($1, 'pending_async')`, name); err != nil { @@ -297,3 +297,20 @@ func TestRunAsyncMigration_ConcurrentSameNameSerialized(t *testing.T) { t.Fatalf("fn invoked %d times, want 1..5 inclusive (bounded by caller count)", got) } } + +func TestAsyncMigrationMinimalSQLiteStore(t *testing.T) { + db, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + db.SetMaxOpenConns(1) + s := &Store{db: db} + if err := s.RunAsyncMigration(context.Background(), "minimal", func(context.Context, *sql.DB) error { return nil }); err != nil { + t.Fatal(err) + } + s.WaitForAsyncMigrations() + if status, err := s.AsyncMigrationStatus("minimal"); err != nil || status != "done" { + t.Fatal(status, err) + } +} diff --git a/cmd/ingestor/client_reception.go b/cmd/ingestor/client_reception.go index 611ed11cb..ecdc7f39d 100644 --- a/cmd/ingestor/client_reception.go +++ b/cmd/ingestor/client_reception.go @@ -128,7 +128,7 @@ func (s *Store) UpsertClientObserver(pubkey, name, ts string) error { return nil } _, err := s.db.Exec(` - INSERT INTO client_observers (pubkey, name, last_seen) VALUES ($1,$2,$3) + INSERT INTO client_observers (pubkey, name, last_seen) VALUES (`+s.parameter(1)+`,`+s.parameter(2)+`,`+s.parameter(3)+`) ON CONFLICT(pubkey) DO UPDATE SET name = excluded.name, last_seen = excluded.last_seen`, strings.ToLower(pubkey), name, ts) return err @@ -258,7 +258,7 @@ func (s *Store) InsertClientReception(r *ClientReception) (bool, error) { res, err := s.db.Exec(` INSERT INTO client_receptions (rx_pubkey, heard_key, heard_keylen, rssi, snr, lat, lon, pos_acc_m, rx_at, ingested_at, src) - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) + VALUES (`+s.parameter(1)+`,`+s.parameter(2)+`,`+s.parameter(3)+`,`+s.parameter(4)+`,`+s.parameter(5)+`,`+s.parameter(6)+`,`+s.parameter(7)+`,`+s.parameter(8)+`,`+s.parameter(9)+`,`+s.parameter(10)+`,`+s.parameter(11)+`) ON CONFLICT(rx_pubkey, heard_key, rx_at) DO NOTHING`, r.RxPubkey, r.HeardKey, r.HeardKeyLen, r.RSSI, r.SNR, r.Lat, r.Lon, r.PosAccM, r.RxAt, r.IngestedAt, r.Src) if err != nil { @@ -299,7 +299,7 @@ func (s *Store) InsertClientRfSample(o *ClientRfSample) (bool, error) { uptime_secs, battery_mv, queue_len, errors, noise_floor, last_rssi, last_snr, tx_air_secs, rx_air_secs, recv, sent, flood_rx, direct_rx, flood_tx, direct_tx, recv_errors) - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23) + VALUES (`+s.parameter(1)+`,`+s.parameter(2)+`,`+s.parameter(3)+`,`+s.parameter(4)+`,`+s.parameter(5)+`,`+s.parameter(6)+`,`+s.parameter(7)+`,`+s.parameter(8)+`,`+s.parameter(9)+`,`+s.parameter(10)+`,`+s.parameter(11)+`,`+s.parameter(12)+`,`+s.parameter(13)+`,`+s.parameter(14)+`,`+s.parameter(15)+`,`+s.parameter(16)+`,`+s.parameter(17)+`,`+s.parameter(18)+`,`+s.parameter(19)+`,`+s.parameter(20)+`,`+s.parameter(21)+`,`+s.parameter(22)+`,`+s.parameter(23)+`) ON CONFLICT(rx_pubkey, sampled_at) DO NOTHING`, o.RxPubkey, o.SampledAt, o.IngestedAt, o.Lat, o.Lon, o.PosAccM, boolToInt(o.Stationary), o.UptimeSecs, o.BatteryMV, o.QueueLen, o.Errors, o.NoiseFloor, o.LastRSSI, @@ -418,7 +418,7 @@ func (s *Store) InsertClientRxObservation(o *ClientRxObservation) (bool, error) (rx_pubkey, rx_at, ingested_at, pkt_hash, route_type, payload_type, code1, code2, scope_name, hash_size, hop_count, path_json, forwarder, snr, rssi, lat, lon, pos_acc_m) - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18) + VALUES (`+s.parameter(1)+`,`+s.parameter(2)+`,`+s.parameter(3)+`,`+s.parameter(4)+`,`+s.parameter(5)+`,`+s.parameter(6)+`,`+s.parameter(7)+`,`+s.parameter(8)+`,`+s.parameter(9)+`,`+s.parameter(10)+`,`+s.parameter(11)+`,`+s.parameter(12)+`,`+s.parameter(13)+`,`+s.parameter(14)+`,`+s.parameter(15)+`,`+s.parameter(16)+`,`+s.parameter(17)+`,`+s.parameter(18)+`) ON CONFLICT(rx_pubkey, pkt_hash, rx_at) DO NOTHING`, o.RxPubkey, o.RxAt, o.IngestedAt, o.PktHash, o.RouteType, o.PayloadType, o.Code1, o.Code2, o.ScopeName, o.HashSize, o.HopCount, o.PathJSON, o.Forwarder, @@ -458,7 +458,7 @@ func (s *Store) InsertClientDeclaredRegions(o *ClientDeclaredRegions) (bool, err res, err := s.db.Exec(` INSERT INTO node_declared_regions (target, rx_pubkey, observed_at, ingested_at, regions_csv, truncated, lat, lon, pos_acc_m, repeater_clock) - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) + VALUES (`+s.parameter(1)+`,`+s.parameter(2)+`,`+s.parameter(3)+`,`+s.parameter(4)+`,`+s.parameter(5)+`,`+s.parameter(6)+`,`+s.parameter(7)+`,`+s.parameter(8)+`,`+s.parameter(9)+`,`+s.parameter(10)+`) ON CONFLICT(target, rx_pubkey, observed_at) DO NOTHING`, o.Target, o.RxPubkey, o.ObservedAt, o.IngestedAt, o.RegionsCSV, boolToInt(o.Truncated), o.Lat, o.Lon, o.PosAccM, o.RepeaterClock) if err != nil { @@ -478,7 +478,7 @@ func (s *Store) CurrentDeclaredRegions(target string) (*ClientDeclaredRegions, e row := s.db.QueryRow(` SELECT target, rx_pubkey, observed_at, ingested_at, regions_csv, truncated, lat, lon, pos_acc_m, repeater_clock FROM node_declared_regions - WHERE target = $1 + WHERE target = `+s.parameter(1)+` ORDER BY observed_at DESC LIMIT 1`, target) var o ClientDeclaredRegions var truncated int diff --git a/cmd/ingestor/client_reception_test.go b/cmd/ingestor/client_reception_test.go index a57724a96..65c5d3d02 100644 --- a/cmd/ingestor/client_reception_test.go +++ b/cmd/ingestor/client_reception_test.go @@ -54,10 +54,12 @@ func TestPruneOldClientReceptions(t *testing.T) { // this is a regression guard. func TestPruneClientRxObservationsUsesIndex(t *testing.T) { s := newTestStore(t) - if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { - t.Fatal(err) + if testBackendValue() == "postgres" { + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } } - rows, err := s.db.Query(`EXPLAIN (COSTS OFF) DELETE FROM client_rx_observations WHERE rx_at < $1`, "2026-01-01T00:00:00Z") + rows, err := s.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN DELETE FROM client_rx_observations WHERE rx_at < $1`, `EXPLAIN (COSTS OFF) DELETE FROM client_rx_observations WHERE rx_at < $1`), "2026-01-01T00:00:00Z") if err != nil { t.Fatal(err) } @@ -65,7 +67,7 @@ func TestPruneClientRxObservationsUsesIndex(t *testing.T) { plan := "" for rows.Next() { var detail string - if err := rows.Scan(&detail); err != nil { + if err := scanTestPlan(rows, &detail); err != nil { t.Fatal(err) } plan += detail + "\n" @@ -141,7 +143,7 @@ func TestPruneOldClientRxObservations(t *testing.T) { func TestClientReceptionsTableExists(t *testing.T) { s := newTestStore(t) cols := map[string]bool{} - rows, err := s.db.Query(`SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='client_receptions' ORDER BY ordinal_position`) + rows, err := s.db.Query(testNativeSQL(`SELECT cid,name,type,"notnull",dflt_value,pk FROM pragma_table_info('client_receptions') ORDER BY cid`, `SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='client_receptions' ORDER BY ordinal_position`)) if err != nil { t.Fatalf("PRAGMA failed: %v", err) } @@ -173,9 +175,11 @@ func crI(i int) *int { return &i } // is "Seq Scan on client_receptions". func TestClientReceptionsCoverageQueryUsesIndex(t *testing.T) { s := newTestStore(t) - q := `EXPLAIN (COSTS OFF) SELECT lat, lon, snr, rssi, heard_key, rx_at + q := testNativeSQL(`EXPLAIN QUERY PLAN SELECT lat, lon, snr, rssi, heard_key, rx_at + FROM client_receptions + WHERE heard_key IN ($1,$2,$3) AND lat BETWEEN $4 AND $5 AND lon BETWEEN $6 AND $7`, `EXPLAIN (COSTS OFF) SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions - WHERE heard_key IN ($1,$2,$3) AND lat BETWEEN $4 AND $5 AND lon BETWEEN $6 AND $7` + WHERE heard_key IN ($1,$2,$3) AND lat BETWEEN $4 AND $5 AND lon BETWEEN $6 AND $7`) s.db.Exec(`SET enable_seqscan=off`) rows, err := s.db.Query(q, "aabbccddeeff00112233", "aabbcc", "aabb", 50.0, 52.0, 3.0, 4.0) if err != nil { @@ -185,7 +189,7 @@ func TestClientReceptionsCoverageQueryUsesIndex(t *testing.T) { plan := "" for rows.Next() { var detail string - if err := rows.Scan(&detail); err != nil { + if err := scanTestPlan(rows, &detail); err != nil { t.Fatal(err) } plan += detail + "\n" @@ -203,10 +207,12 @@ func TestClientReceptionsCoverageQueryUsesIndex(t *testing.T) { // index rather than full-scanning under the writer lock (polish review). func TestClientReceptionsRetentionUsesRxAtIndex(t *testing.T) { s := newTestStore(t) - if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { - t.Fatal(err) + if testBackendValue() == "postgres" { + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } } - rows, err := s.db.Query(`EXPLAIN (COSTS OFF) DELETE FROM client_receptions WHERE rx_at < $1`, "2026-01-01T00:00:00Z") + rows, err := s.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN DELETE FROM client_receptions WHERE rx_at < $1`, `EXPLAIN (COSTS OFF) DELETE FROM client_receptions WHERE rx_at < $1`), "2026-01-01T00:00:00Z") if err != nil { t.Fatal(err) } @@ -214,7 +220,7 @@ func TestClientReceptionsRetentionUsesRxAtIndex(t *testing.T) { plan := "" for rows.Next() { var detail string - if err := rows.Scan(&detail); err != nil { + if err := scanTestPlan(rows, &detail); err != nil { t.Fatal(err) } plan += detail + "\n" @@ -234,16 +240,24 @@ func TestClientReceptionsRetentionUsesRxAtIndex(t *testing.T) { // index-backed). func TestRxLeaderboardQueryIsIndexBacked(t *testing.T) { s := newTestStore(t) - if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { - t.Fatal(err) + if testBackendValue() == "postgres" { + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } } - rows, err := s.db.Query(`EXPLAIN (COSTS OFF) + rows, err := s.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN + SELECT cr.rx_pubkey, COUNT(*), COUNT(DISTINCT cr.heard_key) + FROM client_receptions cr + WHERE cr.rx_at >= $1 + GROUP BY cr.rx_pubkey + ORDER BY COUNT(*) DESC + LIMIT $2`, `EXPLAIN (COSTS OFF) SELECT cr.rx_pubkey, COUNT(*), COUNT(DISTINCT cr.heard_key) FROM client_receptions cr WHERE cr.rx_at >= $1 GROUP BY cr.rx_pubkey ORDER BY COUNT(*) DESC - LIMIT $2`, "2026-01-01T00:00:00Z", 100) + LIMIT $2`), "2026-01-01T00:00:00Z", 100) if err != nil { t.Fatal(err) } @@ -251,7 +265,7 @@ func TestRxLeaderboardQueryIsIndexBacked(t *testing.T) { plan := "" for rows.Next() { var detail string - if err := rows.Scan(&detail); err != nil { + if err := scanTestPlan(rows, &detail); err != nil { t.Fatal(err) } plan += detail + "\n" diff --git a/cmd/ingestor/client_regions_test.go b/cmd/ingestor/client_regions_test.go index 5d7485a9b..7ceb1daa9 100644 --- a/cmd/ingestor/client_regions_test.go +++ b/cmd/ingestor/client_regions_test.go @@ -480,10 +480,12 @@ func TestCurrentDeclaredRegionsNormalizesTargetCase(t *testing.T) { // index seek on idx_ndr_prune rather than a full table scan. func TestPruneClientDeclaredRegionsUsesIndex(t *testing.T) { s := newTestStore(t) - if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { - t.Fatal(err) + if testBackendValue() == "postgres" { + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } } - rows, err := s.db.Query(`EXPLAIN (COSTS OFF) DELETE FROM node_declared_regions WHERE observed_at < $1`, "2026-01-01T00:00:00Z") + rows, err := s.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN DELETE FROM node_declared_regions WHERE observed_at < $1`, `EXPLAIN (COSTS OFF) DELETE FROM node_declared_regions WHERE observed_at < $1`), "2026-01-01T00:00:00Z") if err != nil { t.Fatal(err) } @@ -491,7 +493,7 @@ func TestPruneClientDeclaredRegionsUsesIndex(t *testing.T) { plan := "" for rows.Next() { var detail string - if err := rows.Scan(&detail); err != nil { + if err := scanTestPlan(rows, &detail); err != nil { t.Fatal(err) } plan += detail + "\n" diff --git a/cmd/ingestor/client_rf_sample.go b/cmd/ingestor/client_rf_sample.go index 1ecc4c300..ad307f112 100644 --- a/cmd/ingestor/client_rf_sample.go +++ b/cmd/ingestor/client_rf_sample.go @@ -134,7 +134,7 @@ func (s *Store) ClientRfDeltas(rxPubkey, from, to string) ([]ClientRfDelta, erro LAG(recv) OVER w AS prev_recv, LAG(recv_errors) OVER w AS prev_errs FROM client_rf_samples - WHERE rx_pubkey = $1 AND sampled_at >= $2 AND sampled_at < $3 + WHERE rx_pubkey = `+s.parameter(1)+` AND sampled_at >= `+s.parameter(2)+` AND sampled_at < `+s.parameter(3)+` WINDOW w AS (ORDER BY sampled_at) ORDER BY sampled_at`, rxPubkey, from, to) if err != nil { diff --git a/cmd/ingestor/config.go b/cmd/ingestor/config.go index 7b8a26379..657a3a393 100644 --- a/cmd/ingestor/config.go +++ b/cmd/ingestor/config.go @@ -543,9 +543,6 @@ func LoadConfig(path string) (*Config, error) { if v := os.Getenv("CORESCOPE_STATE_DIR"); v != "" { cfg.StateDir = v } - if cfg.StateDir == "" { - cfg.StateDir = "data" - } if v := os.Getenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL"); v != "" { if cfg.UserManagement == nil { cfg.UserManagement = &UserManagementConfig{} @@ -630,6 +627,7 @@ const autoRegionKeysDefaultRefreshMinutes = 15 // UserManagementConfig is the ingestor's view of userManagement. type UserManagementConfig struct { + DatabaseURL string `json:"databaseURL,omitempty"` Enabled bool `json:"enabled"` ApprovedChannelsDatabaseURL string `json:"approvedChannelsDatabaseURL,omitempty"` DBPath string `json:"dbPath,omitempty"` diff --git a/cmd/ingestor/config_test.go b/cmd/ingestor/config_test.go index 5d9a37144..52657c9c0 100644 --- a/cmd/ingestor/config_test.go +++ b/cmd/ingestor/config_test.go @@ -40,7 +40,7 @@ func TestLoadConfigMissingFile(t *testing.T) { if err != nil { t.Fatalf("missing config should not error (zero-config mode), got: %v", err) } - if cfg.DBPath != "" || cfg.DatabaseURL != "" || cfg.StateDir != "data" { + if cfg.DBPath != "" || cfg.DatabaseURL != "" || cfg.StateDir != "" { t.Errorf("unexpected database/state defaults") } // Should default to localhost MQTT @@ -193,7 +193,7 @@ func TestLoadConfigDefaultDBPath(t *testing.T) { if err != nil { t.Fatal(err) } - if cfg.DBPath != "" || cfg.DatabaseURL != "" || cfg.StateDir != "data" { + if cfg.DBPath != "" || cfg.DatabaseURL != "" || cfg.StateDir != "" { t.Errorf("unexpected database/state defaults") } } diff --git a/cmd/ingestor/coverage_boost_test.go b/cmd/ingestor/coverage_boost_test.go index fb77d5e56..33d86958c 100644 --- a/cmd/ingestor/coverage_boost_test.go +++ b/cmd/ingestor/coverage_boost_test.go @@ -1027,7 +1027,7 @@ func TestApplySchemaMigrationsOnFreshDB(t *testing.T) { // Check that migrations were recorded var count int - if err := store.db.QueryRow("SELECT COUNT(*) FROM corescope_schema WHERE kind='telemetry' AND ready=true").Scan(&count); err != nil { + if err := store.db.QueryRow(testNativeSQL(`SELECT COUNT(*) FROM _migrations WHERE name='observers_identity_autoincrement_v1'`, "SELECT COUNT(*) FROM corescope_schema WHERE kind='telemetry' AND ready=true")).Scan(&count); err != nil { t.Fatal(err) } if count != 1 { @@ -1036,19 +1036,19 @@ func TestApplySchemaMigrationsOnFreshDB(t *testing.T) { // Check observations table exists with dedup index var tblName string - err := store.db.QueryRow("SELECT table_name FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='observations'").Scan(&tblName) + err := store.db.QueryRow(testNativeSQL(`SELECT name FROM sqlite_master WHERE type='table' AND name='observations'`, "SELECT table_name FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='observations'")).Scan(&tblName) if err != nil { t.Error("observations table should exist") } // Check inactive_nodes table exists - err = store.db.QueryRow("SELECT table_name FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='inactive_nodes'").Scan(&tblName) + err = store.db.QueryRow(testNativeSQL(`SELECT name FROM sqlite_master WHERE type='table' AND name='inactive_nodes'`, "SELECT table_name FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='inactive_nodes'")).Scan(&tblName) if err != nil { t.Error("inactive_nodes table should exist") } // Check packets_v view exists - err = store.db.QueryRow("SELECT table_name FROM information_schema.views WHERE table_schema=current_schema() AND table_name='packets_v'").Scan(&tblName) + err = store.db.QueryRow(testNativeSQL(`SELECT name FROM sqlite_master WHERE type='view' AND name='packets_v'`, "SELECT table_name FROM information_schema.views WHERE table_schema=current_schema() AND table_name='packets_v'")).Scan(&tblName) if err != nil { t.Error("packets_v view should exist") } diff --git a/cmd/ingestor/db.go b/cmd/ingestor/db.go index 907cf6512..7f4fd5931 100644 --- a/cmd/ingestor/db.go +++ b/cmd/ingestor/db.go @@ -14,6 +14,8 @@ import ( "sync/atomic" "time" + _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/dbschema" "github.com/meshcore-analyzer/packetpath" "github.com/meshcore-analyzer/pgutil" @@ -65,8 +67,9 @@ func (s *DBStats) SnapshotBackfills() map[string]int64 { return out } -// Store wraps the PostgreSQL database for packet ingestion. +// Store wraps the selected native database for packet ingestion. type Store struct { + backend dbconfig.Backend db *sql.DB path string // compatibility anchor for local file queues; never a database URL stateDir string @@ -123,13 +126,51 @@ const relayTouchDebounce = 5 * time.Minute // an extra UPDATE, never a missed one. const relayTouchedMaxEntries = 50000 -// OpenStore connects with a restricted telemetry writer credential. +// OpenStore preserves native path/URL callers; runtime uses selected Storage. func OpenStore(databaseURL string) (*Store, error) { return OpenStoreWithInterval(databaseURL, 300) } func OpenStoreWithInterval(databaseURL string, sampleIntervalSec int) (*Store, error) { - return OpenStoreWithState(databaseURL, "data", sampleIntervalSec) + return OpenStoreWithState(databaseURL, "", sampleIntervalSec) } -func OpenStoreWithState(databaseURL, stateDir string, sampleIntervalSec int) (*Store, error) { - db, err := pgutil.Open(databaseURL, false) +func OpenStoreWithState(target, stateDir string, sampleIntervalSec int) (*Store, error) { + storage := dbconfig.Storage{Backend: dbconfig.SQLite, DBPath: target, StateDir: stateDir} + if strings.HasPrefix(target, "postgres:") || strings.HasPrefix(target, "postgresql:") { + storage = dbconfig.Storage{Backend: dbconfig.Postgres, WriterDatabaseURL: target, StateDir: stateDir} + } + return openStore(storage, nil, sampleIntervalSec, true) +} +func openStoreStorage(storage dbconfig.Storage, cfg *DBConfig, sampleIntervalSec int) (*Store, error) { + return openStore(storage, cfg, sampleIntervalSec, false) +} +func openStore(storage dbconfig.Storage, cfg *DBConfig, sampleIntervalSec int, create bool) (*Store, error) { + var db *sql.DB + var err error + stateDir := storage.StateDir + switch storage.Backend { + case dbconfig.SQLite: + if storage.DBPath == "" || strings.Contains(storage.DBPath, "://") || strings.HasPrefix(storage.DBPath, "postgres:") || strings.HasPrefix(storage.DBPath, "postgresql:") { + return nil, fmt.Errorf("SQLite telemetry requires a filesystem path") + } + if create { + if err = os.MkdirAll(filepath.Dir(storage.DBPath), 0700); err != nil { + return nil, err + } + } + var dsn string + dsn, err = sqliteTelemetryWriterDSN(storage.DBPath, create) + if err == nil { + db, err = sql.Open("sqlite3", dsn) + } + if stateDir == "" { + stateDir = filepath.Dir(storage.DBPath) + } + case dbconfig.Postgres: + db, err = pgutil.Open(storage.WriterDatabaseURL, false) + if stateDir == "" { + stateDir = "data" + } + default: + return nil, fmt.Errorf("telemetry requires an explicitly selected storage backend") + } if err != nil { return nil, err } @@ -141,19 +182,36 @@ func OpenStoreWithState(databaseURL, stateDir string, sampleIntervalSec int) (*S }() db.SetMaxOpenConns(1) db.SetMaxIdleConns(1) - var synchronous string - if err = db.QueryRow(`SHOW synchronous_commit`).Scan(&synchronous); err != nil { - return nil, err - } - if synchronous != "on" { - return nil, fmt.Errorf("telemetry writer requires synchronous_commit=on") - } + if storage.Backend == dbconfig.SQLite { + if !create { + if err = validateExistingTelemetry(db); err != nil { + return nil, err + } + } + if err = dbschema.ApplySQLite(db, log.Printf); err != nil { + return nil, err + } + if !create { + if _, err = db.Exec(`PRAGMA auto_vacuum=INCREMENTAL; PRAGMA journal_mode=WAL`); err != nil { + return nil, err + } + } - if err = dbschema.AssertReady(db); err != nil { - return nil, err - } - if err = dbschema.AssertWriter(db); err != nil { - return nil, err + } else { + var synchronous string + if err = db.QueryRow(`SHOW synchronous_commit`).Scan(&synchronous); err != nil { + return nil, err + } + if synchronous != "on" { + return nil, fmt.Errorf("telemetry writer requires synchronous_commit=on") + } + + if err = dbschema.AssertReady(db); err != nil { + return nil, err + } + if err = dbschema.AssertWriter(db); err != nil { + return nil, err + } } if stateDir == "" { stateDir = "data" @@ -161,13 +219,40 @@ func OpenStoreWithState(databaseURL, stateDir string, sampleIntervalSec int) (*S if err = os.MkdirAll(stateDir, 0700); err != nil { return nil, fmt.Errorf("create state directory: %w", err) } - s := &Store{db: db, path: filepath.Join(stateDir, "meshcore"), stateDir: stateDir, sampleIntervalSec: sampleIntervalSec} + anchor := filepath.Join(stateDir, "meshcore") + if storage.Backend == dbconfig.SQLite { + anchor = filepath.Join(stateDir, filepath.Base(storage.DBPath)) + } + s := &Store{db: db, path: anchor, stateDir: stateDir, backend: storage.Backend, sampleIntervalSec: sampleIntervalSec} if err = s.prepareStatements(); err != nil { return nil, fmt.Errorf("prepare writer statements: %w", err) } if err = s.LoadScopeMatchTotals(); err != nil { log.Printf("[regions] restoring scope-match tally: %v", err) } + + if s.Backend() == dbconfig.SQLite { + var present int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_observer_idx_timestamp'`).Scan(&present); err != nil { + return nil, err + } + // Completion metadata must not hide a missing physical index after recovery. + if present == 0 { + if _, err := db.Exec(`UPDATE _async_migrations SET status='pending_async' WHERE name='obs_observer_ts_idx_v1' AND status='done'`); err != nil { + return nil, err + } + } + // PREFLIGHT: async=true reason="upstream composite observation index; keep its potentially large build off the startup goroutine" + if err := s.RunAsyncMigration(context.Background(), "obs_observer_ts_idx_v1", func(ctx context.Context, d *sql.DB) error { + if err := dbschema.EnsureSQLiteObserverTimeIndex(ctx, d); err != nil { + return err + } + _, err := d.ExecContext(ctx, `INSERT INTO _migrations(name) VALUES('obs_observer_ts_idx_v1') ON CONFLICT DO NOTHING`) + return err + }); err != nil { + return nil, err + } + } var evidenceStatus string _ = db.QueryRow(`SELECT status FROM _async_migrations WHERE name='advert_route_evidence_v1'`).Scan(&evidenceStatus) s.advertEvidenceComplete.Store(evidenceStatus == "done") @@ -195,20 +280,20 @@ func (s *Store) prepareStatements() error { return err } - s.stmtGetTxByHash, err = s.db.Prepare("SELECT id, first_seen FROM transmissions WHERE hash = $1") + s.stmtGetTxByHash, err = s.db.Prepare("SELECT id, first_seen FROM transmissions WHERE hash = " + s.parameter(1)) if err != nil { return err } s.stmtInsertTransmission, err = s.db.Prepare(` INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, channel_hash, scope_name, from_pubkey, last_seen, code1, code2) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) RETURNING id + VALUES (` + s.parameter(1) + `, ` + s.parameter(2) + `, ` + s.parameter(3) + `, ` + s.parameter(4) + `, ` + s.parameter(5) + `, ` + s.parameter(6) + `, ` + s.parameter(7) + `, ` + s.parameter(8) + `, ` + s.parameter(9) + `, ` + s.parameter(10) + `, ` + s.parameter(11) + `, ` + s.parameter(12) + `, ` + s.parameter(13) + `) RETURNING id `) if err != nil { return err } - s.stmtUpdateTxFirstSeen, err = s.db.Prepare("UPDATE transmissions SET first_seen = $1 WHERE id = $2") + s.stmtUpdateTxFirstSeen, err = s.db.Prepare("UPDATE transmissions SET first_seen = " + s.parameter(1) + " WHERE id = " + s.parameter(2)) if err != nil { return err } @@ -223,19 +308,19 @@ func (s *Store) prepareStatements() error { // "tx_last_seen_backfill_v1". // PREFLIGHT: async=true reason="prepared-statement row-level UPDATE BY PRIMARY KEY (transmissions.id) — single-row touch per observation, indexed by PK, constant-time at any scale. Not a migration." s.stmtTouchNodeLastSeen, err = s.db.Prepare( - "UPDATE nodes SET last_seen = $1 WHERE public_key = $2 AND (last_seen IS NULL OR last_seen < $3)") + "UPDATE nodes SET last_seen = " + s.parameter(1) + " WHERE public_key = " + s.parameter(2) + " AND (last_seen IS NULL OR last_seen < " + s.parameter(3) + ")") if err != nil { return fmt.Errorf("preparing touch node last_seen: %w", err) } - s.stmtBumpTxLastSeen, err = s.db.Prepare("UPDATE transmissions SET last_seen = $1 WHERE id = $2 AND last_seen < $3") + s.stmtBumpTxLastSeen, err = s.db.Prepare("UPDATE transmissions SET last_seen = " + s.parameter(1) + " WHERE id = " + s.parameter(2) + " AND last_seen < " + s.parameter(3)) if err != nil { return err } s.stmtInsertObservation, err = s.db.Prepare(` INSERT INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + VALUES (` + s.parameter(1) + `, ` + s.parameter(2) + `, ` + s.parameter(3) + `, ` + s.parameter(4) + `, ` + s.parameter(5) + `, ` + s.parameter(6) + `, ` + s.parameter(7) + `, ` + s.parameter(8) + `, ` + s.parameter(9) + `, ` + s.parameter(10) + `) ON CONFLICT(transmission_id, observer_idx, COALESCE(path_json, '')) DO UPDATE SET snr = COALESCE(excluded.snr, observations.snr), rssi = COALESCE(excluded.rssi, observations.rssi), @@ -247,50 +332,76 @@ func (s *Store) prepareStatements() error { return err } - s.stmtUpsertNode, err = s.db.Prepare(` + s.stmtUpsertNode, err = s.db.Prepare(s.nativeSQL(` INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen) - VALUES ($1, $2, $3, $4, $5, $6, $7) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) ON CONFLICT(public_key) DO UPDATE SET - name = COALESCE($8, nodes.name), - role = COALESCE($9, nodes.role), - lat = COALESCE($10, nodes.lat), - lon = COALESCE($11, nodes.lon), - last_seen = GREATEST(LEAST(COALESCE(nodes.last_seen, ''), $12), $13) - `) + name = COALESCE(?8, nodes.name), + role = COALESCE(?9, nodes.role), + lat = COALESCE(?10, nodes.lat), + lon = COALESCE(?11, nodes.lon), + last_seen = MAX(MIN(COALESCE(nodes.last_seen, ''), ?12), ?13) + `, ` + INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen) + VALUES (`+s.parameter(1)+`, `+s.parameter(2)+`, `+s.parameter(3)+`, `+s.parameter(4)+`, `+s.parameter(5)+`, `+s.parameter(6)+`, `+s.parameter(7)+`) + ON CONFLICT(public_key) DO UPDATE SET + name = COALESCE(`+s.parameter(8)+`, nodes.name), + role = COALESCE(`+s.parameter(9)+`, nodes.role), + lat = COALESCE(`+s.parameter(10)+`, nodes.lat), + lon = COALESCE(`+s.parameter(11)+`, nodes.lon), + last_seen = GREATEST(LEAST(COALESCE(nodes.last_seen, ''), `+s.parameter(12)+`), `+s.parameter(13)+`) + `)) if err != nil { return err } s.stmtIncrementAdvertCount, err = s.db.Prepare(` - UPDATE nodes SET advert_count = advert_count + 1 WHERE public_key = $1 + UPDATE nodes SET advert_count = advert_count + 1 WHERE public_key = ` + s.parameter(1) + ` `) if err != nil { return err } - s.stmtUpsertObserver, err = s.db.Prepare(` + s.stmtUpsertObserver, err = s.db.Prepare(s.nativeSQL(` INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, can_relay, can_relay_seen) - VALUES ($1, $2, $3, $4, $5, 1, $6, $7, $8, $9, $10, $11, $12, COALESCE($13, 1), CASE WHEN $14::bigint IS NULL THEN 0 ELSE 1 END) + VALUES (?1, ?2, ?3, ?4, ?5, 1, ?6, ?7, ?8, ?9, ?10, ?11, ?12, COALESCE(?13, 1), CASE WHEN CAST(?14 AS INTEGER) IS NULL THEN 0 ELSE 1 END) ON CONFLICT(id) DO UPDATE SET - name = COALESCE($15, observers.name), - iata = COALESCE($16, observers.iata), - last_seen = GREATEST(LEAST(COALESCE(observers.last_seen, ''), $17), $18), + name = COALESCE(?15, observers.name), + iata = COALESCE(?16, observers.iata), + last_seen = MAX(MIN(COALESCE(observers.last_seen, ''), ?17), ?18), packet_count = observers.packet_count + 1, - model = COALESCE($19, observers.model), - firmware = COALESCE($20, observers.firmware), - client_version = COALESCE($21, observers.client_version), - radio = COALESCE($22, observers.radio), - battery_mv = COALESCE($23, observers.battery_mv), - uptime_secs = COALESCE($24, observers.uptime_secs), - noise_floor = COALESCE($25, observers.noise_floor), - can_relay = COALESCE($26, observers.can_relay), - can_relay_seen = CASE WHEN $27::bigint IS NULL THEN observers.can_relay_seen ELSE 1 END - `) + model = COALESCE(?19, observers.model), + firmware = COALESCE(?20, observers.firmware), + client_version = COALESCE(?21, observers.client_version), + radio = COALESCE(?22, observers.radio), + battery_mv = COALESCE(?23, observers.battery_mv), + uptime_secs = COALESCE(?24, observers.uptime_secs), + noise_floor = COALESCE(?25, observers.noise_floor), + can_relay = COALESCE(?26, observers.can_relay), + can_relay_seen = CASE WHEN CAST(?27 AS INTEGER) IS NULL THEN observers.can_relay_seen ELSE 1 END + `, ` + INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, can_relay, can_relay_seen) + VALUES (`+s.parameter(1)+`, `+s.parameter(2)+`, `+s.parameter(3)+`, `+s.parameter(4)+`, `+s.parameter(5)+`, 1, `+s.parameter(6)+`, `+s.parameter(7)+`, `+s.parameter(8)+`, `+s.parameter(9)+`, `+s.parameter(10)+`, `+s.parameter(11)+`, `+s.parameter(12)+`, COALESCE(`+s.parameter(13)+`, 1), CASE WHEN `+s.parameter(14)+`::bigint IS NULL THEN 0 ELSE 1 END) + ON CONFLICT(id) DO UPDATE SET + name = COALESCE(`+s.parameter(15)+`, observers.name), + iata = COALESCE(`+s.parameter(16)+`, observers.iata), + last_seen = GREATEST(LEAST(COALESCE(observers.last_seen, ''), `+s.parameter(17)+`), `+s.parameter(18)+`), + packet_count = observers.packet_count + 1, + model = COALESCE(`+s.parameter(19)+`, observers.model), + firmware = COALESCE(`+s.parameter(20)+`, observers.firmware), + client_version = COALESCE(`+s.parameter(21)+`, observers.client_version), + radio = COALESCE(`+s.parameter(22)+`, observers.radio), + battery_mv = COALESCE(`+s.parameter(23)+`, observers.battery_mv), + uptime_secs = COALESCE(`+s.parameter(24)+`, observers.uptime_secs), + noise_floor = COALESCE(`+s.parameter(25)+`, observers.noise_floor), + can_relay = COALESCE(`+s.parameter(26)+`, observers.can_relay), + can_relay_seen = CASE WHEN `+s.parameter(27)+`::bigint IS NULL THEN observers.can_relay_seen ELSE 1 END + `)) if err != nil { return err } - s.stmtGetObserverRowid, err = s.db.Prepare("SELECT rowid FROM observers WHERE id = $1") + s.stmtGetObserverRowid, err = s.db.Prepare("SELECT rowid FROM observers WHERE id = " + s.parameter(1)) if err != nil { return err } @@ -298,20 +409,24 @@ func (s *Store) prepareStatements() error { // Args: ingestNow, rxTime, ingestNow, rxTime, rowid // MIN(existing, ingestNow) clamps any future value already in the DB before // taking MAX with rxTime, so the guard never locks in a past bug's stale future. - s.stmtUpdateObserverLastSeen, err = s.db.Prepare(` + s.stmtUpdateObserverLastSeen, err = s.db.Prepare(s.nativeSQL(` UPDATE observers SET - last_seen = GREATEST(LEAST(COALESCE(last_seen, ''), $1), $2), - last_packet_at = GREATEST(LEAST(COALESCE(last_packet_at, ''), $3), $4) - WHERE rowid = $5`) + last_seen = MAX(MIN(COALESCE(last_seen, ''), ?1), ?2), + last_packet_at = MAX(MIN(COALESCE(last_packet_at, ''), ?3), ?4) + WHERE rowid = ?5`, ` + UPDATE observers SET + last_seen = GREATEST(LEAST(COALESCE(last_seen, ''), `+s.parameter(1)+`), `+s.parameter(2)+`), + last_packet_at = GREATEST(LEAST(COALESCE(last_packet_at, ''), `+s.parameter(3)+`), `+s.parameter(4)+`) + WHERE rowid = `+s.parameter(5))) if err != nil { return err } s.stmtUpdateNodeTelemetry, err = s.db.Prepare(` UPDATE nodes SET - battery_mv = COALESCE($1, battery_mv), - temperature_c = COALESCE($2, temperature_c) - WHERE public_key = $3 + battery_mv = COALESCE(` + s.parameter(1) + `, battery_mv), + temperature_c = COALESCE(` + s.parameter(2) + `, temperature_c) + WHERE public_key = ` + s.parameter(3) + ` `) if err != nil { return err @@ -319,7 +434,7 @@ func (s *Store) prepareStatements() error { s.stmtUpsertMetrics, err = s.db.Prepare(` INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + VALUES (` + s.parameter(1) + `, ` + s.parameter(2) + `, ` + s.parameter(3) + `, ` + s.parameter(4) + `, ` + s.parameter(5) + `, ` + s.parameter(6) + `, ` + s.parameter(7) + `, ` + s.parameter(8) + `, ` + s.parameter(9) + `) ON CONFLICT(observer_id,timestamp) DO UPDATE SET noise_floor=excluded.noise_floor, tx_air_secs=excluded.tx_air_secs,rx_air_secs=excluded.rx_air_secs,recv_errors=excluded.recv_errors, battery_mv=excluded.battery_mv,packets_sent=excluded.packets_sent,packets_recv=excluded.packets_recv @@ -345,7 +460,7 @@ func (s *Store) InsertTransmission(data *PacketData) (bool, error) { writerMu.Unlock() recordWriterTiming("mqtt_handler", wait, time.Since(holdStart), "InsertTransmission") }() - tx, err := beginWrite(s.db) + tx, err := beginWrite(s.db, s.Backend()) if err != nil { return false, err } @@ -427,15 +542,19 @@ func (s *Store) InsertTransmission(data *PacketData) (bool, error) { // beginWrite serializes ID allocation before any statement can consume an // identity value. A transaction lock releases only after commit or rollback. -func beginWrite(db *sql.DB) (*sql.Tx, error) { return beginWriteContext(context.Background(), db) } -func beginWriteContext(ctx context.Context, db *sql.DB) (*sql.Tx, error) { +func beginWrite(db *sql.DB, backend ...dbconfig.Backend) (*sql.Tx, error) { + return beginWriteContext(context.Background(), db, backend...) +} +func beginWriteContext(ctx context.Context, db *sql.DB, backend ...dbconfig.Backend) (*sql.Tx, error) { tx, err := db.BeginTx(ctx, nil) if err != nil { return nil, err } - if _, err = tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended(current_schema(),$1))`, dbschema.WriterLockKey); err != nil { - tx.Rollback() - return nil, err + if len(backend) == 0 || backend[0] == dbconfig.Postgres { + if _, err = tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended(current_schema(),$1))`, dbschema.WriterLockKey); err != nil { + tx.Rollback() + return nil, err + } } return tx, nil } @@ -492,7 +611,7 @@ func (s *Store) MarkNodeForeign(pubKey string) error { if pubKey == "" { return nil } - _, err := s.db.Exec(`UPDATE nodes SET foreign_advert = 1 WHERE public_key = $1`, pubKey) + _, err := s.db.Exec(`UPDATE nodes SET foreign_advert = 1 WHERE public_key = `+s.parameter(1), pubKey) if err != nil { s.Stats.WriteErrors.Add(1) } @@ -570,7 +689,7 @@ func (s *Store) UpsertObserverAt(id, name, iata string, meta *ObserverMeta, last s.Stats.ObserverUpserts.Add(1) // Reactivate if this observer was previously marked inactive - s.db.Exec(`UPDATE observers SET inactive = 0 WHERE id = $1 AND inactive = 1`, id) + s.db.Exec(`UPDATE observers SET inactive = 0 WHERE id = `+s.parameter(1)+` AND inactive = 1`, id) return nil } @@ -591,20 +710,33 @@ func (s *Store) UpsertObserverRetained(id, name, iata string, meta *ObserverMeta normalizedIATA := strings.TrimSpace(strings.ToUpper(iata)) model, firmware, clientVersion, radio, batteryMv, uptimeSecs, noiseFloor, canRelay := observerMetaColumns(meta) - _, err := s.db.Exec(` + _, err := s.db.Exec(s.nativeSQL(` + UPDATE observers SET + name = COALESCE(?1, name), + iata = COALESCE(?2, iata), + model = COALESCE(?3, model), + firmware = COALESCE(?4, firmware), + client_version = COALESCE(?5, client_version), + radio = COALESCE(?6, radio), + battery_mv = COALESCE(?7, battery_mv), + uptime_secs = COALESCE(?8, uptime_secs), + noise_floor = COALESCE(?9, noise_floor), + can_relay = COALESCE(?10, can_relay), + can_relay_seen = CASE WHEN CAST(?11 AS INTEGER) IS NULL THEN can_relay_seen ELSE 1 END + WHERE id = ?12`, ` UPDATE observers SET - name = COALESCE($1, name), - iata = COALESCE($2, iata), - model = COALESCE($3, model), - firmware = COALESCE($4, firmware), - client_version = COALESCE($5, client_version), - radio = COALESCE($6, radio), - battery_mv = COALESCE($7, battery_mv), - uptime_secs = COALESCE($8, uptime_secs), - noise_floor = COALESCE($9, noise_floor), - can_relay = COALESCE($10, can_relay), - can_relay_seen = CASE WHEN $11::bigint IS NULL THEN can_relay_seen ELSE 1 END - WHERE id = $12`, + name = COALESCE(`+s.parameter(1)+`, name), + iata = COALESCE(`+s.parameter(2)+`, iata), + model = COALESCE(`+s.parameter(3)+`, model), + firmware = COALESCE(`+s.parameter(4)+`, firmware), + client_version = COALESCE(`+s.parameter(5)+`, client_version), + radio = COALESCE(`+s.parameter(6)+`, radio), + battery_mv = COALESCE(`+s.parameter(7)+`, battery_mv), + uptime_secs = COALESCE(`+s.parameter(8)+`, uptime_secs), + noise_floor = COALESCE(`+s.parameter(9)+`, noise_floor), + can_relay = COALESCE(`+s.parameter(10)+`, can_relay), + can_relay_seen = CASE WHEN `+s.parameter(11)+`::bigint IS NULL THEN can_relay_seen ELSE 1 END + WHERE id = `+s.parameter(12)), name, normalizedIATA, model, firmware, clientVersion, radio, batteryMv, uptimeSecs, noiseFloor, canRelay, canRelay, id, ) @@ -725,7 +857,7 @@ func (s *Store) InsertMetrics(data *MetricsData) error { func (s *Store) PruneOldMetrics(retentionDays int) (int64, error) { cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays).Format(time.RFC3339) // Tagged for /api/perf writer-lock visibility (#1340). - result, err := s.instrumentedExec("prune_metrics", `DELETE FROM observer_metrics WHERE timestamp < $1`, cutoff) + result, err := s.instrumentedExec("prune_metrics", `DELETE FROM observer_metrics WHERE timestamp < `+s.parameter(1), cutoff) if err != nil { return 0, fmt.Errorf("prune metrics: %w", err) } @@ -772,7 +904,7 @@ func (s *Store) BackfillPathJSONAsync() { -- to prevent the infinite re-UPDATE loop fixed in #1119. AND (o.path_json IS NULL OR o.path_json = '') AND t.payload_type != 9 - LIMIT $1`, batchSize) + LIMIT `+s.parameter(1), batchSize) if err != nil { log.Printf("[backfill] path_json query error: %v", err) errored = true @@ -796,7 +928,7 @@ func (s *Store) BackfillPathJSONAsync() { for _, r := range batch { hops, err := packetpath.DecodePathFromRawHex(r.rawHex) if err != nil || len(hops) == 0 { - if _, execErr := s.db.Exec(`UPDATE observations SET path_json = '[]' WHERE id = $1`, r.id); execErr != nil { + if _, execErr := s.db.Exec(`UPDATE observations SET path_json = '[]' WHERE id = `+s.parameter(1), r.id); execErr != nil { log.Printf("[backfill] write error (id=%d): %v", r.id, execErr) } else { s.Stats.IncBackfill("path_json") @@ -804,7 +936,7 @@ func (s *Store) BackfillPathJSONAsync() { continue } b, _ := json.Marshal(hops) - if _, execErr := s.db.Exec(`UPDATE observations SET path_json = $1 WHERE id = $2`, string(b), r.id); execErr != nil { + if _, execErr := s.db.Exec(`UPDATE observations SET path_json = `+s.parameter(1)+` WHERE id = `+s.parameter(2), string(b), r.id); execErr != nil { log.Printf("[backfill] write error (id=%d): %v", r.id, execErr) } else { updated++ @@ -935,9 +1067,9 @@ func (s *Store) backfillTransportCodes(batchSize int) (int, error) { for { rows, err := s.db.Query(` SELECT id, raw_hex FROM transmissions - WHERE id > $1 AND code1 IS NULL AND route_type IN (0, 3) + WHERE id > `+s.parameter(1)+` AND code1 IS NULL AND route_type IN (0, 3) ORDER BY id - LIMIT $2`, lastID, batchSize) + LIMIT `+s.parameter(2), lastID, batchSize) if err != nil { return total, fmt.Errorf("transport_codes select: %w", err) } @@ -968,11 +1100,11 @@ func (s *Store) backfillTransportCodes(batchSize int) (int, error) { } if len(items) > 0 { - tx, err := beginWrite(s.db) + tx, err := beginWrite(s.db, s.Backend()) if err != nil { return total, fmt.Errorf("transport_codes begin: %w", err) } - stmt, err := tx.Prepare(`UPDATE transmissions SET code1 = $1, code2 = $2 WHERE id = $3`) + stmt, err := tx.Prepare(`UPDATE transmissions SET code1 = ` + s.parameter(1) + `, code2 = ` + s.parameter(2) + ` WHERE id = ` + s.parameter(3)) if err != nil { tx.Rollback() return total, fmt.Errorf("transport_codes prepare: %w", err) @@ -1095,17 +1227,17 @@ func (s *Store) compactRelayTouched(now time.Time) { // Returns the number of nodes moved. func (s *Store) MoveStaleNodes(nodeDays int) (int64, error) { cutoff := time.Now().UTC().AddDate(0, 0, -nodeDays).Format(time.RFC3339) - tx, err := beginWrite(s.db) + tx, err := beginWrite(s.db, s.Backend()) if err != nil { return 0, fmt.Errorf("begin tx: %w", err) } defer tx.Rollback() - _, err = tx.Exec(`INSERT INTO inactive_nodes (public_key,name,role,lat,lon,last_seen,first_seen,advert_count,battery_mv,temperature_c,foreign_advert,default_scope,configured_scope,configured_scope_at,multibyte_sup,multibyte_evidence) SELECT public_key,name,role,lat,lon,last_seen,first_seen,advert_count,battery_mv,temperature_c,foreign_advert,default_scope,configured_scope,configured_scope_at,multibyte_sup,multibyte_evidence FROM nodes WHERE last_seen < $1 ON CONFLICT(public_key) DO UPDATE SET name=excluded.name,role=excluded.role,lat=excluded.lat,lon=excluded.lon,last_seen=excluded.last_seen,first_seen=excluded.first_seen,advert_count=excluded.advert_count,battery_mv=excluded.battery_mv,temperature_c=excluded.temperature_c,foreign_advert=excluded.foreign_advert,default_scope=excluded.default_scope,configured_scope=excluded.configured_scope,configured_scope_at=excluded.configured_scope_at,multibyte_sup=excluded.multibyte_sup,multibyte_evidence=excluded.multibyte_evidence`, cutoff) + _, err = tx.Exec(`INSERT INTO inactive_nodes (public_key,name,role,lat,lon,last_seen,first_seen,advert_count,battery_mv,temperature_c,foreign_advert,default_scope,configured_scope,configured_scope_at,multibyte_sup,multibyte_evidence) SELECT public_key,name,role,lat,lon,last_seen,first_seen,advert_count,battery_mv,temperature_c,foreign_advert,default_scope,configured_scope,configured_scope_at,multibyte_sup,multibyte_evidence FROM nodes WHERE last_seen < `+s.parameter(1)+` ON CONFLICT(public_key) DO UPDATE SET name=excluded.name,role=excluded.role,lat=excluded.lat,lon=excluded.lon,last_seen=excluded.last_seen,first_seen=excluded.first_seen,advert_count=excluded.advert_count,battery_mv=excluded.battery_mv,temperature_c=excluded.temperature_c,foreign_advert=excluded.foreign_advert,default_scope=excluded.default_scope,configured_scope=excluded.configured_scope,configured_scope_at=excluded.configured_scope_at,multibyte_sup=excluded.multibyte_sup,multibyte_evidence=excluded.multibyte_evidence`, cutoff) if err != nil { return 0, fmt.Errorf("insert inactive: %w", err) } - result, err := tx.Exec(`DELETE FROM nodes WHERE last_seen < $1`, cutoff) + result, err := tx.Exec(`DELETE FROM nodes WHERE last_seen < `+s.parameter(1), cutoff) if err != nil { return 0, fmt.Errorf("delete stale: %w", err) } @@ -1130,7 +1262,7 @@ func (s *Store) RemoveStaleObservers(observerDays int) (int64, error) { } cutoff := time.Now().UTC().AddDate(0, 0, -observerDays).Format(time.RFC3339) // Tagged for /api/perf writer-lock visibility (#1340). - result, err := s.instrumentedExec("prune_observers", `UPDATE observers SET inactive = 1 WHERE last_seen < $1 AND (inactive IS NULL OR inactive = 0)`, cutoff) + result, err := s.instrumentedExec("prune_observers", `UPDATE observers SET inactive = 1 WHERE last_seen < `+s.parameter(1)+` AND (inactive IS NULL OR inactive = 0)`, cutoff) if err != nil { return 0, fmt.Errorf("mark stale observers inactive: %w", err) } @@ -1165,7 +1297,7 @@ func (s *Store) PurgeStaleObservers(purgeDays int) (int64, error) { result, err := s.instrumentedExec("purge_observers", ` DELETE FROM observers WHERE inactive = 1 - AND last_seen < $1 + AND last_seen < `+s.parameter(1)+` AND NOT EXISTS (SELECT 1 FROM observations o WHERE o.observer_idx = observers.rowid) AND NOT EXISTS (SELECT 1 FROM observer_metrics m WHERE m.observer_id = observers.id) AND NOT EXISTS (SELECT 1 FROM dropped_packets d WHERE d.observer_id = observers.id)`, cutoff) @@ -1193,7 +1325,7 @@ type DroppedPacket struct { // InsertDroppedPacket records a rejected packet in the dropped_packets table. func (s *Store) InsertDroppedPacket(dp *DroppedPacket) error { _, err := s.db.Exec( - `INSERT INTO dropped_packets (hash, raw_hex, reason, observer_id, observer_name, node_pubkey, node_name) VALUES ($1, $2, $3, $4, $5, $6, $7)`, + `INSERT INTO dropped_packets (hash, raw_hex, reason, observer_id, observer_name, node_pubkey, node_name) VALUES (`+s.parameter(1)+`, `+s.parameter(2)+`, `+s.parameter(3)+`, `+s.parameter(4)+`, `+s.parameter(5)+`, `+s.parameter(6)+`, `+s.parameter(7)+`)`, dp.Hash, dp.RawHex, dp.Reason, dp.ObserverID, dp.ObserverName, dp.NodePubKey, dp.NodeName, ) if err != nil { @@ -1210,7 +1342,7 @@ func (s *Store) PruneDroppedPackets(retentionDays int) (int64, error) { return 0, nil } cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays).Format(time.RFC3339) - result, err := s.db.Exec(`DELETE FROM dropped_packets WHERE dropped_at < $1`, cutoff) + result, err := s.db.Exec(`DELETE FROM dropped_packets WHERE dropped_at < `+s.parameter(1), cutoff) if err != nil { return 0, fmt.Errorf("prune dropped packets: %w", err) } @@ -1289,15 +1421,15 @@ func (s *Store) UpdateNodeDefaultScope(pubkey, scope string) error { } // Short-circuit: skip if already stored. var cur sql.NullString - row := s.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = $1`, pubkey) + row := s.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = `+s.parameter(1), pubkey) if row.Scan(&cur) == nil && cur.Valid && cur.String == scope { return nil } - if _, err := s.db.Exec(`UPDATE nodes SET default_scope = $1 WHERE public_key = $2`, scope, pubkey); err != nil { + if _, err := s.db.Exec(`UPDATE nodes SET default_scope = `+s.parameter(1)+` WHERE public_key = `+s.parameter(2), scope, pubkey); err != nil { return err } // Mirror to inactive_nodes (node may be there if recently moved by retention). - _, err := s.db.Exec(`UPDATE inactive_nodes SET default_scope = $1 WHERE public_key = $2`, scope, pubkey) + _, err := s.db.Exec(`UPDATE inactive_nodes SET default_scope = `+s.parameter(1)+` WHERE public_key = `+s.parameter(2), scope, pubkey) return err } @@ -1396,19 +1528,19 @@ func (s *Store) UpdateNodeConfiguredScope(pubkey, scope, reportedAt string) erro // Last-write-wins: skip if the stored confirmation is newer-or-equal. if reportedAt != "" { var curAt sql.NullString - row := s.db.QueryRow(`SELECT configured_scope_at FROM nodes WHERE public_key = $1`, pubkey) + row := s.db.QueryRow(`SELECT configured_scope_at FROM nodes WHERE public_key = `+s.parameter(1), pubkey) if row.Scan(&curAt) == nil && curAt.Valid && curAt.String != "" && curAt.String >= reportedAt { return nil } } if _, err := s.db.Exec( - `UPDATE nodes SET configured_scope = $1, configured_scope_at = $2 WHERE public_key = $3`, + `UPDATE nodes SET configured_scope = `+s.parameter(1)+`, configured_scope_at = `+s.parameter(2)+` WHERE public_key = `+s.parameter(3), scope, reportedAt, pubkey); err != nil { return err } // Mirror to inactive_nodes (node may be there if recently moved by retention). _, err := s.db.Exec( - `UPDATE inactive_nodes SET configured_scope = $1, configured_scope_at = $2 WHERE public_key = $3`, + `UPDATE inactive_nodes SET configured_scope = `+s.parameter(1)+`, configured_scope_at = `+s.parameter(2)+` WHERE public_key = `+s.parameter(3), scope, reportedAt, pubkey) return err } @@ -1433,12 +1565,12 @@ func (s *Store) RecordNaiveSkew(observerID string, deltaSec int64, now time.Time // increments it. _, err := s.db.Exec(` INSERT INTO observers (id, clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at) - VALUES ($1, $2, 1, $3) + VALUES (`+s.parameter(1)+`, `+s.parameter(2)+`, 1, `+s.parameter(3)+`) ON CONFLICT(id) DO UPDATE SET clock_skew_seconds = excluded.clock_skew_seconds, clock_last_naive_at = excluded.clock_last_naive_at, clock_skew_count_24h = CASE - WHEN observers.clock_last_naive_at IS NULL OR observers.clock_last_naive_at < $4 + WHEN observers.clock_last_naive_at IS NULL OR observers.clock_last_naive_at < `+s.parameter(4)+` THEN 1 ELSE COALESCE(observers.clock_skew_count_24h, 0) + 1 END @@ -1796,7 +1928,7 @@ func (s *Store) WriterTx(component string, fn func(*sql.Tx) error) error { writerMu.Lock() wait := time.Since(waitStart) holdStart := time.Now() - tx, err := beginWrite(s.db) + tx, err := beginWrite(s.db, s.Backend()) if err != nil { hold := time.Since(holdStart) writerMu.Unlock() diff --git a/cmd/ingestor/db_test.go b/cmd/ingestor/db_test.go index 28f76f57f..5d0ef086b 100644 --- a/cmd/ingestor/db_test.go +++ b/cmd/ingestor/db_test.go @@ -41,7 +41,7 @@ func TestOpenStore(t *testing.T) { defer s.Close() // Verify tables exist - rows, err := s.db.Query("SELECT table_name FROM information_schema.tables WHERE table_schema=current_schema() ORDER BY table_name") + rows, err := s.db.Query(testNativeSQL(`SELECT name FROM sqlite_master WHERE type='table' ORDER BY name`, "SELECT table_name FROM information_schema.tables WHERE table_schema=current_schema() ORDER BY table_name")) if err != nil { t.Fatal(err) } @@ -70,7 +70,7 @@ func TestOpenStore(t *testing.T) { // Verify packets_v view exists var viewCount int - err = s.db.QueryRow("SELECT COUNT(*) FROM information_schema.views WHERE table_schema=current_schema() AND table_name='packets_v'").Scan(&viewCount) + err = s.db.QueryRow(testNativeSQL(`SELECT COUNT(*) FROM sqlite_master WHERE type='view' AND name='packets_v'`, "SELECT COUNT(*) FROM information_schema.views WHERE table_schema=current_schema() AND table_name='packets_v'")).Scan(&viewCount) if err != nil { t.Fatal(err) } @@ -299,15 +299,15 @@ func TestUpsertObserverWithMeta(t *testing.T) { // Verify typeof returns correct SQLite types var typBattery, typUptime, typNoise string - s.db.QueryRow("SELECT pg_typeof(battery_mv)::text, pg_typeof(uptime_secs)::text, pg_typeof(noise_floor)::text FROM observers WHERE id = 'obs1'"). + s.db.QueryRow(testNativeSQL(`SELECT typeof(battery_mv), typeof(uptime_secs), typeof(noise_floor) FROM observers WHERE id = 'obs1'`, "SELECT pg_typeof(battery_mv)::text, pg_typeof(uptime_secs)::text, pg_typeof(noise_floor)::text FROM observers WHERE id = 'obs1'")). Scan(&typBattery, &typUptime, &typNoise) - if typBattery != "bigint" { + if typBattery != testNativeSQL("integer", "bigint") { t.Errorf("typeof(battery_mv)=%s, want integer", typBattery) } - if typUptime != "bigint" { + if typUptime != testNativeSQL("integer", "bigint") { t.Errorf("typeof(uptime_secs)=%s, want integer", typUptime) } - if typNoise != "double precision" { + if typNoise != testNativeSQL("real", "double precision") { t.Errorf("typeof(noise_floor)=%s, want real", typNoise) } } @@ -461,7 +461,7 @@ func TestSchemaNoiseFloorIsReal(t *testing.T) { defer s.Close() // Check column type affinity via PRAGMA - rows, err := s.db.Query(`SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='observers' ORDER BY ordinal_position`) + rows, err := s.db.Query(testNativeSQL(`SELECT cid,name,type,"notnull",dflt_value,pk FROM pragma_table_info('observers') ORDER BY cid`, `SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='observers' ORDER BY ordinal_position`)) if err != nil { t.Fatal(err) } @@ -1014,7 +1014,7 @@ func TestSchemaCompatibility(t *testing.T) { // Verify column names match what Node.js expects expectedTxCols := []string{"id", "raw_hex", "hash", "first_seen", "route_type", "payload_type", "payload_version", "decoded_json", "created_at"} - rows, _ := s.db.Query(`SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='transmissions' ORDER BY ordinal_position`) + rows, _ := s.db.Query(testNativeSQL(`SELECT cid,name,type,"notnull",dflt_value,pk FROM pragma_table_info('transmissions') ORDER BY cid`, `SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='transmissions' ORDER BY ordinal_position`)) var txCols []string for rows.Next() { var cid int @@ -1042,7 +1042,7 @@ func TestSchemaCompatibility(t *testing.T) { // Verify observations columns expectedObsCols := []string{"id", "transmission_id", "observer_idx", "direction", "snr", "rssi", "score", "path_json", "timestamp"} - rows, _ = s.db.Query(`SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='observations' ORDER BY ordinal_position`) + rows, _ = s.db.Query(testNativeSQL(`SELECT cid,name,type,"notnull",dflt_value,pk FROM pragma_table_info('observations') ORDER BY cid`, `SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='observations' ORDER BY ordinal_position`)) var obsCols []string for rows.Next() { var cid int @@ -1438,7 +1438,7 @@ func TestTelemetryMigrationAddsColumns(t *testing.T) { } var count int - s.db.QueryRow("SELECT COUNT(*) FROM corescope_schema WHERE kind='telemetry' AND ready=true").Scan(&count) + s.db.QueryRow(testNativeSQL(`SELECT COUNT(*) FROM _migrations WHERE name='observers_identity_autoincrement_v1'`, "SELECT COUNT(*) FROM corescope_schema WHERE kind='telemetry' AND ready=true")).Scan(&count) if count != 1 { t.Errorf("canonical telemetry schema should be ready, count=%d", count) } @@ -1600,7 +1600,7 @@ func TestObsTimestampIndexMigration(t *testing.T) { var count int err = s.db.QueryRow( - "SELECT COUNT(*) FROM pg_indexes WHERE schemaname=current_schema() AND indexname='idx_observations_timestamp'", + testNativeSQL(`SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_timestamp'`, "SELECT COUNT(*) FROM pg_indexes WHERE schemaname=current_schema() AND indexname='idx_observations_timestamp'"), ).Scan(&count) if err != nil { t.Fatal(err) @@ -1634,6 +1634,18 @@ func TestObsTimestampIndexMigration(t *testing.T) { t.Fatal(err) } store.Close() + if store.Backend() == "sqlite" { + next, err := openPostgresTestStore(t, key) + if err != nil { + t.Fatal(err) + } + defer next.Close() + var count int + if err = next.db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_timestamp'`).Scan(&count); err != nil || count != 1 { + t.Fatalf("SQLite writer did not restore its index: %d %v", count, err) + } + return + } if next, err := openPostgresTestStore(t, key); err == nil { next.Close() t.Fatal("runtime accepted missing canonical index") @@ -2158,7 +2170,7 @@ func TestScopeNameMigration(t *testing.T) { defer store.Close() // Verify column exists - rows, err := store.db.Query(`SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='transmissions' ORDER BY ordinal_position`) + rows, err := store.db.Query(testNativeSQL(`SELECT cid,name,type,"notnull",dflt_value,pk FROM pragma_table_info('transmissions') ORDER BY cid`, `SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='transmissions' ORDER BY ordinal_position`)) if err != nil { t.Fatalf("PRAGMA: %v", err) } @@ -2839,7 +2851,7 @@ func TestSchemaMultibyteSupColumns(t *testing.T) { defer store.Close() for _, table := range []string{"nodes", "inactive_nodes"} { - rows, err := store.db.Query(`SELECT ordinal_position::int,column_name,data_type,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name=$1`, table) + rows, err := store.db.Query(testNativeSQL(`SELECT cid,name,type,"notnull",dflt_value,pk FROM pragma_table_info($1) ORDER BY cid`, `SELECT ordinal_position::int,column_name,data_type,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name=$1`), table) if err != nil { t.Fatalf("PRAGMA table_info(%s): %v", table, err) } @@ -3122,10 +3134,12 @@ func TestInsertClientRfSample(t *testing.T) { // this is a regression guard. func TestPruneClientRfSamplesUsesIndex(t *testing.T) { s := newTestStore(t) - if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { - t.Fatal(err) + if testBackendValue() == "postgres" { + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } } - rows, err := s.db.Query(`EXPLAIN (COSTS OFF) DELETE FROM client_rf_samples WHERE sampled_at < $1`, "2026-01-01T00:00:00Z") + rows, err := s.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN DELETE FROM client_rf_samples WHERE sampled_at < $1`, `EXPLAIN (COSTS OFF) DELETE FROM client_rf_samples WHERE sampled_at < $1`), "2026-01-01T00:00:00Z") if err != nil { t.Fatal(err) } @@ -3133,7 +3147,7 @@ func TestPruneClientRfSamplesUsesIndex(t *testing.T) { plan := "" for rows.Next() { var detail string - if err := rows.Scan(&detail); err != nil { + if err := scanTestPlan(rows, &detail); err != nil { t.Fatal(err) } plan += detail + "\n" diff --git a/cmd/ingestor/go.mod b/cmd/ingestor/go.mod index 526200b38..7f26c9703 100644 --- a/cmd/ingestor/go.mod +++ b/cmd/ingestor/go.mod @@ -49,6 +49,7 @@ require github.com/meshcore-analyzer/mbcapqueue v0.0.0 replace github.com/meshcore-analyzer/mbcapqueue => ../../internal/mbcapqueue require ( + github.com/mattn/go-sqlite3 v1.14.52 github.com/meshcore-analyzer/channel v0.0.0 github.com/meshcore-analyzer/pgutil v0.0.0 ) diff --git a/cmd/ingestor/go.sum b/cmd/ingestor/go.sum index fca6ac2ea..b17071293 100644 --- a/cmd/ingestor/go.sum +++ b/cmd/ingestor/go.sum @@ -13,6 +13,8 @@ github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index cecf47e68..e163f639c 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -8,6 +8,7 @@ import ( "crypto/tls" "encoding/hex" "encoding/json" + "errors" "flag" "fmt" "log" @@ -25,6 +26,7 @@ import ( "time" mqtt "github.com/eclipse/paho.mqtt.golang" + "github.com/meshcore-analyzer/dbconfig" ) func main() { @@ -45,11 +47,9 @@ func main() { configPath := flag.String("config", "config.json", "path to config file") databaseURL := flag.String("database-url", "", "PostgreSQL telemetry writer URL") stateDir := flag.String("state-dir", "", "local queue and statistics directory") - legacyDB := flag.String("db", "", "removed SQLite path; use the offline importer") + legacyDB := flag.String("db", "", "SQLite telemetry path (bootstrap override)") + backend := flag.String("backend", "", "Storage backend: sqlite or postgres (bootstrap choice)") flag.Parse() - if *legacyDB != "" { - log.Fatal("-db is no longer supported; import SQLite offline and configure -database-url") - } log.SetFlags(log.LstdFlags | log.Lmsgprefix) log.SetPrefix("[ingestor] ") @@ -80,21 +80,32 @@ func main() { sources := cfg.ResolvedSources() - if *databaseURL != "" { - cfg.DatabaseURL = *databaseURL + base, err := os.Getwd() + if err != nil { + log.Fatal("resolve working directory") } - if *stateDir != "" { - cfg.StateDir = *stateDir + rawStorage, err := cfg.storageInputs(base, storageFlags{Backend: dbconfig.Backend(*backend), DBPath: *legacyDB, DatabaseURL: *databaseURL, StateDir: *stateDir}, os.Getenv) + if err != nil { + log.Fatalf("storage: %v", err) } - if cfg.DatabaseURL == "" { - log.Fatal("databaseURL or CORESCOPE_DATABASE_URL is required; migrate legacy SQLite files offline") + storage, selectionLease, err := resolveRuntimeStorage(rawStorage) + if errors.Is(err, dbconfig.ErrSelectionMissing) { + if err = adoptLegacyStorage(rawStorage); err == nil { + storage, selectionLease, err = resolveRuntimeStorage(rawStorage) + } } - store, err := OpenStoreWithState(cfg.DatabaseURL, cfg.StateDir, cfg.MetricsSampleInterval()) + if err != nil { + log.Fatalf("storage: %v", err) + } + defer selectionLease.Close() + cfg.StateDir = storage.StateDir + store, err := openStoreStorage(storage, cfg.DB, cfg.MetricsSampleInterval()) + if err != nil { log.Fatalf("db: %v", err) } defer store.Close() - log.Print("PostgreSQL telemetry writer connected") + log.Printf("%s telemetry writer connected", storage.Backend) // Async backfill: path_json from raw_hex (#888) — must not block MQTT startup store.BackfillPathJSONAsync() @@ -109,6 +120,7 @@ func main() { go store.BackfillFromPubkey(5000, 100*time.Millisecond, nil) // Check auto_vacuum mode and optionally migrate (#919) + store.CheckAutoVacuum(cfg) channelKeys := loadChannelKeys(cfg, *configPath) if len(channelKeys) > 0 { @@ -117,9 +129,9 @@ func main() { log.Printf("No channel keys loaded — GRP_TXT packets will not be decrypted") } - keySet := newChannelKeySet(channelKeys, cfg.ApprovedChannelsURL(), cfg.ApprovedChannelsMax()) + keySet := newChannelKeySetStorage(channelKeys, storage, cfg.ApprovedChannelsMax()) if cfg.ApprovedChannelsEnabled() { - logApprovedChannelsSource(cfg.ApprovedChannelsURL()) + log.Printf("[channels] approved channel storage: %s", storage.Backend) keySet.refresh() go func() { t := time.NewTicker(approvedChannelsRefresh) @@ -361,6 +373,9 @@ func main() { } } + vacuumPages := cfg.IncrementalVacuumPages() + store.RunIncrementalVacuum(vacuumPages) + // Gate open: the synchronous startup writes above cannot return until the // single serialized writer is free. WaitForAsyncMigrations() makes that // explicit. Now drain everything the subscription buffered during startup. @@ -384,7 +399,7 @@ func main() { go func() { for range retentionTicker.C { store.MoveStaleNodes(nodeDays) - + store.RunIncrementalVacuum(vacuumPages) } }() @@ -394,10 +409,12 @@ func main() { time.Sleep(90 * time.Second) // stagger after metrics prune store.RemoveStaleObservers(observerDays) store.PurgeStaleObservers(observerPurgeDays) + store.RunIncrementalVacuum(vacuumPages) for range observerRetentionTicker.C { store.RemoveStaleObservers(observerDays) store.PurgeStaleObservers(observerPurgeDays) + store.RunIncrementalVacuum(vacuumPages) } }() @@ -408,7 +425,7 @@ func main() { for range metricsRetentionTicker.C { store.PruneOldMetrics(metricsDays) store.PruneDroppedPackets(metricsDays) - + store.RunIncrementalVacuum(vacuumPages) } }() @@ -421,7 +438,7 @@ func main() { if n, err := store.PruneOldPackets(packetDays); err != nil { log.Printf("[prune] error: %v", err) } else if n > 0 { - + store.RunIncrementalVacuum(vacuumPages) } } }() @@ -441,28 +458,28 @@ func main() { if n, err := store.PruneOldClientReceptions(clientRxDays); err != nil { log.Printf("[prune] error: %v", err) } else if n > 0 { - + store.RunIncrementalVacuum(vacuumPages) } } if clientRxObsDays > 0 { if n, err := store.PruneOldClientRxObservations(clientRxObsDays); err != nil { log.Printf("[prune] client_rx_observations: %v", err) } else if n > 0 { - + store.RunIncrementalVacuum(vacuumPages) } } if clientRfDays > 0 { if n, err := store.PruneOldClientRfSamples(clientRfDays); err != nil { log.Printf("[prune] client_rf_samples: %v", err) } else if n > 0 { - + store.RunIncrementalVacuum(vacuumPages) } } if clientRegionsDays > 0 { if n, err := store.PruneOldClientDeclaredRegions(clientRegionsDays); err != nil { log.Printf("[prune] node_declared_regions: %v", err) } else if n > 0 { - + store.RunIncrementalVacuum(vacuumPages) } } } @@ -497,7 +514,60 @@ func main() { log.Printf("[regions] auto-derived region keys enabled: refreshing every %v, cap %d", interval, cfg.AutoRegionKeysMaxDerived()) } - // PostgreSQL autovacuum, checkpoints and planner statistics are owned by the database service. + // PostgreSQL maintenance remains owned by its database service. + if storage.Backend == dbconfig.SQLite { + // Hourly WAL checkpoint to prevent unbounded WAL growth. + // TRUNCATE resets the WAL file to zero bytes when all frames are flushed; + // if the server's read connection holds frames, remaining pages stay in the + // WAL until the next tick. Staggered 30s after startup to avoid competing + // with the initial burst of ingest writes. + walCheckpointTicker := time.NewTicker(1 * time.Hour) + go func() { + time.Sleep(30 * time.Second) + store.Checkpoint() + for range walCheckpointTicker.C { + store.Checkpoint() + } + }() + log.Printf("[db] WAL checkpoint scheduled every 1h") + + // Daily planner statistics refresh (#2058), in two parts. + // + // The routine refresh is staggered 2 minutes past startup for the same reason + // as the checkpoint above: it takes the write lock, and by then the initial + // ingest burst has passed, so it also sees the rows that burst added. + // + // The build in front of it deliberately does compete with that burst, because + // a database with no statistics at all has nothing better to offer the queries + // arriving in those 2 minutes. It only runs once per database; see + // Store.EnsurePlannerStats, which also carries what that costs. + // + // Bounded by analysis_limit either way, so neither grows with the file the way + // an unbounded ANALYZE does: 2.0s against 242.9s on a 9.4 GB database, both + // timed warm. Cold, on a first start, it is 3m43.9s. + { + analysisLimit := cfg.AnalysisLimit() + if analysisLimit < 0 { + log.Printf("[analyze] planner statistics refresh disabled (db.analysisLimit=%d)", analysisLimit) + } else { + analyzeTicker := time.NewTicker(24 * time.Hour) + go func() { + // Before the stagger, and only on a database that has never been + // analyzed: the stagger is a 2 minute window in which the first + // query would otherwise run on no statistics at all. A restart + // finds sqlite_stat1 already in the file and skips this. + store.EnsurePlannerStats(analysisLimit) + time.Sleep(2 * time.Minute) + store.RefreshPlannerStats(analysisLimit) + for range analyzeTicker.C { + store.RefreshPlannerStats(analysisLimit) + } + }() + log.Printf("[analyze] planner statistics refresh scheduled every 24h (analysis_limit=%d)", analysisLimit) + } + } + + } // Daily neighbor_edges retention (#1287 — moved from cmd/server). { diff --git a/cmd/ingestor/maintenance.go b/cmd/ingestor/maintenance.go index a1afa76ae..43cbb73a6 100644 --- a/cmd/ingestor/maintenance.go +++ b/cmd/ingestor/maintenance.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "log" + "runtime" "time" "github.com/meshcore-analyzer/dbschema" @@ -64,6 +65,9 @@ const ( // returned alongside it — those rows are gone, so reporting 0 would be // wrong. func (s *Store) PruneOldPackets(days int) (int64, error) { + return s.pruneOldPackets(days, runtime.Gosched) +} +func (s *Store) pruneOldPackets(days int, betweenBatches func()) (int64, error) { if days <= 0 { return 0, nil } @@ -94,6 +98,8 @@ func (s *Store) PruneOldPackets(days int) (int64, error) { if batch < pruneBatchTransmissions { break } + // Yield only after the transaction commits and releases the writer. + betweenBatches() } if total > 0 { log.Printf("[prune] deleted %d transmissions older than %d days", total, days) @@ -114,13 +120,13 @@ func (s *Store) PruneOldClientReceptions(days int) (int64, error) { var n int64 err := s.WriterTx("prune_client_receptions", func(tx *sql.Tx) error { - res, err := tx.Exec(`DELETE FROM client_receptions WHERE rx_at < $1`, cutoff) + res, err := tx.Exec(`DELETE FROM client_receptions WHERE rx_at < `+s.parameter(1), cutoff) if err != nil { return fmt.Errorf("prune client_receptions: %w", err) } n, _ = res.RowsAffected() // Drop companion name rows not refreshed within the window. - if _, err := tx.Exec(`DELETE FROM client_observers WHERE last_seen < $1`, cutoff); err != nil { + if _, err := tx.Exec(`DELETE FROM client_observers WHERE last_seen < `+s.parameter(1), cutoff); err != nil { return fmt.Errorf("prune client_observers: %w", err) } return nil @@ -146,7 +152,7 @@ func (s *Store) PruneOldClientRxObservations(days int) (int64, error) { return 0, nil } cutoff := time.Now().UTC().AddDate(0, 0, -days).Format(rxTimeMillisLayout) - res, err := s.db.Exec(`DELETE FROM client_rx_observations WHERE rx_at < $1`, cutoff) + res, err := s.db.Exec(`DELETE FROM client_rx_observations WHERE rx_at < `+s.parameter(1), cutoff) if err != nil { return 0, fmt.Errorf("prune client_rx_observations: %w", err) } @@ -169,7 +175,7 @@ func (s *Store) PruneOldClientRfSamples(days int) (int64, error) { return 0, nil } cutoff := time.Now().UTC().AddDate(0, 0, -days).Format(rxTimeMillisLayout) - res, err := s.db.Exec(`DELETE FROM client_rf_samples WHERE sampled_at < $1`, cutoff) + res, err := s.db.Exec(`DELETE FROM client_rf_samples WHERE sampled_at < `+s.parameter(1), cutoff) if err != nil { return 0, fmt.Errorf("prune client_rf_samples: %w", err) } @@ -207,7 +213,7 @@ func (s *Store) PruneOldClientDeclaredRegions(days int) (int64, error) { // specific side of it. func (s *Store) pruneOldClientDeclaredRegionsAt(cutoffInstant time.Time) (int64, error) { cutoff := cutoffInstant.Format(rxTimeMillisLayout) - res, err := s.db.Exec(`DELETE FROM node_declared_regions WHERE observed_at < $1`, cutoff) + res, err := s.db.Exec(`DELETE FROM node_declared_regions WHERE observed_at < `+s.parameter(1), cutoff) if err != nil { return 0, fmt.Errorf("prune node_declared_regions: %w", err) } @@ -237,7 +243,7 @@ func (s *Store) PruneNeighborEdges(maxAgeDays int) (int64, error) { return 0, nil } cutoff := time.Now().UTC().Add(-time.Duration(maxAgeDays) * 24 * time.Hour).Format(time.RFC3339) - res, err := s.db.Exec("DELETE FROM neighbor_edges WHERE last_seen < $1", cutoff) + res, err := s.db.Exec("DELETE FROM neighbor_edges WHERE last_seen < "+s.parameter(1), cutoff) if err != nil { return 0, fmt.Errorf("prune neighbor_edges: %w", err) } @@ -299,7 +305,7 @@ func (s *Store) BackfillFromPubkey(chunkSize int, yieldDuration time.Duration, p } log.Printf("[backfill] from_pubkey starting: %d ADVERT rows", total) - stmt, err := s.db.Prepare("UPDATE transmissions SET from_pubkey = $1 WHERE id = $2") + stmt, err := s.db.Prepare("UPDATE transmissions SET from_pubkey = " + s.parameter(1) + " WHERE id = " + s.parameter(2)) if err != nil { log.Printf("[backfill] from_pubkey prepare: %v", err) return @@ -309,7 +315,7 @@ func (s *Store) BackfillFromPubkey(chunkSize int, yieldDuration time.Duration, p var processed int64 for { rows, err := s.db.Query( - "SELECT id, decoded_json FROM transmissions WHERE from_pubkey IS NULL AND payload_type = 4 LIMIT $1", + "SELECT id, decoded_json FROM transmissions WHERE from_pubkey IS NULL AND payload_type = 4 LIMIT "+s.parameter(1), chunkSize) if err != nil { log.Printf("[backfill] from_pubkey select: %v", err) @@ -333,7 +339,7 @@ func (s *Store) BackfillFromPubkey(chunkSize int, yieldDuration time.Duration, p break } - tx, err := beginWrite(s.db) + tx, err := beginWrite(s.db, s.Backend()) if err != nil { log.Printf("[backfill] from_pubkey begin tx: %v", err) return diff --git a/cmd/ingestor/multibyte_persist.go b/cmd/ingestor/multibyte_persist.go index 9d51cf498..b24058b4f 100644 --- a/cmd/ingestor/multibyte_persist.go +++ b/cmd/ingestor/multibyte_persist.go @@ -76,7 +76,7 @@ func (s *Store) RunMultibyteCapPersist() (MultibyteCapPersistStats, error) { return stats, nil } - tx, err := beginWrite(s.db) + tx, err := beginWrite(s.db, s.Backend()) if err != nil { return stats, err } @@ -85,19 +85,19 @@ func (s *Store) RunMultibyteCapPersist() (MultibyteCapPersistStats, error) { // inactive_nodes. The pre-#1386 implementation issued one UPDATE // against each table per entry — 50% guaranteed-empty. We now // look up the table once, then issue the matching UPDATE. - stmtN, err := tx.Prepare(`UPDATE nodes SET multibyte_sup=$1, multibyte_evidence=$2 WHERE public_key=$3`) + stmtN, err := tx.Prepare(`UPDATE nodes SET multibyte_sup=` + s.parameter(1) + `, multibyte_evidence=` + s.parameter(2) + ` WHERE public_key=` + s.parameter(3)) if err != nil { return stats, err } defer stmtN.Close() - stmtI, err := tx.Prepare(`UPDATE inactive_nodes SET multibyte_sup=$1, multibyte_evidence=$2 WHERE public_key=$3`) + stmtI, err := tx.Prepare(`UPDATE inactive_nodes SET multibyte_sup=` + s.parameter(1) + `, multibyte_evidence=` + s.parameter(2) + ` WHERE public_key=` + s.parameter(3)) if err != nil { return stats, err } defer stmtI.Close() // Membership probe: one indexed PK lookup. Cheap; avoids the // guaranteed-miss second UPDATE. - stmtProbe, err := tx.Prepare(`SELECT 1 FROM nodes WHERE public_key=$1 LIMIT 1`) + stmtProbe, err := tx.Prepare(`SELECT 1 FROM nodes WHERE public_key=` + s.parameter(1) + ` LIMIT 1`) if err != nil { return stats, err } @@ -188,7 +188,7 @@ func containsCI(s, sub string) bool { // internal/dbschema migration (#1386). func (s *Store) hasMultibyteSupColumns() bool { var found bool - _ = s.db.QueryRow(`SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='nodes' AND column_name='multibyte_sup')`).Scan(&found) + _ = s.db.QueryRow(s.nativeSQL(`SELECT EXISTS(SELECT 1 FROM pragma_table_info('nodes') WHERE name='multibyte_sup')`, `SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='nodes' AND column_name='multibyte_sup')`)).Scan(&found) return found } diff --git a/cmd/ingestor/multibyte_persist_helpers_test.go b/cmd/ingestor/multibyte_persist_helpers_test.go index d82501d37..bee01ce3a 100644 --- a/cmd/ingestor/multibyte_persist_helpers_test.go +++ b/cmd/ingestor/multibyte_persist_helpers_test.go @@ -58,7 +58,7 @@ func logContains(buf *syncBuffer, substr string) bool { func columnExists(t *testing.T, db *sql.DB, table, col string) bool { t.Helper() var found bool - if err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name=$1 AND column_name=$2)`, table, col).Scan(&found); err != nil { + if err := db.QueryRow(testNativeSQL(`SELECT EXISTS(SELECT 1 FROM pragma_table_info(?1) WHERE name=?2)`, `SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name=$1 AND column_name=$2)`), table, col).Scan(&found); err != nil { t.Fatal(err) } return found diff --git a/cmd/ingestor/neighbor_builder.go b/cmd/ingestor/neighbor_builder.go index 8c77d6c6e..01ba7369e 100644 --- a/cmd/ingestor/neighbor_builder.go +++ b/cmd/ingestor/neighbor_builder.go @@ -197,9 +197,9 @@ func (s *Store) buildAndPersistNeighborEdges(trust *packetpath.TrustConfig) (int FROM observations o JOIN transmissions t ON t.id = o.transmission_id LEFT JOIN observers obs ON obs.rowid = o.observer_idx - WHERE o.timestamp > $1 + WHERE o.timestamp > `+s.parameter(1)+` ORDER BY o.timestamp - LIMIT $2`, watermarkEpoch, neighborBuilderMaxBatch) + LIMIT `+s.parameter(2), watermarkEpoch, neighborBuilderMaxBatch) if err != nil { return 0, fmt.Errorf("scan observations: %w", err) } @@ -281,16 +281,18 @@ func (s *Store) buildAndPersistNeighborEdges(trust *packetpath.TrustConfig) (int if len(values) == 0 { return nil } - _, err := tx.Exec(`INSERT INTO neighbor_edges(node_a,node_b,count,last_seen) VALUES `+strings.Join(values, ",")+` + _, err := tx.Exec(`INSERT INTO neighbor_edges(node_a,node_b,count,last_seen) VALUES `+strings.Join(values, ",")+s.nativeSQL(` ON CONFLICT(node_a,node_b) DO UPDATE SET count=neighbor_edges.count+excluded.count, - last_seen=CASE WHEN neighbor_edges.last_seen IS NULL OR excluded.last_seen IS NULL THEN NULL ELSE GREATEST(neighbor_edges.last_seen,excluded.last_seen) END`, args...) + last_seen=CASE WHEN neighbor_edges.last_seen IS NULL OR excluded.last_seen IS NULL THEN NULL ELSE MAX(neighbor_edges.last_seen,excluded.last_seen) END`, ` + ON CONFLICT(node_a,node_b) DO UPDATE SET count=neighbor_edges.count+excluded.count, + last_seen=CASE WHEN neighbor_edges.last_seen IS NULL OR excluded.last_seen IS NULL THEN NULL ELSE GREATEST(neighbor_edges.last_seen,excluded.last_seen) END`), args...) values = values[:0] args = args[:0] return err } for _, e := range counts { n := len(args) - values = append(values, fmt.Sprintf("($%d,$%d,$%d,$%d)", n+1, n+2, n+3, n+4)) + values = append(values, fmt.Sprintf("(%s,%s,%s,%s)", s.parameter(n+1), s.parameter(n+2), s.parameter(n+3), s.parameter(n+4))) args = append(args, e.a, e.b, e.count, e.ts) if len(values) == batchSize { if err := flush(); err != nil { diff --git a/cmd/ingestor/neighbor_builder_delta_test.go b/cmd/ingestor/neighbor_builder_delta_test.go index 3d8742e20..a60792a5f 100644 --- a/cmd/ingestor/neighbor_builder_delta_test.go +++ b/cmd/ingestor/neighbor_builder_delta_test.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "path/filepath" "testing" "time" @@ -60,6 +61,39 @@ func TestNeighborEdgesBuilderDeltaScan(t *testing.T) { // Seed the same 100K logical rows in one transaction without per-row RPCs. seed := func(prefix string, start int64, count int) { t.Helper() + if store.Backend() == "sqlite" { + tx, err := store.db.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + parents, err := tx.Prepare(`INSERT INTO transmissions(raw_hex,hash,first_seen,route_type,payload_type,payload_version,decoded_json,from_pubkey) VALUES('',?1,?2,0,?3,0,'{}','aaaaaaaaaa')`) + if err != nil { + t.Fatal(err) + } + defer parents.Close() + children, err := tx.Prepare(`INSERT INTO observations(transmission_id,observer_idx,path_json,timestamp) VALUES(?1,?2,'["bb"]',?3)`) + if err != nil { + t.Fatal(err) + } + defer children.Close() + for i := 0; i < count; i++ { + stamp := start + int64(i) + res, err := parents.Exec(fmt.Sprint(prefix, i), time.Unix(stamp, 0).UTC().Format(time.RFC3339), payloadADVERT) + if err != nil { + t.Fatal(err) + } + id, _ := res.LastInsertId() + if _, err = children.Exec(id, obsRowid, stamp); err != nil { + t.Fatal(err) + } + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + return + } + _, err := store.db.Exec(`WITH inserted AS ( INSERT INTO transmissions(raw_hex,hash,first_seen,route_type,payload_type,payload_version,decoded_json,from_pubkey) SELECT '',$1::text||n::text,to_char(to_timestamp($2::bigint+n) AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS"Z"'),0,$3,0,'{}','aaaaaaaaaa' diff --git a/cmd/ingestor/postgres_config_test.go b/cmd/ingestor/postgres_config_test.go index b03e867b8..abaec5c01 100644 --- a/cmd/ingestor/postgres_config_test.go +++ b/cmd/ingestor/postgres_config_test.go @@ -1,6 +1,7 @@ package main import ( + "github.com/meshcore-analyzer/dbconfig" "os" "path/filepath" "strings" @@ -33,11 +34,11 @@ func TestPostgresConfigurationPrecedence(t *testing.T) { } } -func TestOpenStoreRejectsLegacyPathsWithoutCreatingFiles(t *testing.T) { +func TestSelectedPostgresCannotFallBackToSQLiteFiles(t *testing.T) { path := filepath.Join(t.TempDir(), "missing", "database.db") - if s, err := OpenStore(path); err == nil { + if s, err := OpenStoreStorage(dbconfig.Storage{Backend: dbconfig.Postgres, DBPath: path}, nil); err == nil { s.Close() - t.Fatal("SQLite path accepted as DSN") + t.Fatal("missing PostgreSQL URL fell back to SQLite") } if _, err := os.Stat(filepath.Dir(path)); !os.IsNotExist(err) { t.Fatal("legacy path created a directory") diff --git a/cmd/ingestor/postgres_test.go b/cmd/ingestor/postgres_test.go index 7dc7a4354..dcb997fff 100644 --- a/cmd/ingestor/postgres_test.go +++ b/cmd/ingestor/postgres_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/dbschema" "github.com/meshcore-analyzer/pgutil" "github.com/meshcore-analyzer/pgutil/pgtest" @@ -22,6 +23,9 @@ func openPostgresTestStore(t testing.TB, key string) (*Store, error) { return openPostgresTestStoreInterval(t, key, 300) } func openPostgresTestStoreInterval(t testing.TB, key string, interval int) (*Store, error) { + if testBackend(t) == dbconfig.SQLite { + return OpenStoreWithInterval(key, interval) + } stateDir := filepath.Dir(key) if strings.HasPrefix(key, "postgres") { stateDir = t.TempDir() @@ -32,6 +36,9 @@ func openPostgresTestStoreInterval(t testing.TB, key string, interval int) (*Sto // A historical fixture path is only a test key. Runtime OpenStore accepts // PostgreSQL URLs exclusively, with a real restricted writer credential. func testPostgresURL(t testing.TB, key string) string { + if testBackend(t) != dbconfig.Postgres { + t.Skip("PostgreSQL matrix not selected") + } t.Helper() if strings.HasPrefix(key, "postgres://") || strings.HasPrefix(key, "postgresql://") { return key @@ -79,6 +86,9 @@ func testPostgresURL(t testing.TB, key string) string { } func testAdmin(t testing.TB, s *Store) *sql.DB { + if s.Backend() == dbconfig.SQLite { + return s.db + } t.Helper() var schema string if err := s.db.QueryRow(`SELECT current_schema()`).Scan(&schema); err != nil { diff --git a/cmd/ingestor/pragma_parity_test.go b/cmd/ingestor/pragma_parity_test.go index 29f11beef..bfa5c48bc 100644 --- a/cmd/ingestor/pragma_parity_test.go +++ b/cmd/ingestor/pragma_parity_test.go @@ -5,12 +5,16 @@ import "testing" // Durable settings are measured through the runtime connection, not logs. func TestOpenStoreDurability(t *testing.T) { s := newTestStore(t) - for _, setting := range []string{"fsync", "full_page_writes", "synchronous_commit"} { + settings := map[string]string{"SHOW fsync": "on", "SHOW full_page_writes": "on", "SHOW synchronous_commit": "on"} + if s.Backend() == "sqlite" { + settings = map[string]string{"PRAGMA journal_mode": "wal", "PRAGMA synchronous": "2", "PRAGMA foreign_keys": "1"} + } + for setting, want := range settings { var got string - if err := s.db.QueryRow("SHOW " + setting).Scan(&got); err != nil { + if err := s.db.QueryRow(setting).Scan(&got); err != nil { t.Fatal(err) } - if got != "on" { + if got != want { t.Errorf("%s=%s; require on", setting, got) } } diff --git a/cmd/ingestor/prune_chunked_test.go b/cmd/ingestor/prune_chunked_test.go index 567f6396c..f3e186e2d 100644 --- a/cmd/ingestor/prune_chunked_test.go +++ b/cmd/ingestor/prune_chunked_test.go @@ -250,17 +250,19 @@ func TestPruneAgedTransmissionIDsHasOrderedIndex(t *testing.T) { "observations delete": pruneObservationsBatch, "transmissions delete": pruneTransmissionsBatch, } { - if _, err := store.db.Exec(`SET enable_seqscan=off; SET enable_bitmapscan=off`); err != nil { - t.Fatal(err) + if testBackendValue() == "postgres" { + if _, err := store.db.Exec(`SET enable_seqscan=off; SET enable_bitmapscan=off`); err != nil { + t.Fatal(err) + } } - rows, err := store.db.Query("EXPLAIN (COSTS OFF) "+q, cutoff, pruneBatchTransmissions) + rows, err := store.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN `, "EXPLAIN (COSTS OFF) ")+q, cutoff, pruneBatchTransmissions) if err != nil { - t.Fatalf("%s: EXPLAIN (COSTS OFF): %v", name, err) + t.Fatalf(testNativeSQL(`%s: EXPLAIN QUERY PLAN: %v`, "%s: EXPLAIN (COSTS OFF): %v"), name, err) } var steps []string for rows.Next() { var detail string - if err := rows.Scan(&detail); err != nil { + if err := scanTestPlan(rows, &detail); err != nil { rows.Close() t.Fatalf("%s: scan plan row: %v", name, err) } diff --git a/cmd/ingestor/prune_geofilter.go b/cmd/ingestor/prune_geofilter.go index 6464c044f..04316415d 100644 --- a/cmd/ingestor/prune_geofilter.go +++ b/cmd/ingestor/prune_geofilter.go @@ -33,7 +33,7 @@ func (s *Store) DeleteNodesByPubkeys(pubkeys []string) (int64, error) { batch := pubkeys[start:end] slots := make([]string, len(batch)) for i := range slots { - slots[i] = fmt.Sprintf("$%d", i+1) + slots[i] = fmt.Sprintf("%s", s.parameter(i+1)) } placeholders := strings.Join(slots, ",") args := make([]interface{}, len(batch)) diff --git a/cmd/ingestor/refresh_planner_stats_test.go b/cmd/ingestor/refresh_planner_stats_test.go index a83c2fb10..a04801709 100644 --- a/cmd/ingestor/refresh_planner_stats_test.go +++ b/cmd/ingestor/refresh_planner_stats_test.go @@ -1,9 +1,12 @@ package main import ( + "bytes" "github.com/meshcore-analyzer/dbconfig" + "log" "os" "path/filepath" + "strings" "testing" ) @@ -59,6 +62,9 @@ func TestAnalysisLimitSurvivesTheConfigFile_Issue2058(t *testing.T) { } func TestPostgresOwnerAnalyzesTelemetry(t *testing.T) { + if testBackend(t) != dbconfig.Postgres { + t.Skip("PostgreSQL planner matrix not selected") + } s := newTestStore(t) owner := testAdmin(t, s) if _, err := owner.Exec(`INSERT INTO transmissions(raw_hex,hash,first_seen) SELECT 'aa', n::text,'2026-01-01T00:00:00Z' FROM generate_series(1,1000) n`); err != nil { @@ -76,8 +82,230 @@ func TestPostgresOwnerAnalyzesTelemetry(t *testing.T) { } } func TestPostgresRuntimeCannotAlterPlannerTargets(t *testing.T) { + if testBackend(t) != dbconfig.Postgres { + t.Skip("PostgreSQL planner matrix not selected") + } s := newTestStore(t) if _, err := s.db.Exec(`ALTER TABLE transmissions ALTER COLUMN payload_type SET STATISTICS 1000`); err == nil { t.Fatal("runtime role changed owner planner policy") } } + +func TestRefreshPlannerStatsWritesStatistics_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + + s := newTestStore(t) + defer s.Close() + + if hasStat1(t, s) { + t.Fatal("a fresh store already carries sqlite_stat1, so this test cannot tell whether the refresh did anything") + } + + if !s.RefreshPlannerStats(10000) { + t.Fatal("RefreshPlannerStats reported no refresh") + } + + if !hasStat1(t, s) { + t.Error("sqlite_stat1 was not created, so the planner still has no statistics") + } +} + +func TestRefreshPlannerStatsAppliesTheLimit_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + + s := newTestStore(t) + defer s.Close() + + s.RefreshPlannerStats(250) + + // analysis_limit is per connection. The store runs SetMaxOpenConns(1) + // (db.go:142), which is the only reason setting it through Exec is sound + // here: on a multi-connection pool the pragma could land on a connection + // the ANALYZE never uses, and the limit would silently not apply. + var limit int + if err := s.db.QueryRow("PRAGMA analysis_limit").Scan(&limit); err != nil { + t.Fatalf("read back analysis_limit: %v", err) + } + if limit != 250 { + t.Errorf("analysis_limit did not reach the connection: want 250, got %d", limit) + } +} + +func TestRefreshPlannerStatsNegativeLimitIsANoop_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + + s := newTestStore(t) + defer s.Close() + + if s.RefreshPlannerStats(-1) { + t.Error("a negative limit must not report a refresh") + } + if hasStat1(t, s) { + t.Error("a negative limit still built sqlite_stat1; the refresh is not actually disabled") + } +} + +func TestRefreshPlannerStatsIsRepeatable_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + + s := newTestStore(t) + defer s.Close() + + // The ticker calls this every 24h for the life of the process. A second + // call must not error or undo the first, which is the part a single-call + // test would not notice. + s.RefreshPlannerStats(10000) + if !s.RefreshPlannerStats(10000) { + t.Fatal("the second refresh reported failure") + } + if !hasStat1(t, s) { + t.Error("sqlite_stat1 disappeared across two refreshes") + } +} + +func TestEnsurePlannerStatsBuildsWhenAbsent_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + + s := newTestStore(t) + defer s.Close() + + if !s.EnsurePlannerStats(10000) { + t.Fatal("EnsurePlannerStats did not build statistics on a database that has none") + } + if !hasStat1(t, s) { + t.Error("sqlite_stat1 absent after EnsurePlannerStats") + } +} + +func TestEnsurePlannerStatsSkipsWhenPresent_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + + s := newTestStore(t) + defer s.Close() + + s.RefreshPlannerStats(10000) + // The point of the skip: on every restart after the first this must cost one + // sqlite_master query, not an ANALYZE. If it ever returns true here it is + // running the 2s refresh on every boot. + if s.EnsurePlannerStats(10000) { + t.Error("EnsurePlannerStats rebuilt statistics that were already there") + } +} + +func TestEnsurePlannerStatsRespectsDisabled_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + + s := newTestStore(t) + defer s.Close() + + if s.EnsurePlannerStats(-1) { + t.Error("a negative limit must not build statistics") + } + if hasStat1(t, s) { + t.Error("sqlite_stat1 built despite the refresh being disabled") + } +} + +func TestPlannerStatsSurviveReopen_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + + dir := t.TempDir() + dbPath := dir + "/reopen.db" + + first, err := OpenStore(dbPath) + if err != nil { + t.Fatal(err) + } + first.WaitForAsyncMigrations() + if !first.EnsurePlannerStats(10000) { + t.Fatal("first store did not build statistics") + } + first.Close() + + second, err := OpenStore(dbPath) + if err != nil { + t.Fatal(err) + } + defer second.Close() + second.WaitForAsyncMigrations() + + if !hasStat1(t, second) { + t.Fatal("statistics did not survive reopening the database") + } + if second.EnsurePlannerStats(10000) { + t.Error("the reopened store rebuilt statistics, so a restart would pay for an ANALYZE it does not need") + } +} + +func TestEnsurePlannerStatsWarnsBeforeBuilding_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + + s := newTestStore(t) + defer s.Close() + + var buf bytes.Buffer + orig := log.Writer() + log.SetOutput(&buf) + defer log.SetOutput(orig) + + s.EnsurePlannerStats(10000) + + out := buf.String() + for _, want := range []string{"no planner statistics", "write connection", "Once per database"} { + if !strings.Contains(out, want) { + t.Errorf("the first-build warning does not mention %q; an operator seeing ingest stall gets no explanation.\ngot: %s", want, out) + } + } + if i, j := strings.Index(out, "no planner statistics"), strings.Index(out, "statistics built in"); i == -1 || j == -1 || i > j { + t.Errorf("the warning must come before the completion line, so it is visible while the write path is held; got: %s", out) + } +} + +func TestEnsurePlannerStatsIsQuietWhenPresent_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + + s := newTestStore(t) + defer s.Close() + s.RefreshPlannerStats(10000) + + var buf bytes.Buffer + orig := log.Writer() + log.SetOutput(&buf) + defer log.SetOutput(orig) + + s.EnsurePlannerStats(10000) + + if out := buf.String(); strings.Contains(out, "no planner statistics") { + t.Errorf("warned about building on a database that already has statistics: %s", out) + } +} + +func hasStat1(t *testing.T, s *Store) bool { + t.Helper() + var n int + if err := s.db.QueryRow( + `SELECT count(*) FROM sqlite_master WHERE type='table' AND name='sqlite_stat1'`).Scan(&n); err != nil { + t.Fatalf("query sqlite_master: %v", err) + } + return n > 0 +} diff --git a/cmd/ingestor/region_keys.go b/cmd/ingestor/region_keys.go index 376affeed..26b9c5f41 100644 --- a/cmd/ingestor/region_keys.go +++ b/cmd/ingestor/region_keys.go @@ -129,7 +129,7 @@ func (s *Store) declaredRegionSources() ([]declaredRegionStat, error) { rows.Close() var present string - if err := s.db.QueryRow(`SELECT table_name FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='node_declared_regions'`).Scan(&present); err == nil && present != "" { + if err := s.db.QueryRow(s.nativeSQL("SELECT name FROM sqlite_master WHERE type='table' AND name='node_declared_regions'", `SELECT table_name FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='node_declared_regions'`)).Scan(&present); err == nil && present != "" { ndr, err := s.db.Query(` WITH ranked AS ( SELECT target, observed_at, regions_csv, diff --git a/cmd/ingestor/retention_yield_test.go b/cmd/ingestor/retention_yield_test.go new file mode 100644 index 000000000..3ad45d8ca --- /dev/null +++ b/cmd/ingestor/retention_yield_test.go @@ -0,0 +1,103 @@ +package main + +import ( + "context" + "fmt" + "testing" + "time" +) + +func TestPruneYieldsAfterCommittedFullBatches(t *testing.T) { + s := openPruneStore(t, "yield.db") + const aged = pruneBatchTransmissions*2 + 7 + seedRetentionFanout(t, s, aged, aged) + yields := 0 + removed, err := s.pruneOldPackets(5, func() { + yields++ + if !writerMu.TryLock() { + t.Error("scheduling action still holds writerMu") + return + } + writerMu.Unlock() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + conn, err := s.db.Conn(ctx) + if err != nil { + t.Error("scheduling action still occupies the single DB connection", err) + return + } + var remaining int + err = conn.QueryRowContext(ctx, `SELECT count(*) FROM transmissions WHERE first_seen < '2021-01-01'`).Scan(&remaining) + conn.Close() + if err != nil || remaining != aged-yields*pruneBatchTransmissions { + t.Error("scheduling action preceded a committed full batch", remaining, err) + return + } + done := make(chan error, 1) + sequence := yields + go func() { + _, err := s.InsertTransmission(&PacketData{ + RawHex: "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976", + Hash: fmt.Sprint("scheduled-live-", sequence), Timestamp: time.Now().UTC().Format(time.RFC3339), + RouteType: 2, PayloadType: 2, PathJSON: "[]", DecodedJSON: "{}", + }) + done <- err + }() + select { + case err := <-done: + if err != nil { + t.Error(err) + } + case <-ctx.Done(): + t.Error("foreground ingestion could not finish between committed batches") + } + }) + if err != nil || removed != aged { + t.Fatalf("removed=%d err=%v", removed, err) + } + if yields != 2 { + t.Errorf("scheduling action called %d times; want once after each of two full committed batches", yields) + } + if countRows(t, s, "transmissions") != 2 || countRows(t, s, "observations") != 2 { + t.Error("foreground rows were not ingested and preserved between batches") + } +} + +func seedRetentionFanout(tb testing.TB, s *Store, transmissions, observations int) { + tb.Helper() + s.WaitForAsyncMigrations() + tx, err := s.db.Begin() + if err != nil { + tb.Fatal(err) + } + defer tx.Rollback() + parents, err := tx.Prepare(`INSERT INTO transmissions(id,raw_hex,hash,first_seen,last_seen) VALUES($1,'00',$2,'2020-01-01T00:00:00Z',1)`) + if err != nil { + tb.Fatal(err) + } + defer parents.Close() + children, err := tx.Prepare(`INSERT INTO observations(transmission_id,observer_idx,path_json,timestamp) VALUES($1,$2,'[]',1)`) + if err != nil { + tb.Fatal(err) + } + defer children.Close() + for i := 1; i <= transmissions; i++ { + if _, err := parents.Exec(i, fmt.Sprint("retention-", i)); err != nil { + tb.Fatal(err) + } + } + for i := 0; i < observations; i++ { + if _, err := children.Exec(i%transmissions+1, i/transmissions); err != nil { + tb.Fatal(err) + } + } + + if err := tx.Commit(); err != nil { + tb.Fatal(err) + } + if s.Backend() == "postgres" { + if _, err := testAdmin(tb, s).Exec(`SELECT setval('transmissions_id_seq',$1,true)`, transmissions); err != nil { + tb.Fatal(err) + } + } +} diff --git a/cmd/ingestor/scope_match_tally.go b/cmd/ingestor/scope_match_tally.go index cb3c2702a..813cbf029 100644 --- a/cmd/ingestor/scope_match_tally.go +++ b/cmd/ingestor/scope_match_tally.go @@ -55,7 +55,7 @@ func (s *Store) SaveScopeMatchTotals() error { _, err := s.db.Exec(` INSERT INTO scope_match_totals (id, since_unix, unique_matches, explicit_over_derived, ambiguous, none_matches, updated_unix) - VALUES (1, $1, $2, $3, $4, $5, $6) + VALUES (1, `+s.parameter(1)+`, `+s.parameter(2)+`, `+s.parameter(3)+`, `+s.parameter(4)+`, `+s.parameter(5)+`, `+s.parameter(6)+`) ON CONFLICT(id) DO UPDATE SET since_unix = excluded.since_unix, unique_matches = excluded.unique_matches, diff --git a/cmd/ingestor/sig_validate_ingest_test.go b/cmd/ingestor/sig_validate_ingest_test.go index 75d15196c..6e7a29f19 100644 --- a/cmd/ingestor/sig_validate_ingest_test.go +++ b/cmd/ingestor/sig_validate_ingest_test.go @@ -296,8 +296,8 @@ func TestPruneDroppedPackets(t *testing.T) { defer store.Close() // Insert an old dropped packet - store.db.Exec(`INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('old', 'test', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC' - interval '60 days','YYYY-MM-DD HH24:MI:SS'))`) - store.db.Exec(`INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('new', 'test', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD HH24:MI:SS'))`) + store.db.Exec(testNativeSQL(`INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('old', 'test', datetime('now','-60 days'))`, `INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('old', 'test', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC' - interval '60 days','YYYY-MM-DD HH24:MI:SS'))`)) + store.db.Exec(testNativeSQL(`INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('new', 'test', datetime('now'))`, `INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('new', 'test', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD HH24:MI:SS'))`)) n, err := store.PruneDroppedPackets(30) if err != nil { diff --git a/cmd/ingestor/sqlite_maintenance.go b/cmd/ingestor/sqlite_maintenance.go new file mode 100644 index 000000000..8a89e921c --- /dev/null +++ b/cmd/ingestor/sqlite_maintenance.go @@ -0,0 +1,141 @@ +package main + +import ( + "fmt" + "github.com/meshcore-analyzer/dbconfig" + "log" + "time" +) + +func (s *Store) CheckAutoVacuum(cfg *Config) { + if s.Backend() != dbconfig.SQLite { + return + } + + var autoVacuum int + if err := s.db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { + log.Printf("[db] warning: could not read auto_vacuum: %v", err) + return + } + + if autoVacuum == 2 { + log.Printf("[db] auto_vacuum=INCREMENTAL") + return + } + + modes := map[int]string{0: "NONE", 1: "FULL", 2: "INCREMENTAL"} + mode := modes[autoVacuum] + if mode == "" { + mode = fmt.Sprintf("UNKNOWN(%d)", autoVacuum) + } + + log.Printf("[db] auto_vacuum=%s — DB needs one-time VACUUM to enable incremental auto-vacuum. "+ + "Set db.vacuumOnStartup: true in config to migrate (will block startup for several minutes on large DBs). "+ + "See https://github.com/Kpa-clawbot/CoreScope/issues/919", mode) + + if cfg.DB != nil && cfg.DB.VacuumOnStartup { + // WARNING: Full VACUUM creates a temporary copy of the entire DB file. + // Requires ~2× the DB file size in free disk space or it will fail. + log.Printf("[db] vacuumOnStartup=true — starting one-time full VACUUM (ensure 2x DB size free disk space)...") + start := time.Now() + + if _, err := s.instrumentedExec("vacuum", "PRAGMA auto_vacuum = INCREMENTAL"); err != nil { + log.Printf("[db] VACUUM failed: could not set auto_vacuum: %v", err) + return + } + if _, err := s.instrumentedExec("vacuum", "VACUUM"); err != nil { + log.Printf("[db] VACUUM failed: %v", err) + return + } + + elapsed := time.Since(start) + log.Printf("[db] VACUUM complete in %v — auto_vacuum is now INCREMENTAL", elapsed.Round(time.Millisecond)) + } +} + +func (s *Store) RunIncrementalVacuum(pages int) { + if s.Backend() != dbconfig.SQLite { + return + } + + // Tagged for /api/perf writer-lock visibility (#1340). + if _, err := s.instrumentedExec("vacuum", fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pages)); err != nil { + log.Printf("[vacuum] incremental_vacuum error: %v", err) + } +} + +func (s *Store) RefreshPlannerStats(analysisLimit int) bool { + if s.Backend() != dbconfig.SQLite { + return false + } + + if analysisLimit < 0 { + return false + } + first := !s.hasPlannerStats() + start := time.Now() + // Tagged for /api/perf writer-lock visibility (#1340). + if _, err := s.instrumentedExec("analyze", fmt.Sprintf("PRAGMA analysis_limit=%d", analysisLimit)); err != nil { + log.Printf("[analyze] could not set analysis_limit: %v", err) + return false + } + if _, err := s.instrumentedExec("analyze", "ANALYZE"); err != nil { + log.Printf("[analyze] ANALYZE failed: %v", err) + return false + } + elapsed := time.Since(start).Round(time.Millisecond) + if first { + log.Printf("[analyze] planner statistics built in %v (analysis_limit=%d, first run against this database)", elapsed, analysisLimit) + } else { + log.Printf("[analyze] planner statistics refreshed in %v (analysis_limit=%d)", elapsed, analysisLimit) + } + return true +} + +func (s *Store) EnsurePlannerStats(analysisLimit int) bool { + if s.Backend() != dbconfig.SQLite { + return false + } + + if s.hasPlannerStats() { + return false + } + log.Printf("[analyze] this database has no planner statistics; building them now. " + + "ANALYZE holds the single write connection until it finishes (3m43.9s measured on 9.4 GB, cold), " + + "so ingest will buffer and catch up. Once per database, not once per restart.") + return s.RefreshPlannerStats(analysisLimit) +} + +func (s *Store) hasPlannerStats() bool { + if s.Backend() != dbconfig.SQLite { + return false + } + + var n int + if err := s.db.QueryRow( + `SELECT count(*) FROM sqlite_master WHERE type='table' AND name='sqlite_stat1'`).Scan(&n); err != nil { + return false + } + return n > 0 +} + +func (s *Store) Checkpoint() int { + if s.Backend() != dbconfig.SQLite { + return 0 + } + waitStart := time.Now() + writerMu.Lock() + wait := time.Since(waitStart) + holdStart := time.Now() + defer func() { writerMu.Unlock(); recordWriterTiming("checkpoint", wait, time.Since(holdStart), "Checkpoint") }() + + var busy, walFrames, checkpointed int + if err := s.db.QueryRow("PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &walFrames, &checkpointed); err != nil { + log.Printf("[db] WAL checkpoint error: %v", err) + return 0 + } + if walFrames > 0 { + log.Printf("[db] WAL checkpoint: %d/%d frames checkpointed (blocked=%v)", checkpointed, walFrames, busy != 0) + } + return checkpointed +} diff --git a/cmd/ingestor/storage_adoption.go b/cmd/ingestor/storage_adoption.go new file mode 100644 index 000000000..9145ee227 --- /dev/null +++ b/cmd/ingestor/storage_adoption.go @@ -0,0 +1,147 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "net/url" + "os" + "path/filepath" + "time" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" + "github.com/meshcore-analyzer/pgutil" +) + +// Adoption never initializes a missing telemetry file. Setup creates a fresh +// installation; this path records only an existing, validated native target. +func adoptLegacyStorage(raw dbconfig.StorageInputs) error { + selectionPath, err := storageSelectionPath(raw) + if err != nil { + return err + } + if _, err := dbconfig.ReadSelection(selectionPath); err == nil { + return nil + } else if !errors.Is(err, dbconfig.ErrSelectionMissing) { + return err + } + if raw.Backend == "" && raw.EnvBackend == "" && raw.DBPath == "" && raw.DatabaseURL == "" && raw.ReaderDatabaseURL == "" && raw.WriterDatabaseURL == "" && raw.UsersDatabaseURL == "" && raw.ApprovedChannelsDatabaseURL == "" { + raw.DBPath = filepath.Join(raw.BaseDir, "data", "meshcore.db") + } + raw.StateDir = filepath.Dir(selectionPath) + storage, err := dbconfig.ResolveStorage(raw) + if err != nil { + return err + } + selected, err := dbconfig.NewSelection(storage) + if err != nil { + return err + } + if storage.Backend == dbconfig.SQLite { + db, err := openExistingSQLite(storage.DBPath) + if err != nil { + return err + } + err = dbconfig.AssertSQLiteImportComplete(db) + if err == nil { + err = legacy.CheckLegacySource(db) + } + db.Close() + if err != nil { + return err + } + if _, err := os.Stat(storage.UsersDBPath); err == nil { + accounts, err := openExistingSQLite(storage.UsersDBPath) + if err != nil { + return err + } + var version int + if err = dbconfig.AssertSQLiteImportComplete(accounts); err != nil { + accounts.Close() + return err + } + err = accounts.QueryRow(`SELECT MAX(version) FROM schema_version`).Scan(&version) + accounts.Close() + if err != nil || version < 1 || version > 6 { + return errors.New("existing SQLite accounts schema is unsupported; use setup or offline recovery") + } + } else if errors.Is(err, os.ErrNotExist) { + // Absent legacy accounts are uninitialized, not an initialized empty store. + // Only explicit setup may create them and fill this selection target. + selected.Accounts = nil + } else { + return err + } + } else { + if storage.UsersDatabaseURL == "" && storage.ApprovedChannelsDatabaseURL == "" { + legacyAccounts := raw.UsersDBPath + if legacyAccounts == "" { + legacyAccounts = filepath.Join(storage.StateDir, "users.db") + } + if _, err := os.Stat(legacyAccounts); err == nil { + return errors.New("existing SQLite accounts require offline conversion before adopting PostgreSQL") + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + } + db, err := pgutil.Open(storage.WriterDatabaseURL, false) + if err != nil { + return err + } + err = dbschema.AssertPostgresReady(db) + if err == nil { + err = dbschema.AssertWriter(db) + } + db.Close() + if err != nil { + return err + } + accountURL := storage.UsersDatabaseURL + if accountURL == "" { + accountURL = storage.ApprovedChannelsDatabaseURL + } + if accountURL != "" { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + same, err := pgutil.SameDatabase(ctx, storage.WriterDatabaseURL, accountURL) + cancel() + if err != nil { + return err + } + if same { + return errors.New("accounts must not use the telemetry database") + } + accounts, err := pgutil.Open(accountURL, true) + if err != nil { + return err + } + var ready bool + err = accounts.QueryRow(`SELECT ready FROM corescope_schema WHERE kind='accounts' AND version=1`).Scan(&ready) + accounts.Close() + if err != nil || !ready { + return errors.New("PostgreSQL accounts import/bootstrap is incomplete") + } + } + } + + _, err = dbconfig.AdoptSelection(selectionPath, selected) + return err +} + +func openExistingSQLite(path string) (*sql.DB, error) { + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + return nil, errors.New("existing SQLite storage is missing or empty; run explicit setup instead of creating a new runtime database") + } + dsn, err := dbconfig.SQLiteURI(path, url.Values{"mode": {"ro"}, "_query_only": {"on"}, "_busy_timeout": {"5000"}}) + if err != nil { + return nil, err + } + db, err := sql.Open("sqlite3", dsn) + if err != nil { + return nil, err + } + db.SetMaxOpenConns(1) + return db, nil +} diff --git a/cmd/ingestor/storage_adoption_test.go b/cmd/ingestor/storage_adoption_test.go new file mode 100644 index 000000000..ce9d882b4 --- /dev/null +++ b/cmd/ingestor/storage_adoption_test.go @@ -0,0 +1,125 @@ +package main + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbconfig" +) + +func TestRuntimeMissingSQLiteCannotCreateReplacement(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "uninitialized", "meshcore.db") + raw := dbconfig.StorageInputs{BaseDir: dir, Backend: dbconfig.SQLite, DBPath: path} + if err := adoptLegacyStorage(raw); err == nil { + t.Fatal("missing source was treated as fresh") + } + if _, err := os.Stat(filepath.Dir(path)); !errors.Is(err, os.ErrNotExist) { + t.Fatal("runtime created replacement storage", err) + } +} + +func TestRuntimeAdoptsExistingSQLiteAndPinsRecordedChoice(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "meshcore.db") + seed, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + seed.Close() + raw := dbconfig.StorageInputs{BaseDir: dir, DBPath: path} + if err := adoptLegacyStorage(raw); err != nil { + t.Fatal(err) + } + raw.Backend = dbconfig.Postgres + raw.EnvBackend = dbconfig.Postgres + raw.DatabaseURL = "postgresql://unused@localhost/unselected" + storage, lease, err := resolveRuntimeStorage(raw) + if err != nil { + t.Fatal(err) + } + defer lease.Close() + if storage.Backend != dbconfig.SQLite || storage.DBPath != path || storage.UsersDBPath != "" { + t.Fatal("obsolete bootstrap choices overrode recorded installation") + } + selected, err := dbconfig.ReadSelection(dbconfig.SelectionPath(dir)) + if err != nil { + t.Fatal(err) + } + if selected.Accounts != nil { + t.Fatal("absent accounts marked initialized") + } + if _, err := dbconfig.BeginSelectionUpdate(dbconfig.SelectionPath(dir), selected.Generation); !errors.Is(err, dbconfig.ErrSelectionBusy) { + t.Fatal("runtime did not hold the selection lease", err) + } +} + +func TestRuntimeCannotAdoptStagedIncompleteSQLite(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "staged.db") + seed, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + if _, err = seed.db.Exec(`CREATE TABLE ` + dbconfig.SQLiteImportPendingTable + `(phase TEXT)`); err != nil { + t.Fatal(err) + } + seed.Close() + raw := dbconfig.StorageInputs{BaseDir: dir, StateDir: dir, DBPath: path} + if err := adoptLegacyStorage(raw); !errors.Is(err, dbconfig.ErrSQLiteImportIncomplete) { + t.Fatal("incomplete target accepted", err) + } + if _, err := dbconfig.ReadSelection(dbconfig.SelectionPath(dir)); !errors.Is(err, dbconfig.ErrSelectionMissing) { + t.Fatal("incomplete target recorded", err) + } +} + +func TestRecordedSQLiteCannotCreateReplacement(t *testing.T) { + for _, empty := range []bool{false, true} { + name := "missing" + if empty { + name = "empty" + } + t.Run(name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "meshcore.db") + seed, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + seed.Close() + raw := dbconfig.StorageInputs{BaseDir: dir, DBPath: path} + if err := adoptLegacyStorage(raw); err != nil { + t.Fatal(err) + } + if err := os.Rename(path, path+".preserved"); err != nil { + t.Fatal(err) + } + if empty { + if err := os.WriteFile(path, nil, 0600); err != nil { + t.Fatal(err) + } + } + storage, lease, err := resolveRuntimeStorage(raw) + if err != nil { + t.Fatal(err) + } + defer lease.Close() + store, err := openStoreStorage(storage, nil, 300) + if err == nil { + store.Close() + t.Error("recorded runtime silently initialized replacement telemetry") + } + info, statErr := os.Stat(path) + if empty { + if statErr != nil || info.Size() != 0 { + t.Error("runtime modified the empty recorded target", statErr) + } + } else if !errors.Is(statErr, os.ErrNotExist) { + t.Error("runtime created a missing recorded target", statErr) + } + }) + } +} diff --git a/cmd/ingestor/storage_backend.go b/cmd/ingestor/storage_backend.go new file mode 100644 index 000000000..120e96fd5 --- /dev/null +++ b/cmd/ingestor/storage_backend.go @@ -0,0 +1,72 @@ +package main + +import ( + "database/sql" + "errors" + "net/url" + "os" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" +) + +func (s *Store) Backend() dbconfig.Backend { + if s.backend == "" { + return dbconfig.SQLite + } + return s.backend +} +func (s *Store) parameter(n int) string { return s.Backend().Parameter(n) } +func (s *Store) nativeSQL(sqlite, postgres string) string { + if s.Backend() == dbconfig.SQLite { + return sqlite + } + return postgres +} + +// OpenStoreStorage is the runtime boundary after the shared selector resolves +// raw inputs. Compatibility constructors still accept paths or explicit URLs. +func OpenStoreStorage(storage dbconfig.Storage, cfg *DBConfig) (*Store, error) { + return openStoreStorage(storage, cfg, 300) +} + +// Retain the shared durability/connection options. Existing runtime targets use +// rw (never rwc); persistent pragmas run only after schema validation succeeds. +func sqliteTelemetryWriterDSN(path string, create bool) (string, error) { + if !create { + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + return "", errors.New("selected SQLite telemetry is missing or empty; restore the initialized store") + } + } + dsn, err := legacy.WriterDSN(path) + if err != nil { + return "", err + } + u, err := url.Parse(dsn) + if err != nil { + return "", err + } + q := u.Query() + q.Set("_txlock", "immediate") + if !create { + q.Set("mode", "rw") + q.Del("_journal_mode") + q.Del("_auto_vacuum") + } + u.RawQuery = q.Encode() + return u.String(), nil +} + +func validateExistingTelemetry(db *sql.DB) error { + if err := dbconfig.AssertSQLiteImportComplete(db); err != nil { + return err + } + if err := dbschema.AssertSQLiteReady(db); err == nil { + return nil + } + // A supported older v3 store may need the normal writer upgrade. This also + // rejects empty/header-only/unrelated databases through the opened handle. + return legacy.CheckLegacySource(db) +} diff --git a/cmd/ingestor/storage_backend_test.go b/cmd/ingestor/storage_backend_test.go new file mode 100644 index 000000000..51d823cfb --- /dev/null +++ b/cmd/ingestor/storage_backend_test.go @@ -0,0 +1,100 @@ +package main + +import ( + "database/sql" + "github.com/meshcore-analyzer/dbconfig" + "os" + "path/filepath" + "testing" +) + +func TestOpenStoreNativeSQLiteWithoutPostgres(t *testing.T) { + t.Setenv("CORESCOPE_TEST_POSTGRES_URL", "") + path := filepath.Join(t.TempDir(), "telemetry.db") + s, err := OpenStore(path) + if err != nil { + t.Fatalf("native SQLite writer must work without PostgreSQL: %v", err) + } + defer s.Close() + s.WaitForAsyncMigrations() + for pragma, want := range map[string]string{"journal_mode": "wal", "synchronous": "2", "foreign_keys": "1"} { + var actual string + if err := s.db.QueryRow("PRAGMA " + pragma).Scan(&actual); err != nil || actual != want { + t.Fatalf("%s=%q want %q: %v", pragma, actual, want, err) + } + } + if s.path != path { + t.Fatalf("legacy queue anchor changed: %q", s.path) + } +} + +func TestSQLiteObserverTimeIndexCompletesThroughAsyncLedger(t *testing.T) { + s, err := OpenStore(filepath.Join(t.TempDir(), "observer-index.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + s.WaitForAsyncMigrations() + var count int + if err = s.db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_observer_idx_timestamp'`).Scan(&count); err != nil || count != 1 { + t.Fatalf("native observer/time index count=%d: %v", count, err) + } + if status, err := s.AsyncMigrationStatus("obs_observer_ts_idx_v1"); err != nil || status != "done" { + t.Fatal("native index migration did not complete", status, err) + } +} + +func TestSQLiteMissingObserverIndexIsNotHiddenByCompletionMarker(t *testing.T) { + path := filepath.Join(t.TempDir(), "recovered-index.db") + s, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + s.WaitForAsyncMigrations() + if _, err = s.db.Exec(`DROP INDEX idx_observations_observer_idx_timestamp`); err != nil { + t.Fatal(err) + } + s.Close() + s, err = OpenStore(path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + s.WaitForAsyncMigrations() + var count int + if err = s.db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_observer_idx_timestamp'`).Scan(&count); err != nil || count != 1 { + t.Fatal("missing index concealed by done marker", count, err) + } +} + +func testBackend(t testing.TB) dbconfig.Backend { + t.Helper() + v := os.Getenv("CORESCOPE_TEST_BACKEND") + if v != "" && v != "sqlite" && v != "postgres" { + t.Fatalf("invalid CORESCOPE_TEST_BACKEND") + } + if v == "postgres" { + return dbconfig.Postgres + } + return dbconfig.SQLite +} +func testBackendValue() dbconfig.Backend { + if os.Getenv("CORESCOPE_TEST_BACKEND") == "postgres" { + return dbconfig.Postgres + } + return dbconfig.SQLite +} +func testNativeSQL(sqlite, postgres string) string { + if testBackendValue() == dbconfig.Postgres { + return postgres + } + return sqlite +} + +func scanTestPlan(rows *sql.Rows, detail *string) error { + if testBackendValue() == dbconfig.SQLite { + var id, parent, unused int + return rows.Scan(&id, &parent, &unused, detail) + } + return rows.Scan(detail) +} diff --git a/cmd/ingestor/storage_config.go b/cmd/ingestor/storage_config.go new file mode 100644 index 000000000..021b1e3d9 --- /dev/null +++ b/cmd/ingestor/storage_config.go @@ -0,0 +1,120 @@ +package main + +import ( + "errors" + "io" + "path/filepath" + "strings" + + "github.com/meshcore-analyzer/dbconfig" +) + +type storageFlags struct { + Backend dbconfig.Backend + DBPath, DatabaseURL, StateDir string +} + +// storageInputs gathers bootstrap values without guessing an installation's +// state. The shared persisted selector applies before ResolveStorage runs. +func (c *Config) storageInputs(baseDir string, flags storageFlags, getenv func(string) string) (dbconfig.StorageInputs, error) { + base, err := filepath.Abs(baseDir) + if err != nil { + return dbconfig.StorageInputs{}, errors.New("resolve storage base directory") + } + in := dbconfig.StorageInputs{BaseDir: base, DBPath: c.DBPath, StateDir: envOrValue(getenv, "CORESCOPE_STATE_DIR", c.StateDir), + EnvBackend: dbconfig.Backend(getenv("CORESCOPE_DB_BACKEND")), + DatabaseURL: envOrValue(getenv, "CORESCOPE_DATABASE_URL", c.DatabaseURL), + ReaderDatabaseURL: getenv("CORESCOPE_READER_DATABASE_URL"), WriterDatabaseURL: getenv("CORESCOPE_WRITER_DATABASE_URL"), + UsersDatabaseURL: getenv("CORESCOPE_USERS_DATABASE_URL"), ApprovedChannelsDatabaseURL: getenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL")} + if p := getenv("DB_PATH"); p != "" { + in.DBPath = p + } + if c.DB != nil { + in.Backend = c.DB.Backend + } + if c.UserManagement != nil { + in.UsersDBPath = strings.TrimSpace(c.UserManagement.DBPath) + in.UsersDatabaseURL = envOrValue(getenv, "CORESCOPE_USERS_DATABASE_URL", c.UserManagement.DatabaseURL) + in.ApprovedChannelsDatabaseURL = envOrValue(getenv, "CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", c.UserManagement.ApprovedChannelsDatabaseURL) + } + if flags.Backend != "" { + in.Backend = flags.Backend + } + if flags.DBPath != "" { + in.DBPath = flags.DBPath + } + if flags.DatabaseURL != "" { + in.DatabaseURL = flags.DatabaseURL + in.WriterDatabaseURL = flags.DatabaseURL + } + if flags.StateDir != "" { + in.StateDir = flags.StateDir + } + // The ingestor receives the process working directory as its base; explicit + // legacy paths retain their historical working-directory interpretation. + for _, p := range []*string{&in.DBPath, &in.UsersDBPath} { + if *p == "" || strings.Contains(*p, "://") { + continue + } + if filepath.VolumeName(*p) != "" && !filepath.IsAbs(*p) { + return in, errors.New("drive-relative storage paths are ambiguous") + } + *p, err = filepath.Abs(*p) + if err != nil { + return in, errors.New("resolve SQLite storage path") + } + } + return in, nil +} + +func envOrValue(getenv func(string) string, key, fallback string) string { + if value := strings.TrimSpace(getenv(key)); value != "" { + return value + } + return fallback +} + +// Discovery uses the stable legacy anchor, including after a backend switch. +func storageSelectionPath(raw dbconfig.StorageInputs) (string, error) { + state := raw.StateDir + if state == "" { + if strings.Contains(raw.DBPath, "://") || strings.HasPrefix(raw.DBPath, "postgres:") || strings.HasPrefix(raw.DBPath, "postgresql:") { + return "", errors.New("a database URL cannot be used as the legacy state directory anchor") + } + path := raw.DBPath + if path == "" { + path = filepath.Join(raw.BaseDir, "data", "meshcore.db") + } + state = filepath.Dir(path) + } + if strings.Contains(state, "://") || strings.ContainsRune(state, 0) { + return "", errors.New("invalid storage state directory") + } + if !filepath.IsAbs(state) { + state = filepath.Join(raw.BaseDir, state) + } + return dbconfig.SelectionPath(filepath.Clean(state)), nil +} +func resolveRuntimeStorage(raw dbconfig.StorageInputs) (dbconfig.Storage, io.Closer, error) { + path, err := storageSelectionPath(raw) + if err != nil { + return dbconfig.Storage{}, nil, err + } + selected, lease, err := dbconfig.OpenSelection(path) + if err != nil { + return dbconfig.Storage{}, nil, err + } + fail := func(err error) (dbconfig.Storage, io.Closer, error) { + lease.Close() + return dbconfig.Storage{}, nil, err + } + raw, err = selected.ApplyTo(raw) + if err != nil { + return fail(err) + } + storage, err := dbconfig.ResolveStorage(raw) + if err != nil { + return fail(err) + } + return storage, lease, nil +} diff --git a/cmd/ingestor/storage_config_test.go b/cmd/ingestor/storage_config_test.go new file mode 100644 index 000000000..390e7d917 --- /dev/null +++ b/cmd/ingestor/storage_config_test.go @@ -0,0 +1,88 @@ +package main + +import ( + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbconfig" +) + +func TestStorageInputsKeepBootstrapRawAndLegacyPrecedence(t *testing.T) { + base := t.TempDir() + cfg := &Config{DBPath: "configured.db", UserManagement: &UserManagementConfig{DBPath: "accounts.db"}} + env := func(k string) string { + if k == "DB_PATH" { + return "environment.db" + } + return "" + } + in, err := cfg.storageInputs(base, storageFlags{}, env) + if err != nil { + t.Fatal(err) + } + want, _ := filepath.Abs("environment.db") + if in.DBPath != want || in.StateDir != "" || in.Backend != "" || in.FreshInstall { + t.Fatalf("raw values/defaults changed: %+v", in) + } + in, err = cfg.storageInputs(base, storageFlags{DBPath: "cli.db"}, env) + if err != nil { + t.Fatal(err) + } + want, _ = filepath.Abs("cli.db") + if in.DBPath != want { + t.Fatal("CLI path did not win") + } + in, err = (&Config{}).storageInputs(base, storageFlags{}, func(string) string { return "" }) + if err != nil { + t.Fatal(err) + } + if _, err = dbconfig.ResolveStorage(in); err == nil { + t.Fatal("unknown existing install was guessed fresh") + } + in.FreshInstall = true + got, err := dbconfig.ResolveStorage(in) + if err != nil { + t.Fatal(err) + } + if got.Backend != dbconfig.SQLite || got.DBPath != filepath.Join(base, "data", "meshcore.db") { + t.Fatalf("fresh default: %+v", got) + } +} + +func TestStorageInputsWriterURLAndStatePrecedence(t *testing.T) { + base := t.TempDir() + cfg := &Config{DB: &DBConfig{Backend: dbconfig.Postgres}, DatabaseURL: "postgresql://reader@localhost/config", StateDir: "configured"} + env := func(k string) string { + switch k { + case "CORESCOPE_DATABASE_URL": + return "postgresql://reader@localhost/generic" + case "CORESCOPE_WRITER_DATABASE_URL": + return "postgresql://reader@localhost/reader" + case "CORESCOPE_STATE_DIR": + return "state" + } + return "" + } + in, err := cfg.storageInputs(base, storageFlags{}, env) + if err != nil { + t.Fatal(err) + } + got, err := dbconfig.ResolveStorage(in) + if err != nil { + t.Fatal(err) + } + if got.WriterDatabaseURL != env("CORESCOPE_WRITER_DATABASE_URL") || got.StateDir != filepath.Join(base, "state") { + t.Fatal("reader environment precedence changed") + } + in, err = cfg.storageInputs(base, storageFlags{DatabaseURL: "postgresql://reader@localhost/cli", StateDir: "cli-state"}, env) + if err != nil { + t.Fatal(err) + } + got, err = dbconfig.ResolveStorage(in) + if err != nil { + t.Fatal(err) + } + if got.ReaderDatabaseURL != "postgresql://reader@localhost/cli" || got.WriterDatabaseURL != "postgresql://reader@localhost/cli" || got.StateDir != filepath.Join(base, "cli-state") { + t.Fatal("CLI overrides lost") + } +} diff --git a/cmd/ingestor/storage_runtime_open_test.go b/cmd/ingestor/storage_runtime_open_test.go new file mode 100644 index 000000000..e182d164b --- /dev/null +++ b/cmd/ingestor/storage_runtime_open_test.go @@ -0,0 +1,121 @@ +package main + +import ( + "crypto/sha256" + "database/sql" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbconfig" +) + +func TestSelectedTelemetryRejectsWrongOrUninitializedSchemaWithoutMutation(t *testing.T) { + for _, query := range []string{`PRAGMA user_version=1`, `CREATE TABLE unrelated(value TEXT); INSERT INTO unrelated VALUES('keep')`} { + path := filepath.Join(t.TempDir(), "existing.db") + db, err := sql.Open("sqlite3", path) + if err != nil { + t.Fatal(err) + } + if _, err = db.Exec(query); err != nil { + t.Fatal(err) + } + db.Close() + before, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if s, err := OpenStoreStorage(dbconfig.Storage{Backend: dbconfig.SQLite, DBPath: path}, nil); err == nil { + s.Close() + t.Fatal("unrelated or uninitialized store accepted") + } + after, err := os.ReadFile(path) + if err != nil || sha256.Sum256(before) != sha256.Sum256(after) { + t.Fatal("rejected telemetry store changed", err) + } + } +} + +func TestSelectedTelemetryCannotRecreateRemovedVolume(t *testing.T) { + dir := filepath.Join(t.TempDir(), "missing-volume") + if s, err := OpenStoreStorage(dbconfig.Storage{Backend: dbconfig.SQLite, DBPath: filepath.Join(dir, "telemetry.db")}, nil); err == nil { + s.Close() + t.Fatal("missing volume accepted") + } + if _, err := os.Stat(dir); !errors.Is(err, os.ErrNotExist) { + t.Fatal("runtime recreated missing volume", err) + } +} + +func TestSQLiteTelemetryTargetRemovedAfterDSNValidation(t *testing.T) { + for _, replaceEmpty := range []bool{false, true} { + path := filepath.Join(t.TempDir(), "telemetry.db") + seed, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + seed.Close() + dsn, err := sqliteTelemetryWriterDSN(path, false) + if err != nil { + t.Fatal(err) + } + if err = os.Rename(path, path+".preserved"); err != nil { + t.Fatal(err) + } + if replaceEmpty { + if err = os.WriteFile(path, nil, 0600); err != nil { + t.Fatal(err) + } + } + db, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal(err) + } + if err = validateExistingTelemetry(db); err == nil { + t.Fatal("raced missing/empty target accepted") + } + db.Close() + info, err := os.Stat(path) + if replaceEmpty { + if err != nil || info.Size() != 0 { + t.Fatal("raced empty file was initialized", err) + } + } else if !errors.Is(err, os.ErrNotExist) { + t.Fatal("raced missing file was recreated", err) + } + } +} + +func TestSelectedTelemetryRetainsRowsSequenceAndDurability(t *testing.T) { + path := filepath.Join(t.TempDir(), "telemetry.db") + seed, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + if _, err = seed.InsertTransmission(&PacketData{Hash: "preserved", RawHex: "AA", Timestamp: "2026-01-01T00:00:00Z", PathJSON: "[]"}); err != nil { + t.Fatal(err) + } + if _, err = seed.db.Exec(`INSERT INTO transmissions(id,raw_hex,hash,first_seen) VALUES(900,'AA','retired','2026-01-01T00:00:00Z'); DELETE FROM transmissions WHERE id=900`); err != nil { + t.Fatal(err) + } + seed.Close() + s, err := OpenStoreStorage(dbconfig.Storage{Backend: dbconfig.SQLite, DBPath: path}, nil) + if err != nil { + t.Fatal(err) + } + defer s.Close() + var count, sequence int + if err = s.db.QueryRow(`SELECT count(*) FROM transmissions WHERE hash='preserved'`).Scan(&count); err != nil || count != 1 { + t.Fatal("stored row lost", count, err) + } + if err = s.db.QueryRow(`SELECT seq FROM sqlite_sequence WHERE name='transmissions'`).Scan(&sequence); err != nil || sequence != 900 { + t.Fatal("allocator reset", sequence, err) + } + for pragma, want := range map[string]string{"journal_mode": "wal", "synchronous": "2", "foreign_keys": "1"} { + var got string + if err = s.db.QueryRow("PRAGMA " + pragma).Scan(&got); err != nil || got != want { + t.Fatalf("%s=%s: %v", pragma, got, err) + } + } +} diff --git a/cmd/migrate/account_setup.go b/cmd/migrate/account_setup.go new file mode 100644 index 000000000..b139b11ca --- /dev/null +++ b/cmd/migrate/account_setup.go @@ -0,0 +1,186 @@ +package main + +import ( + "context" + "errors" + "os" + "reflect" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" +) + +// The existing journal identifies this one same-backend transition without a +// separate initialization flag or sidecar. Its telemetry target never changes. +func accountInitialization(current, next dbconfig.Selection) bool { + return current.Generation != "" && current.Accounts == nil && next.Accounts != nil && current.Backend == next.Backend && current.StateDir == next.StateDir && reflect.DeepEqual(current.Telemetry, next.Telemetry) +} + +func initializeAccounts(ctx context.Context, o storageOptions, current dbconfig.Selection, raw dbconfig.StorageInputs) error { + update, err := dbconfig.BeginSelectionUpdate(o.SelectionFile, current.Generation) + if err != nil { + return err + } + defer update.Close() + // Failed validation before staging has done no account work. A real process + // crash still leaves the journal for an explicit metadata-only abort. + defer func() { + if update.Target().Generation == "" { + _ = update.Abort() + } + }() + if err := validateRuntimeSelection(ctx, current, raw); err != nil { + return err + } + bound, err := current.ApplyTo(raw) + if err != nil { + return err + } + bound.ExistingBackend = "" // This explicit setup, alone, supplies a new account default. + if current.Backend == dbconfig.SQLite { + bound.UsersDBPath = raw.UsersDBPath + } else { + if o.UsersOwnerURL == "" || raw.UsersDatabaseURL == "" { + return errors.New("first PostgreSQL account setup requires both account owner and runtime writer URLs") + } + bound.UsersDatabaseURL = o.UsersOwnerURL + bound.ApprovedChannelsDatabaseURL = "" + } + storage, err := dbconfig.ResolveStorage(bound) + if err != nil { + return err + } + next, err := dbconfig.NewSelection(storage) + if err != nil { + return err + } + if !accountInitialization(current, next) { + return errors.New("account setup cannot change the selected telemetry target") + } + if current.Backend == dbconfig.Postgres { + if err := validateInitialOwnerTargets(ctx, next, storage); err != nil { + return err + } + } + if err := update.Stage(next); err != nil { + return err + } + return finishAccountSetup(ctx, o, update, raw, false) +} + +func finishAccountSetup(ctx context.Context, o storageOptions, update *dbconfig.SelectionUpdate, raw dbconfig.StorageInputs, resume bool) error { + current, next := update.Current(), update.Target() + if !accountInitialization(current, next) { + return errors.New("pending journal is not a supported first account initialization") + } + if err := validateRuntimeSelection(ctx, current, raw); err != nil { + return err + } + if next.Backend == dbconfig.SQLite { + path := next.Accounts.SQLitePath + telemetry, err := os.Stat(current.Telemetry.SQLitePath) + if err != nil { + return err + } + info, statErr := os.Stat(path) + if statErr == nil && os.SameFile(telemetry, info) { + return errors.New("account setup cannot use the telemetry file") + } + if statErr != nil && !errors.Is(statErr, os.ErrNotExist) { + return statErr + } + if resume { + if statErr != nil { + return errors.New("pending account target is missing; restore it or abort this uncommitted initialization before explicit setup; it was not recreated") + } + db, err := openSQLite(path, "ro") + if err != nil { + return err + } + err = assertSQLiteData(ctx, db, "accounts") + db.Close() + if err != nil { + return errors.New("account initialization did not reach a ready target; preserve it, abort the uncommitted update and rerun explicit setup") + } + } else if err := applySQLiteTarget(ctx, path, "accounts"); err != nil { + return err + } + db, err := openSQLite(path, "ro") + if err != nil { + return err + } + err = assertSQLiteData(ctx, db, "accounts") + db.Close() + if err != nil { + return err + } + } else { + if o.UsersOwnerURL == "" || raw.UsersDatabaseURL == "" { + return errors.New("first PostgreSQL account setup requires both account owner and runtime writer URLs") + } + ownerInputs := raw + ownerInputs.UsersDatabaseURL = o.UsersOwnerURL + ownerInputs.ApprovedChannelsDatabaseURL = "" + bound, err := next.ApplyTo(ownerInputs) + if err != nil { + return err + } + owners, err := dbconfig.ResolveStorage(bound) + if err != nil { + return err + } + same, err := pgutil.SameDatabase(ctx, owners.WriterDatabaseURL, owners.UsersDatabaseURL) + if err != nil { + return err + } + if same { + return errors.New("accounts require a separate PostgreSQL database") + } + db, err := pgutil.Open(owners.UsersDatabaseURL, false) + if err != nil { + return err + } + readyErr := users.AssertPostgresReady(db) + if readyErr != nil { + if resume { + db.Close() + return errors.New("account initialization did not reach a ready PostgreSQL target; preserve it, abort the uncommitted update and rerun setup") + } + var objects int + err = db.QueryRowContext(ctx, importDestinationObjects).Scan(&objects) + if err == nil && objects != 0 { + err = errors.New("account target is neither a ready store nor a fresh empty PostgreSQL schema") + } + if err == nil { + err = users.ApplyPostgres(db) + } + } + if err == nil { + tables, layoutErr := layouts("accounts") + if layoutErr != nil { + err = layoutErr + } else { + err = validatePostgresLayout(db, "accounts", tables) + } + } + db.Close() + if err != nil { + return err + } + owners.WriterDatabaseURL = "" // Existing telemetry permissions are not part of first account setup. + if err := grantRuntimeTargets(ctx, next, owners, raw); err != nil { + return err + } + } + if err := validateRuntimeSelection(ctx, next, raw); err != nil { + return err + } + if o.beforeCommit != nil { + if err := o.beforeCommit(); err != nil { + return err + } + } + _, err := update.Commit(next) + return err +} diff --git a/cmd/migrate/account_setup_test.go b/cmd/migrate/account_setup_test.go new file mode 100644 index 000000000..0aeba5a27 --- /dev/null +++ b/cmd/migrate/account_setup_test.go @@ -0,0 +1,244 @@ +package main + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/users" +) + +func TestLegacyAccountAbsenceFirstSetupAndRecordedLoss(t *testing.T) { + for _, key := range []string{"CORESCOPE_DB_BACKEND", "DB_PATH", "CORESCOPE_DATABASE_URL", "CORESCOPE_USERS_OWNER_DATABASE_URL", "CORESCOPE_USERS_DATABASE_URL", "CORESCOPE_APPROVED_CHANNELS_DATABASE_URL"} { + t.Setenv(key, "") + } + source := telemetrySource(t) + accounts := filepath.Join(t.TempDir(), "first-accounts.sqlite") + dir := t.TempDir() + o := storageOptions{Action: "adopt", Backend: "sqlite", Offline: true, SelectionFile: filepath.Join(dir, "storage-selection.json"), SQLitePath: source, UsersSQLitePath: accounts, ConfigDir: t.TempDir()} + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal(err) + } + selected, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil || selected.Accounts != nil { + t.Fatal("legacy absence was initialized", err) + } + if _, err := os.Stat(accounts); !os.IsNotExist(err) { + t.Fatal("legacy adoption created account file", err) + } + _, lease, err := dbconfig.OpenSelection(o.SelectionFile) + if err != nil { + t.Fatal(err) + } + o.Action = "setup" + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); !errors.Is(err, dbconfig.ErrSelectionBusy) { + t.Fatal("first initialization lacked exclusive lease", err) + } + lease.Close() + if _, err := os.Stat(accounts); !os.IsNotExist(err) { + t.Fatal("busy setup created account file", err) + } + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal("first explicit setup", err) + } + initialized, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil || initialized.Accounts == nil || initialized.Accounts.SQLitePath != accounts || initialized.Generation == selected.Generation { + t.Fatal("initialized target not recorded", err) + } + db, err := openSQLite(accounts, "ro") + if err != nil { + t.Fatal(err) + } + if err := users.AssertSQLiteReady(db); err != nil { + t.Fatal(err) + } + db.Close() + if err := os.Remove(accounts); err != nil { + t.Fatal(err) + } + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err == nil { + t.Fatal("recorded missing accounts recreated") + } + if _, err := os.Stat(accounts); !os.IsNotExist(err) { + t.Fatal("lost account file replaced", err) + } + after, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil || after.Generation != initialized.Generation { + t.Fatal("failed setup changed recorded target", err) + } +} + +func TestLegacyDisabledExistingAccountStoreIsRecorded(t *testing.T) { + source, accounts := telemetrySource(t), accountSource(t, 5) + dir := t.TempDir() + config := filepath.Join(dir, "config.json") + if err := os.WriteFile(config, []byte(`{"userManagement":{"enabled":false}}`), 0600); err != nil { + t.Fatal(err) + } + o := storageOptions{Action: "adopt", Backend: "sqlite", Offline: true, SelectionFile: filepath.Join(dir, "storage-selection.json"), SQLitePath: source, UsersSQLitePath: accounts, ConfigFile: config} + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal(err) + } + selected, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil || selected.Accounts == nil || selected.Accounts.SQLitePath != accounts { + t.Fatal("disabled initialized store was omitted", err) + } + db, err := openSQLite(accounts, "ro") + if err != nil { + t.Fatal(err) + } + defer db.Close() + var id int + if err := db.QueryRow(`SELECT id FROM mail_events WHERE mail_id=4`).Scan(&id); err != nil || id != 900 { + t.Fatal("existing account data changed", id, err) + } +} + +func TestSQLiteAccountSetupResumeUsesOnlyItsJournal(t *testing.T) { + for _, missing := range []bool{false, true} { + t.Run(map[bool]string{false: "complete", true: "lost-target"}[missing], func(t *testing.T) { + source := telemetrySource(t) + accounts := filepath.Join(t.TempDir(), "accounts.sqlite") + dir := t.TempDir() + o := storageOptions{Action: "adopt", Backend: "sqlite", Offline: true, SelectionFile: filepath.Join(dir, "storage-selection.json"), SQLitePath: source, UsersSQLitePath: accounts, ConfigDir: t.TempDir()} + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal(err) + } + o.Action = "setup" + o.beforeCommit = func() error { return errors.New("stop before account selection") } + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err == nil { + t.Fatal("injected cutover failure passed") + } + status, err := dbconfig.InspectSelection(o.SelectionFile) + if err != nil || status.State != "pending" || status.SourceBackend != dbconfig.SQLite || status.TargetBackend != dbconfig.SQLite { + t.Fatal("same-backend journal status", status, err) + } + if missing { + if err := os.Remove(accounts); err != nil { + t.Fatal(err) + } + } + t.Setenv("CORESCOPE_DATABASE_URL", "must-not-connect") + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", "must-not-connect") + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "must-not-connect") + o.Action = "resume" + o.JobID = status.JobID + o.beforeCommit = nil + err = runStorageAction(context.Background(), o, &bytes.Buffer{}) + if missing { + if err == nil || !strings.Contains(err.Error(), "not recreated") { + t.Fatal("lost staged account target was recreated", err) + } + if _, err := os.Stat(accounts); !os.IsNotExist(err) { + t.Fatal("missing staged target replaced") + } + o.Action = "abort" + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal(err) + } + } else if err != nil { + t.Fatal("SQLite-only account resume", err) + } + }) + } +} + +func TestUnstagedJournalNeedsNoDatabaseForAbort(t *testing.T) { + source := telemetrySource(t) + if err := applySQLiteTarget(context.Background(), source, "telemetry"); err != nil { + t.Fatal(err) + } + dir := t.TempDir() + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.SQLite, StateDir: dir, DBPath: source}) + if err != nil { + t.Fatal(err) + } + path := dbconfig.SelectionPath(dir) + if _, err := dbconfig.AdoptSelection(path, selected); err != nil { + t.Fatal(err) + } + update, err := dbconfig.BeginSelectionUpdate(path, selected.Generation) + if err != nil { + t.Fatal(err) + } + id := update.ID() + update.Close() + t.Setenv("CORESCOPE_DATABASE_URL", "must-not-connect") + o := storageOptions{Action: "resume", Offline: true, SelectionFile: path, JobID: id, ConfigDir: t.TempDir()} + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err == nil || !strings.Contains(err.Error(), "no staged destination") { + t.Fatal("unstaged job resume guessed a target", err) + } + o.Action = "abort" + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal("metadata-only abort", err) + } +} + +func TestPostgresAccountFirstSetupRequiresCredentialsAndResumes(t *testing.T) { + owner, accountOwner := postgresDatabase(t), postgresDatabase(t) + db := openImportDB(t, owner) + if err := dbschema.ApplyPostgres(db, nil); err != nil { + t.Fatal(err) + } + reader, writer := restrictedRole(t, owner), restrictedRole(t, owner) + dir := t.TempDir() + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.Postgres, StateDir: dir, WriterDatabaseURL: owner}) + if err != nil { + t.Fatal(err) + } + raw := dbconfig.StorageInputs{BaseDir: dir, ReaderDatabaseURL: reader, WriterDatabaseURL: writer} + if err := grantRuntimeTargets(context.Background(), selected, dbconfig.Storage{WriterDatabaseURL: owner}, raw); err != nil { + t.Fatal(err) + } + selection := dbconfig.SelectionPath(dir) + if _, err := dbconfig.AdoptSelection(selection, selected); err != nil { + t.Fatal(err) + } + t.Setenv("CORESCOPE_DATABASE_URL", owner) + t.Setenv("CORESCOPE_READER_DATABASE_URL", reader) + t.Setenv("CORESCOPE_WRITER_DATABASE_URL", writer) + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", "") + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") + t.Setenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", "") + o := storageOptions{Action: "setup", Offline: true, SelectionFile: selection, ConfigDir: t.TempDir()} + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal("ordinary PG setup invented an account target", err) + } + unchanged, err := dbconfig.ReadSelection(selection) + if err != nil || unchanged.Accounts != nil || unchanged.Generation != selected.Generation { + t.Fatal("ordinary PG setup changed absence", err) + } + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", accountOwner) + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err == nil { + t.Fatal("account initialization lacked runtime credentials") + } + unchanged, err = dbconfig.ReadSelection(selection) + if err != nil || unchanged.Accounts != nil { + t.Fatal("failed preflight changed selection", err) + } + t.Setenv("CORESCOPE_USERS_DATABASE_URL", restrictedRole(t, accountOwner)) + o.beforeCommit = func() error { return errors.New("interrupt account commit") } + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err == nil { + t.Fatal("injected account commit passed") + } + status, err := dbconfig.InspectSelection(selection) + if err != nil || status.SourceBackend != dbconfig.Postgres || status.TargetBackend != dbconfig.Postgres { + t.Fatal("PG account journal", status, err) + } + o.Action = "resume" + o.JobID = status.JobID + o.beforeCommit = nil + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal("PG account initialization resume", err) + } + after, err := dbconfig.ReadSelection(selection) + if err != nil || after.Accounts == nil || after.Generation == selected.Generation { + t.Fatal("account target not atomically recorded", err) + } +} diff --git a/cmd/migrate/backend_test.go b/cmd/migrate/backend_test.go new file mode 100644 index 000000000..40e5f7d09 --- /dev/null +++ b/cmd/migrate/backend_test.go @@ -0,0 +1,41 @@ +package main + +import ( + "fmt" + "github.com/meshcore-analyzer/pgutil/pgtest" + "os" + "testing" +) + +func TestMain(m *testing.M) { + backend := os.Getenv("CORESCOPE_TEST_BACKEND") + if backend != "" && backend != "sqlite" && backend != "postgres" { + fmt.Fprintln(os.Stderr, "unsupported CORESCOPE_TEST_BACKEND") + os.Exit(1) + } + if backend == "postgres" && os.Getenv("CORESCOPE_TEST_POSTGRES_URL") == "" { + fmt.Fprintln(os.Stderr, "explicit PostgreSQL matrix requires CORESCOPE_TEST_POSTGRES_URL") + os.Exit(1) + } + os.Exit(m.Run()) +} + +func requirePostgres(t testing.TB) { + t.Helper() + switch os.Getenv("CORESCOPE_TEST_BACKEND") { + case "", "sqlite": + t.Skip("select CORESCOPE_TEST_BACKEND=postgres for the required PostgreSQL matrix") + case "postgres": + if os.Getenv("CORESCOPE_TEST_POSTGRES_URL") == "" { + t.Fatal("explicit PostgreSQL matrix requires CORESCOPE_TEST_POSTGRES_URL") + } + default: + t.Fatal("unsupported CORESCOPE_TEST_BACKEND") + } +} +func postgresSchema(t testing.TB) string { t.Helper(); requirePostgres(t); return pgtest.NewSchema(t) } +func postgresDatabase(t testing.TB) string { + t.Helper() + requirePostgres(t) + return pgtest.NewDatabase(t) +} diff --git a/cmd/migrate/cli_test.go b/cmd/migrate/cli_test.go index e9bfbf3e6..3e79c41c3 100644 --- a/cmd/migrate/cli_test.go +++ b/cmd/migrate/cli_test.go @@ -8,11 +8,10 @@ import ( "testing" "github.com/meshcore-analyzer/dbschema" - "github.com/meshcore-analyzer/pgutil/pgtest" ) func TestCheckReadyDoesNotBootstrap(t *testing.T) { - dsn := pgtest.NewSchema(t) + dsn := postgresSchema(t) t.Setenv("CORESCOPE_DATABASE_URL", dsn) t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") var output bytes.Buffer @@ -39,7 +38,7 @@ func TestCheckReadyDoesNotBootstrap(t *testing.T) { func TestCLIImportRequiresOfflineAndDoesNotFinalizeOnAccountFailure(t *testing.T) { source := telemetrySource(t) - telemetry, accounts := pgtest.NewDatabase(t), pgtest.NewDatabase(t) + telemetry, accounts := postgresDatabase(t), postgresDatabase(t) t.Setenv("CORESCOPE_DATABASE_URL", telemetry) t.Setenv("CORESCOPE_USERS_DATABASE_URL", accounts) var output bytes.Buffer @@ -57,7 +56,7 @@ func TestCLIImportRequiresOfflineAndDoesNotFinalizeOnAccountFailure(t *testing.T } func TestCheckImportMarkerBindsExactSource(t *testing.T) { - dsn, source := pgtest.NewSchema(t), telemetrySource(t) + dsn, source := postgresSchema(t), telemetrySource(t) t.Setenv("CORESCOPE_DATABASE_URL", dsn) t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") var output bytes.Buffer @@ -93,7 +92,7 @@ func TestWALModeCleanShutdownImportAndGuard(t *testing.T) { t.Fatal(err) } db.Close() - dsn := pgtest.NewSchema(t) + dsn := postgresSchema(t) t.Setenv("CORESCOPE_DATABASE_URL", dsn) t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") var output bytes.Buffer diff --git a/cmd/migrate/datetime_test.go b/cmd/migrate/datetime_test.go index 9f0054d3a..f1c2308bf 100644 --- a/cmd/migrate/datetime_test.go +++ b/cmd/migrate/datetime_test.go @@ -8,13 +8,12 @@ import ( "testing" "github.com/meshcore-analyzer/dbschema" - "github.com/meshcore-analyzer/pgutil/pgtest" ) func dateSource(t *testing.T, declared string, values []any) string { t.Helper() source := telemetrySource(t) - db, err := sql.Open(importSQLiteDriver, sqliteURL(source, "rw")) + db, err := openSQLite(source, "rw") if err != nil { t.Fatal(err) } @@ -62,7 +61,7 @@ func TestImportPreservesDeclaredDateTextAndResume(t *testing.T) { if err != nil { t.Fatal(err) } - dsn := pgtest.NewSchema(t) + dsn := postgresSchema(t) target := openImportDB(t, dsn) options := importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 2} options.afterBatch = func() error { @@ -137,7 +136,7 @@ func TestImportRefusesNumericStorageInDeclaredDateColumn(t *testing.T) { if err != nil { t.Fatal(err) } - dsn := pgtest.NewSchema(t) + dsn := postgresSchema(t) report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry"}) if err == nil || !strings.Contains(err.Error(), "non-text value") || report.Verified { t.Fatalf("non-text date storage was silently coerced: verified=%v error=%v", report.Verified, err) diff --git a/cmd/migrate/destination_empty_test.go b/cmd/migrate/destination_empty_test.go index 4b7aa91c9..519e674c9 100644 --- a/cmd/migrate/destination_empty_test.go +++ b/cmd/migrate/destination_empty_test.go @@ -4,12 +4,10 @@ import ( "context" "strings" "testing" - - "github.com/meshcore-analyzer/pgutil/pgtest" ) func TestFreshImportRefusesStandaloneSequence(t *testing.T) { - dsn := pgtest.NewSchema(t) + dsn := postgresSchema(t) db := openImportDB(t, dsn) if _, err := db.Exec(`CREATE SEQUENCE retained_sequence START WITH 77`); err != nil { t.Fatal(err) diff --git a/cmd/migrate/generated_columns_test.go b/cmd/migrate/generated_columns_test.go index a879f2c03..56ed32272 100644 --- a/cmd/migrate/generated_columns_test.go +++ b/cmd/migrate/generated_columns_test.go @@ -7,7 +7,6 @@ import ( "strings" "testing" - "github.com/meshcore-analyzer/pgutil/pgtest" "github.com/meshcore-analyzer/users" ) @@ -50,7 +49,7 @@ func TestImportRejectsGeneratedAccountColumnsBeforeNormalization(t *testing.T) { if err != nil { t.Fatal(err) } - dsn, state := pgtest.NewSchema(t), t.TempDir() + dsn, state := postgresSchema(t), t.TempDir() report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: state, Kind: "accounts"}) if err == nil || !strings.Contains(err.Error(), "generated or hidden") { t.Errorf("generated source must be explicitly refused before normalization: verified=%v error=%v", report.Verified, err) @@ -63,7 +62,7 @@ func TestImportRejectsGeneratedAccountColumnsBeforeNormalization(t *testing.T) { if err := users.AssertReady(target); err == nil { t.Error("unsupported source became runtime-ready") } - normalized, err := sql.Open("sqlite3", sqliteURL(filepath.Join(state, "accounts", "normalized.sqlite"), "ro")) + normalized, err := openSQLite(filepath.Join(state, "accounts", "normalized.sqlite"), "ro") if err != nil { t.Fatal(err) } diff --git a/cmd/migrate/go.mod b/cmd/migrate/go.mod index 563d410ac..abaa9c9b0 100644 --- a/cmd/migrate/go.mod +++ b/cmd/migrate/go.mod @@ -5,11 +5,25 @@ go 1.25.0 require ( github.com/jackc/pgx/v5 v5.7.6 github.com/mattn/go-sqlite3 v1.14.52 + github.com/meshcore-analyzer/dbconfig v0.0.0 github.com/meshcore-analyzer/dbschema v0.0.0 github.com/meshcore-analyzer/pgutil v0.0.0 + github.com/meshcore-analyzer/sqliteutil v0.0.0 github.com/meshcore-analyzer/users v0.0.0 ) +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + modernc.org/libc v1.55.3 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect + modernc.org/sqlite v1.34.5 // indirect +) + require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect @@ -28,3 +42,7 @@ replace github.com/meshcore-analyzer/users => ../../internal/users replace github.com/meshcore-analyzer/pgutil => ../../internal/pgutil replace github.com/meshcore-analyzer/packetpath => ../../internal/packetpath + +replace github.com/meshcore-analyzer/dbconfig => ../../internal/dbconfig + +replace github.com/meshcore-analyzer/sqliteutil => ../../internal/sqliteutil diff --git a/cmd/migrate/go.sum b/cmd/migrate/go.sum index 4915dde04..eb0d9a0d8 100644 --- a/cmd/migrate/go.sum +++ b/cmd/migrate/go.sum @@ -1,6 +1,12 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -9,10 +15,16 @@ github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -20,13 +32,42 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= +modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= +modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= +modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= +modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= +modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= +modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= +modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/cmd/migrate/grants.go b/cmd/migrate/grants.go new file mode 100644 index 000000000..d0f14de14 --- /dev/null +++ b/cmd/migrate/grants.go @@ -0,0 +1,123 @@ +package main + +import ( + "context" + "errors" + "strings" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" +) + +// Same narrow policy as docker/postgres-grants.sql, applied to the verified +// unselected destination before runtime validation and the atomic selection +// commit. Runtime role names come from their explicit credential URLs; no +// password or connection URL enters a SQL statement or child process argv. +func grantRuntimeTargets(ctx context.Context, selected dbconfig.Selection, owners dbconfig.Storage, raw dbconfig.StorageInputs) error { + bound, err := selected.ApplyTo(raw) + if err != nil { + return err + } + runtime, err := dbconfig.ResolveStorage(bound) + if err != nil { + return err + } + role := func(dsn string) (string, error) { + if dsn == "" { + return "", errors.New("prepare the runtime role credentials before backend selection") + } + config, err := pgutil.ParseConfig(dsn) + if err != nil { + return "", err + } + return quote(config.User), nil + } + reader, err := role(runtime.ReaderDatabaseURL) + if err != nil { + return err + } + writer, err := role(runtime.WriterDatabaseURL) + if err != nil { + return err + } + if reader == writer { + return errors.New("PostgreSQL telemetry reader and writer must use separate restricted roles") + } + type grants struct { + kind, owner, schema string + roles []string + queries []string + } + telemetry := grants{kind: "telemetry", owner: owners.WriterDatabaseURL, schema: quote(selected.Telemetry.Postgres.Schema), roles: []string{reader, writer}} + telemetry.queries = []string{ + `GRANT SELECT ON ALL TABLES IN SCHEMA ` + telemetry.schema + ` TO ` + reader, + `GRANT SELECT ON ALL SEQUENCES IN SCHEMA ` + telemetry.schema + ` TO ` + reader, + `GRANT SELECT ON corescope_schema,packets_v TO ` + writer, + `GRANT USAGE,SELECT ON ALL SEQUENCES IN SCHEMA ` + telemetry.schema + ` TO ` + writer, + } + tables, err := layouts("telemetry") + if err != nil { + return err + } + for _, table := range tables { + telemetry.queries = append(telemetry.queries, `GRANT SELECT,INSERT,UPDATE,DELETE ON TABLE `+quote(table.Name)+` TO `+writer) + } + stores := []grants{telemetry} + if selected.Accounts != nil { + account, err := role(runtime.UsersDatabaseURL) + if err != nil { + return err + } + entry := grants{kind: "accounts", owner: owners.UsersDatabaseURL, schema: quote(selected.Accounts.Postgres.Schema), roles: []string{account}} + entry.queries = []string{`GRANT SELECT ON ALL TABLES IN SCHEMA ` + entry.schema + ` TO ` + account, `GRANT USAGE,SELECT ON ALL SEQUENCES IN SCHEMA ` + entry.schema + ` TO ` + account} + tables, err := layouts("accounts") + if err != nil { + return err + } + for _, table := range tables { + entry.queries = append(entry.queries, `GRANT SELECT,INSERT,UPDATE,DELETE ON TABLE `+quote(table.Name)+` TO `+account) + } + if runtime.ApprovedChannelsDatabaseURL != "" { + channel, err := role(runtime.ApprovedChannelsDatabaseURL) + if err != nil { + return err + } + if channel == account { + return errors.New("account writer and channel projection reader must use separate restricted roles") + } + entry.roles = append(entry.roles, channel) + entry.queries = append(entry.queries, `GRANT SELECT ON approved_channels,corescope_schema TO `+channel) + } + stores = append(stores, entry) + } + for _, store := range stores { + if store.owner == "" { + continue + } // Account-only initialization leaves telemetry grants unchanged. + db, err := pgutil.Open(store.owner, false) + if err != nil { + return err + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + db.Close() + return err + } + queries := append([]string{`REVOKE CREATE ON SCHEMA ` + store.schema + ` FROM PUBLIC`, `GRANT USAGE ON SCHEMA ` + store.schema + ` TO ` + strings.Join(store.roles, ",")}, store.queries...) + for _, query := range queries { + if _, err = tx.ExecContext(ctx, query); err != nil { + break + } + } + if err == nil { + err = tx.Commit() + } else { + tx.Rollback() + } + db.Close() + if err != nil { + return err + } + } + return nil +} diff --git a/cmd/migrate/identity_layout_test.go b/cmd/migrate/identity_layout_test.go new file mode 100644 index 000000000..7eed801fb --- /dev/null +++ b/cmd/migrate/identity_layout_test.go @@ -0,0 +1,107 @@ +package main + +import ( + "context" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/users" + "path/filepath" + "testing" +) + +func TestNewSQLiteIdentitySourceLayouts(t *testing.T) { + for _, kind := range []string{"telemetry", "accounts"} { + t.Run(kind, func(t *testing.T) { + source := filepath.Join(t.TempDir(), "identity # % é.sqlite") + db, err := openSQLite(source, "rwc") + if err != nil { + t.Fatal(err) + } + if kind == "telemetry" { + err = dbschema.ApplySQLite(db, nil) + } else { + err = users.ApplySQLite(db) + } + if err != nil { + db.Close() + t.Fatal(err) + } + if kind == "accounts" { + for _, q := range []string{ + `INSERT INTO users(id,email,display_name,password_hash,created_at) VALUES(5,'test@example.invalid','','hash',1)`, + `INSERT INTO mail_log(id,user_id,to_email,purpose,sent_at,last_event_at) VALUES(4,5,'test@example.invalid','test',1,1)`, + `INSERT INTO mail_events(id,mail_id,event,at) VALUES(-9,4,'zero',1),(0,4,'negative',2),(900,4,'delivered',3)`, + `UPDATE sqlite_sequence SET seq=9000 WHERE name='mail_events'`, + } { + if _, err := db.Exec(q); err != nil { + t.Fatal(err) + } + } + } + db.Close() + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + tables, err := layouts(kind) + if err != nil { + t.Fatal(err) + } + _, normalized, err := prepareSource(context.Background(), importOptions{Source: source, StateDir: t.TempDir(), Kind: kind, targetDigest: "test-only"}, tables, true) + if err != nil { + t.Fatal(err) + } + after, err := sourceFingerprint(source) + if err != nil || after != before { + t.Fatal("source changed", err) + } + copyDB, err := openSQLite(normalized, "ro") + if err != nil { + t.Fatal(err) + } + defer copyDB.Close() + if kind == "accounts" { + var n, seq, version int + if err := copyDB.QueryRow(`SELECT count(*) FROM mail_events WHERE id IN (-9,0,900)`).Scan(&n); err != nil || n != 3 { + t.Fatal("explicit event IDs changed", n, err) + } + if err := copyDB.QueryRow(`SELECT seq FROM sqlite_sequence WHERE name='mail_events'`).Scan(&seq); err != nil || seq != 9000 { + t.Fatal("event sequence changed", seq, err) + } + if err := copyDB.QueryRow(`SELECT version FROM schema_version`).Scan(&version); err != nil || version != 6 { + t.Fatal("SQLite source version changed", version, err) + } + } + }) + } +} + +func TestForwardNewSQLiteAccountIDsAndSequence(t *testing.T) { + dsn := postgresSchema(t) + source := accountSource(t, 5) + db, err := openSQLite(source, "rw") + if err != nil { + t.Fatal(err) + } + if err := users.ApplySQLite(db); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`UPDATE sqlite_sequence SET seq=9000 WHERE name='mail_events'`); err != nil { + t.Fatal(err) + } + db.Close() + report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "accounts"}) + if err != nil || !report.Verified { + t.Fatal("forward v6", err) + } + pg := openImportDB(t, dsn) + var id, version int + if err := pg.QueryRow(`SELECT id FROM mail_events WHERE mail_id=4`).Scan(&id); err != nil || id != 900 { + t.Fatal("mail event identity", id, err) + } + if err := pg.QueryRow(`SELECT version FROM schema_version`).Scan(&version); err != nil || version != 5 { + t.Fatal("PG version", version, err) + } + if err := pg.QueryRow(`INSERT INTO mail_events(mail_id,event,at) VALUES(4,'later',44) RETURNING id`).Scan(&id); err != nil || id != 9001 { + t.Fatal("mail sequence", id, err) + } +} diff --git a/cmd/migrate/import.go b/cmd/migrate/import.go index 34a9dc659..ac8552580 100644 --- a/cmd/migrate/import.go +++ b/cmd/migrate/import.go @@ -120,7 +120,7 @@ func prepareSource(ctx context.Context, o importOptions, tables []importTable, a if err := snapshotSource(ctx, raw, normalized); err != nil { return manifest, "", err } - db, err := sql.Open(importSQLiteDriver, sqliteURL(normalized, "rw")) + db, err := openSQLite(normalized, "rw") if err != nil { return manifest, "", err } @@ -240,7 +240,7 @@ func importSQLite(ctx context.Context, o importOptions) (importReport, error) { if err != nil { return report, err } - source, err := sql.Open(importSQLiteDriver, sqliteURL(path, "ro")) + source, err := openSQLite(path, "ro") if err != nil { return report, err } @@ -390,7 +390,7 @@ func columnNames(table importTable) ([]string, []string) { return names, quoted } -func orderBy(db *sql.DB, table importTable, postgres bool) (string, error) { +func orderBy(db *sql.DB, table importTable, postgres, hiddenMailID bool) (string, error) { // Sort by logical primary key with identical byte collation in both engines. rows, err := db.Query(`SELECT a.attname FROM pg_index i JOIN pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=ANY(i.indkey) WHERE i.indrelid=to_regclass($1) AND i.indisprimary ORDER BY array_position(i.indkey,a.attnum)`, table.Name) @@ -421,7 +421,7 @@ func orderBy(db *sql.DB, table importTable, postgres bool) (string, error) { } } expr := quote(name) - if table.Name == "mail_events" && name == "id" && !postgres { + if table.Name == "mail_events" && name == "id" && hiddenMailID { expr = "rowid" } if colType == "TEXT" { @@ -518,7 +518,7 @@ func hashRow(h hash.Hash, values []any) { func digestTarget(ctx context.Context, db *sql.DB, table importTable) (tableReport, error) { report := tableReport{Table: table.Name} _, quoted := columnNames(table) - order, err := orderBy(db, table, true) + order, err := orderBy(db, table, true, false) if err != nil { return report, err } @@ -614,12 +614,20 @@ func copyTable(ctx context.Context, source, owner *sql.DB, conn *pgx.Conn, table return nil } if exists != 0 { - order, err := orderBy(owner, table, false) + hiddenMailID := false + if o.Kind == "accounts" && table.Name == "mail_events" { + var explicit int + if err := source.QueryRow(`SELECT count(*) FROM pragma_table_info('mail_events') WHERE name='id'`).Scan(&explicit); err != nil { + return report, err + } + hiddenMailID = explicit == 0 + } + order, err := orderBy(owner, table, false, hiddenMailID) if err != nil { return report, err } expressions := append([]string(nil), quoted...) - if o.Kind == "accounts" && table.Name == "mail_events" { + if hiddenMailID { expressions[0] = "rowid" } // SQLite's unary + preserves the stored value/type while removing declared @@ -707,17 +715,22 @@ func reseed(ctx context.Context, source *sql.DB, conn *pgx.Conn, tables []import return err } if hasSequence > 0 { - rows, err := source.Query(`SELECT name,seq FROM sqlite_sequence`) + rows, err := source.Query(`SELECT name,MAX(seq),count(*) FROM sqlite_sequence GROUP BY name`) if err != nil { return err } for rows.Next() { var name string var n int64 - if err := rows.Scan(&name, &n); err != nil { + var count int + if err := rows.Scan(&name, &n, &count); err != nil { rows.Close() return err } + if count != 1 || n < 0 { + rows.Close() + return errors.New("source SQLite sequence state is invalid or duplicated") + } highWater[name] = n } if err := rows.Err(); err != nil { @@ -726,15 +739,7 @@ func reseed(ctx context.Context, source *sql.DB, conn *pgx.Conn, tables []import } rows.Close() } - references := map[string][][2]string{} - if kind == "telemetry" { - references["observers"] = [][2]string{{"observations", "observer_idx"}} - references["transmissions"] = [][2]string{{"observations", "transmission_id"}, {"advert_route_evidence", "tx_id"}, {"advert_evidence_backfill", "tx_cursor"}} - references["observations"] = [][2]string{{"advert_evidence_backfill", "obs_cursor"}} - } else { - references["users"] = [][2]string{{"audit_log", "actor_user_id"}, {"audit_log", "target_user_id"}, {"users", "activated_by"}, {"sessions", "user_id"}, {"tokens", "user_id"}, {"mail_log", "user_id"}, {"proposals", "proposer_id"}, {"proposals", "reviewer_id"}} - references["mail_log"] = [][2]string{{"mail_events", "mail_id"}} - } + references := identityReferences(kind) for _, table := range tables { for _, col := range table.Columns { if !col.Identity { diff --git a/cmd/migrate/import_safety_test.go b/cmd/migrate/import_safety_test.go index 422a10c2a..7203fbbbb 100644 --- a/cmd/migrate/import_safety_test.go +++ b/cmd/migrate/import_safety_test.go @@ -10,17 +10,16 @@ import ( "testing" "github.com/meshcore-analyzer/dbschema" - "github.com/meshcore-analyzer/pgutil/pgtest" ) func TestResumeRefusesDifferentDestination(t *testing.T) { - o := importOptions{Source: telemetrySource(t), DatabaseURL: pgtest.NewSchema(t), StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 1, afterBatch: func() error { return context.Canceled }} + o := importOptions{Source: telemetrySource(t), DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 1, afterBatch: func() error { return context.Canceled }} if _, err := importSQLite(context.Background(), o); !errors.Is(err, context.Canceled) { t.Fatal(err) } o.Resume = true o.afterBatch = nil - o.DatabaseURL = pgtest.NewSchema(t) + o.DatabaseURL = postgresSchema(t) if _, err := importSQLite(context.Background(), o); err == nil { t.Fatal("resume accepted a different PostgreSQL destination") } @@ -29,7 +28,7 @@ func TestResumeRefusesDifferentDestination(t *testing.T) { func TestResumeAndFinalizationRejectTargetCorruption(t *testing.T) { for _, partial := range []bool{false, true} { t.Run(map[bool]string{false: "finalization", true: "resume"}[partial], func(t *testing.T) { - o := importOptions{Source: telemetrySource(t), DatabaseURL: pgtest.NewSchema(t), StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 1} + o := importOptions{Source: telemetrySource(t), DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 1} if partial { o.afterBatch = func() error { return context.Canceled } } @@ -75,7 +74,7 @@ func TestImportPreservesRawCaseJSONAndFractionalScore(t *testing.T) { } } db.Close() - dsn := pgtest.NewSchema(t) + dsn := postgresSchema(t) if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry"}); err != nil { t.Fatal(err) } @@ -105,7 +104,7 @@ func TestImportRejectsUnexpectedViewsTriggersAndFutureAccounts(t *testing.T) { t.Fatal(err) } db.Close() - dsn := pgtest.NewSchema(t) + dsn := postgresSchema(t) _, err = importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "accounts"}) if err == nil { t.Fatal("unsupported source accepted") @@ -127,7 +126,7 @@ func TestAccountsReseedDeletedHistoricalReferences(t *testing.T) { t.Fatal(err) } db.Close() - dsn := pgtest.NewSchema(t) + dsn := postgresSchema(t) if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "accounts"}); err != nil { t.Fatal(err) } @@ -156,7 +155,7 @@ func TestImportRefusesLegacyV2WithUpgradeGuidance(t *testing.T) { t.Fatal(err) } db.Close() - _, err = importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: pgtest.NewSchema(t), StateDir: t.TempDir(), Kind: "telemetry"}) + _, err = importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "telemetry"}) if err == nil || !strings.Contains(err.Error(), "upgrade") { t.Fatal("v2 refusal lacks explicit upgrade guidance") } @@ -179,7 +178,7 @@ func TestAccountVersionMatrixAndMissingDeclaredTable(t *testing.T) { } db.Close() } - report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: pgtest.NewSchema(t), StateDir: t.TempDir(), Kind: "accounts"}) + report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "accounts"}) if err != nil || !report.Verified { t.Fatalf("account version %d: %v", version, err) } @@ -195,7 +194,7 @@ func TestAccountVersionMatrixAndMissingDeclaredTable(t *testing.T) { t.Fatal(err) } db.Close() - if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: pgtest.NewSchema(t), StateDir: t.TempDir(), Kind: "accounts"}); err == nil { + if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "accounts"}); err == nil { t.Fatal("incomplete declared account schema was accepted") } }) @@ -216,7 +215,7 @@ func TestNormalizeDoesNotRewriteObservationRows(t *testing.T) { if err := snapshotSource(context.Background(), raw, working); err != nil { t.Fatal(err) } - db, err = sql.Open("sqlite3", sqliteURL(working, "rw")) + db, err = openSQLite(working, "rw") if err != nil { t.Fatal(err) } @@ -243,7 +242,7 @@ func TestNormalizeDoesNotRewriteObservationRows(t *testing.T) { } func TestImportPopulatesPlannerStatsBeforeReadiness(t *testing.T) { - dsn := pgtest.NewSchema(t) + dsn := postgresSchema(t) if _, err := importSQLite(context.Background(), importOptions{Source: telemetrySource(t), DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry"}); err != nil { t.Fatal(err) } @@ -278,7 +277,7 @@ func TestReservedNameDoesNotHideSourceTrigger(t *testing.T) { } } db.Close() - db, err = sql.Open("sqlite3", sqliteURL(source, "ro")) + db, err = openSQLite(source, "ro") if err != nil { t.Fatal(err) } @@ -288,7 +287,7 @@ func TestReservedNameDoesNotHideSourceTrigger(t *testing.T) { t.Fatalf("tampered schema fixture: count=%d err=%v", count, err) } db.Close() - if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: pgtest.NewSchema(t), StateDir: t.TempDir(), Kind: "accounts"}); err == nil { + if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "accounts"}); err == nil { t.Fatal("reserved-name trigger bypassed source validation") } } diff --git a/cmd/migrate/import_test.go b/cmd/migrate/import_test.go index fbea42345..01b6f9bad 100644 --- a/cmd/migrate/import_test.go +++ b/cmd/migrate/import_test.go @@ -10,7 +10,6 @@ import ( "github.com/meshcore-analyzer/dbschema" "github.com/meshcore-analyzer/dbschema/legacy" "github.com/meshcore-analyzer/pgutil" - "github.com/meshcore-analyzer/pgutil/pgtest" "github.com/meshcore-analyzer/users" ) @@ -57,7 +56,7 @@ func TestImportTelemetryPreservesIDsNullsAndHighWater(t *testing.T) { if err != nil { t.Fatal(err) } - dsn := pgtest.NewSchema(t) + dsn := postgresSchema(t) report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 2}) if err != nil { t.Fatalf("import valid telemetry: %v", err) @@ -106,7 +105,7 @@ func TestImportTelemetryPreservesIDsNullsAndHighWater(t *testing.T) { func TestImportInterruptResumeAndWrongSource(t *testing.T) { source := telemetrySource(t) - dsn, state := pgtest.NewSchema(t), t.TempDir() + dsn, state := postgresSchema(t), t.TempDir() options := importOptions{Source: source, DatabaseURL: dsn, StateDir: state, Kind: "telemetry", BatchSize: 1, afterBatch: func() error { return context.Canceled }} if _, err := importSQLite(context.Background(), options); !errors.Is(err, context.Canceled) { t.Fatalf("expected cancellation after committed batch, got %v", err) @@ -154,7 +153,7 @@ func TestImportInterruptResumeAndWrongSource(t *testing.T) { func TestImportRefusesUnknownSourceAndOccupiedTarget(t *testing.T) { for _, unknown := range []bool{false, true} { - source, dsn := telemetrySource(t), pgtest.NewSchema(t) + source, dsn := telemetrySource(t), postgresSchema(t) if unknown { db, err := sql.Open("sqlite3", source) if err != nil { @@ -221,7 +220,7 @@ func accountSource(t *testing.T, version int) string { } func TestImportLegacyAccountsPreservesAuthenticationAndEventOrder(t *testing.T) { - source, dsn := accountSource(t, 1), pgtest.NewSchema(t) + source, dsn := accountSource(t, 1), postgresSchema(t) report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "accounts", BatchSize: 1}) if err != nil { t.Fatal(err) diff --git a/cmd/migrate/layout.go b/cmd/migrate/layout.go index 9e0b566ba..74c8d1c49 100644 --- a/cmd/migrate/layout.go +++ b/cmd/migrate/layout.go @@ -100,13 +100,23 @@ func sqliteColumns(db *sql.DB, table string) ([]importColumn, error) { } func validateSource(db *sql.DB, kind string, tables []importTable) error { + var accountVersion int + if kind == "accounts" { + var n int + if err := db.QueryRow(`SELECT COUNT(*),COALESCE(MAX(version),0) FROM schema_version`).Scan(&n, &accountVersion); err != nil { + return errors.New("account source must contain its supported schema_version") + } + if n != 1 || accountVersion < 0 || accountVersion > users.SQLiteSchemaVersion { + return fmt.Errorf("unsupported account schema version %d (supported 0 through %d)", accountVersion, users.SQLiteSchemaVersion) + } + } allowed := map[string]map[string]bool{} for _, table := range tables { cols := map[string]bool{} for _, col := range table.Columns { cols[col.Name] = true } - if kind == "accounts" && table.Name == "mail_events" { + if kind == "accounts" && table.Name == "mail_events" && accountVersion < users.SQLiteSchemaVersion { delete(cols, "id") } allowed[table.Name] = cols @@ -172,13 +182,10 @@ func validateSource(db *sql.DB, kind string, tables []importTable) error { if kind == "telemetry" { return legacy.CheckLegacySource(db) } - var n, version int - if err := db.QueryRow(`SELECT COUNT(*),COALESCE(MAX(version),0) FROM schema_version`).Scan(&n, &version); err != nil { - return errors.New("account source must contain schema_version (supported versions 0 through 5)") - } - if n != 1 || version < 0 || version > users.CurrentSchemaVersion { - return fmt.Errorf("unsupported account schema version %d (supported 0 through %d)", version, users.CurrentSchemaVersion) + if accountVersion == users.SQLiteSchemaVersion { + return users.AssertSQLiteReady(db) } + version := accountVersion // A declared version must contain its complete historical shape. Otherwise // treating an absent table as empty could conceal a damaged account source. expected, err := sql.Open("sqlite3", ":memory:") @@ -230,6 +237,9 @@ func normalizeSource(db *sql.DB, kind, rawPath string, tables []importTable) err if err := db.QueryRow(`SELECT version FROM schema_version`).Scan(&version); err != nil { return err } + if version == users.SQLiteSchemaVersion { + return nil + } for i, migration := range users.LegacyMigrations()[version:] { tx, err := db.Begin() if err != nil { @@ -257,7 +267,11 @@ func normalizeSource(db *sql.DB, kind, rawPath string, tables []importTable) err // Mutation inventory of legacy.Normalize: only these columns can change // existing rows. Duplicate/invalid-row deletion paths were refused above. // Restore changed tuples by rowid, without rewriting the observation corpus. - if _, err := db.Exec(`ATTACH DATABASE ? AS recovery`, sqliteURL(rawPath, "ro")); err != nil { + recoveryURI, err := sqliteURL(rawPath, "ro") + if err != nil { + return err + } + if _, err := db.Exec(`ATTACH DATABASE ? AS recovery`, recoveryURI); err != nil { return err } defer db.Exec(`DETACH DATABASE recovery`) diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go index e987de673..73db2f73e 100644 --- a/cmd/migrate/main.go +++ b/cmd/migrate/main.go @@ -34,11 +34,19 @@ func runCommand(ctx context.Context, args []string, out io.Writer) error { usersURL := flags.String("users-database-url", "", "PostgreSQL accounts owner URL (prefer CORESCOPE_USERS_DATABASE_URL)") source := flags.String("from-sqlite", "", "offline telemetry SQLite source") accountSource := flags.String("users-from-sqlite", "", "offline account SQLite source") - stateDir := flags.String("state-dir", "state/migration", "private migration snapshots and resume manifests") + stateDir := flags.String("state-dir", "", "private migration snapshots and resume manifests (storage actions default beside selection; legacy imports use state/migration)") offline := flags.Bool("offline", false, "confirm telemetry and account writers are stopped") resume := flags.Bool("resume", false, "resume the same source/state/destination import") checkReady := flags.Bool("check-ready", false, "check readiness without changing any schema or data") checkImportKind := flags.String("check-import-kind", "", "read-only completed-import guard: telemetry or accounts, with its SQLite source path") + action := flags.String("storage-action", "", "installation status, setup, adopt, init, switch, resume or abort") + selectionFile := flags.String("selection-file", "", "absolute stable storage-selection.json path") + backend := flags.String("backend", "", "bootstrap backend, or explicit switch destination: sqlite or postgres") + sqlitePath := flags.String("sqlite-path", "", "bootstrap or new destination telemetry SQLite path") + usersSQLitePath := flags.String("users-sqlite-path", "", "bootstrap or new destination account SQLite path") + jobID := flags.String("job-id", "", "exact pending storage switch job to resume or abort") + configFile := flags.String("config", "", "explicit configuration file for storage setup") + configDir := flags.String("config-dir", ".", "configuration discovery directory; relative data paths still use the working directory") if err := flags.Parse(args); err != nil { if errors.Is(err, flag.ErrHelp) { flags.SetOutput(out) @@ -50,6 +58,15 @@ func runCommand(ctx context.Context, args []string, out io.Writer) error { if flags.NArg() != 0 { return errors.New("unexpected positional migration argument; use -help") } + if *action != "" { + if *source != "" || *accountSource != "" || *resume || *checkReady || *checkImportKind != "" { + return errors.New("storage actions cannot be combined with legacy import flags") + } + return runStorageAction(ctx, storageOptions{Action: *action, SelectionFile: *selectionFile, Backend: *backend, SQLitePath: *sqlitePath, UsersSQLitePath: *usersSQLitePath, OwnerURL: *databaseURL, UsersOwnerURL: *usersURL, StateDir: *stateDir, JobID: *jobID, Offline: *offline, ConfigFile: *configFile, ConfigDir: *configDir}, out) + } + if *stateDir == "" { + *stateDir = "state/migration" + } if *databaseURL == "" { *databaseURL = os.Getenv("CORESCOPE_DATABASE_URL") } diff --git a/cmd/migrate/main_test.go b/cmd/migrate/main_test.go index 3bb4f8161..b08121737 100644 --- a/cmd/migrate/main_test.go +++ b/cmd/migrate/main_test.go @@ -17,10 +17,9 @@ import ( "testing" _ "github.com/mattn/go-sqlite3" - postgresSchema "github.com/meshcore-analyzer/dbschema" + postgresSchemaAPI "github.com/meshcore-analyzer/dbschema" dbschema "github.com/meshcore-analyzer/dbschema/legacy" "github.com/meshcore-analyzer/packetpath" - "github.com/meshcore-analyzer/pgutil/pgtest" ) // fixtureCandidates lists possible locations of the committed e2e @@ -44,7 +43,7 @@ func locateFixture(t *testing.T) string { func TestCommittedFixtureRequiresExplicitDuplicateRepair(t *testing.T) { source := locateFixture(t) - _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: pgtest.NewSchema(t), StateDir: t.TempDir(), Kind: "telemetry"}) + _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "telemetry"}) if err == nil || !strings.Contains(err.Error(), "duplicate observation identity") { t.Fatalf("raw legacy fixture must require explicit duplicate repair: %v", err) } @@ -60,7 +59,7 @@ func TestPostgresImportCommittedFixture(t *testing.T) { if err := snapshotSource(context.Background(), original, source); err != nil { t.Fatal(err) } - legacyDB, err := sql.Open("sqlite3", sqliteURL(source, "rw")) + legacyDB, err := openSQLite(source, "rw") if err != nil { t.Fatal(err) } @@ -79,7 +78,7 @@ func TestPostgresImportCommittedFixture(t *testing.T) { t.Fatal(err) } legacyDB.Close() - dsn := pgtest.NewSchema(t) + dsn := postgresSchema(t) report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 64}) if err != nil { t.Fatal(err) @@ -91,7 +90,7 @@ func TestPostgresImportCommittedFixture(t *testing.T) { t.Fatal(err) } db := openImportDB(t, dsn) - if err := postgresSchema.AssertReady(db); err != nil { + if err := postgresSchemaAPI.AssertReady(db); err != nil { t.Fatal(err) } var transmissions, observations int @@ -131,7 +130,7 @@ func TestFixturePreparationToolKeepsOriginalAndRefusesOverwrite(t *testing.T) { if output, err := exec.Command("go", "run", "../../scripts/migrate-fixture-hashes.go", destination).CombinedOutput(); err != nil { t.Fatalf("prepared fixture hash normalization: %v: %s", err, output) } - db, err := sql.Open("sqlite3", sqliteURL(destination, "ro")) + db, err := openSQLite(destination, "ro") if err != nil { t.Fatal(err) } @@ -168,7 +167,7 @@ func TestFixtureHashToolUsesSharedRuntimeIdentity(t *testing.T) { if err != nil { t.Fatalf("hash fixture: %v: %s", err, output) } - db, err = sql.Open("sqlite3", sqliteURL(file, "ro")) + db, err = openSQLite(file, "ro") if err != nil { t.Fatal(err) } diff --git a/cmd/migrate/operator_paths_test.go b/cmd/migrate/operator_paths_test.go index f10b97208..9649ab2c7 100644 --- a/cmd/migrate/operator_paths_test.go +++ b/cmd/migrate/operator_paths_test.go @@ -7,15 +7,13 @@ import ( "path/filepath" "strings" "testing" - - "github.com/meshcore-analyzer/pgutil/pgtest" ) func TestOperatorCustomSourcePathsAndOptionalAccounts(t *testing.T) { for _, existingAccounts := range []bool{false, true} { t.Run(map[bool]string{false: "no previous accounts", true: "preserve existing accounts"}[existingAccounts], func(t *testing.T) { - t.Setenv("CORESCOPE_DATABASE_URL", pgtest.NewDatabase(t)) - accounts := pgtest.NewDatabase(t) + t.Setenv("CORESCOPE_DATABASE_URL", postgresDatabase(t)) + accounts := postgresDatabase(t) t.Setenv("CORESCOPE_USERS_DATABASE_URL", accounts) dir := filepath.Join(t.TempDir(), "custom state and sources") if err := os.Mkdir(dir, 0o700); err != nil { diff --git a/cmd/migrate/postgres_layout.go b/cmd/migrate/postgres_layout.go new file mode 100644 index 000000000..58fb10f71 --- /dev/null +++ b/cmd/migrate/postgres_layout.go @@ -0,0 +1,125 @@ +package main + +import ( + "database/sql" + _ "embed" + "encoding/json" + "errors" + "slices" + "sort" + "strings" +) + +// Captured from the versioned native ApplyPostgres entrypoints. The regression +// recreates both canonical schemas and compares this manifest, so a physical +// schema change requires an explicit converter compatibility decision. +// +//go:embed postgres_layout.json +var postgresLayoutJSON []byte + +func postgresLayout(db *sql.DB, kind string, tables []importTable) ([]string, error) { + var schema string + if err := db.QueryRow(`SELECT current_schema()`).Scan(&schema); err != nil { + return nil, err + } + allowed := map[string]bool{"corescope_schema": true, "corescope_import": true, "corescope_import_progress": true} + if kind == "accounts" { + allowed["schema_version"] = true + } + var tableNames []string + for _, table := range tables { + tableNames = append(tableNames, `'`+strings.ReplaceAll(table.Name, `'`, `''`)+`'`) + allowed[table.Name] = true + } + view := "packets_v" + if kind == "accounts" { + view = "approved_channels" + } + allowed[view] = true + relationRows, err := db.Query(`SELECT c.relname,c.relkind::text FROM pg_class c WHERE c.relnamespace=current_schema()::regnamespace AND c.relkind IN ('r','p','v','m','S','f') ORDER BY c.relname`) + if err != nil { + return nil, err + } + var relations [][2]string + for relationRows.Next() { + var pair [2]string + if err := relationRows.Scan(&pair[0], &pair[1]); err != nil { + relationRows.Close() + return nil, err + } + relations = append(relations, pair) + } + err = relationRows.Err() + relationRows.Close() + if err != nil { + return nil, err + } + for _, table := range tables { + for _, col := range table.Columns { + if !col.Identity { + continue + } + var name string + if err := db.QueryRow(`SELECT c.relname FROM pg_class c WHERE c.oid=pg_get_serial_sequence($1,$2)::regclass`, table.Name, col.Name).Scan(&name); err != nil { + return nil, err + } + allowed[name] = true + } + } + for _, pair := range relations { + if !allowed[pair[0]] { + return nil, errors.New("unsupported PostgreSQL source object; no source data was discarded") + } + } + names := strings.Join(tableNames, ",") + queries := []string{ + `SELECT json_build_array('column',table_name,ordinal_position,column_name,data_type,is_nullable,column_default,collation_name,is_identity,identity_generation,is_generated,generation_expression)::text FROM information_schema.columns WHERE table_schema=current_schema() AND table_name IN (` + names + `)`, + `SELECT json_build_array('table',c.relname,c.relkind,c.relrowsecurity,c.relforcerowsecurity)::text FROM pg_class c WHERE c.relnamespace=current_schema()::regnamespace AND c.relname IN (` + names + `)`, + `SELECT json_build_array('constraint',c.relname,k.conname,k.convalidated,pg_get_constraintdef(k.oid))::text FROM pg_constraint k JOIN pg_class c ON c.oid=k.conrelid WHERE c.relnamespace=current_schema()::regnamespace AND c.relname IN (` + names + `)`, + `SELECT json_build_array('index',c.relname,pg_get_indexdef(i.indexrelid))::text FROM pg_index i JOIN pg_class c ON c.oid=i.indrelid WHERE c.relnamespace=current_schema()::regnamespace AND c.relname IN (` + names + `)`, + `SELECT json_build_array('trigger',c.relname,pg_get_triggerdef(t.oid))::text FROM pg_trigger t JOIN pg_class c ON c.oid=t.tgrelid WHERE NOT t.tgisinternal AND c.relnamespace=current_schema()::regnamespace AND c.relname IN (` + names + `)`, + `SELECT json_build_array('policy',c.relname,p.polname,pg_get_expr(p.polqual,p.polrelid),pg_get_expr(p.polwithcheck,p.polrelid))::text FROM pg_policy p JOIN pg_class c ON c.oid=p.polrelid WHERE c.relnamespace=current_schema()::regnamespace AND c.relname IN (` + names + `)`, + `SELECT json_build_array('view',c.relname,pg_get_viewdef(c.oid))::text FROM pg_class c WHERE c.relnamespace=current_schema()::regnamespace AND c.relkind='v' AND c.relname='` + view + `'`, + } + var out []string + for _, query := range queries { + rows, err := db.Query(query) + if err != nil { + return nil, err + } + for rows.Next() { + var value string + if err := rows.Scan(&value); err != nil { + rows.Close() + return nil, err + } + // Only catalog qualification varies between installations. User data is + // never normalized, decoded or rewritten by these schema comparisons. + value = strings.ReplaceAll(value, strings.ReplaceAll(quote(schema), `"`, `\"`)+".", "{schema}.") + value = strings.ReplaceAll(value, schema+".", "{schema}.") + out = append(out, value) + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + } + sort.Strings(out) + return out, nil +} + +func validatePostgresLayout(db *sql.DB, kind string, tables []importTable) error { + var expected map[string][]string + if err := json.Unmarshal(postgresLayoutJSON, &expected); err != nil { + return errors.New("invalid compiled PostgreSQL conversion layout") + } + got, err := postgresLayout(db, kind, tables) + if err != nil { + return err + } + if !slices.Equal(got, expected[kind]) { + return errors.New("unsupported PostgreSQL columns, constraints, indexes, policies or views; conversion refused without changing the source") + } + return nil +} diff --git a/cmd/migrate/postgres_layout.json b/cmd/migrate/postgres_layout.json new file mode 100644 index 000000000..bb0c02ff9 --- /dev/null +++ b/cmd/migrate/postgres_layout.json @@ -0,0 +1,575 @@ +{ + "accounts": [ + "[\"column\", \"audit_log\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"audit_log\", 2, \"at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"audit_log\", 3, \"actor_user_id\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"audit_log\", 4, \"action\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"audit_log\", 5, \"target_user_id\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"audit_log\", 6, \"detail\", \"text\", \"NO\", \"'{}'::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_events\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"mail_events\", 2, \"mail_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_events\", 3, \"event\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_events\", 4, \"at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_events\", 5, \"reason\", \"text\", \"NO\", \"''::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"mail_log\", 2, \"user_id\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 3, \"to_email\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 4, \"purpose\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 5, \"provider_message_id\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 6, \"sent_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 7, \"last_event\", \"text\", \"NO\", \"'sent'::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 8, \"last_event_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 9, \"last_reason\", \"text\", \"NO\", \"''::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_prefs\", 1, \"user_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_prefs\", 2, \"enabled\", \"bigint\", \"NO\", \"1\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_prefs\", 3, \"events\", \"text\", \"NO\", \"''::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_prefs\", 4, \"unsub_token\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_prefs\", 5, \"updated_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_state\", 1, \"user_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_state\", 2, \"event\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_state\", 3, \"subject\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_state\", 4, \"state\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_state\", 5, \"changed_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_watches\", 1, \"user_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_watches\", 2, \"pubkey\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_watches\", 3, \"created_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"proposals\", 2, \"kind\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 3, \"subject\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 4, \"status\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 5, \"proposer_id\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 6, \"reviewer_id\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 7, \"note\", \"text\", \"NO\", \"''::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 8, \"created_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 9, \"decided_at\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"sessions\", 2, \"token_hash\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 3, \"user_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 4, \"csrf_token\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 5, \"created_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 6, \"expires_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 7, \"last_seen_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 8, \"user_agent\", \"text\", \"NO\", \"''::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"tokens\", 1, \"token_hash\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"tokens\", 2, \"user_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"tokens\", 3, \"purpose\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"tokens\", 4, \"new_email\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"tokens\", 5, \"expires_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"tokens\", 6, \"used_at\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"user_settings\", 1, \"user_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"user_settings\", 2, \"doc\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"user_settings\", 3, \"revision\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"user_settings\", 4, \"generation\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"user_settings\", 5, \"updated_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"users\", 10, \"last_login_at\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 11, \"email_bouncing\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 2, \"email\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 3, \"display_name\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 4, \"password_hash\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 5, \"role\", \"text\", \"NO\", \"'user'::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 6, \"status\", \"text\", \"NO\", \"'pending'::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 7, \"created_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 8, \"activated_at\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 9, \"activated_by\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"constraint\", \"audit_log\", \"audit_log_action_not_null\", true, \"NOT NULL action\"]", + "[\"constraint\", \"audit_log\", \"audit_log_at_not_null\", true, \"NOT NULL at\"]", + "[\"constraint\", \"audit_log\", \"audit_log_detail_not_null\", true, \"NOT NULL detail\"]", + "[\"constraint\", \"audit_log\", \"audit_log_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"audit_log\", \"audit_log_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"mail_events\", \"mail_events_at_not_null\", true, \"NOT NULL at\"]", + "[\"constraint\", \"mail_events\", \"mail_events_event_not_null\", true, \"NOT NULL event\"]", + "[\"constraint\", \"mail_events\", \"mail_events_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"mail_events\", \"mail_events_mail_id_event_at_key\", true, \"UNIQUE (mail_id, event, at)\"]", + "[\"constraint\", \"mail_events\", \"mail_events_mail_id_fkey\", true, \"FOREIGN KEY (mail_id) REFERENCES mail_log(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"mail_events\", \"mail_events_mail_id_not_null\", true, \"NOT NULL mail_id\"]", + "[\"constraint\", \"mail_events\", \"mail_events_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"mail_events\", \"mail_events_reason_not_null\", true, \"NOT NULL reason\"]", + "[\"constraint\", \"mail_log\", \"mail_log_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"mail_log\", \"mail_log_last_event_at_not_null\", true, \"NOT NULL last_event_at\"]", + "[\"constraint\", \"mail_log\", \"mail_log_last_event_not_null\", true, \"NOT NULL last_event\"]", + "[\"constraint\", \"mail_log\", \"mail_log_last_reason_not_null\", true, \"NOT NULL last_reason\"]", + "[\"constraint\", \"mail_log\", \"mail_log_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"mail_log\", \"mail_log_purpose_not_null\", true, \"NOT NULL purpose\"]", + "[\"constraint\", \"mail_log\", \"mail_log_sent_at_not_null\", true, \"NOT NULL sent_at\"]", + "[\"constraint\", \"mail_log\", \"mail_log_to_email_not_null\", true, \"NOT NULL to_email\"]", + "[\"constraint\", \"mail_log\", \"mail_log_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_enabled_check\", true, \"CHECK ((enabled = ANY (ARRAY[(0)::bigint, (1)::bigint])))\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_enabled_not_null\", true, \"NOT NULL enabled\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_events_not_null\", true, \"NOT NULL events\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_pkey\", true, \"PRIMARY KEY (user_id)\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_unsub_token_key\", true, \"UNIQUE (unsub_token)\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_unsub_token_not_null\", true, \"NOT NULL unsub_token\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_updated_at_not_null\", true, \"NOT NULL updated_at\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_user_id_not_null\", true, \"NOT NULL user_id\"]", + "[\"constraint\", \"notification_state\", \"notification_state_changed_at_not_null\", true, \"NOT NULL changed_at\"]", + "[\"constraint\", \"notification_state\", \"notification_state_event_not_null\", true, \"NOT NULL event\"]", + "[\"constraint\", \"notification_state\", \"notification_state_pkey\", true, \"PRIMARY KEY (user_id, event, subject)\"]", + "[\"constraint\", \"notification_state\", \"notification_state_state_not_null\", true, \"NOT NULL state\"]", + "[\"constraint\", \"notification_state\", \"notification_state_subject_not_null\", true, \"NOT NULL subject\"]", + "[\"constraint\", \"notification_state\", \"notification_state_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"notification_state\", \"notification_state_user_id_not_null\", true, \"NOT NULL user_id\"]", + "[\"constraint\", \"notification_watches\", \"notification_watches_created_at_not_null\", true, \"NOT NULL created_at\"]", + "[\"constraint\", \"notification_watches\", \"notification_watches_pkey\", true, \"PRIMARY KEY (user_id, pubkey)\"]", + "[\"constraint\", \"notification_watches\", \"notification_watches_pubkey_not_null\", true, \"NOT NULL pubkey\"]", + "[\"constraint\", \"notification_watches\", \"notification_watches_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"notification_watches\", \"notification_watches_user_id_not_null\", true, \"NOT NULL user_id\"]", + "[\"constraint\", \"proposals\", \"proposals_created_at_not_null\", true, \"NOT NULL created_at\"]", + "[\"constraint\", \"proposals\", \"proposals_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"proposals\", \"proposals_kind_not_null\", true, \"NOT NULL kind\"]", + "[\"constraint\", \"proposals\", \"proposals_note_not_null\", true, \"NOT NULL note\"]", + "[\"constraint\", \"proposals\", \"proposals_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"proposals\", \"proposals_proposer_id_fkey\", true, \"FOREIGN KEY (proposer_id) REFERENCES users(id) ON DELETE SET NULL\"]", + "[\"constraint\", \"proposals\", \"proposals_reviewer_id_fkey\", true, \"FOREIGN KEY (reviewer_id) REFERENCES users(id) ON DELETE SET NULL\"]", + "[\"constraint\", \"proposals\", \"proposals_status_check\", true, \"CHECK ((status = ANY (ARRAY['pending'::text, 'approved'::text, 'rejected'::text, 'revoked'::text])))\"]", + "[\"constraint\", \"proposals\", \"proposals_status_not_null\", true, \"NOT NULL status\"]", + "[\"constraint\", \"proposals\", \"proposals_subject_not_null\", true, \"NOT NULL subject\"]", + "[\"constraint\", \"sessions\", \"sessions_created_at_not_null\", true, \"NOT NULL created_at\"]", + "[\"constraint\", \"sessions\", \"sessions_csrf_token_not_null\", true, \"NOT NULL csrf_token\"]", + "[\"constraint\", \"sessions\", \"sessions_expires_at_not_null\", true, \"NOT NULL expires_at\"]", + "[\"constraint\", \"sessions\", \"sessions_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"sessions\", \"sessions_last_seen_at_not_null\", true, \"NOT NULL last_seen_at\"]", + "[\"constraint\", \"sessions\", \"sessions_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"sessions\", \"sessions_token_hash_key\", true, \"UNIQUE (token_hash)\"]", + "[\"constraint\", \"sessions\", \"sessions_token_hash_not_null\", true, \"NOT NULL token_hash\"]", + "[\"constraint\", \"sessions\", \"sessions_user_agent_not_null\", true, \"NOT NULL user_agent\"]", + "[\"constraint\", \"sessions\", \"sessions_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"sessions\", \"sessions_user_id_not_null\", true, \"NOT NULL user_id\"]", + "[\"constraint\", \"tokens\", \"tokens_expires_at_not_null\", true, \"NOT NULL expires_at\"]", + "[\"constraint\", \"tokens\", \"tokens_pkey\", true, \"PRIMARY KEY (token_hash)\"]", + "[\"constraint\", \"tokens\", \"tokens_purpose_check\", true, \"CHECK ((purpose = ANY (ARRAY['activate'::text, 'reset'::text, 'email_change'::text])))\"]", + "[\"constraint\", \"tokens\", \"tokens_purpose_not_null\", true, \"NOT NULL purpose\"]", + "[\"constraint\", \"tokens\", \"tokens_token_hash_not_null\", true, \"NOT NULL token_hash\"]", + "[\"constraint\", \"tokens\", \"tokens_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"tokens\", \"tokens_user_id_not_null\", true, \"NOT NULL user_id\"]", + "[\"constraint\", \"user_settings\", \"user_settings_doc_not_null\", true, \"NOT NULL doc\"]", + "[\"constraint\", \"user_settings\", \"user_settings_generation_not_null\", true, \"NOT NULL generation\"]", + "[\"constraint\", \"user_settings\", \"user_settings_pkey\", true, \"PRIMARY KEY (user_id)\"]", + "[\"constraint\", \"user_settings\", \"user_settings_revision_not_null\", true, \"NOT NULL revision\"]", + "[\"constraint\", \"user_settings\", \"user_settings_updated_at_not_null\", true, \"NOT NULL updated_at\"]", + "[\"constraint\", \"user_settings\", \"user_settings_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"user_settings\", \"user_settings_user_id_not_null\", true, \"NOT NULL user_id\"]", + "[\"constraint\", \"users\", \"users_created_at_not_null\", true, \"NOT NULL created_at\"]", + "[\"constraint\", \"users\", \"users_display_name_not_null\", true, \"NOT NULL display_name\"]", + "[\"constraint\", \"users\", \"users_email_bouncing_check\", true, \"CHECK ((email_bouncing = ANY (ARRAY[(0)::bigint, (1)::bigint])))\"]", + "[\"constraint\", \"users\", \"users_email_bouncing_not_null\", true, \"NOT NULL email_bouncing\"]", + "[\"constraint\", \"users\", \"users_email_key\", true, \"UNIQUE (email)\"]", + "[\"constraint\", \"users\", \"users_email_not_null\", true, \"NOT NULL email\"]", + "[\"constraint\", \"users\", \"users_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"users\", \"users_password_hash_not_null\", true, \"NOT NULL password_hash\"]", + "[\"constraint\", \"users\", \"users_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"users\", \"users_role_check\", true, \"CHECK ((role = ANY (ARRAY['user'::text, 'admin'::text])))\"]", + "[\"constraint\", \"users\", \"users_role_not_null\", true, \"NOT NULL role\"]", + "[\"constraint\", \"users\", \"users_status_check\", true, \"CHECK ((status = ANY (ARRAY['pending'::text, 'active'::text, 'disabled'::text])))\"]", + "[\"constraint\", \"users\", \"users_status_not_null\", true, \"NOT NULL status\"]", + "[\"index\", \"audit_log\", \"CREATE INDEX audit_actor ON {schema}.audit_log USING btree (actor_user_id)\"]", + "[\"index\", \"audit_log\", \"CREATE INDEX audit_at ON {schema}.audit_log USING btree (at)\"]", + "[\"index\", \"audit_log\", \"CREATE INDEX audit_target ON {schema}.audit_log USING btree (target_user_id)\"]", + "[\"index\", \"audit_log\", \"CREATE UNIQUE INDEX audit_log_pkey ON {schema}.audit_log USING btree (id)\"]", + "[\"index\", \"mail_events\", \"CREATE UNIQUE INDEX mail_events_mail_id_event_at_key ON {schema}.mail_events USING btree (mail_id, event, at)\"]", + "[\"index\", \"mail_events\", \"CREATE UNIQUE INDEX mail_events_pkey ON {schema}.mail_events USING btree (id)\"]", + "[\"index\", \"mail_log\", \"CREATE INDEX mail_user ON {schema}.mail_log USING btree (user_id)\"]", + "[\"index\", \"mail_log\", \"CREATE UNIQUE INDEX mail_log_pkey ON {schema}.mail_log USING btree (id)\"]", + "[\"index\", \"mail_log\", \"CREATE UNIQUE INDEX mail_msgid ON {schema}.mail_log USING btree (provider_message_id) WHERE (provider_message_id IS NOT NULL)\"]", + "[\"index\", \"notification_prefs\", \"CREATE UNIQUE INDEX notification_prefs_pkey ON {schema}.notification_prefs USING btree (user_id)\"]", + "[\"index\", \"notification_prefs\", \"CREATE UNIQUE INDEX notification_prefs_unsub_token_key ON {schema}.notification_prefs USING btree (unsub_token)\"]", + "[\"index\", \"notification_state\", \"CREATE UNIQUE INDEX notification_state_pkey ON {schema}.notification_state USING btree (user_id, event, subject)\"]", + "[\"index\", \"notification_watches\", \"CREATE UNIQUE INDEX notification_watches_pkey ON {schema}.notification_watches USING btree (user_id, pubkey)\"]", + "[\"index\", \"proposals\", \"CREATE INDEX proposals_status ON {schema}.proposals USING btree (status)\"]", + "[\"index\", \"proposals\", \"CREATE UNIQUE INDEX proposals_kind_subject ON {schema}.proposals USING btree (kind, subject)\"]", + "[\"index\", \"proposals\", \"CREATE UNIQUE INDEX proposals_pkey ON {schema}.proposals USING btree (id)\"]", + "[\"index\", \"sessions\", \"CREATE INDEX sessions_user ON {schema}.sessions USING btree (user_id)\"]", + "[\"index\", \"sessions\", \"CREATE UNIQUE INDEX sessions_pkey ON {schema}.sessions USING btree (id)\"]", + "[\"index\", \"sessions\", \"CREATE UNIQUE INDEX sessions_token_hash_key ON {schema}.sessions USING btree (token_hash)\"]", + "[\"index\", \"tokens\", \"CREATE INDEX tokens_user ON {schema}.tokens USING btree (user_id, purpose)\"]", + "[\"index\", \"tokens\", \"CREATE UNIQUE INDEX tokens_pkey ON {schema}.tokens USING btree (token_hash)\"]", + "[\"index\", \"user_settings\", \"CREATE UNIQUE INDEX user_settings_pkey ON {schema}.user_settings USING btree (user_id)\"]", + "[\"index\", \"users\", \"CREATE UNIQUE INDEX users_email_key ON {schema}.users USING btree (email)\"]", + "[\"index\", \"users\", \"CREATE UNIQUE INDEX users_pkey ON {schema}.users USING btree (id)\"]", + "[\"table\", \"audit_log\", \"r\", false, false]", + "[\"table\", \"mail_events\", \"r\", false, false]", + "[\"table\", \"mail_log\", \"r\", false, false]", + "[\"table\", \"notification_prefs\", \"r\", false, false]", + "[\"table\", \"notification_state\", \"r\", false, false]", + "[\"table\", \"notification_watches\", \"r\", false, false]", + "[\"table\", \"proposals\", \"r\", false, false]", + "[\"table\", \"sessions\", \"r\", false, false]", + "[\"table\", \"tokens\", \"r\", false, false]", + "[\"table\", \"user_settings\", \"r\", false, false]", + "[\"table\", \"users\", \"r\", false, false]", + "[\"view\", \"approved_channels\", \" SELECT kind,\\n subject,\\n decided_at,\\n id\\n FROM proposals\\n WHERE (status = 'approved'::text);\"]" + ], + "telemetry": [ + "[\"column\", \"_async_migrations\", 1, \"name\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"_async_migrations\", 2, \"status\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"_async_migrations\", 3, \"started_at\", \"text\", \"NO\", \"to_char((CURRENT_TIMESTAMP AT TIME ZONE 'UTC'::text), 'YYYY-MM-DD HH24:MI:SS'::text)\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"_async_migrations\", 4, \"ended_at\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"_async_migrations\", 5, \"error\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"_migrations\", 1, \"name\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"advert_evidence_backfill\", 1, \"id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"advert_evidence_backfill\", 2, \"tx_cursor\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"advert_evidence_backfill\", 3, \"obs_cursor\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"advert_route_evidence\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"advert_route_evidence\", 2, \"tx_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"advert_route_evidence\", 3, \"bit\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_observers\", 1, \"pubkey\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_observers\", 2, \"name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_observers\", 3, \"last_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"client_receptions\", 10, \"rx_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 11, \"ingested_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 12, \"src\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 2, \"rx_pubkey\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 3, \"heard_key\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 4, \"heard_keylen\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 5, \"rssi\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 6, \"snr\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 7, \"lat\", \"double precision\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 8, \"lon\", \"double precision\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 9, \"pos_acc_m\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 10, \"battery_mv\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 11, \"queue_len\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 12, \"errors\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 13, \"noise_floor\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 14, \"last_rssi\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 15, \"last_snr\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 16, \"tx_air_secs\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 17, \"rx_air_secs\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 18, \"recv\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 19, \"sent\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 2, \"rx_pubkey\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 20, \"flood_rx\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 21, \"direct_rx\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 22, \"flood_tx\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 23, \"direct_tx\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 24, \"recv_errors\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 3, \"sampled_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 4, \"ingested_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 5, \"lat\", \"double precision\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 6, \"lon\", \"double precision\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 7, \"pos_acc_m\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 8, \"stationary\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 9, \"uptime_secs\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 10, \"scope_name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 11, \"hash_size\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 12, \"hop_count\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 13, \"path_json\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 14, \"forwarder\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 15, \"snr\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 16, \"rssi\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 17, \"lat\", \"double precision\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 18, \"lon\", \"double precision\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 19, \"pos_acc_m\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 2, \"rx_pubkey\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 3, \"rx_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 4, \"ingested_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 5, \"pkt_hash\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 6, \"route_type\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 7, \"payload_type\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 8, \"code1\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 9, \"code2\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 2, \"hash\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 3, \"raw_hex\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 4, \"reason\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 5, \"observer_id\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 6, \"observer_name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 7, \"node_pubkey\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 8, \"node_name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 9, \"dropped_at\", \"text\", \"YES\", \"to_char((CURRENT_TIMESTAMP AT TIME ZONE 'UTC'::text), 'YYYY-MM-DD HH24:MI:SS'::text)\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 1, \"public_key\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 10, \"temperature_c\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 11, \"foreign_advert\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 12, \"default_scope\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 13, \"configured_scope\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 14, \"configured_scope_at\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 15, \"multibyte_sup\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 16, \"multibyte_evidence\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 2, \"name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 3, \"role\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 4, \"lat\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 5, \"lon\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 6, \"last_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 7, \"first_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 8, \"advert_count\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 9, \"battery_mv\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"neighbor_edges\", 1, \"node_a\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"neighbor_edges\", 2, \"node_b\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"neighbor_edges\", 3, \"count\", \"bigint\", \"YES\", \"1\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"neighbor_edges\", 4, \"last_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 10, \"pos_acc_m\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 11, \"repeater_clock\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 2, \"target\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 3, \"rx_pubkey\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 4, \"observed_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 5, \"ingested_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 6, \"regions_csv\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 7, \"truncated\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 8, \"lat\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 9, \"lon\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 1, \"public_key\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 10, \"temperature_c\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 11, \"foreign_advert\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 12, \"default_scope\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 13, \"configured_scope\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 14, \"configured_scope_at\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 15, \"multibyte_sup\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 16, \"multibyte_evidence\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 2, \"name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 3, \"role\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 4, \"lat\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 5, \"lon\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 6, \"last_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 7, \"first_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 8, \"advert_count\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 9, \"battery_mv\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"observations\", 10, \"raw_hex\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 11, \"resolved_path\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 2, \"transmission_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 3, \"observer_idx\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 4, \"direction\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 5, \"snr\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 6, \"rssi\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 7, \"score\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 8, \"path_json\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 9, \"timestamp\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 1, \"observer_id\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 2, \"timestamp\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 3, \"noise_floor\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 4, \"tx_air_secs\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 5, \"rx_air_secs\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 6, \"recv_errors\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 7, \"battery_mv\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 8, \"packets_sent\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 9, \"packets_recv\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 1, \"rowid\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"observers\", 10, \"client_version\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 11, \"radio\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 12, \"battery_mv\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 13, \"uptime_secs\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 14, \"noise_floor\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 15, \"inactive\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 16, \"last_packet_at\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 17, \"clock_skew_seconds\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 18, \"clock_skew_count_24h\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 19, \"clock_last_naive_at\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 2, \"id\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 20, \"can_relay\", \"bigint\", \"YES\", \"1\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 21, \"can_relay_seen\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 3, \"name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 4, \"iata\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 5, \"last_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 6, \"first_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 7, \"packet_count\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 8, \"model\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 9, \"firmware\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"scope_match_totals\", 1, \"id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"scope_match_totals\", 2, \"since_unix\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"scope_match_totals\", 3, \"unique_matches\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"scope_match_totals\", 4, \"explicit_over_derived\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"scope_match_totals\", 5, \"ambiguous\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"scope_match_totals\", 6, \"none_matches\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"scope_match_totals\", 7, \"updated_unix\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"transmissions\", 10, \"last_seen\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 11, \"created_at\", \"text\", \"YES\", \"to_char((CURRENT_TIMESTAMP AT TIME ZONE 'UTC'::text), 'YYYY-MM-DD HH24:MI:SS'::text)\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 12, \"code1\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 13, \"code2\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 14, \"channel_hash\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 15, \"scope_name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 2, \"raw_hex\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 3, \"hash\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 4, \"first_seen\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 5, \"route_type\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 6, \"payload_type\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 7, \"payload_version\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 8, \"decoded_json\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 9, \"from_pubkey\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"constraint\", \"_async_migrations\", \"_async_migrations_name_not_null\", true, \"NOT NULL name\"]", + "[\"constraint\", \"_async_migrations\", \"_async_migrations_pkey\", true, \"PRIMARY KEY (name)\"]", + "[\"constraint\", \"_async_migrations\", \"_async_migrations_started_at_not_null\", true, \"NOT NULL started_at\"]", + "[\"constraint\", \"_async_migrations\", \"_async_migrations_status_not_null\", true, \"NOT NULL status\"]", + "[\"constraint\", \"_migrations\", \"_migrations_name_not_null\", true, \"NOT NULL name\"]", + "[\"constraint\", \"_migrations\", \"_migrations_pkey\", true, \"PRIMARY KEY (name)\"]", + "[\"constraint\", \"advert_evidence_backfill\", \"advert_evidence_backfill_id_check\", true, \"CHECK ((id = 1))\"]", + "[\"constraint\", \"advert_evidence_backfill\", \"advert_evidence_backfill_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"advert_evidence_backfill\", \"advert_evidence_backfill_obs_cursor_not_null\", true, \"NOT NULL obs_cursor\"]", + "[\"constraint\", \"advert_evidence_backfill\", \"advert_evidence_backfill_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"advert_evidence_backfill\", \"advert_evidence_backfill_tx_cursor_not_null\", true, \"NOT NULL tx_cursor\"]", + "[\"constraint\", \"advert_route_evidence\", \"advert_route_evidence_bit_check\", true, \"CHECK ((\\\"bit\\\" = ANY (ARRAY[(1)::bigint, (2)::bigint])))\"]", + "[\"constraint\", \"advert_route_evidence\", \"advert_route_evidence_bit_not_null\", true, \"NOT NULL \\\"bit\\\"\"]", + "[\"constraint\", \"advert_route_evidence\", \"advert_route_evidence_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"advert_route_evidence\", \"advert_route_evidence_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"advert_route_evidence\", \"advert_route_evidence_tx_id_bit_key\", true, \"UNIQUE (tx_id, \\\"bit\\\")\"]", + "[\"constraint\", \"advert_route_evidence\", \"advert_route_evidence_tx_id_fkey\", true, \"FOREIGN KEY (tx_id) REFERENCES transmissions(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"advert_route_evidence\", \"advert_route_evidence_tx_id_not_null\", true, \"NOT NULL tx_id\"]", + "[\"constraint\", \"client_observers\", \"client_observers_pkey\", true, \"PRIMARY KEY (pubkey)\"]", + "[\"constraint\", \"client_observers\", \"client_observers_pubkey_not_null\", true, \"NOT NULL pubkey\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_heard_key_not_null\", true, \"NOT NULL heard_key\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_heard_keylen_not_null\", true, \"NOT NULL heard_keylen\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_ingested_at_not_null\", true, \"NOT NULL ingested_at\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_lat_not_null\", true, \"NOT NULL lat\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_lon_not_null\", true, \"NOT NULL lon\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_rx_at_not_null\", true, \"NOT NULL rx_at\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_rx_pubkey_heard_key_rx_at_key\", true, \"UNIQUE (rx_pubkey, heard_key, rx_at)\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_rx_pubkey_not_null\", true, \"NOT NULL rx_pubkey\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_src_not_null\", true, \"NOT NULL src\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_ingested_at_not_null\", true, \"NOT NULL ingested_at\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_lat_not_null\", true, \"NOT NULL lat\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_lon_not_null\", true, \"NOT NULL lon\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_rx_pubkey_not_null\", true, \"NOT NULL rx_pubkey\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_rx_pubkey_sampled_at_key\", true, \"UNIQUE (rx_pubkey, sampled_at)\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_sampled_at_not_null\", true, \"NOT NULL sampled_at\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_stationary_not_null\", true, \"NOT NULL stationary\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_uptime_secs_not_null\", true, \"NOT NULL uptime_secs\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_hash_size_not_null\", true, \"NOT NULL hash_size\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_hop_count_not_null\", true, \"NOT NULL hop_count\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_ingested_at_not_null\", true, \"NOT NULL ingested_at\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_lat_not_null\", true, \"NOT NULL lat\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_lon_not_null\", true, \"NOT NULL lon\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_payload_type_not_null\", true, \"NOT NULL payload_type\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_pkt_hash_not_null\", true, \"NOT NULL pkt_hash\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_route_type_not_null\", true, \"NOT NULL route_type\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_rx_at_not_null\", true, \"NOT NULL rx_at\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_rx_pubkey_not_null\", true, \"NOT NULL rx_pubkey\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_rx_pubkey_pkt_hash_rx_at_key\", true, \"UNIQUE (rx_pubkey, pkt_hash, rx_at)\"]", + "[\"constraint\", \"dropped_packets\", \"dropped_packets_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"dropped_packets\", \"dropped_packets_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"dropped_packets\", \"dropped_packets_reason_not_null\", true, \"NOT NULL reason\"]", + "[\"constraint\", \"inactive_nodes\", \"inactive_nodes_multibyte_sup_not_null\", true, \"NOT NULL multibyte_sup\"]", + "[\"constraint\", \"inactive_nodes\", \"inactive_nodes_pkey\", true, \"PRIMARY KEY (public_key)\"]", + "[\"constraint\", \"inactive_nodes\", \"inactive_nodes_public_key_not_null\", true, \"NOT NULL public_key\"]", + "[\"constraint\", \"neighbor_edges\", \"neighbor_edges_node_a_not_null\", true, \"NOT NULL node_a\"]", + "[\"constraint\", \"neighbor_edges\", \"neighbor_edges_node_b_not_null\", true, \"NOT NULL node_b\"]", + "[\"constraint\", \"neighbor_edges\", \"neighbor_edges_pkey\", true, \"PRIMARY KEY (node_a, node_b)\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_ingested_at_not_null\", true, \"NOT NULL ingested_at\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_observed_at_not_null\", true, \"NOT NULL observed_at\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_regions_csv_not_null\", true, \"NOT NULL regions_csv\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_rx_pubkey_not_null\", true, \"NOT NULL rx_pubkey\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_target_not_null\", true, \"NOT NULL target\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_target_rx_pubkey_observed_at_key\", true, \"UNIQUE (target, rx_pubkey, observed_at)\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_truncated_not_null\", true, \"NOT NULL truncated\"]", + "[\"constraint\", \"nodes\", \"nodes_multibyte_sup_not_null\", true, \"NOT NULL multibyte_sup\"]", + "[\"constraint\", \"nodes\", \"nodes_pkey\", true, \"PRIMARY KEY (public_key)\"]", + "[\"constraint\", \"nodes\", \"nodes_public_key_not_null\", true, \"NOT NULL public_key\"]", + "[\"constraint\", \"observations\", \"observations_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"observations\", \"observations_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"observations\", \"observations_timestamp_not_null\", true, \"NOT NULL \\\"timestamp\\\"\"]", + "[\"constraint\", \"observations\", \"observations_transmission_id_fkey\", true, \"FOREIGN KEY (transmission_id) REFERENCES transmissions(id)\"]", + "[\"constraint\", \"observations\", \"observations_transmission_id_not_null\", true, \"NOT NULL transmission_id\"]", + "[\"constraint\", \"observer_metrics\", \"observer_metrics_observer_id_not_null\", true, \"NOT NULL observer_id\"]", + "[\"constraint\", \"observer_metrics\", \"observer_metrics_pkey\", true, \"PRIMARY KEY (observer_id, \\\"timestamp\\\")\"]", + "[\"constraint\", \"observer_metrics\", \"observer_metrics_timestamp_not_null\", true, \"NOT NULL \\\"timestamp\\\"\"]", + "[\"constraint\", \"observers\", \"observers_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"observers\", \"observers_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"observers\", \"observers_rowid_key\", true, \"UNIQUE (rowid)\"]", + "[\"constraint\", \"observers\", \"observers_rowid_not_null\", true, \"NOT NULL rowid\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_ambiguous_not_null\", true, \"NOT NULL ambiguous\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_explicit_over_derived_not_null\", true, \"NOT NULL explicit_over_derived\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_id_check\", true, \"CHECK ((id = 1))\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_none_matches_not_null\", true, \"NOT NULL none_matches\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_since_unix_not_null\", true, \"NOT NULL since_unix\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_unique_matches_not_null\", true, \"NOT NULL unique_matches\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_updated_unix_not_null\", true, \"NOT NULL updated_unix\"]", + "[\"constraint\", \"transmissions\", \"transmissions_first_seen_not_null\", true, \"NOT NULL first_seen\"]", + "[\"constraint\", \"transmissions\", \"transmissions_hash_key\", true, \"UNIQUE (hash)\"]", + "[\"constraint\", \"transmissions\", \"transmissions_hash_not_null\", true, \"NOT NULL hash\"]", + "[\"constraint\", \"transmissions\", \"transmissions_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"transmissions\", \"transmissions_last_seen_not_null\", true, \"NOT NULL last_seen\"]", + "[\"constraint\", \"transmissions\", \"transmissions_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"transmissions\", \"transmissions_raw_hex_not_null\", true, \"NOT NULL raw_hex\"]", + "[\"index\", \"_async_migrations\", \"CREATE UNIQUE INDEX _async_migrations_pkey ON {schema}._async_migrations USING btree (name)\"]", + "[\"index\", \"_migrations\", \"CREATE UNIQUE INDEX _migrations_pkey ON {schema}._migrations USING btree (name)\"]", + "[\"index\", \"advert_evidence_backfill\", \"CREATE UNIQUE INDEX advert_evidence_backfill_pkey ON {schema}.advert_evidence_backfill USING btree (id)\"]", + "[\"index\", \"advert_route_evidence\", \"CREATE UNIQUE INDEX advert_route_evidence_pkey ON {schema}.advert_route_evidence USING btree (id)\"]", + "[\"index\", \"advert_route_evidence\", \"CREATE UNIQUE INDEX advert_route_evidence_tx_id_bit_key ON {schema}.advert_route_evidence USING btree (tx_id, \\\"bit\\\")\"]", + "[\"index\", \"client_observers\", \"CREATE UNIQUE INDEX client_observers_pkey ON {schema}.client_observers USING btree (pubkey)\"]", + "[\"index\", \"client_receptions\", \"CREATE INDEX idx_client_recept_heard_geo ON {schema}.client_receptions USING btree (heard_key, heard_keylen, lat, lon)\"]", + "[\"index\", \"client_receptions\", \"CREATE INDEX idx_client_recept_latlon ON {schema}.client_receptions USING btree (lat, lon)\"]", + "[\"index\", \"client_receptions\", \"CREATE INDEX idx_client_recept_rxat ON {schema}.client_receptions USING btree (rx_at)\"]", + "[\"index\", \"client_receptions\", \"CREATE UNIQUE INDEX client_receptions_pkey ON {schema}.client_receptions USING btree (id)\"]", + "[\"index\", \"client_receptions\", \"CREATE UNIQUE INDEX client_receptions_rx_pubkey_heard_key_rx_at_key ON {schema}.client_receptions USING btree (rx_pubkey, heard_key, rx_at)\"]", + "[\"index\", \"client_rf_samples\", \"CREATE INDEX idx_crf_prune ON {schema}.client_rf_samples USING btree (sampled_at)\"]", + "[\"index\", \"client_rf_samples\", \"CREATE INDEX idx_crf_track ON {schema}.client_rf_samples USING btree (rx_pubkey, sampled_at)\"]", + "[\"index\", \"client_rf_samples\", \"CREATE UNIQUE INDEX client_rf_samples_pkey ON {schema}.client_rf_samples USING btree (id)\"]", + "[\"index\", \"client_rf_samples\", \"CREATE UNIQUE INDEX client_rf_samples_rx_pubkey_sampled_at_key ON {schema}.client_rf_samples USING btree (rx_pubkey, sampled_at)\"]", + "[\"index\", \"client_rx_observations\", \"CREATE INDEX idx_cro_forwarder ON {schema}.client_rx_observations USING btree (forwarder, rx_at)\"]", + "[\"index\", \"client_rx_observations\", \"CREATE INDEX idx_cro_hash ON {schema}.client_rx_observations USING btree (pkt_hash, rx_at)\"]", + "[\"index\", \"client_rx_observations\", \"CREATE INDEX idx_cro_prune ON {schema}.client_rx_observations USING btree (rx_at)\"]", + "[\"index\", \"client_rx_observations\", \"CREATE INDEX idx_cro_scope ON {schema}.client_rx_observations USING btree (scope_name, rx_at)\"]", + "[\"index\", \"client_rx_observations\", \"CREATE UNIQUE INDEX client_rx_observations_pkey ON {schema}.client_rx_observations USING btree (id)\"]", + "[\"index\", \"client_rx_observations\", \"CREATE UNIQUE INDEX client_rx_observations_rx_pubkey_pkt_hash_rx_at_key ON {schema}.client_rx_observations USING btree (rx_pubkey, pkt_hash, rx_at)\"]", + "[\"index\", \"dropped_packets\", \"CREATE INDEX idx_dropped_node ON {schema}.dropped_packets USING btree (node_pubkey)\"]", + "[\"index\", \"dropped_packets\", \"CREATE INDEX idx_dropped_observer ON {schema}.dropped_packets USING btree (observer_id)\"]", + "[\"index\", \"dropped_packets\", \"CREATE UNIQUE INDEX dropped_packets_pkey ON {schema}.dropped_packets USING btree (id)\"]", + "[\"index\", \"inactive_nodes\", \"CREATE INDEX idx_inactive_nodes_last_seen ON {schema}.inactive_nodes USING btree (last_seen)\"]", + "[\"index\", \"inactive_nodes\", \"CREATE UNIQUE INDEX inactive_nodes_pkey ON {schema}.inactive_nodes USING btree (public_key)\"]", + "[\"index\", \"neighbor_edges\", \"CREATE UNIQUE INDEX neighbor_edges_pkey ON {schema}.neighbor_edges USING btree (node_a, node_b)\"]", + "[\"index\", \"node_declared_regions\", \"CREATE INDEX idx_ndr_prune ON {schema}.node_declared_regions USING btree (observed_at)\"]", + "[\"index\", \"node_declared_regions\", \"CREATE INDEX idx_ndr_target ON {schema}.node_declared_regions USING btree (target, observed_at)\"]", + "[\"index\", \"node_declared_regions\", \"CREATE UNIQUE INDEX node_declared_regions_pkey ON {schema}.node_declared_regions USING btree (id)\"]", + "[\"index\", \"node_declared_regions\", \"CREATE UNIQUE INDEX node_declared_regions_target_rx_pubkey_observed_at_key ON {schema}.node_declared_regions USING btree (target, rx_pubkey, observed_at)\"]", + "[\"index\", \"nodes\", \"CREATE INDEX idx_nodes_foreign_advert ON {schema}.nodes USING btree (foreign_advert) WHERE (foreign_advert = 1)\"]", + "[\"index\", \"nodes\", \"CREATE INDEX idx_nodes_last_seen ON {schema}.nodes USING btree (last_seen)\"]", + "[\"index\", \"nodes\", \"CREATE UNIQUE INDEX nodes_pkey ON {schema}.nodes USING btree (public_key)\"]", + "[\"index\", \"observations\", \"CREATE INDEX idx_observations_observer_idx ON {schema}.observations USING btree (observer_idx)\"]", + "[\"index\", \"observations\", \"CREATE INDEX idx_observations_observer_idx_timestamp ON {schema}.observations USING btree (observer_idx, \\\"timestamp\\\")\"]", + "[\"index\", \"observations\", \"CREATE INDEX idx_observations_timestamp ON {schema}.observations USING btree (\\\"timestamp\\\")\"]", + "[\"index\", \"observations\", \"CREATE INDEX idx_observations_transmission_id ON {schema}.observations USING btree (transmission_id)\"]", + "[\"index\", \"observations\", \"CREATE INDEX idx_observations_tx_ts ON {schema}.observations USING btree (transmission_id, \\\"timestamp\\\")\"]", + "[\"index\", \"observations\", \"CREATE UNIQUE INDEX idx_observations_dedup ON {schema}.observations USING btree (transmission_id, observer_idx, COALESCE(path_json, ''::text))\"]", + "[\"index\", \"observations\", \"CREATE UNIQUE INDEX observations_pkey ON {schema}.observations USING btree (id)\"]", + "[\"index\", \"observer_metrics\", \"CREATE INDEX idx_observer_metrics_timestamp ON {schema}.observer_metrics USING btree (\\\"timestamp\\\")\"]", + "[\"index\", \"observer_metrics\", \"CREATE UNIQUE INDEX observer_metrics_pkey ON {schema}.observer_metrics USING btree (observer_id, \\\"timestamp\\\")\"]", + "[\"index\", \"observers\", \"CREATE INDEX idx_observers_iata_norm ON {schema}.observers USING btree (upper(TRIM(BOTH FROM iata)))\"]", + "[\"index\", \"observers\", \"CREATE INDEX idx_observers_last_seen ON {schema}.observers USING btree (last_seen)\"]", + "[\"index\", \"observers\", \"CREATE UNIQUE INDEX observers_pkey ON {schema}.observers USING btree (id)\"]", + "[\"index\", \"observers\", \"CREATE UNIQUE INDEX observers_rowid_key ON {schema}.observers USING btree (rowid)\"]", + "[\"index\", \"scope_match_totals\", \"CREATE UNIQUE INDEX scope_match_totals_pkey ON {schema}.scope_match_totals USING btree (id)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_transmissions_channel_hash_payload ON {schema}.transmissions USING btree (channel_hash, payload_type, first_seen)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_transmissions_first_seen ON {schema}.transmissions USING btree (first_seen, id)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_transmissions_from_pubkey ON {schema}.transmissions USING btree (from_pubkey)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_transmissions_hash ON {schema}.transmissions USING btree (hash)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_transmissions_payload_type ON {schema}.transmissions USING btree (payload_type)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_tx_channel_hash ON {schema}.transmissions USING btree (channel_hash) WHERE (payload_type = 5)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_tx_code1 ON {schema}.transmissions USING btree (code1) WHERE (code1 IS NOT NULL)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_tx_last_seen_zero ON {schema}.transmissions USING btree (id) WHERE (last_seen = 0)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_tx_scope_name ON {schema}.transmissions USING btree (scope_name) WHERE (scope_name IS NOT NULL)\"]", + "[\"index\", \"transmissions\", \"CREATE UNIQUE INDEX transmissions_hash_key ON {schema}.transmissions USING btree (hash)\"]", + "[\"index\", \"transmissions\", \"CREATE UNIQUE INDEX transmissions_pkey ON {schema}.transmissions USING btree (id)\"]", + "[\"table\", \"_async_migrations\", \"r\", false, false]", + "[\"table\", \"_migrations\", \"r\", false, false]", + "[\"table\", \"advert_evidence_backfill\", \"r\", false, false]", + "[\"table\", \"advert_route_evidence\", \"r\", false, false]", + "[\"table\", \"client_observers\", \"r\", false, false]", + "[\"table\", \"client_receptions\", \"r\", false, false]", + "[\"table\", \"client_rf_samples\", \"r\", false, false]", + "[\"table\", \"client_rx_observations\", \"r\", false, false]", + "[\"table\", \"dropped_packets\", \"r\", false, false]", + "[\"table\", \"inactive_nodes\", \"r\", false, false]", + "[\"table\", \"neighbor_edges\", \"r\", false, false]", + "[\"table\", \"node_declared_regions\", \"r\", false, false]", + "[\"table\", \"nodes\", \"r\", false, false]", + "[\"table\", \"observations\", \"r\", false, false]", + "[\"table\", \"observer_metrics\", \"r\", false, false]", + "[\"table\", \"observers\", \"r\", false, false]", + "[\"table\", \"scope_match_totals\", \"r\", false, false]", + "[\"table\", \"transmissions\", \"r\", false, false]", + "[\"view\", \"packets_v\", \" SELECT o.id,\\n COALESCE(o.raw_hex, t.raw_hex) AS raw_hex,\\n to_char((to_timestamp((o.\\\"timestamp\\\")::double precision) AT TIME ZONE 'UTC'::text), 'YYYY-MM-DD HH24:MI:SS'::text) AS \\\"timestamp\\\",\\n obs.id AS observer_id,\\n obs.name AS observer_name,\\n o.direction,\\n o.snr,\\n o.rssi,\\n o.score,\\n t.hash,\\n t.route_type,\\n t.payload_type,\\n t.payload_version,\\n o.path_json,\\n t.decoded_json,\\n t.created_at\\n FROM ((observations o\\n JOIN transmissions t ON ((t.id = o.transmission_id)))\\n LEFT JOIN observers obs ON (((obs.rowid = o.observer_idx) AND ((obs.inactive IS NULL) OR (obs.inactive = 0)))));\"]" + ] +} diff --git a/cmd/migrate/postgres_layout_test.go b/cmd/migrate/postgres_layout_test.go new file mode 100644 index 000000000..ebb536765 --- /dev/null +++ b/cmd/migrate/postgres_layout_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "encoding/json" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/users" + "os" + "testing" +) + +func TestPostgresConversionLayoutMatchesNativeSchema(t *testing.T) { + for _, kind := range []string{"telemetry", "accounts"} { + t.Run(kind, func(t *testing.T) { + db := openImportDB(t, postgresSchema(t)) + var err error + if kind == "telemetry" { + err = dbschema.ApplyPostgres(db, nil) + } else { + err = users.ApplyPostgres(db) + } + if err != nil { + t.Fatal(err) + } + tables, err := layouts(kind) + if err != nil { + t.Fatal(err) + } + if err := validatePostgresLayout(db, kind, tables); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`CREATE INDEX unexpected_extension ON ` + quote(tables[0].Name) + ` (` + quote(tables[0].Columns[0].Name) + `)`); err != nil { + t.Fatal(err) + } + if err := validatePostgresLayout(db, kind, tables); err == nil { + t.Fatal("custom source extension would be discarded") + } + }) + } +} + +// This explicit developer mode exports a prospective layout for review. It +// never changes the compiled manifest or skips its normal compatibility test. +func TestExportPostgresConversionLayout(t *testing.T) { + path := os.Getenv("CORESCOPE_EXPORT_POSTGRES_LAYOUT") + if path == "" { + t.Skip("explicit private layout export only") + } + output := map[string][]string{} + for _, kind := range []string{"telemetry", "accounts"} { + db := openImportDB(t, postgresSchema(t)) + var err error + if kind == "telemetry" { + err = dbschema.ApplyPostgres(db, nil) + } else { + err = users.ApplyPostgres(db) + } + if err != nil { + t.Fatal(err) + } + tables, err := layouts(kind) + if err != nil { + t.Fatal(err) + } + if output[kind], err = postgresLayout(db, kind, tables); err != nil { + t.Fatal(err) + } + } + data, err := json.MarshalIndent(output, "", " ") + if err != nil { + t.Fatal(err) + } + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + t.Fatal(err) + } + if _, err = file.Write(append(data, '\n')); err != nil { + file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/cmd/migrate/resume_finalized_test.go b/cmd/migrate/resume_finalized_test.go index 57b60dbbe..6bb8eaaae 100644 --- a/cmd/migrate/resume_finalized_test.go +++ b/cmd/migrate/resume_finalized_test.go @@ -12,7 +12,6 @@ import ( "testing" "github.com/meshcore-analyzer/dbschema" - "github.com/meshcore-analyzer/pgutil/pgtest" "github.com/meshcore-analyzer/users" ) @@ -64,8 +63,8 @@ func mixedImport(t *testing.T, startAccounts, readyTelemetry bool) [2]importOpti t.Helper() state := filepath.Join(t.TempDir(), "migration state") options := [2]importOptions{ - {Source: telemetrySource(t), DatabaseURL: pgtest.NewDatabase(t), StateDir: state, Kind: "telemetry"}, - {Source: accountSource(t, 5), DatabaseURL: pgtest.NewDatabase(t), StateDir: state, Kind: "accounts"}, + {Source: telemetrySource(t), DatabaseURL: postgresDatabase(t), StateDir: state, Kind: "telemetry"}, + {Source: accountSource(t, 5), DatabaseURL: postgresDatabase(t), StateDir: state, Kind: "accounts"}, } for i, o := range options { if i == 1 && !startAccounts { @@ -260,7 +259,7 @@ func TestCLIResumeStartsUntouchedSecondStore(t *testing.T) { } func TestResumePreparationArtifactsArePreserved(t *testing.T) { - o := importOptions{Source: telemetrySource(t), DatabaseURL: pgtest.NewSchema(t), StateDir: t.TempDir(), Kind: "telemetry", Resume: true} + o := importOptions{Source: telemetrySource(t), DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "telemetry", Resume: true} recovery := filepath.Join(o.StateDir, o.Kind, "recovery.sqlite") if err := os.MkdirAll(filepath.Dir(recovery), 0o700); err != nil { t.Fatal(err) @@ -315,7 +314,7 @@ func TestReadyResumeRefusesChangedEvidenceWithoutWrites(t *testing.T) { } for name, statement := range changes { t.Run(name, func(t *testing.T) { - o := importOptions{Source: telemetrySource(t), DatabaseURL: pgtest.NewSchema(t), StateDir: t.TempDir(), Kind: "telemetry"} + o := importOptions{Source: telemetrySource(t), DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "telemetry"} if _, err := importSQLite(context.Background(), o); err != nil { t.Fatal(err) } diff --git a/cmd/migrate/reverse.go b/cmd/migrate/reverse.go new file mode 100644 index 000000000..5c108b519 --- /dev/null +++ b/cmd/migrate/reverse.go @@ -0,0 +1,817 @@ +package main + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net/url" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" +) + +type reverseOptions struct { + DatabaseURL, Destination, StateDir, Kind string + BatchSize int + Resume bool + keepPending bool + afterBatch func() error + source *postgresFence +} + +type identityState struct { + Table string `json:"table"` + Column string `json:"column"` + LastValue int64 `json:"last_value"` + IsCalled bool `json:"is_called"` + SQLiteFloor int64 `json:"sqlite_floor"` +} + +type reverseManifest struct { + Version int `json:"version"` + RunID string `json:"run_id"` + Kind string `json:"kind"` + Destination string `json:"destination"` + SourceIdentity string `json:"source_identity"` + RecoverySHA256 string `json:"recovery_sha256"` + Source []tableReport `json:"source"` + Identities []identityState `json:"identities"` + AddedSQLiteMigrations []string `json:"added_sqlite_migrations"` +} + +type postgresFence struct { + db *sql.DB + tx *sql.Tx + kind string + tables []importTable + identity string +} + +func (f *postgresFence) Close() { + if f.tx != nil { + f.tx.Rollback() + } + if f.db != nil { + f.db.Close() + } +} + +// Local service leases are only one fence. Native table locks and the ingestor's +// commit-order lock also exclude real database writers during the offline copy. +func fencePostgres(ctx context.Context, dsn, kind string) (_ *postgresFence, err error) { + f := &postgresFence{kind: kind} + if f.tables, err = layouts(kind); err != nil { + return nil, err + } + if f.db, err = pgutil.Open(dsn, false); err != nil { + return nil, err + } + defer func() { + if err != nil { + f.Close() + } + }() + if kind == "telemetry" { + err = dbschema.AssertPostgresReady(f.db) + } else { + err = users.AssertPostgresReady(f.db) + } + if err != nil { + return nil, err + } + f.tx, err = f.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelRepeatableRead}) + if err != nil { + return nil, err + } + var locked bool + if err = f.tx.QueryRowContext(ctx, `SELECT pg_try_advisory_xact_lock(hashtextextended(current_schema(),$1))`, dbschema.WriterLockKey).Scan(&locked); err != nil { + return nil, err + } + if !locked { + return nil, errors.New("PostgreSQL source writer is active; stop all writers before conversion") + } + var names []string + for _, table := range f.tables { + names = append(names, quote(table.Name)) + } + if _, err = f.tx.ExecContext(ctx, `LOCK TABLE `+strings.Join(names, ",")+` IN SHARE MODE NOWAIT`); err != nil { + return nil, errors.New("PostgreSQL source is in use; stop all writers before conversion") + } + if err = validatePostgresLayout(f.db, kind, f.tables); err != nil { + return nil, err + } + config, e := pgutil.ParseConfig(dsn) + if e != nil { + return nil, e + } + var database, schema string + var dbOID, schemaOID int64 + if err = f.tx.QueryRowContext(ctx, `SELECT current_database(),current_schema(),(SELECT oid::bigint FROM pg_database WHERE datname=current_database()),(SELECT oid::bigint FROM pg_namespace WHERE nspname=current_schema())`).Scan(&database, &schema, &dbOID, &schemaOID); err != nil { + return nil, err + } + data, _ := json.Marshal([]any{database, schema, dbOID, schemaOID, config.Host, config.Port}) + sum := sha256.Sum256(data) + f.identity = hex.EncodeToString(sum[:]) + return f, nil +} + +func identityReferences(kind string) map[string][][2]string { + if kind == "telemetry" { + return map[string][][2]string{ + "observers": {{"observations", "observer_idx"}}, + "transmissions": {{"observations", "transmission_id"}, {"advert_route_evidence", "tx_id"}, {"advert_evidence_backfill", "tx_cursor"}}, + "observations": {{"advert_evidence_backfill", "obs_cursor"}}, + } + } + return map[string][][2]string{ + "users": {{"audit_log", "actor_user_id"}, {"audit_log", "target_user_id"}, {"users", "activated_by"}, {"sessions", "user_id"}, {"tokens", "user_id"}, {"mail_log", "user_id"}, {"proposals", "proposer_id"}, {"proposals", "reviewer_id"}, {"user_settings", "user_id"}, {"notification_prefs", "user_id"}, {"notification_watches", "user_id"}, {"notification_state", "user_id"}}, + "mail_log": {{"mail_events", "mail_id"}}, + } +} + +func postgresIdentities(ctx context.Context, f *postgresFence) ([]identityState, error) { + var out []identityState + refs := identityReferences(f.kind) + for _, table := range f.tables { + for _, col := range table.Columns { + if !col.Identity { + continue + } + var sequence string + if err := f.db.QueryRowContext(ctx, `SELECT pg_get_serial_sequence($1,$2)`, table.Name, col.Name).Scan(&sequence); err != nil { + return nil, err + } + var increment, min, max int64 + var cycle bool + if err := f.db.QueryRowContext(ctx, `SELECT seqincrement,seqmin,seqmax,seqcycle FROM pg_sequence WHERE seqrelid=$1::regclass`, sequence).Scan(&increment, &min, &max, &cycle); err != nil { + return nil, err + } + if increment != 1 || min != 1 || max != math.MaxInt64 || cycle { + return nil, errors.New("source identity sequence has unsupported custom allocation settings") + } + state := identityState{Table: table.Name, Column: col.Name} + if err := f.db.QueryRowContext(ctx, `SELECT last_value,is_called FROM `+sequence).Scan(&state.LastValue, &state.IsCalled); err != nil { + return nil, err + } + state.SQLiteFloor = state.LastValue + if !state.IsCalled { + state.SQLiteFloor-- + } + if state.SQLiteFloor < 0 { + return nil, errors.New("invalid source identity high water") + } + for _, ref := range append([][2]string{{table.Name, col.Name}}, refs[table.Name]...) { + var high int64 + if err := f.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(`+quote(ref[1])+`),0) FROM `+quote(ref[0])).Scan(&high); err != nil { + return nil, err + } + if high > state.SQLiteFloor { + state.SQLiteFloor = high + } + } + out = append(out, state) + } + } + return out, nil +} + +func digestPostgresSource(ctx context.Context, f *postgresFence) ([]tableReport, error) { + var reports []tableReport + for _, table := range f.tables { + // Bound decompressed text before a large value reaches the Go driver. + sizes := []string{"0"} + for _, col := range table.Columns { + if col.Type == "TEXT" { + sizes = append(sizes, `COALESCE(octet_length(`+quote(col.Name)+`),0)::bigint`) + } else { + sizes = append(sizes, "8") + } + } + var oversized bool + if err := f.db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM `+quote(table.Name)+` WHERE `+strings.Join(sizes, "+")+`>$1)`, maxImportRowBytes).Scan(&oversized); err != nil { + return nil, err + } + if oversized { + return nil, errors.New("a source row exceeds the 64 MiB migration safety limit") + } + report, err := digestTarget(ctx, f.db, table) + if err != nil { + return nil, err + } + reports = append(reports, report) + } + return reports, nil +} + +func writePrivateJSON(path string, value any) error { + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return err + } + _, writeErr := file.Write(append(data, '\n')) + syncErr := file.Sync() + closeErr := file.Close() + return errors.Join(writeErr, syncErr, closeErr) +} +func readPrivateJSON(path string, value any) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + d := json.NewDecoder(io.LimitReader(file, 1024*1024)) + d.DisallowUnknownFields() + if err := d.Decode(value); err != nil { + return errors.New("invalid private conversion manifest") + } + if err := d.Decode(new(any)); err != io.EOF { + return errors.New("invalid trailing conversion manifest") + } + return nil +} +func fileDigest(ctx context.Context, path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + h := sha256.New() + if _, err := io.CopyBuffer(h, contextReader{ctx, file}, make([]byte, 128*1024)); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func prepareReverse(ctx context.Context, o reverseOptions, f *postgresFence) (reverseManifest, error) { + var manifest reverseManifest + dest, err := filepath.Abs(o.Destination) + if err != nil { + return manifest, err + } + if o.Destination == "" || o.StateDir == "" { + return manifest, errors.New("reverse conversion requires a fresh SQLite target and private recovery directory") + } + dir := filepath.Join(o.StateDir, o.Kind) + path := filepath.Join(dir, "reverse.json") + backup := filepath.Join(dir, "recovery.pg_dump") + source, err := digestPostgresSource(ctx, f) + if err != nil { + return manifest, err + } + identities, err := postgresIdentities(ctx, f) + if err != nil { + return manifest, err + } + err = readPrivateJSON(path, &manifest) + if err == nil { + if !o.Resume || manifest.Version != 1 || manifest.Kind != o.Kind || manifest.Destination != dest || manifest.SourceIdentity != f.identity || !slices.Equal(source, manifest.Source) || !slices.Equal(identities, manifest.Identities) { + return manifest, errors.New("reverse resume source, sequence state or destination differs from the immutable manifest") + } + digest, err := fileDigest(ctx, backup) + if err != nil { + return manifest, err + } + if digest != manifest.RecoverySHA256 { + return manifest, errors.New("native PostgreSQL recovery archive changed; resume refused") + } + return manifest, nil + } + if !errors.Is(err, os.ErrNotExist) { + return manifest, err + } + if o.Resume { + return manifest, errors.New("reverse preparation has no complete manifest; preserve recovery files and abort before choosing a new job") + } + if _, err := os.Lstat(dest); !errors.Is(err, os.ErrNotExist) { + return manifest, errors.New("reverse destination already exists or cannot be inspected; existing files were not changed") + } + entries, err := os.ReadDir(dir) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return manifest, err + } + if len(entries) != 0 { + return manifest, errors.New("reverse recovery directory is not empty") + } + if err := os.MkdirAll(dir, 0700); err != nil { + return manifest, err + } + // Recovery is native and contains the whole selected database. The source + // table fences remain held; removing search_path changes no database target. + u, err := url.Parse(o.DatabaseURL) + if err != nil { + return manifest, errors.New("invalid source database URL") + } + q := u.Query() + q.Del("search_path") + u.RawQuery = q.Encode() + if err := pgutil.Dump(ctx, u.String(), backup); err != nil { + return manifest, err + } + digest, err := fileDigest(ctx, backup) + if err != nil { + return manifest, err + } + var random [16]byte + if _, err := rand.Read(random[:]); err != nil { + return manifest, err + } + manifest = reverseManifest{Version: 1, RunID: hex.EncodeToString(random[:]), Kind: o.Kind, Destination: dest, SourceIdentity: f.identity, RecoverySHA256: digest, Source: source, Identities: identities} + if o.Kind == "telemetry" { + template, err := sql.Open(importSQLiteDriver, ":memory:") + if err != nil { + return manifest, err + } + defer template.Close() + template.SetMaxOpenConns(1) + if err := dbschema.ApplySQLite(template, func(string, ...interface{}) {}); err != nil { + return manifest, err + } + rows, err := template.Query(`SELECT name FROM _migrations ORDER BY name`) + if err != nil { + return manifest, err + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + return manifest, err + } + var n int + if err := f.db.QueryRowContext(ctx, `SELECT count(*) FROM _migrations WHERE name=$1`, name).Scan(&n); err != nil { + rows.Close() + return manifest, err + } + if n == 0 { + manifest.AddedSQLiteMigrations = append(manifest.AddedSQLiteMigrations, name) + } + } + err = rows.Err() + rows.Close() + if err != nil { + return manifest, err + } + } + after, err := postgresIdentities(ctx, f) + if err != nil { + return manifest, err + } + if !slices.Equal(identities, after) { + return manifest, errors.New("source sequence allocation changed during backup; stop every writer") + } + return manifest, writePrivateJSON(path, manifest) +} + +func reversePostgres(ctx context.Context, o reverseOptions) (importReport, error) { + report := importReport{Kind: o.Kind} + if o.BatchSize == 0 { + o.BatchSize = 1000 + } + if o.BatchSize < 1 || o.BatchSize > 10000 { + return report, errors.New("copy batch size must be between 1 and 10000") + } + f := o.source + if f == nil { + var err error + f, err = fencePostgres(ctx, o.DatabaseURL, o.Kind) + if err != nil { + return report, err + } + defer f.Close() + } + manifest, err := prepareReverse(ctx, o, f) + if err != nil { + return report, err + } + destination, verifyOnly, err := openReverseTarget(ctx, o, manifest) + if err != nil { + return report, err + } + defer destination.Close() + for i, table := range f.tables { + got, err := copyReverseTable(ctx, f.db, destination, table, o, manifest, verifyOnly) + if err != nil { + return report, fmt.Errorf("reverse %s: %w", table.Name, err) + } + if got != manifest.Source[i] { + return report, errors.New("reverse source count or logical digest changed") + } + report.Tables = append(report.Tables, got) + } + for _, identity := range manifest.Identities { + var count int + var high int64 + if err := destination.QueryRowContext(ctx, `SELECT count(*),COALESCE(MAX(seq),0) FROM sqlite_sequence WHERE name=?`, identity.Table).Scan(&count, &high); err != nil { + return report, err + } + if verifyOnly { + if count != 1 || high != identity.SQLiteFloor { + return report, errors.New("completed SQLite identity allocation changed; resume refused") + } + continue + } + if _, err := destination.ExecContext(ctx, `DELETE FROM sqlite_sequence WHERE name=?`, identity.Table); err != nil { + return report, err + } + if _, err := destination.ExecContext(ctx, `INSERT INTO sqlite_sequence(name,seq) VALUES(?,?)`, identity.Table, identity.SQLiteFloor); err != nil { + return report, err + } + } + if !verifyOnly { + for _, name := range manifest.AddedSQLiteMigrations { + if _, err := destination.ExecContext(ctx, `INSERT OR IGNORE INTO _migrations(name) VALUES(?)`, name); err != nil { + return report, err + } + } + } + for _, name := range manifest.AddedSQLiteMigrations { + var n int + if err := destination.QueryRowContext(ctx, `SELECT count(*) FROM _migrations WHERE name=?`, name).Scan(&n); err != nil || n != 1 { + return report, errors.New("SQLite physical migration marker is missing") + } + } + if o.Kind == "telemetry" { + // A copied ledger can already mark this async task done. Build the real + // index offline; never rewrite its retained status to hide absent work. + if err := dbschema.EnsureSQLiteObserverTimeIndex(ctx, destination); err != nil { + return report, err + } + } + if err := assertSQLiteIntegrity(ctx, destination); err != nil { + return report, err + } + sourceAfter, err := digestPostgresSource(ctx, f) + if err != nil { + return report, err + } + identitiesAfter, err := postgresIdentities(ctx, f) + if err != nil { + return report, err + } + if !slices.Equal(sourceAfter, manifest.Source) || !slices.Equal(identitiesAfter, manifest.Identities) { + return report, errors.New("PostgreSQL source or sequence changed during conversion; target remains unselected") + } + if err := validatePostgresLayout(f.db, o.Kind, f.tables); err != nil { + return report, err + } + if !o.keepPending { + if !verifyOnly { + if _, err := destination.ExecContext(ctx, `DROP TABLE corescope_reverse_progress`); err != nil { + return report, err + } + } + if err := assertSQLiteData(ctx, destination, o.Kind); err != nil { + return report, err + } + } + report.Verified = true + return report, nil +} + +func createReverseTarget(ctx context.Context, destination, kind, runID string) (err error) { + if err := os.MkdirAll(filepath.Dir(destination), 0700); err != nil { + return err + } + file, err := os.CreateTemp(filepath.Dir(destination), ".corescope-reverse-*.sqlite") + if err != nil { + return err + } + temp := file.Name() + if err := file.Close(); err != nil { + return err + } + defer func() { + for _, suffix := range []string{"", "-wal", "-shm", "-journal"} { + _ = os.Remove(temp + suffix) + } + }() + uri, err := legacy.WriterDSN(temp) + if err != nil { + return err + } + db, err := sql.Open(importSQLiteDriver, uri) + if err != nil { + return err + } + defer db.Close() + db.SetMaxOpenConns(1) + if kind == "telemetry" { + err = dbschema.ApplySQLite(db, func(string, ...interface{}) {}) + } else { + err = users.ApplySQLite(db) + } + if err != nil { + return err + } + if _, err := db.ExecContext(ctx, `PRAGMA foreign_keys=OFF`); err != nil { + return err + } + tables, err := layouts(kind) + if err != nil { + return err + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + for i := len(tables) - 1; i >= 0; i-- { + if _, err := tx.ExecContext(ctx, `DELETE FROM `+quote(tables[i].Name)); err != nil { + return err + } + } + if _, err := tx.ExecContext(ctx, `CREATE TABLE corescope_reverse_progress(table_name TEXT PRIMARY KEY,rows_copied INTEGER NOT NULL,sha256 TEXT NOT NULL,complete INTEGER NOT NULL); INSERT INTO corescope_reverse_progress VALUES('',0,?,0)`, runID); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return err + } + var busy, log, checkpointed int + if err := db.QueryRowContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`).Scan(&busy, &log, &checkpointed); err != nil { + return err + } + if busy != 0 { + return errors.New("private SQLite staging checkpoint is busy") + } + if err := db.Close(); err != nil { + return err + } + // Publish the complete native pending marker with the file. Link never + // overwrites an existing path, unlike rename on Unix. Both paths share a + // directory/filesystem and the closed database has no outstanding WAL. + return os.Link(temp, destination) +} + +func openReverseTarget(ctx context.Context, o reverseOptions, m reverseManifest) (*sql.DB, bool, error) { + if _, err := os.Lstat(m.Destination); errors.Is(err, os.ErrNotExist) { + if err := createReverseTarget(ctx, m.Destination, o.Kind, m.RunID); err != nil { + return nil, false, err + } + } else if err != nil { + return nil, false, err + } else if !o.Resume { + return nil, false, errors.New("SQLite destination exists; use the exact verified job to resume") + } + uri, err := legacy.WriterDSN(m.Destination) + if err != nil { + return nil, false, err + } + db, err := sql.Open(importSQLiteDriver, uri) + if err != nil { + return nil, false, err + } + db.SetMaxOpenConns(1) + fail := func(err error) (*sql.DB, bool, error) { db.Close(); return nil, false, err } + var progress int + if err := db.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_schema WHERE name='corescope_reverse_progress' AND type='table'`).Scan(&progress); err != nil { + return fail(err) + } + if progress == 0 { + return db, true, nil + } + var id string + if err := db.QueryRowContext(ctx, `SELECT sha256 FROM corescope_reverse_progress WHERE table_name=''`).Scan(&id); err != nil || id != m.RunID { + return fail(errors.New("SQLite target belongs to another or incomplete preparation job")) + } + if _, err := db.ExecContext(ctx, `PRAGMA foreign_keys=OFF`); err != nil { + return fail(err) + } + return db, false, nil +} +func digestSQLiteTable(ctx context.Context, source, target *sql.DB, table importTable, m reverseManifest) (tableReport, error) { + report := tableReport{Table: table.Name} + _, names := columnNames(table) + for i := range names { + names[i] = "+" + names[i] + } + order, err := orderBy(source, table, false, false) + if err != nil { + return report, err + } + where := "" + var args []any + if table.Name == "_migrations" && len(m.AddedSQLiteMigrations) > 0 { + var params []string + for _, name := range m.AddedSQLiteMigrations { + params = append(params, "?") + args = append(args, name) + } + where = ` WHERE name NOT IN (` + strings.Join(params, ",") + `)` + } + rows, err := target.QueryContext(ctx, `SELECT `+strings.Join(names, ",")+` FROM `+quote(table.Name)+where+` ORDER BY `+order, args...) + if err != nil { + return report, err + } + defer rows.Close() + h := sha256.New() + for rows.Next() { + values, err := scanCanonicalRow(rows, table) + if err != nil { + return report, err + } + hashRow(h, values) + report.Rows++ + } + report.SHA256 = hex.EncodeToString(h.Sum(nil)) + return report, rows.Err() +} + +func scanCanonicalRow(rows *sql.Rows, table importTable) ([]any, error) { + values := make([]any, len(table.Columns)) + dest := make([]any, len(values)) + for i := range dest { + dest[i] = &values[i] + } + if err := rows.Scan(dest...); err != nil { + return nil, err + } + for i, col := range table.Columns { + value, err := canonical(values[i], col.Type) + if err != nil { + return nil, fmt.Errorf("column %s: %w", col.Name, err) + } + if f, ok := value.(float64); ok && math.IsNaN(f) { + return nil, errors.New("SQLite cannot preserve a PostgreSQL NaN; source was not changed") + } + values[i] = value + } + return values, nil +} + +func copyReverseTable(ctx context.Context, source, target *sql.DB, table importTable, o reverseOptions, m reverseManifest, verifyOnly bool) (tableReport, error) { + report := tableReport{Table: table.Name} + existing, err := digestSQLiteTable(ctx, source, target, table, m) + if err != nil { + return report, err + } + var copied int64 + var stored string + var complete bool + if verifyOnly { + copied, stored, complete = existing.Rows, existing.SHA256, true + } else { + err := target.QueryRowContext(ctx, `SELECT rows_copied,sha256,complete FROM corescope_reverse_progress WHERE table_name=?`, table.Name).Scan(&copied, &stored, &complete) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return report, err + } + if stored == "" { + sum := sha256.Sum256(nil) + stored = hex.EncodeToString(sum[:]) + } + if existing.Rows != copied || existing.SHA256 != stored { + return report, errors.New("SQLite destination differs from committed copy progress") + } + } + _, names := columnNames(table) + order, err := orderBy(source, table, true, false) + if err != nil { + return report, err + } + rows, err := source.QueryContext(ctx, `SELECT `+strings.Join(names, ",")+` FROM `+quote(table.Name)+` ORDER BY `+order) + if err != nil { + return report, err + } + defer rows.Close() + placeholders := make([]string, len(names)) + for i := range placeholders { + placeholders[i] = fmt.Sprintf("?%d", i+1) + } + insert := `INSERT INTO ` + quote(table.Name) + ` (` + strings.Join(names, ",") + `) VALUES(` + strings.Join(placeholders, ",") + `)` + h := sha256.New() + var batch [][]any + bytes := 0 + flush := func(done bool) error { + if verifyOnly { + return nil + } + if len(batch) == 0 && !done { + return nil + } + tx, err := target.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + stmt, err := tx.PrepareContext(ctx, insert) + if err != nil { + return err + } + defer stmt.Close() + for _, values := range batch { + if _, err := stmt.ExecContext(ctx, values...); err != nil { + return err + } + } + if _, err := tx.ExecContext(ctx, `INSERT INTO corescope_reverse_progress VALUES(?,?,?,?) ON CONFLICT(table_name) DO UPDATE SET rows_copied=excluded.rows_copied,sha256=excluded.sha256,complete=excluded.complete`, table.Name, report.Rows, hex.EncodeToString(h.Sum(nil)), done); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return err + } + wrote := len(batch) > 0 + batch = nil + bytes = 0 + if wrote && o.afterBatch != nil { + return o.afterBatch() + } + return nil + } + for rows.Next() { + values, err := scanCanonicalRow(rows, table) + if err != nil { + return report, err + } + hashRow(h, values) + report.Rows++ + if report.Rows <= copied { + if report.Rows == copied && hex.EncodeToString(h.Sum(nil)) != stored { + return report, errors.New("source prefix differs from reverse copy progress") + } + continue + } + if complete { + return report, errors.New("completed SQLite table has missing source rows") + } + batch = append(batch, values) + for _, value := range values { + if s, ok := value.(string); ok { + bytes += len(s) + } else { + bytes += 8 + } + } + if len(batch) >= o.BatchSize || bytes >= 8*1024*1024 { + if err := flush(false); err != nil { + return report, err + } + } + } + if err := rows.Err(); err != nil { + return report, err + } + if report.Rows < copied { + return report, errors.New("SQLite target contains extra rows") + } + if err := flush(true); err != nil { + return report, err + } + report.SHA256 = hex.EncodeToString(h.Sum(nil)) + got, err := digestSQLiteTable(ctx, source, target, table, m) + if err != nil { + return report, err + } + if got != report { + return report, errors.New("SQLite count or logical digest verification failed") + } + return report, nil +} + +func assertSQLiteData(ctx context.Context, db *sql.DB, kind string) error { + if err := assertSQLiteIntegrity(ctx, db); err != nil { + return err + } + if kind == "telemetry" { + return dbschema.AssertSQLiteReady(db) + } + return users.AssertSQLiteReady(db) +} + +func assertSQLiteIntegrity(ctx context.Context, db *sql.DB) error { + var integrity string + if err := db.QueryRowContext(ctx, `PRAGMA quick_check`).Scan(&integrity); err != nil { + return err + } + if integrity != "ok" { + return errors.New("SQLite target failed integrity check") + } + rows, err := db.QueryContext(ctx, `PRAGMA foreign_key_check`) + if err != nil { + return err + } + invalid := rows.Next() + rowErr := rows.Err() + rows.Close() + if rowErr != nil { + return rowErr + } + if invalid { + return errors.New("SQLite target relationships failed verification") + } + return nil +} diff --git a/cmd/migrate/reverse_test.go b/cmd/migrate/reverse_test.go new file mode 100644 index 000000000..d0727211c --- /dev/null +++ b/cmd/migrate/reverse_test.go @@ -0,0 +1,190 @@ +package main + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" +) + +func TestReverseTelemetryPreservesRowsAndFutureIDs(t *testing.T) { + dsn := postgresDatabase(t) + ctx := context.Background() + _, err := importSQLite(ctx, importOptions{Source: telemetrySource(t), DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry"}) + if err != nil { + t.Fatal(err) + } + if _, err := finalizeImport(ctx, dsn, "telemetry"); err != nil { + t.Fatal(err) + } + pg := openImportDB(t, dsn) + if _, err := pg.Exec(`INSERT INTO _async_migrations(name,status,started_at,ended_at,error) VALUES('obs_observer_ts_idx_v1','done','raw started','raw ended',NULL)`); err != nil { + t.Fatal(err) + } + for _, q := range []string{`SELECT setval(pg_get_serial_sequence('transmissions','id'),100500,false)`, `UPDATE observations SET score=3.5,resolved_path=' {"raw": true} ' WHERE id=90`, `UPDATE transmissions SET first_seen='malformed date kept as text' WHERE id=80`} { + if _, err := pg.Exec(q); err != nil { + t.Fatal(err) + } + } + dest := filepath.Join(t.TempDir(), "reverse # % é.sqlite") + options := reverseOptions{DatabaseURL: dsn, Destination: dest, StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 2} + injected := false + options.afterBatch = func() error { + if !injected { + injected = true + return errors.New("interrupt after committed batch") + } + return nil + } + if _, err := reversePostgres(ctx, options); err == nil || !injected { + t.Fatal("expected interruption after committed batch", err) + } + options.Resume = true + options.afterBatch = nil + report, err := reversePostgres(ctx, options) + if err != nil || !report.Verified { + t.Fatal("resume reverse", err) + } + db, err := openSQLite(dest, "rw") + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := dbschema.AssertSQLiteReady(db); err != nil { + t.Fatal(err) + } + var indexCount int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_observer_idx_timestamp'`).Scan(&indexCount); err != nil || indexCount != 1 { + t.Fatal("copied done ledger lacks its native index", err) + } + var status, started, ended string + if err := db.QueryRow(`SELECT status,started_at,ended_at FROM _async_migrations WHERE name='obs_observer_ts_idx_v1'`).Scan(&status, &started, &ended); err != nil || status != "done" || started != "raw started" || ended != "raw ended" { + t.Fatal("source migration ledger changed", err) + } + var node, parent, unused int + var plan string + if err := db.QueryRow(`EXPLAIN QUERY PLAN SELECT timestamp FROM observations WHERE observer_idx=999 ORDER BY timestamp`).Scan(&node, &parent, &unused, &plan); err != nil || !strings.Contains(plan, "idx_observations_observer_idx_timestamp") { + t.Fatal("observer/time query cannot use native index", plan, err) + } + var n int + if err := db.QueryRow(`SELECT count(*) FROM observations WHERE (id=0 AND observer_idx=0 AND path_json IS NULL) OR (id=90 AND observer_idx=999 AND path_json='' AND score=3.5 AND resolved_path=' {"raw": true} ') OR (id=91 AND observer_idx IS NULL)`).Scan(&n); err != nil || n != 3 { + t.Fatal("row parity", n, err) + } + var id int64 + if err := db.QueryRow(`INSERT INTO transmissions(raw_hex,hash,first_seen) VALUES('00','new','2026-01-03') RETURNING id`).Scan(&id); err != nil || id != 100500 { + t.Fatal("is_called=false future ID", id, err) + } + if err := db.QueryRow(`INSERT INTO observers(id) VALUES('future') RETURNING rowid`).Scan(&id); err != nil || id <= 999 { + t.Fatal("observer orphan reuse", id, err) + } + if _, err := os.Stat(filepath.Join(options.StateDir, "telemetry", "recovery.pg_dump")); err != nil { + t.Fatal("native recovery missing", err) + } + // Verify the actual native recovery archive, not only its filename/hash. + restored := postgresDatabase(t) + config, err := pgutil.ParseConfig(restored) + if err != nil { + t.Fatal(err) + } + command := exec.Command("pg_restore", "--no-password", "--no-owner", "--no-privileges", "--exit-on-error", "--dbname", config.Database, filepath.Join(options.StateDir, "telemetry", "recovery.pg_dump")) + command.Env, err = pgutil.CommandEnv(restored) + if err != nil { + t.Fatal(err) + } + if err := command.Run(); err != nil { + t.Fatal("native PostgreSQL recovery restore failed") + } + var manifest reverseManifest + if err := readPrivateJSON(filepath.Join(options.StateDir, "telemetry", "reverse.json"), &manifest); err != nil { + t.Fatal(err) + } + recovery := openImportDB(t, restored) + tables, err := layouts("telemetry") + if err != nil { + t.Fatal(err) + } + for i, table := range tables { + got, err := digestTarget(ctx, recovery, table) + if err != nil || got != manifest.Source[i] { + t.Fatal("native recovery data mismatch", table.Name, err) + } + } +} + +func TestReverseSequenceMutationLeavesNativePendingMarker(t *testing.T) { + owner := postgresDatabase(t) + ctx := context.Background() + db := openImportDB(t, owner) + if err := dbschema.ApplyPostgres(db, nil); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO observers(id) VALUES('only-observer')`); err != nil { + t.Fatal(err) + } + dest := filepath.Join(t.TempDir(), "pending.sqlite") + options := reverseOptions{DatabaseURL: owner, Destination: dest, StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 1, afterBatch: func() error { + _, err := db.Exec(`SELECT nextval(pg_get_serial_sequence('observers','rowid'))`) + return err + }} + if _, err := reversePostgres(ctx, options); err == nil { + t.Fatal("changed source sequence accepted") + } + target, err := openSQLite(dest, "ro") + if err != nil { + t.Fatal(err) + } + defer target.Close() + if err := dbconfig.AssertSQLiteImportComplete(target); !errors.Is(err, dbconfig.ErrSQLiteImportIncomplete) { + t.Fatal("partial reverse target can be adopted elsewhere", err) + } +} + +func TestReverseAccountsPreservesAuthenticationAndIdentity(t *testing.T) { + dsn := postgresDatabase(t) + ctx := context.Background() + _, err := importSQLite(ctx, importOptions{Source: accountSource(t, 5), DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "accounts"}) + if err != nil { + t.Fatal(err) + } + if _, err := finalizeImport(ctx, dsn, "accounts"); err != nil { + t.Fatal(err) + } + pg := openImportDB(t, dsn) + for _, q := range []string{`SELECT setval(pg_get_serial_sequence('mail_events','id'),9001,false)`, `INSERT INTO audit_log(at,actor_user_id,action,target_user_id) VALUES(2,888,'deleted',999)`} { + if _, err := pg.Exec(q); err != nil { + t.Fatal(err) + } + } + dest := filepath.Join(t.TempDir(), "accounts.sqlite") + options := reverseOptions{DatabaseURL: dsn, Destination: dest, StateDir: t.TempDir(), Kind: "accounts", BatchSize: 2} + report, err := reversePostgres(ctx, options) + if err != nil || !report.Verified { + t.Fatal("reverse accounts", err) + } + db, err := openSQLite(dest, "rw") + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := users.AssertSQLiteReady(db); err != nil { + t.Fatal(err) + } + var id int64 + if err := db.QueryRow(`SELECT id FROM mail_events WHERE mail_id=4`).Scan(&id); err != nil || id != 900 { + t.Fatal("event id", id, err) + } + if err := db.QueryRow(`INSERT INTO mail_events(mail_id,event,at) VALUES(4,'new',99) RETURNING id`).Scan(&id); err != nil || id != 9001 { + t.Fatal("event sequence", id, err) + } + if err := db.QueryRow(`INSERT INTO users(email,display_name,password_hash,created_at) VALUES('new@example.invalid','','hash',3) RETURNING id`).Scan(&id); err != nil || id != 1000 { + t.Fatal("historical user references", id, err) + } +} diff --git a/cmd/migrate/source.go b/cmd/migrate/source.go index c4668e519..2c723628b 100644 --- a/cmd/migrate/source.go +++ b/cmd/migrate/source.go @@ -5,15 +5,13 @@ import ( "crypto/sha256" "database/sql" "errors" - "fmt" "io" "net/url" "os" - "path/filepath" - "runtime" - "time" "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/sqliteutil" ) const importSQLiteDriver = "corescope_sqlite_import" @@ -28,138 +26,39 @@ func init() { }}) } -// snapshotSource creates an offline recovery copy without modifying the input. -// The backup API preserves hidden rowids used by observer links and mail event -// ordering. VACUUM INTO may renumber them and is not a safe migration snapshot. -func snapshotSource(ctx context.Context, source, destination string) (err error) { - if err := ctx.Err(); err != nil { - return err - } - sourceDB, err := sql.Open(importSQLiteDriver, sqliteURL(source, "ro")) - if err != nil { - return err - } - defer sourceDB.Close() - sourceDB.SetMaxOpenConns(1) - if _, err := sourceDB.ExecContext(ctx, `PRAGMA trusted_schema=OFF`); err != nil { - return err - } - var schemaVersion int - if err := sourceDB.QueryRowContext(ctx, `PRAGMA schema_version`).Scan(&schemaVersion); err != nil { - return fmt.Errorf("read SQLite source: %w", err) - } +// snapshotSource adds offline source immutability checks to the shared native +// backup. Snapshot itself is also used by live backups that permit writers. +func snapshotSource(ctx context.Context, source, destination string) error { before, err := sourceFingerprintContext(ctx, source) if err != nil { return err } - reserved, err := os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) - if err != nil { - return fmt.Errorf("create new recovery snapshot: %w", err) - } - if err := reserved.Close(); err != nil { - _ = os.Remove(destination) + if err := sqliteutil.Snapshot(ctx, source, destination); err != nil { return err } - defer func() { + after, err := sourceFingerprintContext(ctx, source) + if err != nil || before != after { + for _, suffix := range []string{"", "-wal", "-shm", "-journal"} { + _ = os.Remove(destination + suffix) + } if err != nil { - for _, suffix := range []string{"", "-wal", "-shm", "-journal"} { - _ = os.Remove(destination + suffix) - } + return err } - }() - destinationDB, err := sql.Open(importSQLiteDriver, sqliteURL(destination, "rw")) - if err != nil { - return err - } - defer destinationDB.Close() - destinationDB.SetMaxOpenConns(1) - src, err := sourceDB.Conn(ctx) - if err != nil { - return err - } - defer src.Close() - dst, err := destinationDB.Conn(ctx) - if err != nil { - return err - } - defer dst.Close() - err = src.Raw(func(sourceDriver any) error { - return dst.Raw(func(destinationDriver any) (err error) { - backup, err := destinationDriver.(*sqlite3.SQLiteConn).Backup("main", sourceDriver.(*sqlite3.SQLiteConn), "main") - if err != nil { - return err - } - defer func() { - if finishErr := backup.Finish(); err == nil { - err = finishErr - } - }() - remaining, progressAt := -1, time.Now() - for { - if err := ctx.Err(); err != nil { - return err - } - done, err := backup.Step(256) - if err != nil { - return err - } - if done { - return nil - } - if current := backup.Remaining(); current != remaining { - remaining, progressAt = current, time.Now() - continue - } - if time.Since(progressAt) > 10*time.Second { - return errors.New("SQLite snapshot made no progress; stop all instance writers before retrying") - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(10 * time.Millisecond): - } - } - }) - }) - if err != nil { - return fmt.Errorf("copy SQLite recovery snapshot: %w", err) - } - var check string - if err := dst.QueryRowContext(ctx, `PRAGMA quick_check`).Scan(&check); err != nil { - return err - } - if check != "ok" { - return errors.New("SQLite recovery snapshot failed integrity verification") - } - after, err := sourceFingerprintContext(ctx, source) - if err != nil { - return err - } - if after != before { return errors.New("SQLite source changed during snapshot; stop all instance writers before retrying") } - if err := dst.Close(); err != nil { - return err - } - if err := destinationDB.Close(); err != nil { - return err - } return nil } -func sqliteURL(path, mode string) string { - abs, err := filepath.Abs(path) +func sqliteURL(path, mode string) (string, error) { + return dbconfig.SQLiteURI(path, url.Values{"mode": {mode}, "_busy_timeout": {"5000"}}) +} + +func openSQLite(path, mode string) (*sql.DB, error) { + uri, err := sqliteURL(path, mode) if err != nil { - abs = path - } - abs = filepath.ToSlash(abs) - if runtime.GOOS == "windows" { - abs = "/" + abs + return nil, err } - u := url.URL{Scheme: "file", Path: abs} - q := url.Values{"mode": {mode}, "_busy_timeout": {"5000"}} - u.RawQuery = q.Encode() - return u.String() + return sql.Open(importSQLiteDriver, uri) } // Hash the main file and committed-write log without loading either into RAM. diff --git a/cmd/migrate/storage_command.go b/cmd/migrate/storage_command.go new file mode 100644 index 000000000..8b74ac761 --- /dev/null +++ b/cmd/migrate/storage_command.go @@ -0,0 +1,475 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" +) + +type storageOptions struct { + Action, SelectionFile, Backend, SQLitePath, UsersSQLitePath string + OwnerURL, UsersOwnerURL, StateDir, JobID, ConfigFile, ConfigDir string + Offline bool + afterStore func(string) error + beforeCommit func() error +} + +func firstSetting(values ...string) string { + for _, value := range values { + if value != "" { + return value + } + } + return "" +} + +func storageActionInputs(o storageOptions, recorded bool) (dbconfig.StorageInputs, error) { + var raw dbconfig.StorageInputs + cwd, err := os.Getwd() + if err != nil { + return raw, err + } + configDir, err := filepath.Abs(firstSetting(o.ConfigDir, ".")) + if err != nil { + return raw, err + } + if o.ConfigFile != "" { + raw, err = dbconfig.ReadStorageConfig(o.ConfigFile) + if err != nil { + return raw, err + } + } else { + for _, path := range []string{filepath.Join(configDir, "config.json"), filepath.Join(configDir, "data", "config.json")} { + config, e := dbconfig.ReadStorageConfig(path) + if e == nil { + raw = config + err = nil + break + } + if !errors.Is(e, os.ErrNotExist) { + return raw, e + } + } + if err != nil { + return raw, err + } + } + envPath := os.Getenv("DB_PATH") + if !recorded && o.SQLitePath == "" && envPath != "" && raw.DBPath != "" { + a, e := filepath.Abs(envPath) + if e != nil { + return raw, e + } + b, e := filepath.Abs(raw.DBPath) + if e != nil { + return raw, e + } + if a != b { + return raw, errors.New("legacy config dbPath and DB_PATH differ between services; supply one explicit -sqlite-path before adoption") + } + } + raw.BaseDir = cwd + raw.Backend = dbconfig.Backend(firstSetting(o.Backend, string(raw.Backend))) + raw.EnvBackend = dbconfig.Backend(os.Getenv("CORESCOPE_DB_BACKEND")) + if o.Backend != "" { + raw.EnvBackend = raw.Backend + } + raw.DBPath = firstSetting(o.SQLitePath, envPath, raw.DBPath) + raw.UsersDBPath = firstSetting(o.UsersSQLitePath, raw.UsersDBPath) + raw.DatabaseURL = firstSetting(os.Getenv("CORESCOPE_DATABASE_URL"), raw.DatabaseURL) + raw.ReaderDatabaseURL = os.Getenv("CORESCOPE_READER_DATABASE_URL") + raw.WriterDatabaseURL = os.Getenv("CORESCOPE_WRITER_DATABASE_URL") + raw.UsersDatabaseURL = firstSetting(os.Getenv("CORESCOPE_USERS_DATABASE_URL"), raw.UsersDatabaseURL) + raw.ApprovedChannelsDatabaseURL = firstSetting(os.Getenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL"), raw.ApprovedChannelsDatabaseURL) + raw.StateDir = filepath.Dir(o.SelectionFile) // stable discovery, not the recovery directory + raw.FreshInstall = true // only permits resolution; native facts below authorize initialization + sqliteDefault := raw.Backend == dbconfig.SQLite || raw.EnvBackend == dbconfig.SQLite || (raw.Backend == "" && raw.EnvBackend == "" && raw.DatabaseURL == "" && raw.ReaderDatabaseURL == "" && raw.WriterDatabaseURL == "" && raw.UsersDatabaseURL == "" && raw.ApprovedChannelsDatabaseURL == "" && o.OwnerURL == "" && o.UsersOwnerURL == "") + if raw.DBPath == "" && sqliteDefault { + raw.DBPath = filepath.Join(configDir, "data", "meshcore.db") + } + return raw, nil +} + +func runStorageAction(ctx context.Context, o storageOptions, out io.Writer) (err error) { + defer func() { + var pgError *pgconn.PgError + if errors.As(err, &pgError) { + err = fmt.Errorf("storage operation failed (SQLSTATE %s); keep writers stopped and inspect the private recovery job", pgError.Code) + } + }() + if !filepath.IsAbs(o.SelectionFile) || filepath.Clean(o.SelectionFile) != o.SelectionFile || filepath.Base(o.SelectionFile) != "storage-selection.json" { + return errors.New("provide an absolute normalized -selection-file ending in storage-selection.json") + } + status, inspectErr := dbconfig.InspectSelection(o.SelectionFile) + if o.Action == "status" { + if err := json.NewEncoder(out).Encode(status); err != nil { + return err + } + return inspectErr + } + if !o.Offline { + return errors.New("storage setup and switching require -offline after stopping all telemetry and account writers") + } + if o.StateDir == "" { + o.StateDir = filepath.Join(filepath.Dir(o.SelectionFile), "migration") + } + switch o.Action { + case "setup", "adopt", "init", "switch", "resume", "abort": + default: + return errors.New("unsupported storage action; use -help") + } + if inspectErr != nil { + return inspectErr + } + o.OwnerURL = firstSetting(o.OwnerURL, os.Getenv("CORESCOPE_DATABASE_URL")) + o.UsersOwnerURL = firstSetting(o.UsersOwnerURL, os.Getenv("CORESCOPE_USERS_OWNER_DATABASE_URL")) + if o.Action == "resume" || o.Action == "abort" || o.Action == "switch" { + if err := runStorageSwitch(ctx, o); err != nil { + return err + } + } else { + if status.State == "pending" { + return dbconfig.ErrSelectionInProgress + } + if err := setupStorage(ctx, o, status.State == "ready"); err != nil { + return err + } + } + status, err = dbconfig.InspectSelection(o.SelectionFile) + if err != nil { + return err + } + return json.NewEncoder(out).Encode(status) +} + +func setupStorage(ctx context.Context, o storageOptions, recorded bool) error { + if recorded && o.Action == "init" { + return errors.New("installation already has a recorded backend; use a verified switch") + } + raw, err := storageActionInputs(o, recorded) + if err != nil { + return err + } + if recorded { + selection, lease, err := dbconfig.OpenSelection(o.SelectionFile) + if err != nil { + return err + } + defer lease.Close() + if selection.Accounts == nil && o.Action == "setup" && (selection.Backend == dbconfig.SQLite || o.UsersOwnerURL != "" || raw.UsersDatabaseURL != "" || raw.ApprovedChannelsDatabaseURL != "") { + lease.Close() + return initializeAccounts(ctx, o, selection, raw) + } + if selection.Backend == dbconfig.Postgres && o.OwnerURL != "" { + owners, err := bindOwnerURLs(selection, o.OwnerURL, o.UsersOwnerURL) + if err != nil { + return err + } + if err := grantRuntimeTargets(ctx, selection, owners, raw); err != nil { + return err + } + } + return validateRuntimeSelection(ctx, selection, raw) + } + ownerInputs := raw + // Owner endpoints are used only for schema operations. Runtime credentials + // stay separate and are validated against the resulting selected targets. + if o.OwnerURL != "" { + ownerInputs.DatabaseURL = o.OwnerURL + ownerInputs.ReaderDatabaseURL = "" + ownerInputs.WriterDatabaseURL = "" + } + if o.UsersOwnerURL != "" { + ownerInputs.UsersDatabaseURL = o.UsersOwnerURL + ownerInputs.ApprovedChannelsDatabaseURL = "" + } + storage, err := dbconfig.ResolveStorage(ownerInputs) + if err != nil { + return err + } + selected, err := dbconfig.NewSelection(storage) + if err != nil { + return err + } + if storage.Backend == dbconfig.SQLite { + legacyTelemetry, err := sqliteOccupied(ctx, storage.DBPath) + if err != nil { + return err + } + for _, store := range []struct{ kind, path string }{{"telemetry", storage.DBPath}, {"accounts", storage.UsersDBPath}} { + occupied, err := sqliteOccupied(ctx, store.path) + if err != nil { + return err + } + if store.kind == "accounts" && legacyTelemetry && !occupied { + selected.Accounts = nil + storage.UsersDBPath = "" + continue + } + if o.Action == "init" && occupied { + return errors.New("initialization target already contains SQLite schema; use setup/adopt for an existing installation") + } + if o.Action == "adopt" && store.kind == "telemetry" && !occupied { + return errors.New("adoption requires an existing telemetry database") + } + if err := applySQLiteTarget(ctx, store.path, store.kind); err != nil { + return err + } + } + } else { + if o.OwnerURL == "" { + return errors.New("unrecorded PostgreSQL setup requires its telemetry owner URL through CORESCOPE_DATABASE_URL") + } + if (raw.UsersDatabaseURL != "" || raw.ApprovedChannelsDatabaseURL != "") && o.UsersOwnerURL == "" { + return errors.New("PostgreSQL account setup requires CORESCOPE_USERS_OWNER_DATABASE_URL; runtime credentials are never owner credentials") + } + if err := validateInitialOwnerTargets(ctx, selected, storage); err != nil { + return err + } + storage, err = bindOwnerURLs(selected, o.OwnerURL, o.UsersOwnerURL) + if err != nil { + return err + } + if storage.UsersDatabaseURL != "" { + same, err := pgutil.SameDatabase(ctx, storage.WriterDatabaseURL, storage.UsersDatabaseURL) + if err != nil { + return err + } + if same { + return errors.New("telemetry and accounts require separate PostgreSQL databases") + } + } + knownSQLite := firstSetting(raw.DBPath, filepath.Join(filepath.Dir(o.SelectionFile), "meshcore.db")) + knownUsers := firstSetting(raw.UsersDBPath, filepath.Join(filepath.Dir(knownSQLite), "users.db")) + for _, store := range []struct{ kind, url, oldPath string }{{"telemetry", storage.WriterDatabaseURL, knownSQLite}, {"accounts", storage.UsersDatabaseURL, knownUsers}} { + occupied, err := sqliteOccupied(ctx, store.oldPath) + if err != nil { + return err + } + if store.url == "" { + if occupied { + return errors.New("an existing SQLite account store must be preserved; supply its PostgreSQL owner target and use a verified switch") + } + continue + } + db, err := pgutil.Open(store.url, false) + if err != nil { + return err + } + readyErr := assertPostgresReady(db, store.kind) + if readyErr == nil { + db.Close() + if o.Action == "init" { + return errors.New("PostgreSQL target is already initialized; use setup/adopt") + } + if occupied && o.Action != "adopt" { + if err := checkImportedSource(ctx, store.url, store.kind, store.oldPath); err != nil { + return errors.New("retained SQLite data needs its verified import or explicit adoption of the existing ready PostgreSQL store") + } + } + continue + } + var objects int + err = db.QueryRowContext(ctx, importDestinationObjects).Scan(&objects) + if err != nil { + db.Close() + return err + } + if objects != 0 || o.Action == "adopt" { + db.Close() + return errors.New("PostgreSQL target is not a ready supported store or a fresh empty schema") + } + if occupied { + db.Close() + return errors.New("refusing to initialize empty PostgreSQL while primary SQLite data exists; use a verified offline switch") + } + if raw.Backend != dbconfig.Postgres && raw.EnvBackend != dbconfig.Postgres { + db.Close() + return errors.New("fresh PostgreSQL initialization requires an explicit backend choice") + } + if store.kind == "telemetry" { + err = dbschema.ApplyPostgres(db, nil) + } else { + err = users.ApplyPostgres(db) + } + db.Close() + if err != nil { + return err + } + } + } + if selected.Backend == dbconfig.Postgres { + if err := grantRuntimeTargets(ctx, selected, storage, raw); err != nil { + return err + } + } + if err := validateRuntimeSelection(ctx, selected, raw); err != nil { + return err + } + _, err = dbconfig.AdoptSelection(o.SelectionFile, selected) + return err +} + +func sqliteOccupied(ctx context.Context, path string) (bool, error) { + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + return false, nil + } else if err != nil { + return false, err + } + db, err := openSQLite(path, "ro") + if err != nil { + return false, err + } + defer db.Close() + var count int + if err := db.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%'`).Scan(&count); err != nil { + return false, errors.New("existing SQLite target cannot be validated; preserve it instead of selecting an empty database") + } + return count != 0, nil +} + +func applySQLiteTarget(ctx context.Context, path, kind string) error { + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return err + } + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return err + } + if err := f.Close(); err != nil { + return err + } + } + uri, err := legacy.WriterDSN(path) + if err != nil { + return err + } + db, err := sql.Open(importSQLiteDriver, uri) + if err != nil { + return err + } + defer db.Close() + db.SetMaxOpenConns(1) + if err := dbconfig.AssertSQLiteImportComplete(db); err != nil { + return err + } + if kind == "telemetry" { + return dbschema.ApplySQLite(db, func(string, ...interface{}) {}) + } + return users.ApplySQLite(db) +} + +func assertPostgresReady(db *sql.DB, kind string) error { + if kind == "telemetry" { + return dbschema.AssertPostgresReady(db) + } + return users.AssertPostgresReady(db) +} + +func validateRuntimeSelection(ctx context.Context, selection dbconfig.Selection, raw dbconfig.StorageInputs) error { + bound, err := selection.ApplyTo(raw) + if err != nil { + return err + } + storage, err := dbconfig.ResolveStorage(bound) + if err != nil { + return err + } + if storage.Backend == dbconfig.SQLite { + a, err := os.Stat(storage.DBPath) + if err != nil { + return err + } + stores := []struct{ kind, path string }{{"telemetry", storage.DBPath}} + if selection.Accounts != nil { + b, err := os.Stat(storage.UsersDBPath) + if err != nil { + return err + } + if os.SameFile(a, b) { + return errors.New("telemetry and accounts cannot share the same SQLite file") + } + stores = append(stores, struct{ kind, path string }{"accounts", storage.UsersDBPath}) + } + for _, store := range stores { + db, err := openSQLite(store.path, "ro") + if err != nil { + return err + } + if err := dbconfig.AssertSQLiteImportComplete(db); err != nil { + db.Close() + return err + } + if store.kind == "telemetry" { + err = dbschema.AssertSQLiteReady(db) + } else { + err = users.AssertSQLiteReady(db) + } + db.Close() + if err != nil { + return err + } + } + return nil + } + if storage.ReaderDatabaseURL == "" || storage.WriterDatabaseURL == "" { + return errors.New("prepare both PostgreSQL reader and writer role URLs before selecting this backend") + } + reader, err := pgutil.Open(storage.ReaderDatabaseURL, true) + if err != nil { + return err + } + if err = dbschema.AssertPostgresReady(reader); err == nil { + err = pgutil.AssertReadOnly(reader) + } + reader.Close() + if err != nil { + return err + } + writer, err := pgutil.Open(storage.WriterDatabaseURL, false) + if err != nil { + return err + } + if err = dbschema.AssertPostgresReady(writer); err == nil { + err = dbschema.AssertWriter(writer) + } + writer.Close() + if err != nil { + return err + } + if selection.Accounts != nil { + if storage.UsersDatabaseURL == "" { + return errors.New("prepare the selected PostgreSQL account runtime URL before cutover") + } + store, err := users.OpenStorage(storage) + if err != nil { + return err + } + store.Close() + } + if storage.ApprovedChannelsDatabaseURL != "" { + db, err := pgutil.Open(storage.ApprovedChannelsDatabaseURL, true) + if err != nil { + return err + } + err = pgutil.AssertReadOnly(db) + db.Close() + if err != nil { + return err + } + } + return nil +} diff --git a/cmd/migrate/storage_command_test.go b/cmd/migrate/storage_command_test.go new file mode 100644 index 000000000..6f45e3074 --- /dev/null +++ b/cmd/migrate/storage_command_test.go @@ -0,0 +1,149 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" +) + +func TestStorageStatusNeedsNoCredentialsOrDatabase(t *testing.T) { + t.Setenv("CORESCOPE_DATABASE_URL", "unreachable-private-secret") + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "unreachable-private-secret") + selection := filepath.Join(t.TempDir(), "storage-selection.json") + var out bytes.Buffer + if err := runCommand(context.Background(), []string{"-storage-action=status", "-selection-file", selection}, &out); err != nil { + t.Fatal(err) + } + var status dbconfig.SelectionStatus + if err := json.Unmarshal(out.Bytes(), &status); err != nil { + t.Fatal(err) + } + if status.State != "unrecorded" || status.Backend != "" { + t.Fatalf("status=%+v", status) + } + if bytes.Contains(out.Bytes(), []byte("secret")) { + t.Fatal("status exposed connection input") + } +} + +func TestRecordedPostgresPortCannotDriftThroughEnvironment(t *testing.T) { + t.Setenv("PGPORT", "") + t.Setenv("PGSERVICE", "") + t.Setenv("PGOPTIONS", "") + base := t.TempDir() + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.Postgres, StateDir: base, WriterDatabaseURL: "postgres://owner:secret@localhost/data"}) + if err != nil { + t.Fatal(err) + } + t.Setenv("PGPORT", "6543") + raw, err := selected.ApplyTo(dbconfig.StorageInputs{BaseDir: base, DatabaseURL: "postgres://rotated:secret@localhost/data"}) + if err != nil { + t.Fatal(err) + } + config, err := pgutil.ParseConfig(raw.WriterDatabaseURL) + if err != nil || config.Port != 5432 || config.Database != "data" || config.RuntimeParams["search_path"] != `"public"` { + t.Fatal("native parser drifted from recorded target", err) + } +} + +func TestSetupRefusesMalformedPrimaryBeforeValidFallback(t *testing.T) { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "data"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{broken`), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "data", "config.json"), []byte(`{"db":{"backend":"sqlite"}}`), 0600); err != nil { + t.Fatal(err) + } + selection := filepath.Join(dir, "data", "storage-selection.json") + if err := runCommand(context.Background(), []string{"-storage-action=setup", "-selection-file", selection, "-config-dir", dir, "-offline"}, &bytes.Buffer{}); err == nil { + t.Fatal("bad primary config silently chose a fallback") + } + if _, err := os.Stat(selection); !os.IsNotExist(err) { + t.Fatal("bad primary initialized an installation", err) + } +} + +func TestStorageSetupDefaultsSQLiteAndPreservesRecordedTargets(t *testing.T) { + for _, name := range []string{"CORESCOPE_DB_BACKEND", "DB_PATH", "CORESCOPE_DATABASE_URL", "CORESCOPE_READER_DATABASE_URL", "CORESCOPE_WRITER_DATABASE_URL", "CORESCOPE_USERS_DATABASE_URL", "CORESCOPE_USERS_OWNER_DATABASE_URL", "CORESCOPE_APPROVED_CHANNELS_DATABASE_URL"} { + t.Setenv(name, "") + } + root := t.TempDir() + selection := filepath.Join(root, "data", "storage-selection.json") + var out bytes.Buffer + args := []string{"-storage-action=setup", "-selection-file", selection, "-config-dir", root, "-offline"} + if err := runCommand(context.Background(), args, &out); err != nil { + t.Fatal(err) + } + got, err := dbconfig.ReadSelection(selection) + if err != nil { + t.Fatal(err) + } + if got.Backend != dbconfig.SQLite || got.Telemetry.SQLitePath != filepath.Join(root, "data", "meshcore.db") || got.Accounts == nil { + t.Fatal("fresh default or account target wrong") + } + db, err := openSQLite(got.Telemetry.SQLitePath, "rw") + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO nodes(public_key,name) VALUES('retained','still here')`); err != nil { + t.Fatal(err) + } + db.Close() + before, err := sourceFingerprint(got.Telemetry.SQLitePath) + if err != nil { + t.Fatal(err) + } + t.Setenv("CORESCOPE_DB_BACKEND", "postgres") + t.Setenv("DB_PATH", filepath.Join(root, "obsolete.sqlite")) + out.Reset() + if err := runCommand(context.Background(), args, &out); err != nil { + t.Fatal(err) + } + after, err := dbconfig.ReadSelection(selection) + if err != nil || after.Generation != got.Generation || after.Backend != dbconfig.SQLite { + t.Fatal("update changed selection", err) + } + if current, err := sourceFingerprint(got.Telemetry.SQLitePath); err != nil || current != before { + t.Fatal("ready setup changed working data", err) + } + out.Reset() + args[0] = "-storage-action=init" + if err := runCommand(context.Background(), args, &out); err == nil { + t.Fatal("init replaced existing installation") + } +} + +func TestUnrecordedConfigPathConflictRequiresOverride(t *testing.T) { + t.Setenv("CORESCOPE_DB_BACKEND", "") + t.Setenv("CORESCOPE_DATABASE_URL", "") + root := t.TempDir() + config := filepath.Join(root, "custom.json") + if err := os.WriteFile(config, []byte(`{"dbPath":"one.sqlite","userManagement":{"enabled":false,"dbPath":"saved-users.sqlite"}}`), 0600); err != nil { + t.Fatal(err) + } + t.Setenv("DB_PATH", "two.sqlite") + args := []string{"-storage-action=setup", "-selection-file", filepath.Join(root, "storage-selection.json"), "-config", config, "-offline"} + var out bytes.Buffer + if err := runCommand(context.Background(), args, &out); err == nil { + t.Fatal("conflicting server/ingestor paths silently adopted") + } + chosen := filepath.Join(root, "chosen.sqlite") + accounts := filepath.Join(root, "chosen-users.sqlite") + args = append(args, "-sqlite-path", chosen, "-users-sqlite-path", accounts) + if err := runCommand(context.Background(), args, &out); err != nil { + t.Fatal(err) + } + got, err := dbconfig.ReadSelection(filepath.Join(root, "storage-selection.json")) + if err != nil || got.Telemetry.SQLitePath != chosen || got.Accounts.SQLitePath != accounts { + t.Fatal("explicit target override failed", err) + } +} diff --git a/cmd/migrate/switch.go b/cmd/migrate/switch.go new file mode 100644 index 000000000..d698df217 --- /dev/null +++ b/cmd/migrate/switch.go @@ -0,0 +1,425 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" +) + +type switchJob struct { + Version int `json:"version"` + ID string `json:"id"` + Source dbconfig.Selection `json:"source"` + Next dbconfig.Selection `json:"next"` +} + +type sqliteFence struct { + read, writer *sql.DB + conn *sql.Conn + path, digest string +} + +func (f *sqliteFence) Close() { + if f.conn != nil { + f.conn.ExecContext(context.Background(), `ROLLBACK`) + f.conn.Close() + } + if f.writer != nil { + f.writer.Close() + } + if f.read != nil { + f.read.Close() + } +} +func (f *sqliteFence) unchanged(ctx context.Context) error { + got, err := fingerprint(ctx, f.path) + if err != nil { + return err + } + if got != f.digest { + return errors.New("original SQLite source changed; keep writers stopped and preserve recovery files") + } + return nil +} + +func fenceSQLite(ctx context.Context, path, kind string) (_ *sqliteFence, err error) { + f := &sqliteFence{path: path} + defer func() { + if err != nil { + f.Close() + } + }() + // The read-only keeper outlives the writer handle, preventing a last-writer + // close from checkpointing the source's pre-existing WAL into its main file. + if f.read, err = openSQLite(path, "ro"); err != nil { + return nil, err + } + f.read.SetMaxOpenConns(1) + if err = dbconfig.AssertSQLiteImportComplete(f.read); err != nil { + return nil, err + } + if kind == "telemetry" { + err = dbschema.AssertSQLiteReady(f.read) + } else { + err = users.AssertSQLiteReady(f.read) + } + if err != nil { + return nil, err + } + if f.digest, err = fingerprint(ctx, path); err != nil { + return nil, err + } + if f.writer, err = openSQLite(path, "rw"); err != nil { + return nil, err + } + f.writer.SetMaxOpenConns(1) + if f.conn, err = f.writer.Conn(ctx); err != nil { + return nil, err + } + if _, err = f.conn.ExecContext(ctx, `PRAGMA busy_timeout=0`); err != nil { + return nil, err + } + if _, err = f.conn.ExecContext(ctx, `BEGIN IMMEDIATE`); err != nil { + return nil, errors.New("SQLite source writer is active; stop all writers before conversion") + } + if err = f.unchanged(ctx); err != nil { + return nil, err + } + return f, nil +} + +func bindOwnerURLs(selected dbconfig.Selection, owner, accounts string) (dbconfig.Storage, error) { + raw := dbconfig.StorageInputs{BaseDir: selected.StateDir, DatabaseURL: owner, UsersDatabaseURL: accounts} + bound, err := selected.ApplyTo(raw) + if err != nil { + return dbconfig.Storage{}, err + } + storage, err := dbconfig.ResolveStorage(bound) + if err != nil { + return storage, err + } + if selected.Backend == dbconfig.Postgres && selected.Accounts != nil && storage.UsersDatabaseURL == "" { + return storage, errors.New("the existing account store requires its PostgreSQL owner URL even when accounts are disabled") + } + return storage, nil +} + +func validateInitialOwnerTargets(ctx context.Context, selected dbconfig.Selection, storage dbconfig.Storage) error { + checks := []struct { + url string + target *dbconfig.PostgresTarget + }{{storage.WriterDatabaseURL, selected.Telemetry.Postgres}} + if selected.Accounts != nil { + checks = append(checks, struct { + url string + target *dbconfig.PostgresTarget + }{storage.UsersDatabaseURL, selected.Accounts.Postgres}) + } + for _, check := range checks { + if check.target == nil || check.url == "" { + return errors.New("initial PostgreSQL owner targets are incomplete") + } + db, err := pgutil.Open(check.url, true) + if err != nil { + return err + } + var database, schema string + err = db.QueryRowContext(ctx, `SELECT current_database(),current_schema()`).Scan(&database, &schema) + db.Close() + if err != nil || database != check.target.Database || schema != check.target.Schema { + return errors.New("initial PostgreSQL database/schema differs from its proposed selection; specify the intended database and search_path explicitly") + } + } + return nil +} + +func runStorageSwitch(ctx context.Context, o storageOptions) error { + state, err := filepath.Abs(o.StateDir) + if err != nil { + return err + } + if o.Action == "abort" { + if o.JobID == "" { + return errors.New("abort requires the exact -job-id") + } + update, err := dbconfig.ResumeSelectionUpdate(o.SelectionFile, o.JobID) + if err != nil { + return err + } + defer update.Close() + return update.Abort() // Staged targets and every recovery copy are preserved. + } + var job switchJob + var update *dbconfig.SelectionUpdate + raw, err := storageActionInputs(o, true) + if err != nil { + return err + } + if o.Action == "switch" { + if o.Backend != "sqlite" && o.Backend != "postgres" { + return errors.New("switch requires an explicit -backend=sqlite or postgres") + } + current, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil { + return err + } + if string(current.Backend) == o.Backend { + return errors.New("the requested backend is already selected") + } + targetInputs := raw + targetInputs.Backend = dbconfig.Backend(o.Backend) + targetInputs.EnvBackend = targetInputs.Backend + targetInputs.ExistingBackend = "" + targetInputs.StateDir = current.StateDir + if targetInputs.Backend == dbconfig.SQLite { + if o.SQLitePath == "" { + return errors.New("reverse switch requires a new explicit -sqlite-path; retained original files are never overwritten") + } + targetInputs.DBPath = o.SQLitePath + targetInputs.UsersDBPath = o.UsersSQLitePath + } else { + if o.OwnerURL == "" { + return errors.New("PostgreSQL destination requires its owner URL through the private environment") + } + if current.Accounts != nil && o.UsersOwnerURL == "" { + return errors.New("the existing account store must also be converted; configure CORESCOPE_USERS_OWNER_DATABASE_URL") + } + targetInputs.DatabaseURL = o.OwnerURL + targetInputs.ReaderDatabaseURL = "" + targetInputs.WriterDatabaseURL = "" + targetInputs.UsersDatabaseURL = o.UsersOwnerURL + targetInputs.ApprovedChannelsDatabaseURL = "" + } + target, err := dbconfig.ResolveStorage(targetInputs) + if err != nil { + return err + } + if current.Accounts == nil { + target.UsersDBPath = "" + target.UsersDatabaseURL = "" + target.ApprovedChannelsDatabaseURL = "" + } + next, err := dbconfig.NewSelection(target) + if err != nil { + return err + } + if next.Backend == dbconfig.Postgres { + if err := validateInitialOwnerTargets(ctx, next, target); err != nil { + return err + } + } + if next.Backend == dbconfig.SQLite { + paths := []string{next.Telemetry.SQLitePath} + if next.Accounts != nil { + paths = append(paths, next.Accounts.SQLitePath) + } + for _, path := range paths { + if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) { + return errors.New("switch SQLite targets must be fresh absent paths") + } + } + } + update, err = dbconfig.BeginSelectionUpdate(o.SelectionFile, current.Generation) + if err != nil { + return err + } + defer update.Close() + job = switchJob{Version: 1, ID: update.ID(), Source: current, Next: next} + if err := update.Stage(next); err != nil { + return err + } + dir := filepath.Join(state, job.ID) + if err := os.MkdirAll(dir, 0700); err != nil { + return err + } + if err := writePrivateJSON(filepath.Join(dir, "job.json"), job); err != nil { + return err + } + } else { + if o.JobID == "" { + return errors.New("resume requires the exact -job-id") + } + update, err = dbconfig.ResumeSelectionUpdate(o.SelectionFile, o.JobID) + if err != nil { + return err + } + defer update.Close() + if update.Target().Generation == "" { + return errors.New("pending job has no staged destination; abort this metadata-only update and start a new job") + } + if accountInitialization(update.Current(), update.Target()) { + return finishAccountSetup(ctx, o, update, raw, true) + } + if err := readPrivateJSON(filepath.Join(state, o.JobID, "job.json"), &job); err != nil { + return errors.New("pending switch has no complete readable job manifest; preserve any recovery files, abort the uncommitted update and start a new job") + } + if job.Version != 1 || job.ID != o.JobID || !reflect.DeepEqual(job.Source, update.Current()) || !reflect.DeepEqual(job.Next, update.Target()) || job.Next.StateDir != job.Source.StateDir || job.Next.Backend == job.Source.Backend { + return errors.New("switch recovery manifest does not match the pending installation generation") + } + } + dir := filepath.Join(state, job.ID) + ownerSelection := job.Next + if job.Source.Backend == dbconfig.Postgres { + ownerSelection = job.Source + } + owners, err := bindOwnerURLs(ownerSelection, o.OwnerURL, o.UsersOwnerURL) + if err != nil { + return err + } + if ownerSelection.Accounts != nil { + same, err := pgutil.SameDatabase(ctx, owners.WriterDatabaseURL, owners.UsersDatabaseURL) + if err != nil { + return err + } + if same { + return errors.New("telemetry and account databases must remain separate") + } + } + var sqliteSources []*sqliteFence + var postgresSources []*postgresFence + stores := []struct { + kind string + source dbconfig.Target + target dbconfig.Target + url string + }{{"telemetry", job.Source.Telemetry, job.Next.Telemetry, owners.WriterDatabaseURL}} + if job.Source.Accounts != nil { + if job.Next.Accounts == nil { + return errors.New("switch manifest would omit an existing account store") + } + stores = append(stores, struct { + kind string + source, target dbconfig.Target + url string + }{"accounts", *job.Source.Accounts, *job.Next.Accounts, owners.UsersDatabaseURL}) + } + // Fence both sources before copying either one. Every defer runs on all + // failure paths; the durable selection journal remains to block startup. + for _, store := range stores { + if job.Source.Backend == dbconfig.SQLite { + f, err := fenceSQLite(ctx, store.source.SQLitePath, store.kind) + if err != nil { + return err + } + sqliteSources = append(sqliteSources, f) + defer f.Close() + } else { + f, err := fencePostgres(ctx, store.url, store.kind) + if err != nil { + return err + } + postgresSources = append(postgresSources, f) + defer f.Close() + } + } + var reports []importReport + for i, store := range stores { + var report importReport + if job.Source.Backend == dbconfig.SQLite { + resume := o.Action == "resume" + if resume { + if _, err := os.Stat(filepath.Join(dir, store.kind, "manifest.json")); errors.Is(err, os.ErrNotExist) { + resume = false + } + } + report, err = importSQLite(ctx, importOptions{Source: store.source.SQLitePath, DatabaseURL: store.url, StateDir: dir, Kind: store.kind, Resume: resume}) + } else { + resume := o.Action == "resume" + if resume { + if _, err := os.Stat(filepath.Join(dir, store.kind, "reverse.json")); errors.Is(err, os.ErrNotExist) { + resume = false + } + } + report, err = reversePostgres(ctx, reverseOptions{DatabaseURL: store.url, Destination: store.target.SQLitePath, StateDir: dir, Kind: store.kind, Resume: resume, source: postgresSources[i], keepPending: true}) + } + if err != nil { + return err + } + if !report.Verified { + return errors.New("store copy returned without complete verification") + } + reports = append(reports, report) + if o.afterStore != nil { + if err := o.afterStore(store.kind); err != nil { + return err + } + } + } + for _, source := range sqliteSources { + if err := source.unchanged(ctx); err != nil { + return err + } + } + verification := struct { + JobID string `json:"job_id"` + Reports []importReport `json:"reports"` + }{job.ID, reports} + if err := writePrivateJSON(filepath.Join(dir, "verified.json"), verification); err != nil { + if !errors.Is(err, os.ErrExist) { + return err + } + previous := verification + previous.Reports = nil + if err := readPrivateJSON(filepath.Join(dir, "verified.json"), &previous); err != nil { + return err + } + if !reflect.DeepEqual(previous, verification) { + return errors.New("saved verification receipt differs from the reverified stores") + } + } + if job.Next.Backend == dbconfig.Postgres { + for _, store := range stores { + if _, err := finalizeImport(ctx, store.url, store.kind); err != nil { + return err + } + } + if err := grantRuntimeTargets(ctx, job.Next, owners, raw); err != nil { + return err + } + } else { + targets := []struct{ kind, path string }{{"telemetry", job.Next.Telemetry.SQLitePath}} + if job.Next.Accounts != nil { + targets = append(targets, struct{ kind, path string }{"accounts", job.Next.Accounts.SQLitePath}) + } + for _, target := range targets { + db, err := openSQLite(target.path, "rw") + if err != nil { + return err + } + _, err = db.ExecContext(ctx, `DROP TABLE IF EXISTS corescope_reverse_progress`) + if err == nil { + err = assertSQLiteData(ctx, db, target.kind) + } + db.Close() + if err != nil { + return err + } + } + } + if err := validateRuntimeSelection(ctx, job.Next, raw); err != nil { + return err + } + if o.beforeCommit != nil { + if err := o.beforeCommit(); err != nil { + return err + } + } + for _, source := range sqliteSources { + if err := source.unchanged(ctx); err != nil { + return err + } + } + if _, err := update.Commit(job.Next); err != nil { + return fmt.Errorf("selection commit needs recovery for job %s: %w", job.ID, err) + } + return nil +} diff --git a/cmd/migrate/switch_test.go b/cmd/migrate/switch_test.go new file mode 100644 index 000000000..eb2c79b68 --- /dev/null +++ b/cmd/migrate/switch_test.go @@ -0,0 +1,454 @@ +package main + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" +) + +func restrictedRole(t *testing.T, owner string) string { + t.Helper() + db := openImportDB(t, owner) + var random [12]byte + if _, err := rand.Read(random[:]); err != nil { + t.Fatal(err) + } + role := "corescope_convert_" + hex.EncodeToString(random[:]) + password := "test_" + hex.EncodeToString(random[:]) + if _, err := db.Exec(`CREATE ROLE ` + quote(role) + ` LOGIN PASSWORD '` + password + `'`); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if _, err := db.Exec(`DROP OWNED BY ` + quote(role) + `;DROP ROLE ` + quote(role)); err != nil { + t.Error(err) + } + }) + u, err := url.Parse(owner) + if err != nil { + t.Fatal("invalid test owner URL") + } + u.User = url.UserPassword(role, password) + return u.String() +} + +func switchFixture(t *testing.T) (storageOptions, dbconfig.Selection) { + t.Helper() + telemetry, accounts := postgresDatabase(t), postgresDatabase(t) + t.Setenv("CORESCOPE_DATABASE_URL", telemetry) + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", accounts) + t.Setenv("CORESCOPE_READER_DATABASE_URL", restrictedRole(t, telemetry)) + t.Setenv("CORESCOPE_WRITER_DATABASE_URL", restrictedRole(t, telemetry)) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", restrictedRole(t, accounts)) + t.Setenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", restrictedRole(t, accounts)) + t.Setenv("CORESCOPE_DB_BACKEND", "") + t.Setenv("DB_PATH", "") + source, account := telemetrySource(t), accountSource(t, 5) + if err := applySQLiteTarget(context.Background(), source, "telemetry"); err != nil { + t.Fatal(err) + } + if err := applySQLiteTarget(context.Background(), account, "accounts"); err != nil { + t.Fatal(err) + } + dir := t.TempDir() + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.SQLite, StateDir: dir, DBPath: source, UsersDBPath: account}) + if err != nil { + t.Fatal(err) + } + selection := dbconfig.SelectionPath(dir) + if _, err := dbconfig.AdoptSelection(selection, selected); err != nil { + t.Fatal(err) + } + return storageOptions{Action: "switch", SelectionFile: selection, Backend: "postgres", OwnerURL: telemetry, UsersOwnerURL: accounts, StateDir: t.TempDir(), Offline: true, ConfigDir: t.TempDir()}, selected +} + +func TestVerifiedTwoStoreSwitchRoundTripAndRuntimeGrants(t *testing.T) { + o, source := switchFixture(t) + ctx := context.Background() + before, err := sourceFingerprint(source.Telemetry.SQLitePath) + if err != nil { + t.Fatal(err) + } + beforeAccount, err := sourceFingerprint(source.Accounts.SQLitePath) + if err != nil { + t.Fatal(err) + } + var out bytes.Buffer + if err := runStorageAction(ctx, o, &out); err != nil { + t.Fatal("forward switch", err) + } + selected, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil || selected.Backend != dbconfig.Postgres { + t.Fatal("PG selection", err) + } + if got, err := sourceFingerprint(source.Telemetry.SQLitePath); err != nil || got != before { + t.Fatal("telemetry source changed", err) + } + if got, err := sourceFingerprint(source.Accounts.SQLitePath); err != nil || got != beforeAccount { + t.Fatal("account source changed", err) + } + for _, pair := range [][2]string{{"CORESCOPE_WRITER_DATABASE_URL", "corescope_schema"}, {"CORESCOPE_USERS_DATABASE_URL", "schema_version"}} { + db, err := pgutil.Open(os.Getenv(pair[0]), false) + if err != nil { + t.Fatal(err) + } + var allowed bool + if err := db.QueryRow(`SELECT has_table_privilege($1,'INSERT,UPDATE,DELETE,TRUNCATE')`, pair[1]).Scan(&allowed); err != nil || allowed { + t.Fatal("metadata DML granted", allowed, err) + } + db.Close() + } + channel, err := pgutil.Open(os.Getenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL"), true) + if err != nil { + t.Fatal(err) + } + if _, err := channel.Exec(`SELECT password_hash FROM users`); err == nil { + t.Fatal("projection reader can access account secrets") + } + channel.Close() + o.Backend = "sqlite" + o.SQLitePath = filepath.Join(t.TempDir(), "telemetry.sqlite") + o.UsersSQLitePath = filepath.Join(t.TempDir(), "accounts.sqlite") + out.Reset() + if err := runStorageAction(ctx, o, &out); err != nil { + t.Fatal("reverse switch", err) + } + selected, err = dbconfig.ReadSelection(o.SelectionFile) + if err != nil || selected.Backend != dbconfig.SQLite || selected.StateDir != source.StateDir || selected.Telemetry.SQLitePath != o.SQLitePath { + t.Fatal("reverse selection", err) + } + db, err := openSQLite(o.UsersSQLitePath, "ro") + if err != nil { + t.Fatal(err) + } + defer db.Close() + var id int + if err := db.QueryRow(`SELECT id FROM mail_events WHERE mail_id=4`).Scan(&id); err != nil || id != 900 { + t.Fatal("round trip event", id, err) + } + if strings.Contains(out.String(), "postgres://") { + t.Fatal("CLI leaked credentials") + } +} + +func TestSwitchFailureKeepsOldSelectionAndResumeVerifiesAgain(t *testing.T) { + o, source := switchFixture(t) + ctx := context.Background() + o.afterStore = func(kind string) error { + if kind == "telemetry" { + return errors.New("injected failure before accounts") + } + return nil + } + if err := runStorageAction(ctx, o, &bytes.Buffer{}); err == nil { + t.Fatal("partial switch succeeded") + } + status, err := dbconfig.InspectSelection(o.SelectionFile) + if err != nil || status.State != "pending" || status.Backend != dbconfig.SQLite || status.Generation != source.Generation { + t.Fatal("partial switch selected target", status, err) + } + if _, _, err := dbconfig.OpenSelection(o.SelectionFile); !errors.Is(err, dbconfig.ErrSelectionInProgress) { + t.Fatal("partial switch allowed runtime", err) + } + o.Action = "resume" + o.JobID = status.JobID + o.afterStore = nil + o.beforeCommit = func() error { return errors.New("injected cutover failure") } + if err := runStorageAction(ctx, o, &bytes.Buffer{}); err == nil { + t.Fatal("cutover failure passed") + } + status, err = dbconfig.InspectSelection(o.SelectionFile) + if err != nil || status.Generation != source.Generation || status.State != "pending" { + t.Fatal("failed cutover changed selection", err) + } + o.beforeCommit = nil + if err := runStorageAction(ctx, o, &bytes.Buffer{}); err != nil { + t.Fatal("verified resume", err) + } + final, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil || final.Backend != dbconfig.Postgres { + t.Fatal("resume failed to select", err) + } +} + +func TestSQLiteSourceFencePreservesWALAndRejectsWriters(t *testing.T) { + source := telemetrySource(t) + ctx := context.Background() + if err := applySQLiteTarget(ctx, source, "telemetry"); err != nil { + t.Fatal(err) + } + writer, err := openSQLite(source, "rw") + if err != nil { + t.Fatal(err) + } + if _, err := writer.Exec(`PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0; INSERT INTO observers(id) VALUES('wal-retained')`); err != nil { + t.Fatal(err) + } + keeper, err := openSQLite(source, "ro") + if err != nil { + t.Fatal(err) + } + defer keeper.Close() + if err := keeper.Ping(); err != nil { + t.Fatal(err) + } + writer.Close() + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + f, err := fenceSQLite(ctx, source, "telemetry") + if err != nil { + t.Fatal(err) + } + other, err := openSQLite(source, "rw") + if err != nil { + t.Fatal(err) + } + if _, err := other.Exec(`PRAGMA busy_timeout=1; INSERT INTO observers(id) VALUES('must-not-write')`); err == nil { + t.Fatal("source accepted a concurrent writer") + } + other.Close() + f.Close() + keeper.Close() + after, err := sourceFingerprint(source) + if err != nil || after != before { + t.Fatal("source fence checkpointed or modified original WAL", err) + } +} + +func TestSwitchSourceMutationAndDestinationSubstitutionRefuse(t *testing.T) { + o, source := switchFixture(t) + ctx := context.Background() + o.afterStore = func(string) error { return errors.New("stop after first store") } + if err := runStorageAction(ctx, o, &bytes.Buffer{}); err == nil { + t.Fatal("expected interruption") + } + status, err := dbconfig.InspectSelection(o.SelectionFile) + if err != nil { + t.Fatal(err) + } + var job switchJob + jobPath := filepath.Join(o.StateDir, status.JobID, "job.json") + if err := readPrivateJSON(jobPath, &job); err != nil { + t.Fatal(err) + } + original := job + job.Next.Telemetry.Postgres = &dbconfig.PostgresTarget{Host: "substituted.invalid", Port: 5432, Database: "other", Schema: "public"} + data, err := json.Marshal(job) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(jobPath, data, 0600); err != nil { + t.Fatal(err) + } + o.Action = "resume" + o.JobID = status.JobID + o.afterStore = nil + if err := runStorageAction(ctx, o, &bytes.Buffer{}); err == nil || !strings.Contains(err.Error(), "manifest") { + t.Fatal("changed target accepted", err) + } + data, err = json.Marshal(original) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(jobPath, data, 0600); err != nil { + t.Fatal(err) + } + changed, err := openSQLite(source.Telemetry.SQLitePath, "rw") + if err != nil { + t.Fatal(err) + } + if _, err := changed.Exec(`UPDATE observations SET score=7.5 WHERE id=90`); err != nil { + t.Fatal(err) + } + changed.Close() + if err := runStorageAction(ctx, o, &bytes.Buffer{}); err == nil { + t.Fatal("changed original source accepted") + } + o.Action = "abort" + if err := runStorageAction(ctx, o, &bytes.Buffer{}); err != nil { + t.Fatal("abort", err) + } + selected, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil || selected.Generation != source.Generation { + t.Fatal("abort changed original selection", err) + } + if _, err := os.Stat(filepath.Join(o.StateDir, status.JobID, "telemetry", "recovery.sqlite")); err != nil { + t.Fatal("abort removed recovery", err) + } +} + +func TestSetupPostgresCannotAbandonExistingSQLite(t *testing.T) { + owner := postgresDatabase(t) + t.Setenv("CORESCOPE_DATABASE_URL", owner) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", "") + t.Setenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", "") + t.Setenv("CORESCOPE_READER_DATABASE_URL", "") + t.Setenv("CORESCOPE_WRITER_DATABASE_URL", "") + t.Setenv("CORESCOPE_DB_BACKEND", "sqlite") + source := telemetrySource(t) + t.Setenv("DB_PATH", source) + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + config := filepath.Join(dir, "config.json") + if err := os.WriteFile(config, []byte(`{"db":{"backend":"sqlite"}}`), 0600); err != nil { + t.Fatal(err) + } + args := []string{"-storage-action=setup", "-backend=postgres", "-selection-file", filepath.Join(dir, "storage-selection.json"), "-config", config, "-offline"} + err = runCommand(context.Background(), args, &bytes.Buffer{}) + if err == nil || !strings.Contains(err.Error(), "primary SQLite data") { + t.Fatal("fresh PG abandoned source or explicit choice did not override hints", err) + } + db := openImportDB(t, owner) + var n int + if err := db.QueryRow(importDestinationObjects).Scan(&n); err != nil || n != 0 { + t.Fatal("empty PG target was initialized", n, err) + } + if after, err := sourceFingerprint(source); err != nil || before != after { + t.Fatal("SQLite source was changed", err) + } +} + +func TestFreshPostgresSetupOverridesBootstrapDefaultAndGrantsBeforeSelection(t *testing.T) { + owner, accounts := postgresDatabase(t), postgresDatabase(t) + t.Setenv("CORESCOPE_DATABASE_URL", owner) + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", accounts) + t.Setenv("CORESCOPE_READER_DATABASE_URL", restrictedRole(t, owner)) + t.Setenv("CORESCOPE_WRITER_DATABASE_URL", restrictedRole(t, owner)) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", restrictedRole(t, accounts)) + t.Setenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", restrictedRole(t, accounts)) + t.Setenv("CORESCOPE_DB_BACKEND", "sqlite") + t.Setenv("DB_PATH", "") + dir := t.TempDir() + config := filepath.Join(dir, "config.json") + if err := os.WriteFile(config, []byte(`{"db":{"backend":"sqlite"},"userManagement":{"enabled":false}}`), 0600); err != nil { + t.Fatal(err) + } + selection := filepath.Join(dir, "storage-selection.json") + args := []string{"-storage-action=setup", "-backend=postgres", "-selection-file", selection, "-config", config, "-offline"} + var out bytes.Buffer + if err := runCommand(context.Background(), args, &out); err != nil { + t.Fatal(err) + } + selected, err := dbconfig.ReadSelection(selection) + if err != nil || selected.Backend != dbconfig.Postgres || selected.Accounts == nil { + t.Fatal("explicit PG setup", err) + } + if bytes.Contains(out.Bytes(), []byte("postgres://")) { + t.Fatal("owner credentials in setup output") + } + // Updates retain PG although the unchanged bootstrap config/env still say SQLite. + args = []string{"-storage-action=setup", "-selection-file", selection, "-config", config, "-offline"} + if err := runCommand(context.Background(), args, &bytes.Buffer{}); err != nil { + t.Fatal("PG update", err) + } + after, err := dbconfig.ReadSelection(selection) + if err != nil || after.Generation != selected.Generation { + t.Fatal("update replaced installation", err) + } +} + +func TestInitialAdoptionRefusesImplicitRoleSchemaDrift(t *testing.T) { + owner := postgresDatabase(t) + config, err := pgutil.ParseConfig(owner) + if err != nil { + t.Fatal(err) + } + db := openImportDB(t, owner) + if _, err := db.Exec(`CREATE SCHEMA ` + quote(config.User)); err != nil { + t.Fatal(err) + } + t.Setenv("CORESCOPE_DATABASE_URL", owner) + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", "") + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") + t.Setenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", "") + t.Setenv("CORESCOPE_READER_DATABASE_URL", "") + t.Setenv("CORESCOPE_WRITER_DATABASE_URL", "") + t.Setenv("DB_PATH", "") + dir := t.TempDir() + selection := filepath.Join(dir, "storage-selection.json") + err = runStorageAction(context.Background(), storageOptions{Action: "setup", Backend: "postgres", OwnerURL: owner, Offline: true, SelectionFile: selection, ConfigDir: dir}, &bytes.Buffer{}) + if err == nil || !strings.Contains(err.Error(), "database/schema differs") { + t.Fatal("implicit owner schema drift was accepted", err) + } + if _, err := os.Stat(selection); !os.IsNotExist(err) { + t.Fatal("ambiguous schema was selected", err) + } + var tables int + if err := db.QueryRow(`SELECT count(*) FROM information_schema.tables WHERE table_schema NOT IN ('pg_catalog','information_schema')`).Scan(&tables); err != nil || tables != 0 { + t.Fatal("ambiguous schema bootstrap wrote tables", tables, err) + } +} + +func TestReverseWithoutAccountSourceResumesAfterCutoverFailure(t *testing.T) { + owner := postgresDatabase(t) + db := openImportDB(t, owner) + if err := dbschema.ApplyPostgres(db, nil); err != nil { + t.Fatal(err) + } + t.Setenv("CORESCOPE_DATABASE_URL", owner) + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", "") + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") + t.Setenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", "") + dir := t.TempDir() + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.Postgres, StateDir: dir, WriterDatabaseURL: owner}) + if err != nil { + t.Fatal(err) + } + selection := dbconfig.SelectionPath(dir) + if _, err := dbconfig.AdoptSelection(selection, selected); err != nil { + t.Fatal(err) + } + o := storageOptions{Action: "switch", Backend: "sqlite", Offline: true, SelectionFile: selection, OwnerURL: owner, StateDir: t.TempDir(), ConfigDir: t.TempDir(), SQLitePath: filepath.Join(t.TempDir(), "telemetry.sqlite"), UsersSQLitePath: filepath.Join(t.TempDir(), "accounts.sqlite"), beforeCommit: func() error { return errors.New("cutover interrupted") }} + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err == nil { + t.Fatal("injected failure passed") + } + status, err := dbconfig.InspectSelection(selection) + if err != nil || status.State != "pending" { + t.Fatal("pending job absent", err) + } + o.Action = "resume" + o.JobID = status.JobID + o.beforeCommit = nil + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal("empty optional account resume", err) + } + after, err := dbconfig.ReadSelection(selection) + if err != nil || after.Backend != dbconfig.SQLite || after.Accounts != nil { + t.Fatal("resume did not preserve uninitialized accounts", err) + } + if _, err := os.Stat(o.UsersSQLitePath); !os.IsNotExist(err) { + t.Fatal("reverse switch invented an account database", err) + } + newOwner := postgresDatabase(t) + t.Setenv("CORESCOPE_DATABASE_URL", newOwner) + t.Setenv("CORESCOPE_READER_DATABASE_URL", restrictedRole(t, newOwner)) + t.Setenv("CORESCOPE_WRITER_DATABASE_URL", restrictedRole(t, newOwner)) + o.Action = "switch" + o.Backend = "postgres" + o.OwnerURL = newOwner + o.JobID = "" + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal("switch nil accounts forward", err) + } + after, err = dbconfig.ReadSelection(selection) + if err != nil || after.Backend != dbconfig.Postgres || after.Accounts != nil { + t.Fatal("forward switch changed account absence", err) + } +} diff --git a/cmd/migrate/target_identity_test.go b/cmd/migrate/target_identity_test.go index 4175b81a1..af4325690 100644 --- a/cmd/migrate/target_identity_test.go +++ b/cmd/migrate/target_identity_test.go @@ -11,11 +11,10 @@ import ( "testing" "github.com/meshcore-analyzer/pgutil" - "github.com/meshcore-analyzer/pgutil/pgtest" ) func TestImportIdentityUsesConfiguredEndpoint(t *testing.T) { - dsn := pgtest.NewSchema(t) + dsn := postgresSchema(t) u, err := url.Parse(dsn) if err != nil { t.Fatal("parse test URL") diff --git a/cmd/server/account_test.go b/cmd/server/account_test.go index 837e7ba38..80dc75b3e 100644 --- a/cmd/server/account_test.go +++ b/cmd/server/account_test.go @@ -135,7 +135,7 @@ func TestAuthResetTokenStoreFailureEndsSessionsFirst(t *testing.T) { c := f.registerAndActivate(t, "pia@example.org", "Pia", pw) thief := f.login(t, "pia@example.org", pw) _, reset := pendingLinks(t, f, c, "pia@example.org", "attacker@example.org") - f.execDB(t, `CREATE FUNCTION down() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'token store down'; END $$; CREATE TRIGGER down BEFORE UPDATE ON tokens FOR EACH ROW WHEN (OLD.purpose='email_change') EXECUTE FUNCTION down()`) + f.execDB(t, testNativeSQL(`CREATE TRIGGER down BEFORE UPDATE ON tokens WHEN OLD.purpose='email_change' BEGIN SELECT RAISE(ABORT,'token store down'); END`, `CREATE FUNCTION down() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'token store down'; END $$; CREATE TRIGGER down BEFORE UPDATE ON tokens FOR EACH ROW WHEN (OLD.purpose='email_change') EXECUTE FUNCTION down()`)) expectStatus(t, f.do("POST", "/api/auth/reset", resetRequest{Token: reset, Password: "a brand new secret"}), 500) expectStatus(t, f.do("GET", "/api/auth/me", nil, as(thief)), 401) expectStatus(t, f.do("GET", "/api/auth/me", nil, as(c)), 401) diff --git a/cmd/server/advert_route_evidence.go b/cmd/server/advert_route_evidence.go index 15bfe041a..9858483d5 100644 --- a/cmd/server/advert_route_evidence.go +++ b/cmd/server/advert_route_evidence.go @@ -16,7 +16,7 @@ func (db *DB) advertEvidencePresent() bool { return true } var exists int - if err := db.conn.QueryRow(`SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='advert_route_evidence'`).Scan(&exists); err != nil { + if err := db.conn.QueryRow(db.nativeSQL("SELECT 1 FROM sqlite_master WHERE type='table' AND name='advert_route_evidence'", `SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='advert_route_evidence'`)).Scan(&exists); err != nil { return false } db.advertEvidenceTable.Store(true) @@ -39,7 +39,7 @@ func (db *DB) advertEvidenceForIDs(ids []int) (map[int]uint8, error) { for i, id := range ids[start:end] { args[i] = id } - query := `SELECT tx_id,SUM(bit) FROM advert_route_evidence WHERE tx_id IN (` + sqlPlaceholders(len(args)) + `) GROUP BY tx_id` + query := `SELECT tx_id,SUM(bit) FROM advert_route_evidence WHERE tx_id IN (` + db.sqlPlaceholders(len(args)) + `) GROUP BY tx_id` if db.advertEvidenceReadHook != nil { db.advertEvidenceReadHook() } @@ -121,7 +121,7 @@ func (s *PacketStore) pollAdvertEvidence(limit int) error { if limit <= 0 || limit > 500 { limit = 500 } - rows, err := s.db.conn.Query(`SELECT id,tx_id,bit FROM advert_route_evidence WHERE id>$1 ORDER BY id LIMIT $2`, s.advertEvidenceCursor, limit) + rows, err := s.db.conn.Query(`SELECT id,tx_id,bit FROM advert_route_evidence WHERE id>`+s.db.parameter(1)+` ORDER BY id LIMIT `+s.db.parameter(2), s.advertEvidenceCursor, limit) if err != nil { return err } diff --git a/cmd/server/advert_route_evidence_test.go b/cmd/server/advert_route_evidence_test.go index 983ae96d5..59041b3fc 100644 --- a/cmd/server/advert_route_evidence_test.go +++ b/cmd/server/advert_route_evidence_test.go @@ -22,7 +22,7 @@ func advertEvidenceFixture(t *testing.T, raw string, bits ...int) (*DB, *sql.DB) t.Cleanup(func() { w.Close() }) for _, stmt := range []string{ `ALTER TABLE transmissions ADD COLUMN from_pubkey TEXT`, - `CREATE TABLE advert_route_evidence (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, tx_id INTEGER NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, bit INTEGER NOT NULL CHECK (bit IN (1,2)), UNIQUE(tx_id,bit))`, + testNativeSQL(`CREATE TABLE advert_route_evidence (id INTEGER PRIMARY KEY AUTOINCREMENT, tx_id INTEGER NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, bit INTEGER NOT NULL CHECK (bit IN (1,2)), UNIQUE(tx_id,bit))`, `CREATE TABLE advert_route_evidence (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, tx_id INTEGER NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, bit INTEGER NOT NULL CHECK (bit IN (1,2)), UNIQUE(tx_id,bit))`), } { if _, err := w.Exec(stmt); err != nil { t.Fatal(err) @@ -88,12 +88,14 @@ func BenchmarkAdvertEvidence30K(b *testing.B) { } conn.SetMaxOpenConns(1) defer conn.Close() - if _, err := conn.Exec(`CREATE TABLE advert_route_evidence(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit)); + if _, err := conn.Exec(testNativeSQL(`CREATE TABLE advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit)); WITH RECURSIVE ids(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM ids WHERE n<30000) - INSERT INTO advert_route_evidence(tx_id,bit) SELECT n,1 FROM ids UNION ALL SELECT n,2 FROM ids`); err != nil { + INSERT INTO advert_route_evidence(tx_id,bit) SELECT n,1 FROM ids UNION ALL SELECT n,2 FROM ids`, `CREATE TABLE advert_route_evidence(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit)); + WITH RECURSIVE ids(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM ids WHERE n<30000) + INSERT INTO advert_route_evidence(tx_id,bit) SELECT n,1 FROM ids UNION ALL SELECT n,2 FROM ids`)); err != nil { b.Fatal(err) } - db := &DB{conn: conn} + db := &DB{backend: testBackendValue(), conn: conn} db.advertEvidenceTable.Store(true) ids := make([]int, 30000) for i := range ids { @@ -369,7 +371,7 @@ func TestAdvertRouteEvidenceMissingTableIsUnknown(t *testing.T) { } assertAdvertEvidenceViews(t, s, "other") var count int - if err := w.QueryRow(`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='advert_route_evidence'`).Scan(&count); err != nil { + if err := w.QueryRow(testNativeSQL(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='advert_route_evidence'`, `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='advert_route_evidence'`)).Scan(&count); err != nil { t.Fatal(err) } if count != 0 { @@ -379,7 +381,7 @@ func TestAdvertRouteEvidenceMissingTableIsUnknown(t *testing.T) { func TestAdvertRouteEvidenceTableAppearsAfterStartup(t *testing.T) { db, w := advertEvidenceFixture(t, "1100aa") - const schema = `CREATE TABLE advert_route_evidence (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, tx_id INTEGER NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, bit INTEGER NOT NULL CHECK (bit IN (1,2)), UNIQUE(tx_id,bit))` + var schema = testNativeSQL(`CREATE TABLE advert_route_evidence (id INTEGER PRIMARY KEY AUTOINCREMENT, tx_id INTEGER NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, bit INTEGER NOT NULL CHECK (bit IN (1,2)), UNIQUE(tx_id,bit))`, `CREATE TABLE advert_route_evidence (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, tx_id INTEGER NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, bit INTEGER NOT NULL CHECK (bit IN (1,2)), UNIQUE(tx_id,bit))`) if _, err := w.Exec(`DROP TABLE advert_route_evidence`); err != nil { t.Fatal(err) } @@ -411,7 +413,7 @@ func TestAdvertRouteEvidencePollRejectsPartialBatch(t *testing.T) { t.Fatal(err) } before := s.advertEvidenceCursor - if _, err := w.Exec(`ALTER TABLE advert_route_evidence DROP CONSTRAINT advert_route_evidence_bit_check; INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,2),(1,3)`); err != nil { + if _, err := w.Exec(testNativeSQL(`PRAGMA ignore_check_constraints=ON; INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,2),(1,3)`, `ALTER TABLE advert_route_evidence DROP CONSTRAINT advert_route_evidence_bit_check; INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,2),(1,3)`)); err != nil { t.Fatal(err) } if err := s.pollAdvertEvidence(500); err == nil { diff --git a/cmd/server/advert_stats.go b/cmd/server/advert_stats.go index 835c877b5..10a14e0ba 100644 --- a/cmd/server/advert_stats.go +++ b/cmd/server/advert_stats.go @@ -63,10 +63,13 @@ func (db *DB) CountFloodAdvertsForNode(pubkey string, windowHours float64, rowCa } floor := time.Now().UTC().Add(-time.Duration(windowHours*float64(time.Hour))).AddDate(0, 0, -1).Format("2006-01-02") rows, err := db.conn.Query( - `SELECT COALESCE(first_seen, ''), $1::integer, COALESCE(hash, '') FROM transmissions t - WHERE from_pubkey=$2 AND payload_type=$3 AND first_seen>=$4 - AND EXISTS(SELECT 1 FROM advert_route_evidence e WHERE e.tx_id=t.id AND e.bit=$5) - ORDER BY id DESC LIMIT $6`, + db.nativeSQL(`SELECT COALESCE(first_seen, ''), CAST(?1 AS INTEGER), COALESCE(hash, '') FROM transmissions t + WHERE from_pubkey=?2 AND payload_type=?3 AND first_seen>=?4 + AND EXISTS(SELECT 1 FROM advert_route_evidence e WHERE e.tx_id=t.id AND e.bit=?5) + ORDER BY id DESC LIMIT ?6`, `SELECT COALESCE(first_seen, ''), `+db.parameter(1)+`::integer, COALESCE(hash, '') FROM transmissions t + WHERE from_pubkey=`+db.parameter(2)+` AND payload_type=`+db.parameter(3)+` AND first_seen>=`+db.parameter(4)+` + AND EXISTS(SELECT 1 FROM advert_route_evidence e WHERE e.tx_id=t.id AND e.bit=`+db.parameter(5)+`) + ORDER BY id DESC LIMIT `+db.parameter(6)), packetpath.AdvertFlood, pubkey, payloadTypeAdvert, floor, packetpath.AdvertFlood, rowCap) if err != nil { return 0, err diff --git a/cmd/server/advert_stats_test.go b/cmd/server/advert_stats_test.go index 7332cfd86..9ef3038fc 100644 --- a/cmd/server/advert_stats_test.go +++ b/cmd/server/advert_stats_test.go @@ -20,7 +20,7 @@ func insertAdvertTx(t *testing.T, db *DB, pubkey, hash string, rt int, ts time.T t.Fatalf("insert transmission: %v", err) } // Fixtures represent ingestor output: canonical routing alone is not evidence. - if _, err := db.conn.Exec(`CREATE TABLE IF NOT EXISTS advert_route_evidence(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit))`); err != nil { + if _, err := db.conn.Exec(testNativeSQL(`CREATE TABLE IF NOT EXISTS advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit))`, `CREATE TABLE IF NOT EXISTS advert_route_evidence(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit))`)); err != nil { t.Fatal(err) } if rt == RouteFlood { @@ -64,7 +64,7 @@ func TestCountFloodAdverts(t *testing.T) { // counting recent flood adverts only (zero-hop and out-of-window excluded). func TestNodeDetailIncludesFloodAdvertCount(t *testing.T) { srv, router := setupTestServer(t) - if _, err := srv.db.conn.Exec(`CREATE TABLE advert_route_evidence(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit))`); err != nil { + if _, err := srv.db.conn.Exec(testNativeSQL(`CREATE TABLE advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit))`, `CREATE TABLE advert_route_evidence(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit))`)); err != nil { t.Fatal(err) } now := time.Now().UTC() @@ -131,7 +131,7 @@ func TestNodeDetailFloodCountAgreesWithRouteEvidence(t *testing.T) { for _, firstRoute := range []int{RouteDirect, RouteFlood, RouteTransportFlood} { t.Run(fmt.Sprint(firstRoute), func(t *testing.T) { srv, router := setupTestServer(t) - if _, err := srv.db.conn.Exec(`CREATE TABLE IF NOT EXISTS advert_route_evidence(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, tx_id INTEGER, bit INTEGER, UNIQUE(tx_id,bit))`); err != nil { + if _, err := srv.db.conn.Exec(testNativeSQL(`CREATE TABLE IF NOT EXISTS advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT, tx_id INTEGER, bit INTEGER, UNIQUE(tx_id,bit))`, `CREATE TABLE IF NOT EXISTS advert_route_evidence(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, tx_id INTEGER, bit INTEGER, UNIQUE(tx_id,bit))`)); err != nil { t.Fatal(err) } const pubkey = "aabbccdd11223344" diff --git a/cmd/server/auth_fixture_test.go b/cmd/server/auth_fixture_test.go index bf0173e79..c2c6da393 100644 --- a/cmd/server/auth_fixture_test.go +++ b/cmd/server/auth_fixture_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/gorilla/mux" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/mailer" "github.com/meshcore-analyzer/pgutil/pgtest" "github.com/meshcore-analyzer/users" @@ -39,29 +40,32 @@ type client struct { } func newTestAuthService(t *testing.T, adminEmails ...string) (*authService, *mailer.Fake) { - a, fake, _ := newTestAuthServiceWithURL(t, pgtest.NewSchema(t), adminEmails...) + a, fake, _ := newTestAuthServiceWithURL(t, postgresTestDSN(t), adminEmails...) return a, fake } func newTestBackupAuthService(t *testing.T, adminEmails ...string) (*authService, *mailer.Fake) { - a, fake, _ := newTestAuthServiceWithURL(t, pgtest.NewDatabase(t), adminEmails...) + a, fake, _ := newTestAuthServiceWithURL(t, testDatabaseDSN(t), adminEmails...) return a, fake } func newTestAuthServiceWithURL(t *testing.T, ownerURL string, adminEmails ...string) (*authService, *mailer.Fake, string) { t.Helper() - owner, err := openFixtureSQL(ownerURL) - if err != nil { - t.Fatal(err) - } - defer owner.Close() - if err := users.Apply(owner); err != nil { - t.Fatal(err) - } - runtimeURL := pgtest.Writer(t, ownerURL) - u, _ := url.Parse(runtimeURL) - if _, err := owner.Exec(`REVOKE INSERT,UPDATE,DELETE ON corescope_schema,schema_version FROM "` + u.User.Username() + `"`); err != nil { - t.Fatal(err) + runtimeURL := ownerURL + if testBackend(t) == dbconfig.Postgres { + owner, err := openFixtureSQL(ownerURL) + if err != nil { + t.Fatal(err) + } + defer owner.Close() + if err := users.Apply(owner); err != nil { + t.Fatal(err) + } + runtimeURL = pgtest.Writer(t, ownerURL) + u, _ := url.Parse(runtimeURL) + if _, err := owner.Exec(`REVOKE INSERT,UPDATE,DELETE ON corescope_schema,schema_version FROM "` + u.User.Username() + `"`); err != nil { + t.Fatal(err) + } } set := &userMgmtSettings{ databaseURL: runtimeURL, adminEmails: map[string]bool{}, @@ -88,11 +92,11 @@ func newTestAuthServiceWithURL(t *testing.T, ownerURL string, adminEmails ...str // newAuthFixture builds a Server with auth on and only the auth routes. func newAuthFixture(t *testing.T, adminEmails ...string) *authFixture { - return newAuthFixtureWithURL(t, pgtest.NewSchema(t), adminEmails...) + return newAuthFixtureWithURL(t, postgresTestDSN(t), adminEmails...) } func newBackupAuthFixture(t *testing.T, adminEmails ...string) *authFixture { - return newAuthFixtureWithURL(t, pgtest.NewDatabase(t), adminEmails...) + return newAuthFixtureWithURL(t, testDatabaseDSN(t), adminEmails...) } func newAuthFixtureWithURL(t *testing.T, databaseURL string, adminEmails ...string) *authFixture { diff --git a/cmd/server/auth_flow_test.go b/cmd/server/auth_flow_test.go index 5154b313d..ef270c2ef 100644 --- a/cmd/server/auth_flow_test.go +++ b/cmd/server/auth_flow_test.go @@ -223,7 +223,7 @@ func TestAuthActivateHashChangedMidwayIs409(t *testing.T) { f := newAuthFixture(t) expectStatus(t, f.do("POST", "/api/auth/register", registerRequest{Email: "ned@example.org", DisplayName: "Ned", Password: pw}), 200) tok := f.lastToken(t) - f.execDB(t, `CREATE FUNCTION race() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN UPDATE users SET password_hash='replaced-by-a-re-register' WHERE id=NEW.user_id; RETURN NEW; END $$; CREATE TRIGGER race AFTER UPDATE OF used_at ON tokens FOR EACH ROW EXECUTE FUNCTION race()`) + f.execDB(t, testNativeSQL(`CREATE TRIGGER race AFTER UPDATE OF used_at ON tokens BEGIN UPDATE users SET password_hash='replaced-by-a-re-register' WHERE id=NEW.user_id; END`, `CREATE FUNCTION race() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN UPDATE users SET password_hash='replaced-by-a-re-register' WHERE id=NEW.user_id; RETURN NEW; END $$; CREATE TRIGGER race AFTER UPDATE OF used_at ON tokens FOR EACH ROW EXECUTE FUNCTION race()`)) w := f.do("POST", "/api/auth/activate", activateRequest{Token: tok, Password: pw}) expectStatus(t, w, 409) if !strings.Contains(w.Body.String(), "account changed, try again") { diff --git a/cmd/server/auth_login_audit_test.go b/cmd/server/auth_login_audit_test.go index 668c2252a..ff2dbf24a 100644 --- a/cmd/server/auth_login_audit_test.go +++ b/cmd/server/auth_login_audit_test.go @@ -143,7 +143,7 @@ func TestLoginAnswersBeforeTheAuditWrite(t *testing.T) { t.Fatal(err) } defer conn.Close() - if _, err := conn.ExecContext(context.Background(), "BEGIN; LOCK TABLE audit_log IN SHARE MODE"); err != nil { + if _, err := conn.ExecContext(context.Background(), testNativeSQL("BEGIN IMMEDIATE", "BEGIN; LOCK TABLE audit_log IN SHARE MODE")); err != nil { t.Fatal(err) } diff --git a/cmd/server/auth_mail_test.go b/cmd/server/auth_mail_test.go index 3ef6c2713..23677dba0 100644 --- a/cmd/server/auth_mail_test.go +++ b/cmd/server/auth_mail_test.go @@ -67,7 +67,7 @@ func TestLogStartupRedactsDatabaseURL(t *testing.T) { log.SetOutput(&buf) defer log.SetOutput(prev) a.logStartup() - if !strings.Contains(buf.String(), "PostgreSQL") || strings.Contains(buf.String(), "secret") || strings.Contains(buf.String(), "hidden") { + if !strings.Contains(buf.String(), string(a.st.Backend())) || strings.Contains(buf.String(), "secret") || strings.Contains(buf.String(), "hidden") { t.Fatalf("unsafe startup log: %q", buf.String()) } } diff --git a/cmd/server/auth_service.go b/cmd/server/auth_service.go index 32c4600da..cd5e04549 100644 --- a/cmd/server/auth_service.go +++ b/cmd/server/auth_service.go @@ -1,9 +1,11 @@ package main import ( + "github.com/meshcore-analyzer/dbconfig" "log" "os" "path/filepath" + "strings" "sync" "sync/atomic" "time" @@ -60,7 +62,7 @@ func newAuthService(set *userMgmtSettings, st *users.Store, m mailer.Mailer) *au // initUserManagement builds s.auth when the feature is on. Call it after // NewServer and before RegisterRoutes. measurementDatabaseURL is passed to // users.Open as a forbidden database, so accounts cannot share telemetry storage. -func (s *Server) initUserManagement(measurementDatabaseURL string) error { +func (s *Server) initUserManagement(measurementTarget string) error { if !s.cfg.UserManagementEnabled() { return nil } @@ -68,7 +70,21 @@ func (s *Server) initUserManagement(measurementDatabaseURL string) error { if err != nil { return err } - st, err := users.Open(set.databaseURL, measurementDatabaseURL) + storage := dbconfig.Storage{Backend: dbconfig.SQLite, DBPath: measurementTarget, UsersDBPath: set.databaseURL, StateDir: runtimeStateDir} + if strings.HasPrefix(measurementTarget, "postgres://") || strings.HasPrefix(measurementTarget, "postgresql://") { + storage = dbconfig.Storage{Backend: dbconfig.Postgres, ReaderDatabaseURL: measurementTarget, UsersDatabaseURL: set.databaseURL, StateDir: runtimeStateDir} + } + return s.initUserManagementStorage(storage) +} +func (s *Server) initUserManagementStorage(storage dbconfig.Storage) error { + if !s.cfg.UserManagementEnabled() { + return nil + } + set, err := resolveUserManagement(s.cfg.UserManagement, storage.StateDir, os.Getenv) + if err != nil { + return err + } + st, err := users.OpenStorage(storage) if err != nil { return err } @@ -114,8 +130,8 @@ func (s *Server) closeUserManagement() { } func (a *authService) logStartup() { - log.Printf("[users] user management enabled: PostgreSQL, %d config admin(s), webhook=%v", - len(a.set.adminEmails), a.set.webhookSecret != "") + log.Printf("[users] user management enabled: %s, %d config admin(s), webhook=%v", + a.st.Backend(), len(a.set.adminEmails), a.set.webhookSecret != "") admins, err := a.st.List(users.ListFilter{Role: users.RoleAdmin}) if err != nil { log.Printf("[users] list admins: %v", err) diff --git a/cmd/server/backup.go b/cmd/server/backup.go index d79c35190..a362118da 100644 --- a/cmd/server/backup.go +++ b/cmd/server/backup.go @@ -9,10 +9,12 @@ import ( "path/filepath" "time" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/sqliteutil" ) -// handleBackup stages a consistent PostgreSQL custom-format pg_dump snapshot. +// handleBackup stages a consistent native SQLite file or PostgreSQL archive. // pgutil passes credentials through the child environment, bounds its lifetime, // removes partial output, and returns redacted errors. func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request) { @@ -41,9 +43,18 @@ func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request) { } }() - snapshotPath := filepath.Join(tmpDir, fmt.Sprintf("corescope-backup-%d.dump", ts)) + extension := ".dump" + if s.db.Backend() == dbconfig.SQLite { + extension = ".db" + } + snapshotPath := filepath.Join(tmpDir, fmt.Sprintf("corescope-backup-%d%s", ts, extension)) - if err := pgutil.Dump(r.Context(), s.db.path, snapshotPath); err != nil { + if s.db.Backend() == dbconfig.SQLite { + err = sqliteutil.Snapshot(r.Context(), s.db.path, snapshotPath) + } else { + err = pgutil.Dump(r.Context(), s.db.path, snapshotPath) + } + if err != nil { writeError(w, http.StatusInternalServerError, "snapshot failed: "+err.Error()) return } @@ -60,7 +71,7 @@ func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Length", fmt.Sprintf("%d", stat.Size())) } w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"corescope-backup-%d.dump\"", ts)) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"corescope-backup-%d%s\"", ts, extension)) w.Header().Set("X-Content-Type-Options", "nosniff") w.Header().Set("Cache-Control", "no-store") w.WriteHeader(http.StatusOK) diff --git a/cmd/server/backup_test.go b/cmd/server/backup_test.go index b988a5809..5a565aea1 100644 --- a/cmd/server/backup_test.go +++ b/cmd/server/backup_test.go @@ -7,12 +7,10 @@ import ( "path/filepath" "strings" "testing" - - "github.com/meshcore-analyzer/pgutil/pgtest" ) // postgresMagic is the signature of a PostgreSQL custom-format archive. -const postgresMagic = "PGDMP" +var postgresMagic = testNativeSQL("SQLite format 3\x00", "PGDMP") func TestBackupRequiresAPIKey(t *testing.T) { _, router := setupTestServerWithAPIKey(t, "test-secret-key-strong-enough") @@ -28,7 +26,7 @@ func TestBackupRequiresAPIKey(t *testing.T) { func TestBackupRestoresNativePostgresSnapshot(t *testing.T) { const apiKey = "test-secret-key-strong-enough" srv, router := setupTestServerWithAPIKey(t, apiKey) - srv.db = setupTestDBAtURL(t, pgtest.NewDatabase(t)) + srv.db = setupTestDBAtURL(t, testDatabaseDSN(t)) seedTestData(t, srv.db) writerURL := srv.db.path reader, err := openFixtureReader(t, writerURL) @@ -53,7 +51,7 @@ func TestBackupRestoresNativePostgresSnapshot(t *testing.T) { } cd := w.Header().Get("Content-Disposition") - if !strings.HasPrefix(cd, "attachment;") || !strings.Contains(cd, "filename=\"corescope-backup-") || !strings.HasSuffix(cd, ".dump\"") { + if !strings.HasPrefix(cd, "attachment;") || !strings.Contains(cd, "filename=\"corescope-backup-") || !strings.HasSuffix(cd, testNativeSQL(".db\"", ".dump\"")) { t.Errorf("expected Content-Disposition attachment with corescope-backup-.dump filename, got %q", cd) } diff --git a/cmd/server/bounded_load_test.go b/cmd/server/bounded_load_test.go index 0d8963879..639405547 100644 --- a/cmd/server/bounded_load_test.go +++ b/cmd/server/bounded_load_test.go @@ -148,9 +148,9 @@ func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string { t.Fatalf("setup: %v\nSQL: %s", err, s) } } - execOrFail(`CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`) - execOrFail(`CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp BIGINT, raw_hex TEXT)`) - execOrFail(`CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + execOrFail(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`)) + execOrFail(testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp BIGINT, raw_hex TEXT)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp BIGINT, raw_hex TEXT)`)) + execOrFail(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION)`) execOrFail(`CREATE TABLE schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) @@ -302,19 +302,29 @@ func createTestDBAt(tb testing.TB, dbPath string, numTx int) { tb.Fatalf("test DB setup exec failed: %v\nSQL: %s", err, sql) } } - execOrFail(`CREATE TABLE IF NOT EXISTS transmissions ( + execOrFail(testNativeSQL(`CREATE TABLE IF NOT EXISTS transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, + payload_version INTEGER, decoded_json TEXT + )`, `CREATE TABLE IF NOT EXISTS transmissions ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) - execOrFail(`CREATE TABLE IF NOT EXISTS observations ( + )`)) + execOrFail(testNativeSQL(`CREATE TABLE IF NOT EXISTS observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, raw_hex TEXT + )`, `CREATE TABLE IF NOT EXISTS observations ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT - )`) - execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + )`)) + execOrFail(testNativeSQL(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) execOrFail(`CREATE TABLE IF NOT EXISTS nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION @@ -348,15 +358,21 @@ func createTestDBWithObs(tb testing.TB, dbPath string, numTx int) { tb.Fatalf("test DB setup exec failed: %v\nSQL: %s", err, sqlStr) } } - execOrFail(`CREATE TABLE IF NOT EXISTS transmissions ( + execOrFail(testNativeSQL(`CREATE TABLE IF NOT EXISTS transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT + )`, `CREATE TABLE IF NOT EXISTS transmissions ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) - execOrFail(`CREATE TABLE IF NOT EXISTS observations ( + )`)) + execOrFail(testNativeSQL(`CREATE TABLE IF NOT EXISTS observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT + )`, `CREATE TABLE IF NOT EXISTS observations ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT - )`) - execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + )`)) + execOrFail(testNativeSQL(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) execOrFail(`CREATE TABLE IF NOT EXISTS nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION diff --git a/cmd/server/cache_refresh.go b/cmd/server/cache_refresh.go new file mode 100644 index 000000000..f8d410ddb --- /dev/null +++ b/cmd/server/cache_refresh.go @@ -0,0 +1,50 @@ +package main + +import "sync" + +// cacheRefresher runs at most one background refresh per key. The node, +// region-observer and area-node caches serve their stale value while it runs, +// so callers holding PacketStore.mu never wait on SQL (#2146). The zero value +// is ready to use. +type cacheRefresher struct { + mu sync.Mutex + inFlight map[string]bool +} + +// start runs fn in a new goroutine unless a refresh for key is already +// running. +func (r *cacheRefresher) start(key string, fn func()) { + r.mu.Lock() + if r.inFlight[key] { + r.mu.Unlock() + return + } + if r.inFlight == nil { + r.inFlight = make(map[string]bool) + } + r.inFlight[key] = true + r.mu.Unlock() + + go func() { + defer func() { + r.mu.Lock() + delete(r.inFlight, key) + r.mu.Unlock() + }() + fn() + }() +} + +func (r *cacheRefresher) running(key string) bool { + r.mu.Lock() + defer r.mu.Unlock() + return r.inFlight[key] +} + +// beforeCacheLoad calls the test hook, if any, right before a cache runs its +// SQL. +func (s *PacketStore) beforeCacheLoad(kind string) { + if s.cacheLoadHook != nil { + s.cacheLoadHook(kind) + } +} diff --git a/cmd/server/cache_swr_test.go b/cmd/server/cache_swr_test.go new file mode 100644 index 000000000..828098f9d --- /dev/null +++ b/cmd/server/cache_swr_test.go @@ -0,0 +1,240 @@ +package main + +import ( + "sync" + "sync/atomic" + "testing" + "time" +) + +// #2146: the node, region-observer and area-node caches are read by callers +// that hold s.mu. A cache past its TTL must be served as-is while a single +// background refresh runs the SQL, so no caller holding s.mu waits on the +// 4-connection read pool. + +// blockingLoadHook returns a cacheLoadHook that counts calls per kind and +// blocks every call for kind until release is closed. +func blockingLoadHook(kind string) (hook func(string), calls *int64, entered chan struct{}, release chan struct{}) { + calls = new(int64) + entered = make(chan struct{}, 64) + release = make(chan struct{}) + hook = func(k string) { + if k != kind { + return + } + atomic.AddInt64(calls, 1) + entered <- struct{}{} + <-release + } + return hook, calls, entered, release +} + +// returnsWithin runs fn and fails the test if it does not return within d. +func returnsWithin(t *testing.T, d time.Duration, what string, fn func()) { + t.Helper() + done := make(chan struct{}) + go func() { fn(); close(done) }() + select { + case <-done: + case <-time.After(d): + t.Fatalf("%s blocked for more than %v", what, d) + } +} + +func waitUntil(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(5 * time.Millisecond) + } +} + +func TestNodeCacheServesStaleWhileRefreshing(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + s := &PacketStore{db: db} + + stale := []nodeInfo{{PublicKey: "stale0001", Name: "Stale"}} + s.nodeCache, s.nodePM = stale, buildPrefixMap(stale) + s.nodeCacheTime = time.Now().Add(-time.Hour) + + hook, calls, entered, release := blockingLoadHook("nodes") + s.cacheLoadHook = hook + + var got []nodeInfo + returnsWithin(t, time.Second, "stale getCachedNodesAndPM", func() { + got, _ = s.getCachedNodesAndPM() + }) + if len(got) != 1 || got[0].PublicKey != "stale0001" { + t.Fatalf("want the stale cache served, got %v", got) + } + <-entered // the background refresh reached the SQL + + // Further stale callers while the refresh is in flight: served from the + // stale cache, and they do not start a second refresh. + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { defer wg.Done(); s.getCachedNodesAndPM() }() + } + returnsWithin(t, time.Second, "concurrent stale callers", wg.Wait) + if n := atomic.LoadInt64(calls); n != 1 { + t.Fatalf("want exactly 1 background refresh, got %d", n) + } + + close(release) + waitUntil(t, "refreshed node cache", func() bool { + nodes, _ := s.getCachedNodesAndPM() + return len(nodes) > 0 && nodes[0].PublicKey != "stale0001" + }) +} + +func TestNodeCacheInvalidatedBuildsInline(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + s := &PacketStore{db: db} + + stale := []nodeInfo{{PublicKey: "stale0001", Name: "Stale"}} + s.nodeCache, s.nodePM = stale, buildPrefixMap(stale) + s.nodeCacheTime = time.Now() + s.InvalidateNodeCache() + + var calls int64 + s.cacheLoadHook = func(k string) { + if k == "nodes" { + atomic.AddInt64(&calls, 1) + } + } + nodes, _ := s.getCachedNodesAndPM() + if atomic.LoadInt64(&calls) != 1 { + t.Fatalf("an invalidated cache must rebuild inline, hook calls = %d", calls) + } + for _, n := range nodes { + if n.PublicKey == "stale0001" { + t.Fatal("an invalidated cache must not serve the old node list") + } + } + if len(nodes) == 0 { + t.Fatal("want the seeded nodes after an inline rebuild") + } +} + +// A refresh that started before InvalidateNodeCache read the DB before the +// change the invalidation announces, so it must not mark the cache fresh. +func TestNodeCacheRefreshDoesNotOverrideInvalidation(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + s := &PacketStore{db: db} + + stale := []nodeInfo{{PublicKey: "stale0001", Name: "Stale"}} + s.nodeCache, s.nodePM = stale, buildPrefixMap(stale) + s.nodeCacheTime = time.Now().Add(-time.Hour) + + hook, _, entered, release := blockingLoadHook("nodes") + s.cacheLoadHook = hook + s.getCachedNodesAndPM() + <-entered + + s.InvalidateNodeCache() + close(release) + waitUntil(t, "background refresh to finish", func() bool { return !s.cacheRefresh.running("nodes") }) + + s.cacheMu.Lock() + builtAt := s.nodeCacheTime + s.cacheMu.Unlock() + if !builtAt.IsZero() { + t.Fatal("a refresh that began before InvalidateNodeCache marked the cache fresh") + } +} + +func TestRegionObsServesStaleWhileRefreshing(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + s := &PacketStore{db: db} + + s.regionObsCache = map[string]map[string]bool{"SJC": {"stale-obs": true}} + s.regionObsCacheTime = time.Now().Add(-time.Hour) + + hook, calls, entered, release := blockingLoadHook("regionObs") + s.cacheLoadHook = hook + + var got map[string]bool + returnsWithin(t, time.Second, "stale resolveRegionObservers", func() { + got = s.resolveRegionObservers("SJC") + }) + if !got["stale-obs"] { + t.Fatalf("want the stale region set served, got %v", got) + } + <-entered + returnsWithin(t, time.Second, "second stale caller", func() { s.resolveRegionObservers("SJC") }) + if n := atomic.LoadInt64(calls); n != 1 { + t.Fatalf("want exactly 1 background refresh, got %d", n) + } + + close(release) + waitUntil(t, "refreshed region cache", func() bool { + m := s.resolveRegionObservers("SJC") + return len(m) > 0 && !m["stale-obs"] + }) +} + +// A cold region still queries inline, but it must not hold regionObsMu while +// it does: other regions are served from the cache meanwhile. +func TestRegionObsColdMissDoesNotBlockCachedRegion(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + s := &PacketStore{db: db} + + s.regionObsCache = map[string]map[string]bool{"SJC": {"obs-sjc": true}} + s.regionObsCacheTime = time.Now() + + hook, _, entered, release := blockingLoadHook("regionObs") + s.cacheLoadHook = hook + go s.resolveRegionObservers("LAX") + <-entered + + returnsWithin(t, time.Second, "cached region during a cold miss", func() { + if !s.resolveRegionObservers("SJC")["obs-sjc"] { + t.Error("want the cached SJC set") + } + }) + close(release) +} + +func TestAreaNodesServesStaleWhileRefreshing(t *testing.T) { + db := setupTestDBv2(t) + mustExecDB(t, db, `INSERT INTO nodes (public_key, lat, lon) VALUES ('pk-fresh', 50.85, 4.35)`) + cfg := &Config{Areas: map[string]AreaEntry{ + "BEL": {Label: "Belgium", Polygon: [][2]float64{{50.0, 2.5}, {51.5, 2.5}, {51.5, 6.4}, {50.0, 6.4}}}, + }} + s := newTestStoreWithDB(t, db, cfg) + s.areaNodeCache["BEL"] = map[string]bool{"pk-stale": true} + s.areaNodeCacheTimes["BEL"] = time.Now().Add(-time.Hour) + + hook, calls, entered, release := blockingLoadHook("areaNodes") + s.cacheLoadHook = hook + + var got map[string]bool + returnsWithin(t, time.Second, "stale resolveAreaNodes", func() { + got = s.resolveAreaNodes("BEL") + }) + if !got["pk-stale"] { + t.Fatalf("want the stale area set served, got %v", got) + } + <-entered + returnsWithin(t, time.Second, "second stale caller", func() { s.resolveAreaNodes("BEL") }) + if n := atomic.LoadInt64(calls); n != 1 { + t.Fatalf("want exactly 1 background refresh, got %d", n) + } + + close(release) + waitUntil(t, "refreshed area cache", func() bool { return s.resolveAreaNodes("BEL")["pk-fresh"] }) +} diff --git a/cmd/server/chunked_load.go b/cmd/server/chunked_load.go index f13ab3c4f..079e0a305 100644 --- a/cmd/server/chunked_load.go +++ b/cmd/server/chunked_load.go @@ -414,10 +414,13 @@ func (s *PacketStore) LoadChunked(chunkSize int) error { var chunkSQL string if s.db.isV3 { - chunkSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, + chunkSQL = s.db.nativeSQL(`SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, + t.payload_type, t.payload_version, t.decoded_json, + o.id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, + o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')`, `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, t.payload_type, t.payload_version, t.decoded_json, o.id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, - o.snr, o.rssi, o.score, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')` + obsRawHexCol + rpCol + scopeNameCol + ` + o.snr, o.rssi, o.score, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`) + obsRawHexCol + rpCol + scopeNameCol + ` FROM (SELECT * FROM transmissions t2 ` + whereClause + ` ORDER BY t2.id ASC LIMIT ` + fmt.Sprintf("%d", chunkSize) + `) AS t LEFT JOIN observations o ON o.transmission_id = t.id LEFT JOIN observers obs ON obs.rowid = o.observer_idx diff --git a/cmd/server/chunked_load_id_zero_test.go b/cmd/server/chunked_load_id_zero_test.go index 002c9f11a..47b7c6ec2 100644 --- a/cmd/server/chunked_load_id_zero_test.go +++ b/cmd/server/chunked_load_id_zero_test.go @@ -34,19 +34,29 @@ func createTestDBWithIDZero(tb testing.TB, dbPath string, extraTx int) { defer conn.Close() stmts := []string{ - `CREATE TABLE IF NOT EXISTS transmissions ( + testNativeSQL(`CREATE TABLE IF NOT EXISTS transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, + payload_version INTEGER, decoded_json TEXT + )`, `CREATE TABLE IF NOT EXISTS transmissions ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`, - `CREATE TABLE IF NOT EXISTS observations ( + )`), + testNativeSQL(`CREATE TABLE IF NOT EXISTS observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, raw_hex TEXT + )`, `CREATE TABLE IF NOT EXISTS observations ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT - )`, - `CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, + )`), + testNativeSQL(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`), `CREATE TABLE IF NOT EXISTS nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION diff --git a/cmd/server/chunked_load_oldest_test.go b/cmd/server/chunked_load_oldest_test.go index fe235223c..f45a08a1d 100644 --- a/cmd/server/chunked_load_oldest_test.go +++ b/cmd/server/chunked_load_oldest_test.go @@ -36,19 +36,29 @@ func createTestDBReverseTime(tb testing.TB, dbPath string, numTx int) { defer conn.Close() stmts := []string{ - `CREATE TABLE IF NOT EXISTS transmissions ( + testNativeSQL(`CREATE TABLE IF NOT EXISTS transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, + payload_version INTEGER, decoded_json TEXT + )`, `CREATE TABLE IF NOT EXISTS transmissions ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`, - `CREATE TABLE IF NOT EXISTS observations ( + )`), + testNativeSQL(`CREATE TABLE IF NOT EXISTS observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, raw_hex TEXT + )`, `CREATE TABLE IF NOT EXISTS observations ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT - )`, - `CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, + )`), + testNativeSQL(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`), `CREATE TABLE IF NOT EXISTS nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION diff --git a/cmd/server/config.go b/cmd/server/config.go index d0de5abf2..8f08e1964 100644 --- a/cmd/server/config.go +++ b/cmd/server/config.go @@ -39,7 +39,7 @@ type ListLimitsConfig struct { type Config struct { Port int `json:"port"` APIKey string `json:"apiKey"` - DBPath string `json:"dbPath,omitempty"` // rejected legacy SQLite setting + DBPath string `json:"dbPath,omitempty"` // legacy SQLite path; an installed selection is authoritative DatabaseURL string `json:"databaseURL"` StateDir string `json:"stateDir"` ListLimits *ListLimitsConfig `json:"listLimits"` @@ -495,10 +495,13 @@ func LoadConfig(baseDirs ...string) (*Config, error) { for _, p := range paths { data, err := os.ReadFile(p) if err != nil { - continue + if os.IsNotExist(err) { + continue + } + return nil, fmt.Errorf("cannot read %s; refusing configuration fallback", filepath.Base(p)) } if err := json.Unmarshal(data, cfg); err != nil { - continue + return nil, fmt.Errorf("%s is malformed or has invalid field types; refusing configuration fallback", filepath.Base(p)) } cfg.NormalizeTimestampConfig() cfg.migrateDeprecatedConfig() diff --git a/cmd/server/config_test.go b/cmd/server/config_test.go index aa0727e1b..b241e445d 100644 --- a/cmd/server/config_test.go +++ b/cmd/server/config_test.go @@ -104,15 +104,47 @@ func TestLoadConfigNoFiles(t *testing.T) { func TestLoadConfigInvalidJSON(t *testing.T) { dir := t.TempDir() - os.WriteFile(filepath.Join(dir, "config.json"), []byte("{invalid"), 0644) - - cfg, err := LoadConfig(dir) - if err != nil { + if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte("{invalid"), 0600); err != nil { t.Fatal(err) } - // Should return defaults when JSON is invalid - if cfg.Port != 3000 { - t.Errorf("expected default port 3000, got %d", cfg.Port) + if cfg, err := LoadConfig(dir); err == nil || cfg != nil { + t.Fatal("malformed primary config fell back to defaults") + } +} + +func TestLoadConfigNeverUsesFallbackAfterExistingConfigError(t *testing.T) { + for _, kind := range []string{"invalid JSON", "invalid field type", "unreadable file"} { + t.Run(kind, func(t *testing.T) { + dir := t.TempDir() + if err := os.Mkdir(filepath.Join(dir, "data"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "data", "config.json"), []byte(`{"stateDir":"unintended-installation","dbPath":"other.db"}`), 0600); err != nil { + t.Fatal(err) + } + primary := filepath.Join(dir, "config.json") + switch kind { + case "invalid JSON": + if err := os.WriteFile(primary, []byte(`{"stateDir":`), 0600); err != nil { + t.Fatal(err) + } + case "invalid field type": + if err := os.WriteFile(primary, []byte(`{"databaseURL":"postgresql://user:private-password@host/database","port":"invalid"}`), 0600); err != nil { + t.Fatal(err) + } + case "unreadable file": + if err := os.Mkdir(primary, 0700); err != nil { + t.Fatal(err) + } + } + cfg, err := LoadConfig(dir) + if err == nil || cfg != nil { + t.Fatal("existing config error selected fallback storage") + } + if strings.Contains(err.Error(), "private-password") { + t.Fatal("config error exposed credentials") + } + }) } } diff --git a/cmd/server/coverage_test.go b/cmd/server/coverage_test.go index 99be2e2b9..83a3a2d8b 100644 --- a/cmd/server/coverage_test.go +++ b/cmd/server/coverage_test.go @@ -27,7 +27,35 @@ func setupTestDBv2(t *testing.T) *DB { } // Force single connection so all goroutines share the same in-memory DB conn.SetMaxOpenConns(1) - schema := ` + schema := testNativeSQL(` + CREATE TABLE nodes ( + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, + lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, + battery_mv INTEGER, temperature_c DOUBLE PRECISION, foreign_advert INTEGER DEFAULT 0 + ); + CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT, + packet_count INTEGER DEFAULT 0, model TEXT, firmware TEXT, + client_version TEXT, radio TEXT, battery_mv INTEGER, uptime_secs INTEGER, noise_floor DOUBLE PRECISION, + inactive INTEGER DEFAULT 0 + ); + CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT NOT NULL, + hash TEXT NOT NULL UNIQUE, first_seen TEXT NOT NULL, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, + decoded_json TEXT, channel_hash TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL, created_at TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ','now')) + ); + CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER NOT NULL REFERENCES transmissions(id), + observer_id TEXT, observer_name TEXT, direction TEXT, + snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp INTEGER NOT NULL, raw_hex TEXT + ); + CREATE TRIGGER test_from_pubkey_advert AFTER INSERT ON transmissions + WHEN NEW.from_pubkey IS NULL AND NEW.payload_type=4 AND + CASE WHEN json_valid(NEW.decoded_json) THEN json_type(NEW.decoded_json)='object' ELSE 0 END + BEGIN UPDATE transmissions SET from_pubkey=NULLIF(json_extract(NEW.decoded_json,'$.pubKey'),'') WHERE id=NEW.id; END; + CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey); + `, ` CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, @@ -61,11 +89,11 @@ func setupTestDBv2(t *testing.T) *DB { CREATE TRIGGER test_from_pubkey_advert BEFORE INSERT ON transmissions FOR EACH ROW EXECUTE FUNCTION test_from_pubkey_advert(); CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey); - ` + `) if _, err := conn.Exec(schema); err != nil { t.Fatal(err) } - return &DB{conn: conn, isV3: false} + return &DB{backend: testBackendValue(), conn: conn, isV3: false} } func seedV2Data(t *testing.T, db *DB) { @@ -1371,7 +1399,7 @@ func TestBuildTransmissionWhereRFC3339(t *testing.T) { // PR #1187 r2: RFC3339 since/until MUST use observations.timestamp // subquery so re-observed packets (older first_seen but recent // observation) are still included. Anything else breaks semantics. - if !strings.Contains(where[0], "observations") || !strings.Contains(where[0], "timestamp >= $1") { + if !strings.Contains(where[0], "observations") || !strings.Contains(where[0], "timestamp >= "+db.parameter(1)+"") { t.Errorf("expected observations.timestamp subquery for RFC3339 since, got %q", where[0]) } }) @@ -1385,7 +1413,7 @@ func TestBuildTransmissionWhereRFC3339(t *testing.T) { if len(args) != 1 { t.Errorf("expected 1 arg, got %d", len(args)) } - if !strings.Contains(where[0], "observations") || !strings.Contains(where[0], "timestamp <= $1") { + if !strings.Contains(where[0], "observations") || !strings.Contains(where[0], "timestamp <= "+db.parameter(1)+"") { t.Errorf("expected observations.timestamp subquery for RFC3339 until, got %q", where[0]) } }) @@ -1871,7 +1899,7 @@ func TestEnrichObs(t *testing.T) { t.Skip("no observations loaded") } - enriched := store.enrichObs(obs) + enriched := store.enrichObsWithTx(obs, store.byTxID[obs.TransmissionID]) if enriched["observer_id"] == nil { t.Error("expected observer_id") } @@ -4234,7 +4262,7 @@ func TestGetDBSizeStatsMemory(t *testing.T) { seedTestData(t, db) stats := db.GetDBSizeStats() - if stats["dbSizeMB"].(float64) <= 0 { + if stats["dbSizeMB"].(float64) < 0 || (testBackendValue() == "postgres" && stats["dbSizeMB"].(float64) == 0) { t.Errorf("expected positive PostgreSQL database size, got %v", stats["dbSizeMB"]) } } @@ -4556,7 +4584,7 @@ func TestBuildTransmissionWhereMultiObserver(t *testing.T) { t.Fatalf("expected 1 WHERE clause, got %d", len(where)) } clause := where[0] - if !strings.Contains(clause, "IN ($1,$2)") { + if !strings.Contains(clause, "IN ("+db.parameter(1)+","+db.parameter(2)+")") { t.Errorf("expected IN ($1,$2) in clause, got: %s", clause) } if len(args) != 2 { @@ -4584,7 +4612,7 @@ func TestBuildTransmissionWhereMultiObserver(t *testing.T) { if len(where) != 1 { t.Fatalf("expected 1 WHERE clause, got %d", len(where)) } - if !strings.Contains(where[0], "IN ($1)") { + if !strings.Contains(where[0], "IN ("+db.parameter(1)+")") { t.Errorf("expected IN ($1) for single observer, got: %s", where[0]) } if len(args) != 1 || args[0] != "obs1" { diff --git a/cmd/server/db.go b/cmd/server/db.go index 4dcaa54af..6c7fadc56 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -13,10 +13,9 @@ import ( "sync/atomic" "time" - "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/geofilter" "github.com/meshcore-analyzer/packetpath" - "github.com/meshcore-analyzer/pgutil" "golang.org/x/sync/singleflight" ) @@ -35,8 +34,9 @@ const routeTypeTransportSQL = "route_type IN (0, 3)" // per MeshCore protocol (#1838). const routeTypeNonTransportSQL = "route_type IN (1, 2)" -// DB wraps a read-only connection to the CoreScope PostgreSQL database. +// DB wraps a read-only connection to the selected telemetry database. type DB struct { + backend dbconfig.Backend advertEvidenceTable atomic.Bool advertEvidenceReadHook func() // test-only: immediately before a bulk mask query conn *sql.DB @@ -135,38 +135,14 @@ type channelMessagesCacheEntry struct { exp time.Time } -// OpenDB opens a restricted PostgreSQL reader. Schema creation and writes are -// exclusively bootstrap/ingestor responsibilities. The session flag is defense -// in depth; effective grants are checked so changing it cannot enable writes. -func OpenDB(databaseURL string) (*DB, error) { - conn, err := pgutil.Open(databaseURL, true) - if err != nil { - return nil, err - } - conn.SetMaxOpenConns(4) - if err := pgutil.AssertReadOnly(conn); err != nil { - conn.Close() - return nil, err - } - d := &DB{conn: conn, path: databaseURL} - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - sc, err := conn.Conn(ctx) - if err != nil { - conn.Close() - return nil, fmt.Errorf("schema detection: acquire connection: %w", err) +// OpenDB preserves unambiguous legacy path/URL callers. Runtime main uses +// explicit Storage from the shared selection record. +func OpenDB(target string) (*DB, error) { + storage := dbconfig.Storage{Backend: dbconfig.SQLite, DBPath: target} + if strings.HasPrefix(target, "postgres://") || strings.HasPrefix(target, "postgresql://") { + storage = dbconfig.Storage{Backend: dbconfig.Postgres, ReaderDatabaseURL: target} } - derr := d.detectSchema(ctx, sc) - _ = sc.Close() - if derr != nil { - conn.Close() - return nil, fmt.Errorf("schema detection failed: %w", derr) - } - if err := d.prepareStatements(); err != nil { - conn.Close() - return nil, fmt.Errorf("prepare statements: %w", err) - } - return d, nil + return OpenStorage(storage) } // stmtQueryRow returns a QueryRow-like helper that uses the prepared statement @@ -194,15 +170,15 @@ func (db *DB) prepareStatements() error { db.stmtCountTransmissions = prepare("SELECT COUNT(*) FROM transmissions") db.stmtCountObservations = prepare("SELECT COUNT(*) FROM observations") - db.stmtCountNodesActive = prepare("SELECT COUNT(*) FROM nodes WHERE last_seen > $1") + db.stmtCountNodesActive = prepare("SELECT COUNT(*) FROM nodes WHERE last_seen > " + db.parameter(1)) db.stmtCountNodesAll = prepare("SELECT COUNT(*) FROM nodes") db.stmtCountObservers = prepare("SELECT COUNT(*) FROM observers WHERE inactive IS NULL OR inactive = 0") - db.stmtCountObsLastHour = prepare("SELECT COUNT(*) FROM observations WHERE timestamp > $1") - db.stmtCountObsLastDay = prepare("SELECT COUNT(*) FROM observations WHERE timestamp > $1") - db.stmtNodeLookup = prepare("SELECT public_key FROM nodes WHERE public_key = $1 OR name = $2 LIMIT 1") - db.stmtTxByHash = prepare("SELECT id FROM transmissions WHERE hash = $1") - db.stmtCountNodesByRole = prepare("SELECT COUNT(*) FROM nodes WHERE role = $1 AND last_seen > $2") - db.stmtCountNodesByRoleAll = prepare("SELECT COUNT(*) FROM nodes WHERE role = $1") + db.stmtCountObsLastHour = prepare("SELECT COUNT(*) FROM observations WHERE timestamp > " + db.parameter(1)) + db.stmtCountObsLastDay = prepare("SELECT COUNT(*) FROM observations WHERE timestamp > " + db.parameter(1)) + db.stmtNodeLookup = prepare("SELECT public_key FROM nodes WHERE public_key = " + db.parameter(1) + " OR name = " + db.parameter(2) + " LIMIT 1") + db.stmtTxByHash = prepare("SELECT id FROM transmissions WHERE hash = " + db.parameter(1)) + db.stmtCountNodesByRole = prepare("SELECT COUNT(*) FROM nodes WHERE role = " + db.parameter(1) + " AND last_seen > " + db.parameter(2)) + db.stmtCountNodesByRoleAll = prepare("SELECT COUNT(*) FROM nodes WHERE role = " + db.parameter(1)) db.stmtMaxTxID = prepare("SELECT COALESCE(MAX(id), 0) FROM transmissions") db.stmtMaxObsID = prepare("SELECT COALESCE(MAX(id), 0) FROM observations") return err @@ -235,7 +211,7 @@ func (db *DB) Close() error { // restarted (#1901). Detection is now all-or-nothing — on any probe error the // caller aborts startup so the supervisor can retry. func (db *DB) detectSchema(ctx context.Context, q rowQuerier) error { - obs, err := schemaColumns(ctx, q, "observations") + obs, err := schemaColumns(ctx, q, "observations", db.Backend()) if err != nil { return fmt.Errorf("probe observations: %w", err) } @@ -243,14 +219,14 @@ func (db *DB) detectSchema(ctx context.Context, q rowQuerier) error { db.hasResolvedPath = obs["resolved_path"] db.hasObsRawHex = obs["raw_hex"] - tx, err := schemaColumns(ctx, q, "transmissions") + tx, err := schemaColumns(ctx, q, "transmissions", db.Backend()) if err != nil { return fmt.Errorf("probe transmissions: %w", err) } db.hasScopeName = tx["scope_name"] db.hasLastSeen = tx["last_seen"] - nodes, err := schemaColumns(ctx, q, "nodes") + nodes, err := schemaColumns(ctx, q, "nodes", db.Backend()) if err != nil { return fmt.Errorf("probe nodes: %w", err) } @@ -262,7 +238,7 @@ func (db *DB) detectSchema(ctx context.Context, q rowQuerier) error { // install, so schemaColumns returns nothing and the flag stays false; // present on deployments that collect the same fact by another route. // A missing table is not an error here. - ndr, ndrErr := schemaColumns(ctx, q, "node_declared_regions") + ndr, ndrErr := schemaColumns(ctx, q, "node_declared_regions", db.Backend()) db.hasDeclaredRegionsTable = ndrErr == nil && len(ndr) > 0 if db.isV3 { @@ -280,10 +256,14 @@ type rowQuerier interface { QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) } -// schemaColumns reads only the selected PostgreSQL schema's visible columns. -func schemaColumns(ctx context.Context, q rowQuerier, table string) (map[string]bool, error) { - rows, err := q.QueryContext(ctx, `SELECT column_name FROM information_schema.columns - WHERE table_schema = current_schema() AND table_name = $1`, table) +// schemaColumns reads visible columns through the selected native catalog. +func schemaColumns(ctx context.Context, q rowQuerier, table string, backend ...dbconfig.Backend) (map[string]bool, error) { + query := `SELECT column_name FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = $1` + if len(backend) > 0 && backend[0] == dbconfig.SQLite { + query = `SELECT name FROM pragma_table_info(?1)` + } + rows, err := q.QueryContext(ctx, query, table) if err != nil { return nil, err } @@ -476,6 +456,16 @@ type Stats struct { PacketsLast24h int `json:"packetsLast24h"` } +// CountActiveNodes counts nodes seen in the last 7 days: GetStats' totalNodes +// on its own, without GetStats' whole-table counts. +func (db *DB) CountActiveNodes() (int, error) { + // Node.js uses 7-day active nodes for totalNodes + sevenDaysAgo := time.Now().Add(-7 * 24 * time.Hour).Format(time.RFC3339) + var n int + err := db.stmtQueryRow(db.stmtCountNodesActive, "SELECT COUNT(*) FROM nodes WHERE last_seen > "+db.parameter(1), sevenDaysAgo).Scan(&n) + return n, err +} + // GetStats returns aggregate counts (matches Node.js db.getStats shape). func (db *DB) GetStats() (*Stats, error) { s := &Stats{} @@ -486,22 +476,20 @@ func (db *DB) GetStats() (*Stats, error) { s.TotalPackets = s.TotalTransmissions db.stmtQueryRow(db.stmtCountObservations, "SELECT COUNT(*) FROM observations").Scan(&s.TotalObservations) - // Node.js uses 7-day active nodes for totalNodes - sevenDaysAgo := time.Now().Add(-7 * 24 * time.Hour).Format(time.RFC3339) - db.stmtQueryRow(db.stmtCountNodesActive, "SELECT COUNT(*) FROM nodes WHERE last_seen > $1", sevenDaysAgo).Scan(&s.TotalNodes) + s.TotalNodes, _ = db.CountActiveNodes() db.stmtQueryRow(db.stmtCountNodesAll, "SELECT COUNT(*) FROM nodes").Scan(&s.TotalNodesAllTime) db.stmtQueryRow(db.stmtCountObservers, "SELECT COUNT(*) FROM observers WHERE inactive IS NULL OR inactive = 0").Scan(&s.TotalObservers) oneHourAgo := time.Now().Add(-1 * time.Hour).Unix() - db.stmtQueryRow(db.stmtCountObsLastHour, "SELECT COUNT(*) FROM observations WHERE timestamp > $1", oneHourAgo).Scan(&s.PacketsLastHour) + db.stmtQueryRow(db.stmtCountObsLastHour, "SELECT COUNT(*) FROM observations WHERE timestamp > "+db.parameter(1), oneHourAgo).Scan(&s.PacketsLastHour) oneDayAgo := time.Now().Add(-24 * time.Hour).Unix() - db.stmtQueryRow(db.stmtCountObsLastDay, "SELECT COUNT(*) FROM observations WHERE timestamp > $1", oneDayAgo).Scan(&s.PacketsLast24h) + db.stmtQueryRow(db.stmtCountObsLastDay, "SELECT COUNT(*) FROM observations WHERE timestamp > "+db.parameter(1), oneDayAgo).Scan(&s.PacketsLast24h) return s, nil } -// GetDBSizeStats retains the legacy map caller while using PostgreSQL metrics. +// GetDBSizeStats retains the legacy map caller with native database metrics. func (db *DB) GetDBSizeStats() map[string]interface{} { stats := db.GetDBSizeStatsTyped() var size any @@ -514,12 +502,17 @@ func (db *DB) GetDBSizeStats() map[string]interface{} { "observations": stats.Rows.Observations, "nodes": stats.Rows.Nodes, "observers": stats.Rows.Observers} } - return map[string]interface{}{ + result := map[string]interface{}{ "engine": stats.Engine, "dbSizeMB": size, "rows": rows, "error": stats.Error, } + if stats.Engine == "sqlite" { + result["walSizeMB"] = stats.WalSizeMB + result["freelistMB"] = stats.FreelistMB + } + return result } -// GetDBSizeStatsTyped reports native PostgreSQL database size and table counts. +// GetDBSizeStatsTyped reports native database size and table counts. // WAL is cluster-wide and cannot be attributed to this database by a reader. func (db *DB) GetDBSizeStatsTyped() PostgresStats { sample, err := db.postgresStats(context.Background()) @@ -527,12 +520,20 @@ func (db *DB) GetDBSizeStatsTyped() PostgresStats { Engine: "postgresql", SampledAt: postgresSampleStamp(sample), SampleIntervalSeconds: int(postgresStatsTTL / time.Second), Stale: err != nil, } + if db.Backend() == dbconfig.SQLite { + result.Engine = "sqlite" + } if sample.SampledAt.IsZero() { result.Error = "database diagnostics unavailable" return result } mb := math.Round(float64(sample.DatabaseBytes)/1048576*10) / 10 result.DbSizeMB, result.Rows = &mb, &sample.Rows + if sample.SQLite != nil { + walMB := float64(sample.SQLite.WalSize) / 1048576 + freeMB := float64(sample.SQLite.FreelistCount*sample.SQLite.PageSize) / 1048576 + result.WalSizeMB, result.FreelistMB = &walMB, &freeMB + } return result } @@ -542,7 +543,7 @@ func (db *DB) GetRoleCounts() map[string]int { counts := map[string]int{} for _, role := range []string{"repeater", "room", "companion", "sensor"} { var c int - db.stmtQueryRow(db.stmtCountNodesByRole, "SELECT COUNT(*) FROM nodes WHERE role = $1 AND last_seen > $2", role, sevenDaysAgo).Scan(&c) + db.stmtQueryRow(db.stmtCountNodesByRole, "SELECT COUNT(*) FROM nodes WHERE role = "+db.parameter(1)+" AND last_seen > "+db.parameter(2), role, sevenDaysAgo).Scan(&c) counts[role+"s"] = c } return counts @@ -553,7 +554,7 @@ func (db *DB) GetAllRoleCounts() map[string]int { counts := map[string]int{} for _, role := range []string{"repeater", "room", "companion", "sensor"} { var c int - db.stmtQueryRow(db.stmtCountNodesByRoleAll, "SELECT COUNT(*) FROM nodes WHERE role = $1", role).Scan(&c) + db.stmtQueryRow(db.stmtCountNodesByRoleAll, "SELECT COUNT(*) FROM nodes WHERE role = "+db.parameter(1), role).Scan(&c) counts[role+"s"] = c } return counts @@ -617,8 +618,8 @@ func (db *DB) QueryPackets(q PacketQuery) (*PacketResult, error) { // is what the packets page is showing. The `since=` filter still uses // first_seen / observation timestamp, preserving "received-by-radio since X." selectCols, observerJoin := db.transmissionBaseSQL() - querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s %s ORDER BY t.id %s LIMIT $%d OFFSET $%d", - selectCols, observerJoin, w, q.Order, len(args)+1, len(args)+2) + querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s %s ORDER BY t.id %s LIMIT %s OFFSET %s", + selectCols, observerJoin, w, q.Order, db.parameter(len(args)+1), db.parameter(len(args)+2)) qArgs := make([]interface{}, len(args)) copy(qArgs, args) @@ -676,35 +677,47 @@ func (db *DB) QueryGroupedPackets(q PacketQuery) (*PacketResult, error) { } var querySQL string if db.isV3 { - querySQL = fmt.Sprintf(`SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type, + querySQL = fmt.Sprintf(db.nativeSQL(`SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type, + COALESCE((SELECT COUNT(*) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS count, + COALESCE((SELECT COUNT(DISTINCT oi.observer_idx) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS observer_count, + COALESCE((SELECT MAX(strftime('%%Y-%%m-%%dT%%H:%%M:%%fZ', oi.timestamp, 'unixepoch')) FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen) AS latest, + obs.id AS observer_id, obs.name AS observer_name, COALESCE(obs.iata, '') AS observer_iata, + o.snr, o.rssi, o.path_json, + COALESCE((SELECT group_concat(DISTINCT obi.iata) FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas`, `SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type, COALESCE((SELECT COUNT(*) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS count, COALESCE((SELECT COUNT(DISTINCT oi.observer_idx) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS observer_count, COALESCE((SELECT MAX(to_char(to_timestamp(oi.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')) FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen) AS latest, obs.id AS observer_id, obs.name AS observer_name, COALESCE(obs.iata, '') AS observer_iata, o.snr, o.rssi, o.path_json, - COALESCE((SELECT string_agg(DISTINCT obi.iata, ',') FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas`+scopeNameCol+` + COALESCE((SELECT string_agg(DISTINCT obi.iata, ',') FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas`)+scopeNameCol+` FROM transmissions t LEFT JOIN observations o ON o.id = ( SELECT id FROM observations WHERE transmission_id = t.id ORDER BY length(COALESCE(path_json,'')) DESC NULLS LAST LIMIT 1 ) LEFT JOIN observers obs ON obs.rowid = o.observer_idx - %s ORDER BY latest DESC NULLS LAST LIMIT $%d OFFSET $%d`, w, len(args)+1, len(args)+2) + %s ORDER BY latest DESC NULLS LAST LIMIT %s OFFSET %s`, w, db.parameter(len(args)+1), db.parameter(len(args)+2)) } else { - querySQL = fmt.Sprintf(`SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type, + querySQL = fmt.Sprintf(db.nativeSQL(`SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type, + COALESCE((SELECT COUNT(*) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS count, + COALESCE((SELECT COUNT(DISTINCT oi.observer_id) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS observer_count, + COALESCE((SELECT CAST(MAX(oi.timestamp) AS TEXT) FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen) AS latest, + o.observer_id, o.observer_name, COALESCE(obs2.iata, '') AS observer_iata, + o.snr, o.rssi, o.path_json, + COALESCE((SELECT group_concat(DISTINCT obi.iata) FROM observations oi JOIN observers obi ON obi.id = oi.observer_id WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas`, `SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type, COALESCE((SELECT COUNT(*) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS count, COALESCE((SELECT COUNT(DISTINCT oi.observer_id) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS observer_count, COALESCE((SELECT MAX(oi.timestamp)::text FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen) AS latest, o.observer_id, o.observer_name, COALESCE(obs2.iata, '') AS observer_iata, o.snr, o.rssi, o.path_json, - COALESCE((SELECT string_agg(DISTINCT obi.iata, ',') FROM observations oi JOIN observers obi ON obi.id = oi.observer_id WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas`+scopeNameCol+` + COALESCE((SELECT string_agg(DISTINCT obi.iata, ',') FROM observations oi JOIN observers obi ON obi.id = oi.observer_id WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas`)+scopeNameCol+` FROM transmissions t LEFT JOIN observations o ON o.id = ( SELECT id FROM observations WHERE transmission_id = t.id ORDER BY length(COALESCE(path_json,'')) DESC NULLS LAST LIMIT 1 ) LEFT JOIN observers obs2 ON obs2.id = o.observer_id - %s ORDER BY latest DESC NULLS LAST LIMIT $%d OFFSET $%d`, w, len(args)+1, len(args)+2) + %s ORDER BY latest DESC NULLS LAST LIMIT %s OFFSET %s`, w, db.parameter(len(args)+1), db.parameter(len(args)+2)) } qArgs := make([]interface{}, len(args)) @@ -788,38 +801,38 @@ func (db *DB) buildPacketWhere(q PacketQuery) ([]string, []interface{}) { var args []interface{} if q.Type != nil { - where = append(where, fmt.Sprintf("payload_type = $%d", len(args)+1)) + where = append(where, fmt.Sprintf("payload_type = %s", db.parameter(len(args)+1))) args = append(args, *q.Type) } if q.Route != nil { - where = append(where, fmt.Sprintf("route_type = $%d", len(args)+1)) + where = append(where, fmt.Sprintf("route_type = %s", db.parameter(len(args)+1))) args = append(args, *q.Route) } if q.Observer != "" { - where = append(where, fmt.Sprintf("observer_id = $%d", len(args)+1)) + where = append(where, fmt.Sprintf("observer_id = %s", db.parameter(len(args)+1))) args = append(args, q.Observer) } if q.Hash != "" { - where = append(where, fmt.Sprintf("hash = $%d", len(args)+1)) + where = append(where, fmt.Sprintf("hash = %s", db.parameter(len(args)+1))) args = append(args, strings.ToLower(q.Hash)) } if q.Since != "" { - where = append(where, fmt.Sprintf("timestamp > $%d", len(args)+1)) + where = append(where, fmt.Sprintf("timestamp > %s", db.parameter(len(args)+1))) args = append(args, q.Since) } if q.Until != "" { - where = append(where, fmt.Sprintf("timestamp < $%d", len(args)+1)) + where = append(where, fmt.Sprintf("timestamp < %s", db.parameter(len(args)+1))) args = append(args, q.Until) } if q.Region != "" { - where = append(where, fmt.Sprintf("observer_id IN (SELECT id FROM observers WHERE iata = $%d)", len(args)+1)) + where = append(where, fmt.Sprintf("observer_id IN (SELECT id FROM observers WHERE iata = %s)", db.parameter(len(args)+1))) args = append(args, q.Region) } if q.Node != "" { pk := db.resolveNodePubkey(q.Node) // #1143: exact-match on the dedicated from_pubkey column instead of // LIKE-on-JSON substring (adversarial spoof + same-name false positives). - where = append(where, fmt.Sprintf("from_pubkey = $%d", len(args)+1)) + where = append(where, fmt.Sprintf("from_pubkey = %s", db.parameter(len(args)+1))) args = append(args, pk) } return where, args @@ -832,15 +845,15 @@ func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) { var args []interface{} if q.Type != nil { - where = append(where, fmt.Sprintf("t.payload_type = $%d", len(args)+1)) + where = append(where, fmt.Sprintf("t.payload_type = %s", db.parameter(len(args)+1))) args = append(args, *q.Type) } if q.Route != nil { - where = append(where, fmt.Sprintf("t.route_type = $%d", len(args)+1)) + where = append(where, fmt.Sprintf("t.route_type = %s", db.parameter(len(args)+1))) args = append(args, *q.Route) } if q.Hash != "" { - where = append(where, fmt.Sprintf("t.hash = $%d", len(args)+1)) + where = append(where, fmt.Sprintf("t.hash = %s", db.parameter(len(args)+1))) args = append(args, strings.ToLower(q.Hash)) } if q.Since != "" { @@ -849,36 +862,36 @@ func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) { // but which have observations inside the window) are still included. // Non-RFC3339 falls back to t.first_seen string compare. if ts, err := time.Parse(time.RFC3339Nano, q.Since); err == nil { - where = append(where, fmt.Sprintf("t.id IN (SELECT DISTINCT transmission_id FROM observations WHERE timestamp >= $%d)", len(args)+1)) + where = append(where, fmt.Sprintf("t.id IN (SELECT DISTINCT transmission_id FROM observations WHERE timestamp >= %s)", db.parameter(len(args)+1))) args = append(args, ts.Unix()) } else { - where = append(where, fmt.Sprintf("t.first_seen > $%d", len(args)+1)) + where = append(where, fmt.Sprintf("t.first_seen > %s", db.parameter(len(args)+1))) args = append(args, q.Since) } } if q.Until != "" { if ts, err := time.Parse(time.RFC3339Nano, q.Until); err == nil { - where = append(where, fmt.Sprintf("t.id IN (SELECT DISTINCT transmission_id FROM observations WHERE timestamp <= $%d)", len(args)+1)) + where = append(where, fmt.Sprintf("t.id IN (SELECT DISTINCT transmission_id FROM observations WHERE timestamp <= %s)", db.parameter(len(args)+1))) args = append(args, ts.Unix()) } else { - where = append(where, fmt.Sprintf("t.first_seen < $%d", len(args)+1)) + where = append(where, fmt.Sprintf("t.first_seen < %s", db.parameter(len(args)+1))) args = append(args, q.Until) } } if q.Node != "" { pk := db.resolveNodePubkey(q.Node) // #1143: exact-match on dedicated from_pubkey column. - where = append(where, fmt.Sprintf("t.from_pubkey = $%d", len(args)+1)) + where = append(where, fmt.Sprintf("t.from_pubkey = %s", db.parameter(len(args)+1))) args = append(args, pk) } if q.Channel != "" { // channel_hash column is indexed for payload_type = 5; filter is exact match. - where = append(where, fmt.Sprintf("t.channel_hash = $%d", len(args)+1)) + where = append(where, fmt.Sprintf("t.channel_hash = %s", db.parameter(len(args)+1))) args = append(args, q.Channel) } if q.Observer != "" { ids := strings.Split(q.Observer, ",") - placeholders := sqlPlaceholders(len(ids), len(args)+1) + placeholders := db.sqlPlaceholders(len(ids), len(args)+1) if db.isV3 { where = append(where, "EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.id IN ("+placeholders+"))") } else { @@ -890,9 +903,9 @@ func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) { } if q.Region != "" { if db.isV3 { - where = append(where, fmt.Sprintf("EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.iata = $%d)", len(args)+1)) + where = append(where, fmt.Sprintf("EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.iata = %s)", db.parameter(len(args)+1))) } else { - where = append(where, fmt.Sprintf("EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.id = oi.observer_id WHERE oi.transmission_id = t.id AND obi.iata = $%d)", len(args)+1)) + where = append(where, fmt.Sprintf("EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.id = oi.observer_id WHERE oi.transmission_id = t.id AND obi.iata = %s)", db.parameter(len(args)+1))) } args = append(args, q.Region) } @@ -901,7 +914,7 @@ func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) { func (db *DB) resolveNodePubkey(nodeIDOrName string) string { var pk string - err := db.stmtQueryRow(db.stmtNodeLookup, "SELECT public_key FROM nodes WHERE public_key = $1 OR name = $2 LIMIT 1", nodeIDOrName, nodeIDOrName).Scan(&pk) + err := db.stmtQueryRow(db.stmtNodeLookup, "SELECT public_key FROM nodes WHERE public_key = "+db.parameter(1)+" OR name = "+db.parameter(2)+" LIMIT 1", nodeIDOrName, nodeIDOrName).Scan(&pk) if err != nil { return nodeIDOrName } @@ -911,7 +924,7 @@ func (db *DB) resolveNodePubkey(nodeIDOrName string) string { // GetTransmissionByID fetches from transmissions table with observer data. func (db *DB) GetTransmissionByID(id int) (map[string]interface{}, error) { selectCols, observerJoin := db.transmissionBaseSQL() - querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.id = $1", selectCols, observerJoin) + querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.id = "+db.parameter(1), selectCols, observerJoin) rows, err := db.conn.Query(querySQL, id) if err != nil { @@ -927,7 +940,7 @@ func (db *DB) GetTransmissionByID(id int) (map[string]interface{}, error) { // GetPacketByHash fetches a transmission by content hash with observer data. func (db *DB) GetPacketByHash(hash string) (map[string]interface{}, error) { selectCols, observerJoin := db.transmissionBaseSQL() - querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.hash = $1", selectCols, observerJoin) + querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.hash = "+db.parameter(1), selectCols, observerJoin) rows, err := db.conn.Query(querySQL, strings.ToLower(hash)) if err != nil { @@ -945,7 +958,7 @@ func (db *DB) GetPacketByHash(hash string) (map[string]interface{}, error) { // when the in-memory PacketStore has pruned the entry but the DB still has it. func (db *DB) GetObservationsForHash(hash string) []map[string]interface{} { var txID int - err := db.stmtQueryRow(db.stmtTxByHash, "SELECT id FROM transmissions WHERE hash = $1", strings.ToLower(hash)).Scan(&txID) + err := db.stmtQueryRow(db.stmtTxByHash, "SELECT id FROM transmissions WHERE hash = "+db.parameter(1), strings.ToLower(hash)).Scan(&txID) if err != nil { return nil } @@ -975,13 +988,13 @@ func (db *DB) ObservationRawHexForHash(hash string) map[int]string { return nil } var txID int - if err := db.stmtQueryRow(db.stmtTxByHash, "SELECT id FROM transmissions WHERE hash = $1", + if err := db.stmtQueryRow(db.stmtTxByHash, "SELECT id FROM transmissions WHERE hash = "+db.parameter(1), strings.ToLower(hash)).Scan(&txID); err != nil { return nil } rows, err := db.conn.Query( `SELECT id, raw_hex FROM observations - WHERE transmission_id = $1 AND raw_hex IS NOT NULL AND raw_hex <> ''`, txID) + WHERE transmission_id = `+db.parameter(1)+` AND raw_hex IS NOT NULL AND raw_hex <> ''`, txID) if err != nil { return nil } @@ -1030,15 +1043,15 @@ func (db *DB) GetNodes(nq NodeQuery) ([]map[string]interface{}, int, map[string] var args []interface{} if role != "" { - where = append(where, fmt.Sprintf("role = $%d", len(args)+1)) + where = append(where, fmt.Sprintf("role = %s", db.parameter(len(args)+1))) args = append(args, role) } if search != "" { - where = append(where, fmt.Sprintf("name ILIKE $%d ESCAPE ''", len(args)+1)) + where = append(where, fmt.Sprintf(db.nativeSQL("name LIKE %s", "name ILIKE %s ESCAPE ''"), db.parameter(len(args)+1))) args = append(args, "%"+search+"%") } if before != "" { - where = append(where, fmt.Sprintf("first_seen <= $%d", len(args)+1)) + where = append(where, fmt.Sprintf("first_seen <= %s", db.parameter(len(args)+1))) args = append(args, before) } if lastHeard != "" { @@ -1048,7 +1061,7 @@ func (db *DB) GetNodes(nq NodeQuery) ([]map[string]interface{}, int, map[string] } if ms, ok := durations[lastHeard]; ok { since := time.Now().Add(-time.Duration(ms) * time.Millisecond).Format(time.RFC3339) - where = append(where, fmt.Sprintf("last_seen > $%d", len(args)+1)) + where = append(where, fmt.Sprintf("last_seen > %s", db.parameter(len(args)+1))) args = append(args, since) } } @@ -1059,7 +1072,7 @@ func (db *DB) GetNodes(nq NodeQuery) ([]map[string]interface{}, int, map[string] placeholders := make([]string, len(codes)) regionArgs := make([]interface{}, len(codes)) for i, c := range codes { - placeholders[i] = fmt.Sprintf("$%d", len(args)+i+1) + placeholders[i] = fmt.Sprintf("%s", db.parameter(len(args)+i+1)) regionArgs[i] = c } joinCond := "obs.rowid = o.observer_idx" @@ -1092,7 +1105,7 @@ func (db *DB) GetNodes(nq NodeQuery) ([]map[string]interface{}, int, map[string] if err != nil { return nil, 0, nil, err } - where = append(where, fmt.Sprintf("public_key IN (SELECT value FROM json_array_elements_text($%d::json))", len(args)+1)) + where = append(where, fmt.Sprintf(db.nativeSQL("public_key IN (SELECT value FROM json_each(%s))", "public_key IN (SELECT value FROM json_array_elements_text(%s::json))"), db.parameter(len(args)+1))) args = append(args, string(keysJSON)) } } @@ -1117,7 +1130,7 @@ func (db *DB) GetNodes(nq NodeQuery) ([]map[string]interface{}, int, map[string] var total int db.conn.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM nodes %s", w), args...).Scan(&total) - querySQL := fmt.Sprintf("SELECT %s FROM nodes %s ORDER BY %s LIMIT $%d OFFSET $%d", db.nodeSelectCols(), w, order, len(args)+1, len(args)+2) + querySQL := fmt.Sprintf("SELECT %s FROM nodes %s ORDER BY %s LIMIT %s OFFSET %s", db.nodeSelectCols(), w, order, db.parameter(len(args)+1), db.parameter(len(args)+2)) qArgs := append(args, limit, offset) rows, err := db.conn.Query(querySQL, qArgs...) @@ -1143,7 +1156,7 @@ func (db *DB) SearchNodes(query string, limit int) ([]map[string]interface{}, er if limit <= 0 { limit = 10 } - rows, err := db.conn.Query(fmt.Sprintf("SELECT %s FROM nodes WHERE name ILIKE $1 ESCAPE '' OR public_key ILIKE $2 ESCAPE '' ORDER BY last_seen DESC NULLS LAST LIMIT $3", db.nodeSelectCols()), + rows, err := db.conn.Query(fmt.Sprintf(db.nativeSQL("SELECT %s FROM nodes WHERE name LIKE ?1 OR public_key LIKE ?2 ORDER BY last_seen DESC NULLS LAST LIMIT ?3", "SELECT %s FROM nodes WHERE name ILIKE "+db.parameter(1)+" ESCAPE '' OR public_key ILIKE "+db.parameter(2)+" ESCAPE '' ORDER BY last_seen DESC NULLS LAST LIMIT "+db.parameter(3)), db.nodeSelectCols()), "%"+query+"%", query+"%", limit) if err != nil { return nil, err @@ -1181,7 +1194,7 @@ func (db *DB) GetNodeByPrefix(prefix string) (map[string]interface{}, bool, erro } } rows, err := db.conn.Query( - fmt.Sprintf("SELECT %s FROM nodes WHERE public_key ILIKE $1 ESCAPE '' LIMIT 2", db.nodeSelectCols()), + fmt.Sprintf(db.nativeSQL("SELECT %s FROM nodes WHERE public_key LIKE ?1 LIMIT 2", "SELECT %s FROM nodes WHERE public_key ILIKE "+db.parameter(1)+" ESCAPE '' LIMIT 2"), db.nodeSelectCols()), prefix+"%", ) if err != nil { @@ -1210,7 +1223,7 @@ func (db *DB) GetNodeByPrefix(prefix string) (map[string]interface{}, bool, erro // GetNodeByPubkey returns a single node. func (db *DB) GetNodeByPubkey(pubkey string) (map[string]interface{}, error) { - rows, err := db.conn.Query(fmt.Sprintf("SELECT %s FROM nodes WHERE public_key = $1", db.nodeSelectCols()), pubkey) + rows, err := db.conn.Query(fmt.Sprintf("SELECT %s FROM nodes WHERE public_key = "+db.parameter(1), db.nodeSelectCols()), pubkey) if err != nil { return nil, err } @@ -1237,7 +1250,7 @@ func (db *DB) GetRecentTransmissionsForNode(pubkey string, limit int) ([]map[str // #1345: order by ingest id, not first_seen (=rxTime). Buffered observer // uploads with old rxTime would otherwise displace fresh activity from // the "recent transmissions for node" list. - querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.from_pubkey = $1 ORDER BY t.id DESC LIMIT $2", + querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.from_pubkey = "+db.parameter(1)+" ORDER BY t.id DESC LIMIT "+db.parameter(2), selectCols, observerJoin) args := []interface{}{pubkey, limit} @@ -1302,18 +1315,23 @@ func (db *DB) getObservationsForTransmissions(txIDs []int) map[int][]map[string] placeholders := make([]string, len(txIDs)) args := make([]interface{}, len(txIDs)) for i, id := range txIDs { - placeholders[i] = fmt.Sprintf("$%d", i+1) + placeholders[i] = fmt.Sprintf("%s", db.parameter(i+1)) args[i] = id } var querySQL string if db.isV3 { - querySQL = fmt.Sprintf(`SELECT o.transmission_id, o.id, obs.id AS observer_id, obs.name AS observer_name, COALESCE(obs.iata, '') AS observer_iata, + querySQL = fmt.Sprintf(db.nativeSQL(`SELECT o.transmission_id, o.id, obs.id AS observer_id, obs.name AS observer_name, COALESCE(obs.iata, '') AS observer_iata, + o.direction, o.snr, o.rssi, o.path_json, strftime('%%Y-%%m-%%dT%%H:%%M:%%fZ', o.timestamp, 'unixepoch') AS obs_timestamp + FROM observations o + LEFT JOIN observers obs ON obs.rowid = o.observer_idx + WHERE o.transmission_id IN (%s) + ORDER BY o.timestamp DESC`, `SELECT o.transmission_id, o.id, obs.id AS observer_id, obs.name AS observer_name, COALESCE(obs.iata, '') AS observer_iata, o.direction, o.snr, o.rssi, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') AS obs_timestamp FROM observations o LEFT JOIN observers obs ON obs.rowid = o.observer_idx WHERE o.transmission_id IN (%s) - ORDER BY o.timestamp DESC`, strings.Join(placeholders, ",")) + ORDER BY o.timestamp DESC`), strings.Join(placeholders, ",")) } else { querySQL = fmt.Sprintf(`SELECT o.transmission_id, o.id, o.observer_id, o.observer_name, COALESCE(obs.iata, '') AS observer_iata, o.direction, o.snr, o.rssi, o.path_json, o.timestamp AS obs_timestamp @@ -1371,10 +1389,10 @@ func (db *DB) GetObservers() ([]Observer, error) { // and the server returns CanRelay=nil so the UI shows no badge. canRelayClause := "COALESCE(can_relay, 1)" canRelaySeenClause := "0" - if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay"); !hasCol { + if hasCol, _ := db.tableHasColumn("observers", "can_relay"); !hasCol { canRelayClause = "1" } - if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay_seen"); hasCol { + if hasCol, _ := db.tableHasColumn("observers", "can_relay_seen"); hasCol { canRelaySeenClause = "COALESCE(can_relay_seen, 0)" } rows, err := db.conn.Query(`SELECT id, name, iata, last_seen, first_seen, packet_count, @@ -1434,7 +1452,7 @@ func (db *DB) GetObservers() ([]Observer, error) { func (db *DB) GetNonRelayObserverPubkeys() ([]string, error) { // Graceful no-op when can_relay column is absent (legacy DB / older // test fixture). Avoids noisy schema-degradation log spam. - if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay"); !hasCol { + if hasCol, _ := db.tableHasColumn("observers", "can_relay"); !hasCol { return nil, nil } rows, err := db.conn.Query(`SELECT LOWER(id) FROM observers @@ -1460,7 +1478,7 @@ func (db *DB) GetNonRelayObserverPubkeys() ([]string, error) { // this to render tri-state — observers NOT in this set are "unknown" // and the UI shows no badge. func (db *DB) GetCanRelaySeenObserverPubkeys() ([]string, error) { - if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay_seen"); !hasCol { + if hasCol, _ := db.tableHasColumn("observers", "can_relay_seen"); !hasCol { return nil, nil } rows, err := db.conn.Query(`SELECT LOWER(id) FROM observers @@ -1489,17 +1507,17 @@ func (db *DB) GetObserverByID(id string) (*Observer, error) { var canRelay, canRelaySeen int canRelayClause := "COALESCE(can_relay, 1)" canRelaySeenClause := "0" - if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay"); !hasCol { + if hasCol, _ := db.tableHasColumn("observers", "can_relay"); !hasCol { canRelayClause = "1" } - if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay_seen"); hasCol { + if hasCol, _ := db.tableHasColumn("observers", "can_relay_seen"); hasCol { canRelaySeenClause = "COALESCE(can_relay_seen, 0)" } err := db.conn.QueryRow(`SELECT id, name, iata, last_seen, first_seen, packet_count, model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, last_packet_at, clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at, `+canRelayClause+`, `+canRelaySeenClause+` - FROM observers WHERE id = $1`, id). + FROM observers WHERE id = `+db.parameter(1), id). Scan(&o.ID, &o.Name, &o.IATA, &o.LastSeen, &o.FirstSeen, &o.PacketCount, &o.Model, &o.Firmware, &o.ClientVersion, &o.Radio, &batteryMv, &uptimeSecs, &noiseFloor, &o.LastPacketAt, &clockSkewSec, &clockSkewCount, &o.ClockLastNaiveAt, &canRelay, &canRelaySeen) @@ -1539,7 +1557,7 @@ func (db *DB) GetObserverIdsForRegion(regionParam string) ([]string, error) { placeholders := make([]string, len(codes)) args := make([]interface{}, len(codes)) for i, c := range codes { - placeholders[i] = fmt.Sprintf("$%d", i+1) + placeholders[i] = fmt.Sprintf("%s", db.parameter(i+1)) args[i] = c } rows, err := db.conn.Query(fmt.Sprintf("SELECT id FROM observers WHERE UPPER(TRIM(iata)) IN (%s)", strings.Join(placeholders, ",")), args...) @@ -1658,22 +1676,35 @@ func (db *DB) GetNetworkStatus(healthThresholds HealthThresholds) (map[string]in func (db *DB) GetTraces(hash string) ([]map[string]interface{}, error) { var querySQL string if db.isV3 { - querySQL = `SELECT obs.id AS observer_id, obs.name AS observer_name, + querySQL = db.nativeSQL(`SELECT obs.id AS observer_id, obs.name AS observer_name, + strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch') AS timestamp, + o.snr, o.rssi, o.path_json + FROM observations o + JOIN transmissions t ON t.id = o.transmission_id + LEFT JOIN observers obs ON obs.rowid = o.observer_idx + WHERE t.hash = ?1 + ORDER BY o.timestamp ASC`, `SELECT obs.id AS observer_id, obs.name AS observer_name, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') AS timestamp, o.snr, o.rssi, o.path_json FROM observations o JOIN transmissions t ON t.id = o.transmission_id LEFT JOIN observers obs ON obs.rowid = o.observer_idx - WHERE t.hash = $1 - ORDER BY o.timestamp ASC` + WHERE t.hash = `+db.parameter(1)+` + ORDER BY o.timestamp ASC`) } else { - querySQL = `SELECT o.observer_id, o.observer_name, + querySQL = db.nativeSQL(`SELECT o.observer_id, o.observer_name, + strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch') AS timestamp, + o.snr, o.rssi, o.path_json + FROM observations o + JOIN transmissions t ON t.id = o.transmission_id + WHERE t.hash = ?1 + ORDER BY o.timestamp ASC`, `SELECT o.observer_id, o.observer_name, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') AS timestamp, o.snr, o.rssi, o.path_json FROM observations o JOIN transmissions t ON t.id = o.transmission_id - WHERE t.hash = $1 - ORDER BY o.timestamp ASC` + WHERE t.hash = `+db.parameter(1)+` + ORDER BY o.timestamp ASC`) } rows, err := db.conn.Query(querySQL, strings.ToLower(hash)) if err != nil { @@ -1775,7 +1806,7 @@ func (db *DB) GetChannels(region ...string) ([]map[string]interface{}, error) { if len(regionCodes) > 0 { placeholders := make([]string, len(regionCodes)) for i, code := range regionCodes { - placeholders[i] = fmt.Sprintf("$%d", i+1) + placeholders[i] = fmt.Sprintf("%s", db.parameter(i+1)) args = append(args, code) } regionPlaceholder := strings.Join(placeholders, ",") @@ -1924,7 +1955,7 @@ func (db *DB) GetEncryptedChannels(region ...string) ([]map[string]interface{}, if len(regionCodes) > 0 { placeholders := make([]string, len(regionCodes)) for i, code := range regionCodes { - placeholders[i] = fmt.Sprintf("$%d", i+1) + placeholders[i] = fmt.Sprintf("%s", db.parameter(i+1)) args = append(args, code) } regionPlaceholder := strings.Join(placeholders, ",") @@ -2061,7 +2092,7 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . if len(regionCodes) > 0 { placeholders := make([]string, len(regionCodes)) for i, code := range regionCodes { - placeholders[i] = fmt.Sprintf("$%d", i+2) + placeholders[i] = fmt.Sprintf("%s", db.parameter(i+2)) regionArgs = append(regionArgs, code) } regionPlaceholders = strings.Join(placeholders, ",") @@ -2088,7 +2119,7 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . // 1) Total count (after region filter, before pagination). countSQL := `SELECT COUNT(*) FROM transmissions t - WHERE t.channel_hash = $1 AND t.payload_type = 5` + regionFilter + WHERE t.channel_hash = ` + db.parameter(1) + ` AND t.payload_type = 5` + regionFilter countArgs := []interface{}{channelHash} countArgs = append(countArgs, regionArgs...) var total int @@ -2118,16 +2149,16 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . pageSQL := `SELECT t.id, COALESCE((SELECT MAX(timestamp) FROM observations WHERE transmission_id = t.id), 0) AS latest_obs_epoch FROM transmissions t - WHERE t.channel_hash = $1 AND t.payload_type = 5 + WHERE t.channel_hash = ` + db.parameter(1) + ` AND t.payload_type = 5 ORDER BY latest_obs_epoch DESC NULLS LAST, t.id DESC - LIMIT ` + fmt.Sprintf("$%d OFFSET $%d", len(regionArgs)+2, len(regionArgs)+3) + LIMIT ` + fmt.Sprintf("%s OFFSET %s", db.parameter(len(regionArgs)+2), db.parameter(len(regionArgs)+3)) if len(regionCodes) > 0 { pageSQL = `SELECT t.id, COALESCE((SELECT MAX(timestamp) FROM observations WHERE transmission_id = t.id), 0) AS latest_obs_epoch FROM transmissions t - WHERE t.channel_hash = $1 AND t.payload_type = 5` + regionFilter + ` + WHERE t.channel_hash = ` + db.parameter(1) + ` AND t.payload_type = 5` + regionFilter + ` ORDER BY latest_obs_epoch DESC NULLS LAST, t.id DESC - LIMIT ` + fmt.Sprintf("$%d OFFSET $%d", len(regionArgs)+2, len(regionArgs)+3) + LIMIT ` + fmt.Sprintf("%s OFFSET %s", db.parameter(len(regionArgs)+2), db.parameter(len(regionArgs)+3)) } pageArgs := []interface{}{channelHash} pageArgs = append(pageArgs, regionArgs...) @@ -2159,7 +2190,7 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . idPlaceholders := make([]string, len(pageIDs)) obsArgs := make([]interface{}, len(pageIDs)) for i, id := range pageIDs { - idPlaceholders[i] = fmt.Sprintf("$%d", i+1) + idPlaceholders[i] = fmt.Sprintf("%s", db.parameter(i+1)) obsArgs[i] = id } // #1851: scope_name lives on the transmission row, so appending it as the @@ -2379,12 +2410,12 @@ func (db *DB) GetObserverPacketCounts(sinceEpoch int64) map[string]int { rows, err = db.conn.Query(`SELECT obs.id, COUNT(*) as cnt FROM observations o JOIN observers obs ON obs.rowid = o.observer_idx - WHERE o.timestamp > $1 + WHERE o.timestamp > `+db.parameter(1)+` GROUP BY obs.id`, sinceEpoch) } else { rows, err = db.conn.Query(`SELECT o.observer_id, COUNT(*) as cnt FROM observations o - WHERE o.observer_id IS NOT NULL AND o.timestamp > $1 + WHERE o.observer_id IS NOT NULL AND o.timestamp > `+db.parameter(1)+` GROUP BY o.observer_id`, sinceEpoch) } if err != nil { @@ -2432,7 +2463,7 @@ func (db *DB) GetNodeLocationsByKeys(keys []string) map[string]map[string]interf placeholders := make([]string, len(keys)) args := make([]interface{}, len(keys)) for i, k := range keys { - placeholders[i] = fmt.Sprintf("$%d", i+1) + placeholders[i] = fmt.Sprintf("%s", db.parameter(i+1)) args[i] = strings.ToLower(k) } // #1481 P0-3: drop LOWER(public_key) — that wrap is non-sargable and @@ -2478,17 +2509,17 @@ func (db *DB) QueryMultiNodePackets(pubkeys []string, limit, offset int, order, for _, pk := range pubkeys { resolved := db.resolveNodePubkey(pk) args = append(args, resolved) - placeholders = append(placeholders, fmt.Sprintf("$%d", len(args))) + placeholders = append(placeholders, fmt.Sprintf("%s", db.parameter(len(args)))) } pkWhere := "t.from_pubkey IN (" + strings.Join(placeholders, ",") + ")" var timeFilters []string if since != "" { - timeFilters = append(timeFilters, fmt.Sprintf("t.first_seen >= $%d", len(args)+1)) + timeFilters = append(timeFilters, fmt.Sprintf("t.first_seen >= %s", db.parameter(len(args)+1))) args = append(args, since) } if until != "" { - timeFilters = append(timeFilters, fmt.Sprintf("t.first_seen <= $%d", len(args)+1)) + timeFilters = append(timeFilters, fmt.Sprintf("t.first_seen <= %s", db.parameter(len(args)+1))) args = append(args, until) } @@ -2502,8 +2533,8 @@ func (db *DB) QueryMultiNodePackets(pubkeys []string, limit, offset int, order, selectCols, observerJoin := db.transmissionBaseSQL() // #1345: order by ingest id (see QueryPackets comment above). - querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s %s ORDER BY t.id %s LIMIT $%d OFFSET $%d", - selectCols, observerJoin, w, order, len(args)+1, len(args)+2) + querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s %s ORDER BY t.id %s LIMIT %s OFFSET %s", + selectCols, observerJoin, w, order, db.parameter(len(args)+1), db.parameter(len(args)+2)) qArgs := make([]interface{}, len(args)) copy(qArgs, args) @@ -2714,31 +2745,41 @@ func (db *DB) GetObserverMetrics(observerID, since, until, resolution string, sa // Use LAST value per bucket (latest timestamp) instead of MAX to preserve // reboot semantics: if a device reboots mid-bucket, the last sample is the // post-reboot baseline, not the pre-reboot high-water mark. - query = `SELECT ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv FROM ( + query = db.nativeSQL(`SELECT ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv FROM ( + SELECT + strftime('%Y-%m-%dT%H:00:00Z', timestamp) as ts, + noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv, + ROW_NUMBER() OVER (PARTITION BY observer_id, strftime('%Y-%m-%dT%H:00:00Z', timestamp) ORDER BY timestamp DESC) as rn + FROM observer_metrics WHERE observer_id = ?1`, `SELECT ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv FROM ( SELECT to_char(timestamp::timestamptz AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:00:00"Z"') as ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv, ROW_NUMBER() OVER (PARTITION BY observer_id, to_char(timestamp::timestamptz AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:00:00"Z"') ORDER BY timestamp DESC) as rn - FROM observer_metrics WHERE observer_id = $1` + FROM observer_metrics WHERE observer_id = `+db.parameter(1)) case "1d": bucketSizeSec = 86400 - query = `SELECT ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv FROM ( + query = db.nativeSQL(`SELECT ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv FROM ( + SELECT + strftime('%Y-%m-%dT00:00:00Z', timestamp) as ts, + noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv, + ROW_NUMBER() OVER (PARTITION BY observer_id, strftime('%Y-%m-%dT00:00:00Z', timestamp) ORDER BY timestamp DESC) as rn + FROM observer_metrics WHERE observer_id = ?1`, `SELECT ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv FROM ( SELECT to_char(timestamp::timestamptz AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"00:00:00"Z"') as ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv, ROW_NUMBER() OVER (PARTITION BY observer_id, to_char(timestamp::timestamptz AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"00:00:00"Z"') ORDER BY timestamp DESC) as rn - FROM observer_metrics WHERE observer_id = $1` + FROM observer_metrics WHERE observer_id = `+db.parameter(1)) default: // "5m" or raw query = `SELECT timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv - FROM observer_metrics WHERE observer_id = $1` + FROM observer_metrics WHERE observer_id = ` + db.parameter(1) } if since != "" { - query += fmt.Sprintf(" AND timestamp >= $%d", len(args)+1) + query += fmt.Sprintf(" AND timestamp >= %s", db.parameter(len(args)+1)) args = append(args, since) } if until != "" { - query += fmt.Sprintf(" AND timestamp <= $%d", len(args)+1) + query += fmt.Sprintf(" AND timestamp <= %s", db.parameter(len(args)+1)) args = append(args, until) } @@ -2914,7 +2955,7 @@ func (db *DB) GetMetricsSummary(since string) ([]MetricsSummaryRow, error) { SELECT observer_id, noise_floor, battery_mv, ROW_NUMBER() OVER (PARTITION BY observer_id ORDER BY timestamp DESC) as rn FROM observer_metrics - WHERE timestamp >= $1 + WHERE timestamp >= ` + db.parameter(1) + ` ) SELECT m.observer_id, o.name, COALESCE(o.iata, '') as iata, r.noise_floor as current_nf, @@ -2925,7 +2966,7 @@ func (db *DB) GetMetricsSummary(since string) ([]MetricsSummaryRow, error) { FROM observer_metrics m LEFT JOIN observers o ON o.id = m.observer_id LEFT JOIN ranked r ON r.observer_id = m.observer_id AND r.rn = 1 - WHERE m.timestamp >= $2 + WHERE m.timestamp >= ` + db.parameter(2) + ` GROUP BY m.observer_id, o.name, o.iata, r.noise_floor, r.battery_mv ORDER BY max_nf DESC NULLS LAST ` @@ -2950,7 +2991,7 @@ func (db *DB) GetMetricsSummary(since string) ([]MetricsSummaryRow, error) { // Fetch sparkline data (noise_floor series) for all observers in one query if len(result) > 0 { sparkQuery := `SELECT observer_id, noise_floor FROM observer_metrics - WHERE timestamp >= $1 ORDER BY observer_id, timestamp ASC` + WHERE timestamp >= ` + db.parameter(1) + ` ORDER BY observer_id, timestamp ASC` sparkRows, err := db.conn.Query(sparkQuery, since) if err != nil { return nil, err @@ -2992,17 +3033,17 @@ func (db *DB) GetDroppedPackets(limit int, observerID, nodePubkey string) ([]map var conditions []string var args []interface{} if observerID != "" { - conditions = append(conditions, fmt.Sprintf("observer_id = $%d", len(args)+1)) + conditions = append(conditions, fmt.Sprintf("observer_id = %s", db.parameter(len(args)+1))) args = append(args, observerID) } if nodePubkey != "" { - conditions = append(conditions, fmt.Sprintf("node_pubkey = $%d", len(args)+1)) + conditions = append(conditions, fmt.Sprintf("node_pubkey = %s", db.parameter(len(args)+1))) args = append(args, nodePubkey) } if len(conditions) > 0 { query += " WHERE " + strings.Join(conditions, " AND ") } - query += fmt.Sprintf(" ORDER BY dropped_at DESC NULLS LAST LIMIT $%d", len(args)+1) + query += fmt.Sprintf(" ORDER BY dropped_at DESC NULLS LAST LIMIT %s", db.parameter(len(args)+1)) args = append(args, limit) rows, err := db.conn.Query(query, args...) @@ -3100,16 +3141,16 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { case "1h": since = time.Now().Add(-1 * time.Hour).UTC().Format(time.RFC3339) // 5-minute buckets - bucketExpr = `to_char(date_bin('5 minutes', (CASE WHEN pg_input_is_valid(first_seen, 'timestamp with time zone') THEN first_seen::timestamptz END), '2000-01-01T00:00:00Z'::timestamptz) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')` + bucketExpr = db.nativeSQL(`strftime('%Y-%m-%dT%H:', first_seen) || printf('%02d', (CAST(strftime('%M', first_seen) AS INTEGER) / 5) * 5) || ':00Z'`, `to_char(date_bin('5 minutes', (CASE WHEN pg_input_is_valid(first_seen, 'timestamp with time zone') THEN first_seen::timestamptz END), '2000-01-01T00:00:00Z'::timestamptz) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')`) case "7d": since = time.Now().Add(-7 * 24 * time.Hour).UTC().Format(time.RFC3339) // 6-hour buckets - bucketExpr = `to_char(date_bin('6 hours', (CASE WHEN pg_input_is_valid(first_seen, 'timestamp with time zone') THEN first_seen::timestamptz END), '2000-01-01T00:00:00Z'::timestamptz) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')` + bucketExpr = db.nativeSQL(`strftime('%Y-%m-%dT', first_seen) || printf('%02d', (CAST(strftime('%H', first_seen) AS INTEGER) / 6) * 6) || ':00:00Z'`, `to_char(date_bin('6 hours', (CASE WHEN pg_input_is_valid(first_seen, 'timestamp with time zone') THEN first_seen::timestamptz END), '2000-01-01T00:00:00Z'::timestamptz) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')`) default: // "24h" window = "24h" since = time.Now().Add(-24 * time.Hour).UTC().Format(time.RFC3339) // 1-hour buckets - bucketExpr = `to_char((CASE WHEN pg_input_is_valid(first_seen, 'timestamp with time zone') THEN first_seen::timestamptz END) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:00:00"Z"')` + bucketExpr = db.nativeSQL(`strftime('%Y-%m-%dT%H:00:00Z', first_seen)`, `to_char((CASE WHEN pg_input_is_valid(first_seen, 'timestamp with time zone') THEN first_seen::timestamptz END) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:00:00"Z"')`) } resp := &ScopeStatsResponse{Window: window} @@ -3122,7 +3163,7 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { COALESCE(SUM(CASE WHEN scope_name IS NULL THEN 1 ELSE 0 END), 0) AS unscoped, COALESCE(SUM(CASE WHEN scope_name = '' THEN 1 ELSE 0 END), 0) AS unknown_scope FROM transmissions - WHERE `+routeTypeTransportSQL+` AND first_seen >= $1 + WHERE `+routeTypeTransportSQL+` AND first_seen >= `+db.parameter(1)+` `, since) if err := row.Scan( &resp.Summary.TransportTotal, @@ -3140,7 +3181,7 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { var nonTransportUnscoped int if err := db.conn.QueryRow(` SELECT COUNT(*) FROM transmissions - WHERE `+routeTypeNonTransportSQL+` AND first_seen >= $1 + WHERE `+routeTypeNonTransportSQL+` AND first_seen >= `+db.parameter(1)+` `, since).Scan(&nonTransportUnscoped); err != nil { return nil, fmt.Errorf("scope non-transport count query: %w", err) } @@ -3150,7 +3191,7 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { rows, err := db.conn.Query(` SELECT scope_name, COUNT(*) AS cnt FROM transmissions - WHERE `+routeTypeTransportSQL+` AND scope_name IS NOT NULL AND scope_name != '' AND first_seen >= $1 + WHERE `+routeTypeTransportSQL+` AND scope_name IS NOT NULL AND scope_name != '' AND first_seen >= `+db.parameter(1)+` GROUP BY scope_name ORDER BY cnt DESC NULLS LAST `, since) @@ -3177,7 +3218,7 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { COUNT(scope_name) AS scoped, SUM(CASE WHEN scope_name IS NULL THEN 1 ELSE 0 END) AS unscoped FROM transmissions - WHERE `+routeTypeTransportSQL+` AND first_seen >= $1 + WHERE `+routeTypeTransportSQL+` AND first_seen >= `+db.parameter(1)+` GROUP BY bucket ORDER BY bucket `, bucketExpr) @@ -3212,7 +3253,7 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { SUM(CASE WHEN t.scope_name != '' THEN 1 ELSE 0 END) AS named FROM transmissions t LEFT JOIN nodes n ON n.public_key = t.from_pubkey - WHERE +t.payload_type = $1 AND t.route_type IN (0, 1) AND t.first_seen >= $2 + WHERE +t.payload_type = `+db.parameter(1)+` AND t.route_type IN (0, 1) AND t.first_seen >= `+db.parameter(2)+` GROUP BY 1 ORDER BY COUNT(*) DESC NULLS LAST, 1 `, payloadTypeAdvert, since) diff --git a/cmd/server/db_test.go b/cmd/server/db_test.go index dd1703cec..895e48a27 100644 --- a/cmd/server/db_test.go +++ b/cmd/server/db_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "reflect" + "strconv" "strings" "testing" "time" @@ -28,7 +29,119 @@ func setupTestDBAtURL(t testing.TB, dsn string) *DB { conn.SetMaxOpenConns(1) // Create schema matching MeshCore Analyzer v3 - schema := ` + schema := testNativeSQL(` + CREATE TABLE nodes ( + public_key TEXT PRIMARY KEY, + name TEXT, + role TEXT, + lat REAL, + lon REAL, + last_seen TEXT, + first_seen TEXT, + advert_count INTEGER DEFAULT 0, + battery_mv INTEGER, + temperature_c REAL, + foreign_advert INTEGER DEFAULT 0 + ); + + CREATE TABLE observers ( + id TEXT PRIMARY KEY, + name TEXT, + iata TEXT, + last_seen TEXT, + first_seen TEXT, + packet_count INTEGER DEFAULT 0, + model TEXT, + firmware TEXT, + client_version TEXT, + radio TEXT, + battery_mv INTEGER, + uptime_secs INTEGER, + noise_floor REAL, + inactive INTEGER DEFAULT 0, + last_packet_at TEXT DEFAULT NULL, + clock_skew_seconds INTEGER DEFAULT NULL, + clock_skew_count_24h INTEGER DEFAULT 0, + clock_last_naive_at TEXT DEFAULT NULL + ); + + CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_hex TEXT NOT NULL, + hash TEXT NOT NULL UNIQUE, + first_seen TEXT NOT NULL, + route_type INTEGER, + payload_type INTEGER, + payload_version INTEGER, + decoded_json TEXT, + channel_hash TEXT DEFAULT NULL, + from_pubkey TEXT DEFAULT NULL, + created_at TEXT DEFAULT (datetime('now')) + ); + + CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER NOT NULL REFERENCES transmissions(id), + observer_idx INTEGER, + direction TEXT, + snr REAL, + rssi REAL, + score INTEGER, + path_json TEXT, + timestamp INTEGER NOT NULL, + resolved_path TEXT, + raw_hex TEXT + ); + + CREATE TABLE IF NOT EXISTS observer_metrics ( + observer_id TEXT NOT NULL, + timestamp TEXT NOT NULL, + noise_floor REAL, + tx_air_secs INTEGER, + rx_air_secs INTEGER, + recv_errors INTEGER, + battery_mv INTEGER, + packets_sent INTEGER, + packets_recv INTEGER, + PRIMARY KEY (observer_id, timestamp) + ); + + CREATE INDEX IF NOT EXISTS idx_observer_metrics_timestamp ON observer_metrics(timestamp); + + -- Auto-populate from_pubkey for ADVERT rows so existing test fixtures + -- (which only set decoded_json) still attribute correctly under #1143's + -- exact-match column. Production migration handles legacy data; the + -- ingestor sets the column at write time. + -- + -- m4 alignment: prod ingest leaves from_pubkey NULL when pubKey is + -- missing or empty (cmd/ingestor/db.go ~1289 guards PubKey != empty-string). + -- The trigger mirrors that: only assign when json_extract yields a + -- non-empty string. json_extract returns NULL for missing keys, so + -- the explicit IS NOT NULL AND <> empty-string guard catches the empty-string + -- case too. UPDATE only when we have something to write. + CREATE TRIGGER IF NOT EXISTS test_from_pubkey_advert + AFTER INSERT ON transmissions + FOR EACH ROW + WHEN NEW.from_pubkey IS NULL AND NEW.payload_type = 4 AND NEW.decoded_json IS NOT NULL + AND json_extract(NEW.decoded_json, '$.pubKey') IS NOT NULL + AND json_extract(NEW.decoded_json, '$.pubKey') <> '' + BEGIN + UPDATE transmissions + SET from_pubkey = json_extract(NEW.decoded_json, '$.pubKey') + WHERE id = NEW.id; + END; + CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey); + + -- Mirror prod indexes from internal/dbschema/dbschema.go so query plans + -- in tests match prod. idx_observations_transmission_id is required by + -- GetChannelMessages's grouped MAX(timestamp) per tx aggregate + -- (issue #1366 / PR #1368): without it the perf test on 1500 tx × 50 obs + -- blows the 1.5s budget under -race. + CREATE INDEX IF NOT EXISTS idx_observations_transmission_id ON observations(transmission_id); + CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp); + CREATE INDEX IF NOT EXISTS idx_observations_tx_ts ON observations(transmission_id, timestamp); + CREATE INDEX IF NOT EXISTS idx_transmissions_channel_hash ON transmissions(channel_hash); + `, ` CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, @@ -138,12 +251,12 @@ func setupTestDBAtURL(t testing.TB, dsn string) *DB { CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp); CREATE INDEX IF NOT EXISTS idx_observations_tx_ts ON observations(transmission_id, timestamp); CREATE INDEX IF NOT EXISTS idx_transmissions_channel_hash ON transmissions(channel_hash); - ` + `) if _, err := conn.Exec(schema); err != nil { t.Fatal(err) } - return &DB{conn: conn, path: dsn, stateDir: t.TempDir(), isV3: true, hasResolvedPath: true} + return &DB{backend: testBackendValue(), conn: conn, path: dsn, stateDir: t.TempDir(), isV3: true, hasResolvedPath: true} } func seedTestData(t *testing.T, db *DB) { @@ -191,6 +304,33 @@ func seedTestData(t *testing.T, db *DB) { VALUES (3, 1, 10.0, -92, '["cc"]', $1, '["1122334455667788"]')`, yesterdayEpoch) } +// The topology endpoint takes uniqueNodes from CountActiveNodes, so it must +// count what GetStats reports as totalNodes: nodes seen in the last 7 days. +func TestCountActiveNodes(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + now := time.Now().UTC() + for i, seen := range []time.Time{now.Add(-time.Hour), now.Add(-6 * 24 * time.Hour), now.Add(-8 * 24 * time.Hour)} { + db.conn.Exec(`INSERT INTO nodes (public_key, name, last_seen) VALUES (`+db.parameter(1)+`, `+db.parameter(2)+`, `+db.parameter(3)+`)`, + "pk"+strconv.Itoa(i), "n"+strconv.Itoa(i), seen.Format(time.RFC3339)) + } + + got, err := db.CountActiveNodes() + if err != nil { + t.Fatal(err) + } + if got != 2 { + t.Errorf("CountActiveNodes = %d, want 2 (the node seen 8 days ago is not active)", got) + } + stats, err := db.GetStats() + if err != nil { + t.Fatal(err) + } + if got != stats.TotalNodes { + t.Errorf("CountActiveNodes = %d, GetStats().TotalNodes = %d", got, stats.TotalNodes) + } +} + func TestGetStats(t *testing.T) { db := setupTestDB(t) defer db.Close() @@ -239,7 +379,7 @@ func TestGetDBSizeStats(t *testing.T) { stats := db.GetDBSizeStats() // In-memory DB has dbSizeMB=0 and walSizeMB=0 - if stats["dbSizeMB"].(float64) <= 0 { + if stats["dbSizeMB"].(float64) < 0 || (testBackendValue() == "postgres" && stats["dbSizeMB"].(float64) == 0) { t.Errorf("expected positive PostgreSQL database size, got %v", stats["dbSizeMB"]) } @@ -260,10 +400,13 @@ func TestGetDBSizeStats(t *testing.T) { t.Errorf("expected 2 observers rows, got %d", rows["observers"]) } - if stats["engine"] != "postgresql" { + if stats["engine"] != testNativeSQL("sqlite", "postgresql") { t.Fatal("database engine not identified") } for _, obsolete := range []string{"freelistMB", "walPages", "walSizeMB"} { + if testBackendValue() == "sqlite" { + continue + } if _, ok := stats[obsolete]; ok { t.Fatalf("invented SQLite metric %s", obsolete) } @@ -1200,7 +1343,73 @@ func setupTestDBV2(t *testing.T) *DB { } conn.SetMaxOpenConns(1) - schema := ` + schema := testNativeSQL(` + CREATE TABLE nodes ( + public_key TEXT PRIMARY KEY, + name TEXT, + role TEXT, + lat REAL, + lon REAL, + last_seen TEXT, + first_seen TEXT, + advert_count INTEGER DEFAULT 0, + battery_mv INTEGER, + temperature_c REAL, + foreign_advert INTEGER DEFAULT 0 + ); + + CREATE TABLE observers ( + id TEXT PRIMARY KEY, + name TEXT, + iata TEXT, + last_seen TEXT, + first_seen TEXT, + packet_count INTEGER DEFAULT 0, + last_packet_at TEXT DEFAULT NULL, + inactive INTEGER DEFAULT 0 + ); + + CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_hex TEXT NOT NULL, + hash TEXT NOT NULL UNIQUE, + first_seen TEXT NOT NULL, + route_type INTEGER, + payload_type INTEGER, + payload_version INTEGER, + decoded_json TEXT, + channel_hash TEXT DEFAULT NULL, + from_pubkey TEXT DEFAULT NULL, + created_at TEXT DEFAULT (datetime('now')) + ); + + CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER NOT NULL REFERENCES transmissions(id), + observer_id TEXT, + observer_name TEXT, + direction TEXT, + snr REAL, + rssi REAL, + score INTEGER, + path_json TEXT, + timestamp INTEGER NOT NULL, + raw_hex TEXT + ); + + CREATE TRIGGER IF NOT EXISTS test_from_pubkey_advert + AFTER INSERT ON transmissions + FOR EACH ROW + WHEN NEW.from_pubkey IS NULL AND NEW.payload_type = 4 AND NEW.decoded_json IS NOT NULL + AND json_extract(NEW.decoded_json, '$.pubKey') IS NOT NULL + AND json_extract(NEW.decoded_json, '$.pubKey') <> '' + BEGIN + UPDATE transmissions + SET from_pubkey = json_extract(NEW.decoded_json, '$.pubKey') + WHERE id = NEW.id; + END; + CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey); + `, ` CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, @@ -1264,12 +1473,12 @@ func setupTestDBV2(t *testing.T) *DB { CREATE TRIGGER test_from_pubkey_advert BEFORE INSERT ON transmissions FOR EACH ROW EXECUTE FUNCTION test_from_pubkey_advert(); CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey); - ` + `) if _, err := conn.Exec(schema); err != nil { t.Fatal(err) } - return &DB{conn: conn, isV3: false} + return &DB{backend: testBackendValue(), conn: conn, isV3: false} } func TestGetNodesRegionFilterV2(t *testing.T) { @@ -1452,7 +1661,7 @@ func TestOpenDBValid(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = conn.Exec(`CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`) + _, err = conn.Exec(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`)) if err != nil { conn.Close() t.Fatal(err) @@ -1494,7 +1703,7 @@ func TestDetectSchemaScopeName(t *testing.T) { t.Fatal(err) } conn.SetMaxOpenConns(1) - if _, err := conn.Exec(`CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT, scope_name TEXT)`); err != nil { + if _, err := conn.Exec(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT, scope_name TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT, scope_name TEXT)`)); err != nil { conn.Close() t.Fatalf("create transmissions: %v", err) } @@ -1502,7 +1711,7 @@ func TestDetectSchemaScopeName(t *testing.T) { conn.Close() t.Fatalf("create nodes: %v", err) } - if _, err := conn.Exec(`CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY)`); err != nil { + if _, err := conn.Exec(testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY)`)); err != nil { conn.Close() t.Fatalf("create observations: %v", err) } @@ -1529,9 +1738,9 @@ func TestDetectSchemaScopeName(t *testing.T) { t.Fatal(err) } conn2.SetMaxOpenConns(1) - conn2.Exec(`CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`) + conn2.Exec(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`)) conn2.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) - conn2.Exec(`CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY)`) + conn2.Exec(testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY)`)) ensurePreparable(t, conn2) conn2.Close() @@ -2215,7 +2424,7 @@ func TestPerObservationRawHexEnrich(t *testing.T) { // Check enriched observations for _, obs := range tx.Observations { - m := store.enrichObs(obs) + m := store.enrichObsWithTx(obs, store.byTxID[obs.TransmissionID]) rh, _ := m["raw_hex"].(string) if obs.RawHex != "" { // Observer A: should get per-observation raw_hex @@ -2237,16 +2446,21 @@ func TestGetScopeStats(t *testing.T) { t.Fatalf("sql.Open: %v", err) } conn.SetMaxOpenConns(1) - db := &DB{conn: conn} + db := &DB{backend: testBackendValue(), conn: conn} defer db.conn.Close() // Create minimal schema - db.conn.Exec(`CREATE TABLE IF NOT EXISTS transmissions ( + db.conn.Exec(testNativeSQL(`CREATE TABLE IF NOT EXISTS transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, + payload_type INTEGER, payload_version INTEGER, decoded_json TEXT, + scope_name TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL + )`, `CREATE TABLE IF NOT EXISTS transmissions ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT, scope_name TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL - )`) + )`)) db.conn.Exec(`CREATE TABLE IF NOT EXISTS nodes (public_key TEXT PRIMARY KEY, role TEXT)`) // Manually set hasScopeName since we bypassed the detector db.hasScopeName = true @@ -2297,15 +2511,19 @@ func TestGetScopeStatsAdvertsByRole(t *testing.T) { t.Fatalf("sql.Open: %v", err) } conn.SetMaxOpenConns(1) - db := &DB{conn: conn, hasScopeName: true} + db := &DB{backend: testBackendValue(), conn: conn, hasScopeName: true} defer db.conn.Close() for _, stmt := range []string{ - `CREATE TABLE transmissions ( + testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, + scope_name TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL + )`, `CREATE TABLE transmissions ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, scope_name TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL - )`, + )`), `CREATE TABLE nodes (public_key TEXT PRIMARY KEY, role TEXT)`, `INSERT INTO nodes (public_key, role) VALUES ('rpt', 'repeater'), ('cmp', 'companion'), ('sns', 'sensor'), ('norole', '')`, } { @@ -2381,9 +2599,9 @@ func TestGetScopeStatsAdvertsByRoleEmpty(t *testing.T) { t.Fatalf("sql.Open: %v", err) } conn.SetMaxOpenConns(1) - db := &DB{conn: conn, hasScopeName: true} + db := &DB{backend: testBackendValue(), conn: conn, hasScopeName: true} defer db.conn.Close() - db.conn.Exec(`CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, first_seen TEXT, route_type INTEGER, payload_type INTEGER, scope_name TEXT, from_pubkey TEXT)`) + db.conn.Exec(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, scope_name TEXT, from_pubkey TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, first_seen TEXT, route_type INTEGER, payload_type INTEGER, scope_name TEXT, from_pubkey TEXT)`)) db.conn.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, role TEXT)`) stats, err := db.GetScopeStats("1h") @@ -2468,11 +2686,11 @@ func TestDetectSchemaFailsLoudOnProbeError(t *testing.T) { } conn.SetMaxOpenConns(1) // v3-shaped: observations carries observer_idx. - if _, err := conn.Exec(`CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, observer_idx INTEGER)`); err != nil { + if _, err := conn.Exec(testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, observer_idx INTEGER)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, observer_idx INTEGER)`)); err != nil { conn.Close() t.Fatal(err) } - conn.Exec(`CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY)`) + conn.Exec(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY)`)) conn.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) conn.Close() @@ -2502,8 +2720,8 @@ func TestDetectSchemaV3AndV2(t *testing.T) { t.Fatal(err) } c.SetMaxOpenConns(1) - c.Exec(`CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, observer_idx INTEGER)`) - c.Exec(`CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`) + c.Exec(testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, observer_idx INTEGER)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, observer_idx INTEGER)`)) + c.Exec(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`)) c.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) ensurePreparable(t, c) c.Close() @@ -2522,8 +2740,8 @@ func TestDetectSchemaV3AndV2(t *testing.T) { t.Fatal(err) } c2.SetMaxOpenConns(1) - c2.Exec(`CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, observer_id INTEGER)`) - c2.Exec(`CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`) + c2.Exec(testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, observer_id INTEGER)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, observer_id INTEGER)`)) + c2.Exec(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`)) c2.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) ensurePreparable(t, c2) c2.Close() diff --git a/cmd/server/go.mod b/cmd/server/go.mod index b0e5a9ddf..072136857 100644 --- a/cmd/server/go.mod +++ b/cmd/server/go.mod @@ -57,16 +57,26 @@ require ( ) require ( + github.com/dustin/go-humanize v1.0.1 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect golang.org/x/text v0.24.0 // indirect + modernc.org/libc v1.55.3 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect + modernc.org/sqlite v1.34.5 // indirect ) replace github.com/meshcore-analyzer/channel => ../../internal/channel require ( + github.com/mattn/go-sqlite3 v1.14.52 github.com/meshcore-analyzer/pgutil v0.0.0 + github.com/meshcore-analyzer/sqliteutil v0.0.0 golang.org/x/crypto v0.37.0 // indirect golang.org/x/sys v0.32.0 // indirect ) @@ -74,3 +84,5 @@ require ( replace github.com/meshcore-analyzer/mailer => ../../internal/mailer replace github.com/meshcore-analyzer/pgutil => ../../internal/pgutil + +replace github.com/meshcore-analyzer/sqliteutil => ../../internal/sqliteutil diff --git a/cmd/server/go.sum b/cmd/server/go.sum index 4bd844326..489fcca40 100644 --- a/cmd/server/go.sum +++ b/cmd/server/go.sum @@ -1,6 +1,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= @@ -13,8 +15,16 @@ github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -24,6 +34,7 @@ golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= @@ -32,3 +43,11 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= +modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= diff --git a/cmd/server/hash_migrate_test.go b/cmd/server/hash_migrate_test.go index 60a61a656..3df5c46f1 100644 --- a/cmd/server/hash_migrate_test.go +++ b/cmd/server/hash_migrate_test.go @@ -14,8 +14,9 @@ func TestVerifyContentHashesDoesNotWrite(t *testing.T) { correctHash := ComputeContentHash(rawHex) wrongHash := "deadbeef12345678" - _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES ($1, $2, to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), 0, 2)`, rawHex, wrongHash) + _, err := db.conn.Exec(testNativeSQL(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) + VALUES ($1, $2, strftime('%Y-%m-%dT%H:%M:%SZ','now'), 0, 2)`, `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) + VALUES ($1, $2, to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), 0, 2)`), rawHex, wrongHash) if err != nil { t.Fatal(err) } @@ -57,8 +58,9 @@ func TestMigrateContentHashesAsync_NoOp(t *testing.T) { rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976" correctHash := ComputeContentHash(rawHex) - _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES ($1, $2, to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), 0, 2)`, rawHex, correctHash) + _, err := db.conn.Exec(testNativeSQL(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) + VALUES ($1, $2, strftime('%Y-%m-%dT%H:%M:%SZ','now'), 0, 2)`, `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) + VALUES ($1, $2, to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), 0, 2)`), rawHex, correctHash) if err != nil { t.Fatal(err) } @@ -93,8 +95,9 @@ func TestMigrateContentHashesAsyncDoesNotClaimCompletionWhenWritesFail(t *testin rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976" wrongHash := "deadbeef12345678" - if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES ($1, $2, to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), 0, 2)`, rawHex, wrongHash); err != nil { + if _, err := db.conn.Exec(testNativeSQL(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) + VALUES ($1, $2, strftime('%Y-%m-%dT%H:%M:%SZ','now'), 0, 2)`, `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) + VALUES ($1, $2, to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), 0, 2)`), rawHex, wrongHash); err != nil { t.Fatal(err) } if err := store.Load(); err != nil { diff --git a/cmd/server/hot_startup_test.go b/cmd/server/hot_startup_test.go index f30ed72f0..362344835 100644 --- a/cmd/server/hot_startup_test.go +++ b/cmd/server/hot_startup_test.go @@ -31,9 +31,9 @@ func createTestDBMultiDay(t *testing.T, numDays, txPerDay int) string { t.Fatalf("createTestDBMultiDay setup: %v", err) } } - execOrFail(`CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`) - execOrFail(`CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`) - execOrFail(`CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + execOrFail(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`)) + execOrFail(testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`)) + execOrFail(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION)`) execOrFail(`CREATE TABLE schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) diff --git a/cmd/server/issue1809_bg_load_race_test.go b/cmd/server/issue1809_bg_load_race_test.go index 6a0eea56a..df799c824 100644 --- a/cmd/server/issue1809_bg_load_race_test.go +++ b/cmd/server/issue1809_bg_load_race_test.go @@ -147,21 +147,31 @@ func seedTestDBRows(t *testing.T, dbPath string, numTx, obsPerTx int, rowTimes f } } // PREFLIGHT: async=true reason="unit-test fixture seeder; in-memory ephemeral SQLite; all CREATE/INSERT below covered by this block annotation (pr-preflight 25-line lookback window)" - execOrFail(`CREATE TABLE transmissions ( + execOrFail(testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, + payload_version INTEGER, decoded_json TEXT, + last_seen INTEGER NOT NULL DEFAULT 0 + )`, `CREATE TABLE transmissions ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT, last_seen INTEGER NOT NULL DEFAULT 0 - )`) + )`)) // PREFLIGHT: async=true reason="unit-test fixture seeder" - execOrFail(`CREATE TABLE observations ( + execOrFail(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, raw_hex TEXT + )`, `CREATE TABLE observations ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT - )`) + )`)) // PREFLIGHT: async=true reason="unit-test fixture seeder" - execOrFail(`CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + execOrFail(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) // PREFLIGHT: async=true reason="unit-test fixture seeder" execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION)`) // PREFLIGHT: async=true reason="unit-test fixture seeder" diff --git a/cmd/server/issue810_repro_test.go b/cmd/server/issue810_repro_test.go index 2bb34f357..d24b3d118 100644 --- a/cmd/server/issue810_repro_test.go +++ b/cmd/server/issue810_repro_test.go @@ -10,7 +10,7 @@ import ( ) // TestRepro810 reproduces #810: when the longest-path observation has NULL -// resolved_path but a shorter-path observation has one, fetchResolvedPathForTxBest +// resolved_path but a shorter-path observation has one, bestResolvedPath // returns nil → /api/nodes/{pk}/health.recentPackets[].resolved_path is missing // while /api/packets shows it. func TestRepro810(t *testing.T) { diff --git a/cmd/server/load_graph_invariant_test.go b/cmd/server/load_graph_invariant_test.go index 511db94bd..32cd88320 100644 --- a/cmd/server/load_graph_invariant_test.go +++ b/cmd/server/load_graph_invariant_test.go @@ -30,21 +30,31 @@ func TestLoad_PanicsWhenGraphNotLoadedAndEdgesExist(t *testing.T) { } // Minimal CoreScope schema. PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE transmissions ( + exec(testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, + decoded_json TEXT + )`, `CREATE TABLE transmissions ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) + )`)) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observations ( + exec(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, + observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, raw_hex TEXT, resolved_path TEXT + )`, `CREATE TABLE observations ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT, resolved_path TEXT - )`) + )`)) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + exec(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" exec(`CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, diff --git a/cmd/server/loadchunk_ambiguous_prefix_test.go b/cmd/server/loadchunk_ambiguous_prefix_test.go index 250ff5705..c4ab913cf 100644 --- a/cmd/server/loadchunk_ambiguous_prefix_test.go +++ b/cmd/server/loadchunk_ambiguous_prefix_test.go @@ -32,14 +32,27 @@ func createTestDBAmbiguousPrefix(t *testing.T, relayA, relayB, hop, firstSeen st } // PREFLIGHT: async=true reason="test fixture: in-memory t.TempDir SQLite, never touches a real DB." - exec(`CREATE TABLE transmissions ( + exec(testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, + decoded_json TEXT + )`, `CREATE TABLE transmissions ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) + )`)) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observations ( + exec(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER, + observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, + raw_hex TEXT, + resolved_path TEXT + )`, `CREATE TABLE observations ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, @@ -47,9 +60,9 @@ func createTestDBAmbiguousPrefix(t *testing.T, relayA, relayB, hop, firstSeen st path_json TEXT, timestamp TEXT, raw_hex TEXT, resolved_path TEXT - )`) + )`)) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + exec(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" exec(`CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, diff --git a/cmd/server/loadchunk_pathjson_fallback_test.go b/cmd/server/loadchunk_pathjson_fallback_test.go index 3a63213ff..70757cdd8 100644 --- a/cmd/server/loadchunk_pathjson_fallback_test.go +++ b/cmd/server/loadchunk_pathjson_fallback_test.go @@ -33,15 +33,28 @@ func createTestDBPathJSONNoResolvedPath(t *testing.T, relayPubkey, hopPrefix, fi } // PREFLIGHT: async=true reason="test fixture: in-memory t.TempDir SQLite, never touches a real DB. Tables are CREATE-from-empty in a one-shot OpenDB call, not a schema migration over existing data." - exec(`CREATE TABLE transmissions ( + exec(testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, + decoded_json TEXT + )`, `CREATE TABLE transmissions ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) + )`)) // resolved_path column present (matches live schema) but left NULL. // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observations ( + exec(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER, + observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, + raw_hex TEXT, + resolved_path TEXT + )`, `CREATE TABLE observations ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, @@ -49,9 +62,9 @@ func createTestDBPathJSONNoResolvedPath(t *testing.T, relayPubkey, hopPrefix, fi path_json TEXT, timestamp TEXT, raw_hex TEXT, resolved_path TEXT - )`) + )`)) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + exec(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) // Production nodes schema uses public_key (not pubkey) — getAllNodes / // buildPrefixMap reads public_key, role, advert_count, first_seen. // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" diff --git a/cmd/server/loadchunk_resolved_path_1558_test.go b/cmd/server/loadchunk_resolved_path_1558_test.go index 99d5fbeb1..81373a899 100644 --- a/cmd/server/loadchunk_resolved_path_1558_test.go +++ b/cmd/server/loadchunk_resolved_path_1558_test.go @@ -27,13 +27,26 @@ func createTestDBWithResolvedPath(t *testing.T, numTx int, relayPubkeys []string } } - exec(`CREATE TABLE transmissions ( + exec(testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, + decoded_json TEXT + )`, `CREATE TABLE transmissions ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) - exec(`CREATE TABLE observations ( + )`)) + exec(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER, + observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, + raw_hex TEXT, + resolved_path TEXT + )`, `CREATE TABLE observations ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, @@ -41,8 +54,8 @@ func createTestDBWithResolvedPath(t *testing.T, numTx int, relayPubkeys []string path_json TEXT, timestamp TEXT, raw_hex TEXT, resolved_path TEXT - )`) - exec(`CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + )`)) + exec(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION)`) exec(`CREATE TABLE schema_version (version INTEGER)`) exec(`INSERT INTO schema_version (version) VALUES (1)`) diff --git a/cmd/server/main.go b/cmd/server/main.go index 3356fb4f4..b9a8cdca5 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -17,7 +17,7 @@ import ( "time" "github.com/gorilla/mux" - "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbconfig" ) // Set via -ldflags at build time @@ -88,6 +88,7 @@ func main() { configDir string port int legacyDB string + backend string databaseURL string stateDir string publicDir string @@ -96,7 +97,8 @@ func main() { flag.StringVar(&configDir, "config-dir", ".", "Directory containing config.json") flag.IntVar(&port, "port", 0, "HTTP port (overrides config)") - flag.StringVar(&legacyDB, "db", "", "Removed: import SQLite and use -database-url") + flag.StringVar(&legacyDB, "db", "", "SQLite telemetry path (bootstrap override)") + flag.StringVar(&backend, "backend", "", "Storage backend: sqlite or postgres (bootstrap choice)") flag.StringVar(&databaseURL, "database-url", "", "PostgreSQL telemetry reader URL (overrides config/env)") flag.StringVar(&stateDir, "state-dir", "", "Local directory for queues and sidecars") flag.StringVar(&publicDir, "public", "public", "Directory to serve static files from") @@ -106,7 +108,7 @@ func main() { // Load config cfg, err := LoadConfig(configDir) if err != nil { - log.Printf("[config] warning: %v (using defaults)", err) + log.Fatalf("[config] %v", err) } // CLI flags override config @@ -116,9 +118,7 @@ func main() { if cfg.Port == 0 { cfg.Port = 3000 } - if legacyDB != "" { - log.Fatal("[config] -db is no longer supported; import SQLite and use -database-url") - } + if cfg.APIKey == "" { log.Printf("[security] WARNING: no apiKey configured — write endpoints are BLOCKED (set apiKey in config.json to enable them)") } else if IsWeakAPIKey(cfg.APIKey) { @@ -160,19 +160,18 @@ func main() { warnIfMemlimitUnderprovisioned(limit) } - // Resolve connection separately from local state. Never log a database URL. - resolvedDB, err := cfg.ResolveDatabaseURL() - if databaseURL != "" && cfg.DBPath == "" && os.Getenv("DB_PATH") == "" { - resolvedDB, err = databaseURL, nil - } + rawStorage, err := cfg.storageInputs(configDir, storageFlags{Backend: dbconfig.Backend(backend), DBPath: legacyDB, DatabaseURL: databaseURL, StateDir: stateDir}, os.Getenv) if err != nil { log.Fatalf("[config] %v", err) } - runtimeStateDir = cfg.ResolveStateDir(configDir) - if stateDir != "" { - runtimeStateDir = stateDir + storage, selectionLease, err := resolveRuntimeStorage(rawStorage) + if err != nil { + log.Fatalf("[storage] %v", err) } - log.Printf("[config] port=%d database=postgresql public=%s", cfg.Port, publicDir) + defer selectionLease.Close() + runtimeStateDir = storage.StateDir + log.Printf("[config] port=%d database=%s public=%s", cfg.Port, storage.Backend, publicDir) + if len(cfg.NodeBlacklist) > 0 { log.Printf("[config] nodeBlacklist: %d node(s) will be hidden from API", len(cfg.NodeBlacklist)) for _, pk := range cfg.NodeBlacklist { @@ -183,9 +182,9 @@ func main() { } // Open database - database, err := OpenDB(resolvedDB) + database, err := OpenStorage(storage) if err != nil { - log.Fatalf("[db] failed to open PostgreSQL: %v", err) + log.Fatalf("[db] failed to open selected database: %v", err) } database.stateDir = runtimeStateDir var dbCloseOnce sync.Once @@ -211,8 +210,8 @@ func main() { // (#1287). The server NEVER migrates — it only reads. If a required // column/index/table is missing, the operator must restart the // ingestor (which owns dbschema.Apply) before this server can start. - if err := dbschema.AssertReady(database.conn); err != nil { - log.Fatalf("[db] schema not ready (run the PostgreSQL migration/bootstrap command first): %v", err) + if err := database.AssertReady(); err != nil { + log.Fatalf("[db] schema not ready (run the selected backend setup/writer first): %v", err) } // In-memory packet store @@ -375,7 +374,7 @@ func main() { srv.store = store // Optional user management (off by default). Fails startup on a bad // config rather than running with registration that cannot work. - if err := srv.initUserManagement(resolvedDB); err != nil { + if err := srv.initUserManagementStorage(storage); err != nil { log.Fatalf("[users] %v", err) } router := mux.NewRouter() diff --git a/cmd/server/multibyte_capability_test.go b/cmd/server/multibyte_capability_test.go index d447e5701..11327d8f0 100644 --- a/cmd/server/multibyte_capability_test.go +++ b/cmd/server/multibyte_capability_test.go @@ -27,13 +27,17 @@ func setupCapabilityTestDB(t *testing.T) *DB { lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c DOUBLE PRECISION )`) - conn.Exec(`CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, + conn.Exec(testNativeSQL(`CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, + first_seen TEXT, packet_count INTEGER DEFAULT 0, model TEXT, + firmware TEXT, client_version TEXT, radio TEXT, battery_mv INTEGER, + uptime_secs INTEGER + )`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT, packet_count INTEGER DEFAULT 0, model TEXT, firmware TEXT, client_version TEXT, radio TEXT, battery_mv INTEGER, uptime_secs INTEGER - )`) - return &DB{conn: conn} + )`)) + return &DB{backend: testBackendValue(), conn: conn} } // addTestPacket adds a StoreTx to the store's internal structures including @@ -462,7 +466,7 @@ func setupPersistTestDB(t *testing.T) *DB { foreign_advert INTEGER DEFAULT 0, default_scope TEXT, multibyte_sup INTEGER NOT NULL DEFAULT 0, multibyte_evidence TEXT )`) - return &DB{conn: conn, hasMultibyteSupCols: true} + return &DB{backend: testBackendValue(), conn: conn, hasMultibyteSupCols: true} } // TestMultibyteCapGetMultibyteCapForO1 verifies that GetMultibyteCapFor returns diff --git a/cmd/server/neighbor_api_test.go b/cmd/server/neighbor_api_test.go index b5ccb9725..78e1b5740 100644 --- a/cmd/server/neighbor_api_test.go +++ b/cmd/server/neighbor_api_test.go @@ -473,7 +473,7 @@ func TestBuildNodeInfoMap_ObserverEnrichment(t *testing.T) { // Create tables for _, stmt := range []string{ "CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION)", - "CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, name TEXT, iata TEXT, inactive INTEGER)", + testNativeSQL(`CREATE TABLE observers (id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, "CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, name TEXT, iata TEXT, inactive INTEGER)"), "INSERT INTO nodes VALUES ('AAAA1111', 'Repeater-1', 'repeater', 0, 0)", "INSERT INTO observers (id, name, iata) VALUES ('BBBB2222', 'Observer-Alpha', '')", "INSERT INTO observers (id, name, iata) VALUES ('AAAA1111', 'Obs-also-repeater', '')", @@ -546,7 +546,7 @@ func TestBuildNodeInfoMap_FirstSeenIsCached(t *testing.T) { defer rw.Close() for _, stmt := range []string{ "CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER)", - "CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, name TEXT, iata TEXT, inactive INTEGER)", + testNativeSQL(`CREATE TABLE observers (id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, "CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, name TEXT, iata TEXT, inactive INTEGER)"), "INSERT INTO nodes VALUES ('AAAA1111', 'Repeater-1', 'repeater', 0, 0, '', '2024-01-01T00:00:00Z', 0)", } { if _, err := rw.Exec(stmt); err != nil { @@ -612,7 +612,7 @@ func TestGetAllNodes_FirstSeenSchemaFallback(t *testing.T) { defer rw.Close() for _, stmt := range []string{ "CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, advert_count INTEGER)", - "CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, name TEXT, iata TEXT, inactive INTEGER)", + testNativeSQL(`CREATE TABLE observers (id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, "CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, name TEXT, iata TEXT, inactive INTEGER)"), "INSERT INTO nodes VALUES ('BBBB2222', 'Repeater-2', 'repeater', 0, 0, '2024-02-02T00:00:00Z', 3)", } { if _, err := rw.Exec(stmt); err != nil { diff --git a/cmd/server/neighbor_recomputer_test.go b/cmd/server/neighbor_recomputer_test.go index 1673de975..80a986883 100644 --- a/cmd/server/neighbor_recomputer_test.go +++ b/cmd/server/neighbor_recomputer_test.go @@ -95,9 +95,9 @@ func TestServerStartupRequiresMigratedSchema(t *testing.T) { } defer rw.Close() for _, s := range []string{ - `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT, payload_type INTEGER)`, - `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER)`, - `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`, + testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT, payload_type INTEGER)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT, payload_type INTEGER)`), + testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER)`), + testNativeSQL(`CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`), `CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`, `CREATE TABLE inactive_nodes (public_key TEXT PRIMARY KEY)`, } { diff --git a/cmd/server/node_battery.go b/cmd/server/node_battery.go index c8e8f4ff7..9c1ca3943 100644 --- a/cmd/server/node_battery.go +++ b/cmd/server/node_battery.go @@ -53,9 +53,9 @@ func (db *DB) GetNodeBatteryHistory(pubkey, since string) ([]NodeBatterySample, rows, err := db.conn.Query(` SELECT timestamp, battery_mv FROM observer_metrics - WHERE LOWER(observer_id) = $1 + WHERE LOWER(observer_id) = `+db.parameter(1)+` AND battery_mv IS NOT NULL - AND timestamp >= $2 + AND timestamp >= `+db.parameter(2)+` ORDER BY timestamp ASC`, pk, since) if err != nil { return nil, err diff --git a/cmd/server/node_blacklist_test.go b/cmd/server/node_blacklist_test.go index bc1d6ba4c..b22d7f0a8 100644 --- a/cmd/server/node_blacklist_test.go +++ b/cmd/server/node_blacklist_test.go @@ -71,8 +71,8 @@ func TestConfigBlacklistEmptyEntries(t *testing.T) { func TestBlacklistFiltersHandleNodes(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING") - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING") + db.conn.Exec(testNativeSQL(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) + db.conn.Exec(testNativeSQL(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{ NodeBlacklist: []string{"badnode"}, @@ -105,7 +105,7 @@ func TestBlacklistFiltersHandleNodes(t *testing.T) { func TestBlacklistFiltersNodeDetail(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING") + db.conn.Exec(testNativeSQL(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{ NodeBlacklist: []string{"badnode"}, @@ -124,8 +124,8 @@ func TestBlacklistFiltersNodeDetail(t *testing.T) { func TestBlacklistFiltersNodeSearch(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'TrollNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING") - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING") + db.conn.Exec(testNativeSQL(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'TrollNode', 'companion', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'TrollNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) + db.conn.Exec(testNativeSQL(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{ NodeBlacklist: []string{"badnode"}, @@ -155,7 +155,7 @@ func TestBlacklistFiltersNodeSearch(t *testing.T) { func TestNoBlacklistPassesAll(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('somenode', 'SomeNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING") + db.conn.Exec(testNativeSQL(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('somenode', 'SomeNode', 'companion', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('somenode', 'SomeNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{} srv := NewServer(db, cfg, NewHub()) @@ -234,7 +234,7 @@ func TestBlacklistFiltersNeighborGraph(t *testing.T) { func TestBlacklistFiltersResolveHops(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING") + db.conn.Exec(testNativeSQL(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{ NodeBlacklist: []string{"badnode"}, diff --git a/cmd/server/node_reach.go b/cmd/server/node_reach.go index e4e4d1daa..20798e44c 100644 --- a/cmd/server/node_reach.go +++ b/cmd/server/node_reach.go @@ -218,7 +218,7 @@ func (s *Server) reachNodeScopes(ctx context.Context, pubkey string) (string, st args[i] = &vals[i] } row := s.db.conn.QueryRowContext(ctx, - "SELECT "+strings.Join(cols, ", ")+" FROM nodes WHERE public_key = $1", pubkey) + "SELECT "+strings.Join(cols, ", ")+" FROM nodes WHERE public_key = "+s.db.parameter(1), pubkey) if err := row.Scan(args...); err != nil { return "", "", "" } @@ -736,6 +736,23 @@ func (s *Server) getDegreeSnapshot(ctx context.Context) *degreeSnapshot { return snap } +// reachScanSQL builds the windowed reach scan. Arguments are the window +// start, one LIKE pattern per token, and the hard row limit. +// SQLite CROSS JOIN keeps the timestamp window as the outer loop (#2155). +// PostgreSQL retains its ordinary JOIN syntax and native planner. +func (db *DB) reachScanSQL(nTokens int) string { + likes := make([]string, nTokens) + for i := range likes { + likes[i] = fmt.Sprintf(db.nativeSQL("o.path_json LIKE %s", "o.path_json ILIKE %s ESCAPE ''"), db.parameter(i+2)) + } + return `SELECT LOWER(COALESCE(obs.id,'')), LOWER(COALESCE(t.from_pubkey,'')), COALESCE(t.payload_type,0), o.path_json, o.snr + FROM observations o + ` + db.nativeSQL("CROSS JOIN", "JOIN") + ` transmissions t ON t.id = o.transmission_id + LEFT JOIN observers obs ON obs.rowid = o.observer_idx + WHERE o.timestamp >= ` + db.parameter(1) + ` AND (` + strings.Join(likes, " OR ") + `) + LIMIT ` + db.parameter(nTokens+2) +} + // scanReachRows reads windowed observations whose path contains any reliable // token, with the originator + observer + snr needed for attribution. Observer // id and originator pubkey are lowercased in SQL (not per row), the path slice @@ -750,7 +767,6 @@ func (s *Server) scanReachRows(ctx context.Context, tokens map[string]bool, sinc if len(tokens) == 0 { return nil, nil // defensive: an empty LIKE chain would render `AND ()` (SQL error) } - likes := make([]string, 0, len(tokens)) args := []interface{}{sinceEpoch} // Sort tokens so the generated SQL text is byte-stable across requests // with the same token set — preserves the driver's prepared-statement @@ -761,17 +777,10 @@ func (s *Server) scanReachRows(ctx context.Context, tokens map[string]bool, sinc } sort.Strings(toks) for _, tok := range toks { - likes = append(likes, fmt.Sprintf("o.path_json ILIKE $%d ESCAPE ''", len(args)+1)) args = append(args, "%\""+tok+"\"%") } - q := `SELECT LOWER(COALESCE(obs.id,'')), LOWER(COALESCE(t.from_pubkey,'')), COALESCE(t.payload_type,0), o.path_json, o.snr - FROM observations o - JOIN transmissions t ON t.id = o.transmission_id - LEFT JOIN observers obs ON obs.rowid = o.observer_idx - WHERE o.timestamp >= $1 AND (` + strings.Join(likes, " OR ") + `) - LIMIT ` + fmt.Sprintf("$%d", len(args)+1) args = append(args, reachScanRowLimit) - rows, err := s.db.conn.QueryContext(ctx, q, args...) + rows, err := s.db.conn.QueryContext(ctx, s.db.reachScanSQL(len(toks)), args...) if err != nil { log.Printf("[reach] scan query failed: %v", err) return nil, err diff --git a/cmd/server/node_reach_bench_test.go b/cmd/server/node_reach_bench_test.go index 5397553de..12db04bbb 100644 --- a/cmd/server/node_reach_bench_test.go +++ b/cmd/server/node_reach_bench_test.go @@ -26,9 +26,9 @@ func benchReachDB(b *testing.B, nObs, matchEvery int, lowerHops bool) *DB { b.Fatal(err) } schema := []string{ - `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT, first_seen TEXT, payload_type INTEGER, from_pubkey TEXT)`, - `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`, - `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`, + testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT, first_seen TEXT, payload_type INTEGER, from_pubkey TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT, first_seen TEXT, payload_type INTEGER, from_pubkey TEXT)`), + testNativeSQL(`CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`), + testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`), `CREATE INDEX idx_obs_ts ON observations(timestamp)`, } for _, s := range schema { @@ -60,7 +60,7 @@ func benchReachDB(b *testing.B, nObs, matchEvery int, lowerHops bool) *DB { if err := tx.Commit(); err != nil { b.Fatal(err) } - return &DB{conn: conn} + return &DB{backend: testBackendValue(), conn: conn} } // BenchmarkNodeReachScan measures the windowed scan + path-decode at increasing @@ -161,7 +161,7 @@ func TestScanReachRows_ErrorReturn(t *testing.T) { t.Fatalf("schema: %v", err) } conn.Close() // force QueryContext to fail - srv := &Server{db: &DB{conn: conn}} + srv := &Server{db: &DB{backend: testBackendValue(), conn: conn}} rows, err := srv.scanReachRows(context.Background(), map[string]bool{"01FA": true}, 0) if err == nil { t.Fatalf("expected error from closed DB, got nil (rows=%d)", len(rows)) diff --git a/cmd/server/node_reach_endpoint_test.go b/cmd/server/node_reach_endpoint_test.go index d189c263a..14aa8eb87 100644 --- a/cmd/server/node_reach_endpoint_test.go +++ b/cmd/server/node_reach_endpoint_test.go @@ -65,9 +65,9 @@ func newReachIntegrationDB(t *testing.T, obsPath string) (*DB, string) { now := time.Now().Unix() stmts := []string{ `CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER)`, - `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, from_pubkey TEXT, payload_type INTEGER)`, - `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, inactive INTEGER)`, - `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`, + testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, from_pubkey TEXT, payload_type INTEGER)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, from_pubkey TEXT, payload_type INTEGER)`), + testNativeSQL(`CREATE TABLE observers (id TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, inactive INTEGER)`), + testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`), `CREATE TABLE neighbor_edges (node_a TEXT, node_b TEXT, count INTEGER)`, } for _, s := range stmts { @@ -91,7 +91,7 @@ func newReachIntegrationDB(t *testing.T, obsPath string) (*DB, string) { t.Fatal(err) } } - return &DB{conn: conn, isV3: true}, n + return &DB{backend: testBackendValue(), conn: conn, isV3: true}, n } func TestClampDays(t *testing.T) { @@ -230,14 +230,21 @@ func TestNodeReach_ShapeAndClamp(t *testing.T) { // shape (empty — no rows needed for shape/clamp assertions). mustExecDB(t, db, `DROP TABLE observations`) // PREFLIGHT: async=true reason="test-only in-memory schema rebuild; not a production migration" - mustExecDB(t, db, `CREATE TABLE observations ( + mustExecDB(t, db, testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER, + observer_idx INTEGER, + snr DOUBLE PRECISION, + path_json TEXT, + timestamp INTEGER + )`, `CREATE TABLE observations ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER - )`) + )`)) cfg := &Config{} srv := &Server{store: newTestStoreWithDB(t, db, cfg), db: db, cfg: cfg, perfStats: NewPerfStats()} diff --git a/cmd/server/node_reach_test.go b/cmd/server/node_reach_test.go index 4179c7506..4ea0454fd 100644 --- a/cmd/server/node_reach_test.go +++ b/cmd/server/node_reach_test.go @@ -3,7 +3,10 @@ package main import ( "context" "strconv" + "strings" "testing" + + "github.com/meshcore-analyzer/dbconfig" ) // newReachScanTestDB builds a minimal observer_idx-schema DB with two rows whose @@ -15,9 +18,9 @@ func newReachScanTestDB(t *testing.T) *DB { t.Fatal(err) } stmts := []string{ - `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, from_pubkey TEXT, payload_type INTEGER)`, - `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, inactive INTEGER)`, - `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`, + testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, from_pubkey TEXT, payload_type INTEGER)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, from_pubkey TEXT, payload_type INTEGER)`), + testNativeSQL(`CREATE TABLE observers (id TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, inactive INTEGER)`), + testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`), `INSERT INTO observers (id) VALUES ('OBS1')`, // rowid 1 `INSERT INTO transmissions (id, from_pubkey, payload_type) VALUES (1,'FF00',4),(2,'',5),(3,'',5)`, `INSERT INTO observations (id, transmission_id, observer_idx, snr, path_json, timestamp) VALUES @@ -30,7 +33,7 @@ func newReachScanTestDB(t *testing.T) *DB { t.Fatal(err) } } - return &DB{conn: conn} + return &DB{backend: testBackendValue(), conn: conn} } // resolver that only resolves the exact tokens it's told are unique. @@ -286,3 +289,71 @@ func TestScanReachRows_DecodesRows(t *testing.T) { t.Fatalf("snr not decoded: valid=%v val=%v", got.snrValid, got.snr) } } + +// Upstream #2155 / #2146: plan using the ANALYZE statistics captured from an +// 18.3M-observation, 1.5M-transmission database, without building that corpus. +func TestReachScanStartsFromObservationsTimeIndex(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner regression; PostgreSQL uses its native join planner") + } + db := newReachScanTestDB(t) + db.conn.SetMaxOpenConns(1) + for _, query := range []string{ + `CREATE INDEX idx_observations_timestamp ON observations(timestamp)`, + `CREATE INDEX idx_observations_tx_ts ON observations(transmission_id, timestamp)`, + `ANALYZE`, + `DELETE FROM sqlite_stat1`, + `INSERT INTO sqlite_stat1 (tbl, idx, stat) VALUES + ('observations', 'idx_observations_timestamp', '18270402 4'), + ('observations', 'idx_observations_tx_ts', '18270402 11 3'), + ('observations', NULL, '18270402'), + ('transmissions', NULL, '1517599'), + ('observers', NULL, '88')`, + `ANALYZE sqlite_master`, + } { + if _, err := db.conn.Exec(query); err != nil { + t.Fatal(err) + } + } + rows, err := db.conn.Query("EXPLAIN QUERY PLAN "+db.reachScanSQL(2), int64(0), `%"01FA"%`, `%"01FA00"%`, 10) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var steps []string + for rows.Next() { + var id, parent, unused int + var detail string + if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { + t.Fatal(err) + } + steps = append(steps, detail) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + if len(steps) == 0 || !strings.Contains(steps[0], "o USING INDEX idx_observations_timestamp") { + t.Fatalf("reach scan must start from observations by timestamp; plan: %q", steps) + } + t.Logf("reach plan: %q", steps) +} + +func TestScanReachRows_WindowAndMultipleTokens(t *testing.T) { + db := newReachScanTestDB(t) + for _, query := range []string{ + `INSERT INTO transmissions (id, from_pubkey, payload_type) VALUES (4,NULL,5),(5,'',5)`, + `INSERT INTO observations (id, transmission_id, observer_idx, snr, path_json, timestamp) VALUES + (4,4,NULL,NULL,'["cafe"]',2000), (5,5,1,2,'["CAFE"]',999)`, + } { + if _, err := db.conn.Exec(query); err != nil { + t.Fatal(err) + } + } + rows, err := (&Server{db: db}).scanReachRows(context.Background(), map[string]bool{"01FA": true, "CAFE": true}, 1500) + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 || len(rows[0].path) != 1 || rows[0].path[0] != "CAFE" || rows[0].observerPK != "" || rows[0].fromPubkey != "" || rows[0].snrValid { + t.Fatalf("window, token binding or nullable metadata changed: %+v", rows) + } +} diff --git a/cmd/server/node_resolve.go b/cmd/server/node_resolve.go index 6567dac49..2dcf43600 100644 --- a/cmd/server/node_resolve.go +++ b/cmd/server/node_resolve.go @@ -41,7 +41,7 @@ func (s *Server) handleResolvePrefix(w http.ResponseWriter, r *http.Request) { } // LIMIT 2: we only need to know unique vs ambiguous. nodes.public_key is the // PK and stored lowercase; pfx is validated hex so the LIKE pattern is safe. - rows, err := s.db.conn.Query(`SELECT public_key, COALESCE(name,'') FROM nodes WHERE public_key LIKE $1 LIMIT 2`, pfx+"%") + rows, err := s.db.conn.Query(`SELECT public_key, COALESCE(name,'') FROM nodes WHERE public_key LIKE `+s.db.parameter(1)+` LIMIT 2`, pfx+"%") if err != nil { http.Error(w, "query failed", http.StatusInternalServerError) return diff --git a/cmd/server/notify_service_test.go b/cmd/server/notify_service_test.go index 1804516cf..69ac180ff 100644 --- a/cmd/server/notify_service_test.go +++ b/cmd/server/notify_service_test.go @@ -10,7 +10,6 @@ import ( "github.com/gorilla/mux" "github.com/meshcore-analyzer/mailer" - "github.com/meshcore-analyzer/pgutil/pgtest" "github.com/meshcore-analyzer/users" ) @@ -91,7 +90,7 @@ type notifyFixture struct { // admin@example.org is a config admin. func newNotifyFixture(t *testing.T, ns notifySettings) *notifyFixture { t.Helper() - a, fake, ownerURL := newTestAuthServiceWithURL(t, pgtest.NewSchema(t), "admin@example.org") + a, fake, ownerURL := newTestAuthServiceWithURL(t, postgresTestDSN(t), "admin@example.org") a.set.notify = ns clk := ¬ifyClock{t: time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)} a.st.SetClock(clk.Now) @@ -312,13 +311,13 @@ func TestNotifierStateWriteFailureSendsNothing(t *testing.T) { f.watcher(t, "pat@example.org", "Pat", evPkA) f.setNode(evPkA, "Alpha", "companion", time.Hour, nil) f.tick() - f.execDB(t, `CREATE FUNCTION notify_state_fail() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'boom'; END $$; CREATE TRIGGER notify_state_fail BEFORE UPDATE ON notification_state FOR EACH ROW EXECUTE FUNCTION notify_state_fail()`) + f.execDB(t, testNativeSQL(`CREATE TRIGGER notify_state_fail BEFORE UPDATE ON notification_state BEGIN SELECT RAISE(ABORT,'boom'); END`, `CREATE FUNCTION notify_state_fail() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'boom'; END $$; CREATE TRIGGER notify_state_fail BEFORE UPDATE ON notification_state FOR EACH ROW EXECUTE FUNCTION notify_state_fail()`)) f.clk.Advance(25 * time.Hour) f.tick() if m := f.notifyMails(); len(m) != 0 { t.Fatalf("mailed although the states were not written: %+v", m) } - f.execDB(t, `DROP TRIGGER notify_state_fail ON notification_state`) + f.execDB(t, testNativeSQL(`DROP TRIGGER notify_state_fail`, `DROP TRIGGER notify_state_fail ON notification_state`)) f.tick() if m := f.notifyMails(); len(m) != 1 { t.Fatalf("mails after the write works again = %d; want 1", len(m)) @@ -508,7 +507,7 @@ func TestInitUserManagementStartsTheNotifierOnlyWhenEnabled(t *testing.T) { um.Notifications = &NotificationsConfig{Enabled: true} } srv := &Server{cfg: &Config{UserManagement: um}} - if err := srv.initUserManagement(pgtest.NewDatabase(t)); err != nil { + if err := srv.initUserManagement(testDatabaseDSN(t)); err != nil { t.Fatal(err) } if (srv.auth.notify != nil) != on { diff --git a/cmd/server/notify_source.go b/cmd/server/notify_source.go index d069318a2..d9a3409d2 100644 --- a/cmd/server/notify_source.go +++ b/cmd/server/notify_source.go @@ -143,7 +143,7 @@ func (db *DB) NotifyNodes(pubkeys []string, withForeign bool) (map[string]notify for j, pk := range chunk { args[j] = pk } - rows, err := db.conn.Query(notifyNodeCols+` WHERE public_key IN (`+sqlPlaceholders(len(chunk))+`)`, args...) + rows, err := db.conn.Query(notifyNodeCols+` WHERE public_key IN (`+db.sqlPlaceholders(len(chunk))+`)`, args...) if err != nil { return nil, err } diff --git a/cmd/server/observer_blacklist_test.go b/cmd/server/observer_blacklist_test.go index 0afea186e..0457e25a7 100644 --- a/cmd/server/observer_blacklist_test.go +++ b/cmd/server/observer_blacklist_test.go @@ -48,8 +48,8 @@ func TestConfigIsObserverBlacklistedNil(t *testing.T) { func TestObserverBlacklistFiltersHandleObservers(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT INTO observers (id, name, iata, last_seen) VALUES ('goodobs', 'GoodObs', 'SFO', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING") - db.conn.Exec("INSERT INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING") + db.conn.Exec(testNativeSQL(`INSERT INTO observers (id, name, iata, last_seen) VALUES ('goodobs', 'GoodObs', 'SFO', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO observers (id, name, iata, last_seen) VALUES ('goodobs', 'GoodObs', 'SFO', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) + db.conn.Exec(testNativeSQL(`INSERT INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{ ObserverBlacklist: []string{"badobs"}, @@ -89,7 +89,7 @@ func TestObserverBlacklistFiltersHandleObservers(t *testing.T) { func TestObserverBlacklistFiltersObserverDetail(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING") + db.conn.Exec(testNativeSQL(`INSERT INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{ ObserverBlacklist: []string{"badobs"}, @@ -108,7 +108,7 @@ func TestObserverBlacklistFiltersObserverDetail(t *testing.T) { func TestNoObserverBlacklistPassesAll(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT INTO observers (id, name, iata, last_seen) VALUES ('someobs', 'SomeObs', 'SFO', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING") + db.conn.Exec(testNativeSQL(`INSERT INTO observers (id, name, iata, last_seen) VALUES ('someobs', 'SomeObs', 'SFO', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO observers (id, name, iata, last_seen) VALUES ('someobs', 'SomeObs', 'SFO', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{} srv := NewServer(db, cfg, NewHub()) diff --git a/cmd/server/openapi.go b/cmd/server/openapi.go index 1dc807db4..92af6072f 100644 --- a/cmd/server/openapi.go +++ b/cmd/server/openapi.go @@ -52,6 +52,7 @@ func routeDescriptions() map[string]routeMeta { "GET /api/health": {Summary: "Health check", Description: "Returns server health, uptime, and memory stats.", Tag: "admin"}, "GET /api/stats": {Summary: "Network statistics", Description: "Returns aggregate stats (node counts, packet counts, observer counts). Cached for 10s.", Tag: "admin"}, "GET /api/perf": {Summary: "Performance statistics", Description: "Returns per-endpoint request timing and slow query log.", Tag: "admin"}, + "GET /api/perf/database": {Summary: "Selected database diagnostics", Description: "Native metrics for the selected SQLite or PostgreSQL backend. engine identifies the backend; unavailable metrics stay null. Database and row samples refresh every 30 seconds, while connection pool use remains live. SQLite reports WAL bytes, pages, free space, and planner statistics availability without checkpointing or changing the database.", Tag: "admin"}, "GET /api/perf/postgres": {Summary: "PostgreSQL database statistics", Description: "Returns databaseBytes and PostgreSQL block-cache hit ratio (cacheHitRate, 0..1 or null without available reads), sampled with table counts once per 30 seconds and shared with /api/perf. sampledAt and sampleIntervalSeconds disclose freshness; stale marks a failed refresh. Before a first successful sample, unavailable size/counts are null and error is set. After a refresh failure, the last storage/count sample remains with stale=true and the unavailable block ratio is null. Pool usage and waits remain live. Block-cache counters are cumulative since the PostgreSQL statistics reset and exclude the operating-system cache.", Tag: "admin"}, "GET /api/mqtt/status": {Summary: "MQTT source status", Description: "Returns per-MQTT-source connection state and counters (lastConnectUnix, lastPacketUnix, packetsTotal, etc.). Broker URL passwords are masked. Sourced from the ingestor stats file; empty list when unavailable. (#1043)", Tag: "admin"}, "POST /api/perf/reset": {Summary: "Reset performance stats", Tag: "admin", Auth: true, Session: true}, diff --git a/cmd/server/parity_test.go b/cmd/server/parity_test.go index 23d707f08..e4ea2cf2b 100644 --- a/cmd/server/parity_test.go +++ b/cmd/server/parity_test.go @@ -42,6 +42,12 @@ func loadShapes(t *testing.T) map[string]shapeSpec { if err := json.Unmarshal(data, &shapes); err != nil { t.Fatalf("cannot parse shapes.json: %v", err) } + if testBackendValue() == "sqlite" { + perf := shapes["perf"] + perf.Keys["sqlite"] = perf.Keys["postgres"] + delete(perf.Keys, "postgres") + shapes["perf"] = perf + } return shapes } diff --git a/cmd/server/paths_anchor_bias_test.go b/cmd/server/paths_anchor_bias_test.go index a50f88744..3f3cadd9a 100644 --- a/cmd/server/paths_anchor_bias_test.go +++ b/cmd/server/paths_anchor_bias_test.go @@ -13,7 +13,7 @@ import ( // TestHandleNodePaths_AnchorBiasInconsistency_Issue1278 reproduces #1278: // /api/nodes/{pk}/paths returns a tx whose CANONICAL persisted resolved_path -// (the one the packets page reads via fetchResolvedPathForTxBest) does NOT +// (the one the packets page reads via bestResolvedPath) does NOT // contain the queried pubkey. // // Two nodes share the 1-byte prefix "c0": @@ -63,7 +63,7 @@ func TestHandleNodePaths_AnchorBiasInconsistency_Issue1278(t *testing.T) { t.Fatalf("insert obs1: %v", err) } // obs2: LONGER path_json (two hops, first hop is what packets page shows - // as resolved_path[0]). fetchResolvedPathForTxBest picks this obs as + // as resolved_path[0]). bestResolvedPath picks this obs as // canonical because it has the longer path_json. Its resolved_path // picks nodeGPS for "c0", NOT nodeNoGPS. if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) diff --git a/cmd/server/paths_observations_race_test.go b/cmd/server/paths_observations_race_test.go new file mode 100644 index 000000000..c4fbe840c --- /dev/null +++ b/cmd/server/paths_observations_race_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "net/http/httptest" + "sync" + "testing" + + "github.com/gorilla/mux" +) + +// handleNodePaths reads each surviving candidate's canonical resolved_path +// after releasing s.mu. That lookup used to walk tx.Observations there, which +// ingest appends to under the write lock. Run with -race. +func TestNodePathsDoesNotReadObservationsUnlocked(t *testing.T) { + store := newResolvedPathStore(t) + srv := NewServer(store.db, &Config{Port: 3000}, NewHub()) + srv.store = store + router := mux.NewRouter() + srv.RegisterRoutes(router) + + store.mu.RLock() + tx := store.byHash["testhash00000001"] + base := tx.Observations + store.mu.RUnlock() + + stop := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + go func() { // stands in for ingest adding an observation to the tx + defer wg.Done() + for i := 0; ; i++ { + select { + case <-stop: + return + default: + } + // A fresh slice each time, one longer than base, so the writes + // race with an unlocked reader without the slice growing. + store.mu.Lock() + tx.Observations = append(base[:len(base):len(base)], &StoreObs{ID: 1_000_000 + i, TransmissionID: tx.ID, PathJSON: `["aa"]`}) + store.mu.Unlock() + } + }() + + for i := 0; i < 50; i++ { + clearResolvedPathLRU(store) + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest("GET", "/api/nodes/aabbccdd11223344/paths", nil)) + if w.Code != 200 { + t.Fatalf("paths: status %d: %s", w.Code, w.Body.String()) + } + } + close(stop) + wg.Wait() +} diff --git a/cmd/server/perf_io.go b/cmd/server/perf_io.go index f11894ba8..a9acc7c77 100644 --- a/cmd/server/perf_io.go +++ b/cmd/server/perf_io.go @@ -10,6 +10,7 @@ import ( "sync/atomic" "time" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/perfio" ) @@ -39,6 +40,13 @@ type PerfIOSample = perfio.Sample // PerfPostgresResponse combines sampled database metrics with live pool usage. type PerfPostgresResponse struct { + WalSize *int64 `json:"walSize,omitempty"` + WalSizeMB *float64 `json:"walSizeMB,omitempty"` + PageCount *int64 `json:"pageCount,omitempty"` + PageSize *int64 `json:"pageSize,omitempty"` + CacheSize *int64 `json:"cacheSize,omitempty"` + JournalMode *string `json:"journalMode,omitempty"` + PlannerStats *bool `json:"plannerStats,omitempty"` Engine string `json:"engine"` DatabaseBytes *int64 `json:"databaseBytes"` OpenConnections int `json:"openConnections"` @@ -257,7 +265,18 @@ func readIngestorIOSample() *PerfIOSample { func (s *Server) handlePerfPostgres(w http.ResponseWriter, r *http.Request) { resp := PerfPostgresResponse{Engine: "postgresql", SampleIntervalSeconds: int(postgresStatsTTL / time.Second), Stale: true, Error: "database diagnostics unavailable"} if s.db != nil && s.db.conn != nil { + if s.db.Backend() == dbconfig.SQLite { + resp.Engine = "sqlite" + } sample, err := s.db.postgresStats(r.Context()) + if sample.SQLite != nil { + native := sample.SQLite + walMB := float64(native.WalSize) / 1048576 + resp.WalSize, resp.WalSizeMB = &native.WalSize, &walMB + resp.PageCount, resp.PageSize, resp.CacheSize = &native.PageCount, &native.PageSize, &native.CacheSize + resp.JournalMode, resp.PlannerStats = &native.JournalMode, &native.PlannerStats + } + resp.CacheHitRate = sample.CacheHitRate resp.SampledAt, resp.Stale = postgresSampleStamp(sample), err != nil if !sample.SampledAt.IsZero() { diff --git a/cmd/server/postgres_fixture_test.go b/cmd/server/postgres_fixture_test.go index 68367bfbd..46ae13d55 100644 --- a/cmd/server/postgres_fixture_test.go +++ b/cmd/server/postgres_fixture_test.go @@ -3,12 +3,17 @@ package main import ( "context" "database/sql" + "net/url" "os/exec" + "path/filepath" + "strings" "testing" "time" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/stdlib" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" "github.com/meshcore-analyzer/pgutil" "github.com/meshcore-analyzer/pgutil/pgtest" ) @@ -18,6 +23,31 @@ import ( // settings are changed and there is no INSERT round trip per fixture row. func copyTestRows(t testing.TB, db *sql.DB, table string, columns []string, n int, row func(int) []any) { t.Helper() + if testBackend(t) == dbconfig.SQLite { + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + slots := make([]string, len(columns)) + for i := range slots { + slots[i] = dbconfig.SQLite.Parameter(i + 1) + } + stmt, err := tx.Prepare("INSERT INTO " + table + "(" + strings.Join(columns, ",") + ") VALUES(" + strings.Join(slots, ",") + ")") + if err != nil { + t.Fatal(err) + } + defer stmt.Close() + for i := 0; i < n; i++ { + if _, err = stmt.Exec(row(i)...); err != nil { + t.Fatal(err) + } + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + return + } ctx := context.Background() conn, err := db.Conn(ctx) if err != nil { @@ -39,10 +69,22 @@ func copyTestRows(t testing.TB, db *sql.DB, table string, columns []string, n in func postgresTestDSN(t testing.TB) string { t.Helper() + if testBackend(t) == dbconfig.SQLite { + return filepath.Join(t.TempDir(), "fixture.db") + } return pgtest.NewSchema(t) } -func openFixtureSQL(dsn string) (*sql.DB, error) { return pgutil.Open(dsn, false) } +func openFixtureSQL(dsn string) (*sql.DB, error) { + if strings.HasPrefix(dsn, "postgres://") || strings.HasPrefix(dsn, "postgresql://") { + return pgutil.Open(dsn, false) + } + uri, err := dbconfig.SQLiteURI(dsn, url.Values{"_journal_mode": {"WAL"}, "_synchronous": {"FULL"}, "_busy_timeout": {"5000"}}) + if err != nil { + return nil, err + } + return sql.Open("sqlite3", uri) +} func openPostgresTestDB(t testing.TB) (*sql.DB, error) { t.Helper() @@ -55,10 +97,21 @@ func openPostgresTestDB(t testing.TB) (*sql.DB, error) { func openFixtureReader(t testing.TB, dsn string) (*DB, error) { t.Helper() + if testBackend(t) == dbconfig.SQLite { + return OpenDB(dsn) + } return OpenDB(pgtest.ReadOnly(t, dsn)) } func restorePostgresTestBackup(t testing.TB, path string) *sql.DB { + if testBackend(t) == dbconfig.SQLite { + db, err := openFixtureSQL(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + return db + } t.Helper() dsn := pgtest.NewDatabase(t) config, err := pgutil.ParseConfig(dsn) @@ -83,3 +136,24 @@ func restorePostgresTestBackup(t testing.TB, path string) *sql.DB { t.Cleanup(func() { db.Close() }) return db } + +func testDatabaseDSN(t testing.TB) string { + t.Helper() + if testBackend(t) == dbconfig.SQLite { + return filepath.Join(t.TempDir(), "fixture.db") + } + return pgtest.NewDatabase(t) +} +func postgresOnly(t testing.TB) { + t.Helper() + if testBackend(t) != dbconfig.Postgres { + t.Skip("PostgreSQL matrix not selected") + } +} +func applyTestSchema(t testing.TB, db *sql.DB) error { + t.Helper() + if testBackend(t) == dbconfig.SQLite { + return dbschema.ApplySQLite(db, nil) + } + return dbschema.ApplyPostgres(db, nil) +} diff --git a/cmd/server/postgres_stats.go b/cmd/server/postgres_stats.go index 2cc68ad29..3d19b7b36 100644 --- a/cmd/server/postgres_stats.go +++ b/cmd/server/postgres_stats.go @@ -2,6 +2,9 @@ package main import ( "context" + "errors" + "github.com/meshcore-analyzer/dbconfig" + "os" "time" ) @@ -10,7 +13,15 @@ const postgresBlockCacheRateSQL = `blks_hit::double precision / NULLIF(blks_hit: // Storage traversal and full row counts are diagnostic samples, independent of // ingest and PacketStore invalidation. Keep one bounded immutable sample per DB. +type sqliteDatabaseSample struct { + WalSize int64 + PageCount, PageSize, CacheSize, FreelistCount int64 + JournalMode string + PlannerStats bool +} + type postgresDatabaseSample struct { + SQLite *sqliteDatabaseSample DatabaseBytes int64 CacheHitRate *float64 Rows DatabaseRowCounts @@ -44,12 +55,40 @@ func (db *DB) postgresStats(ctx context.Context) (postgresDatabaseSample, error) queryCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() var sample postgresDatabaseSample - err := db.conn.QueryRowContext(queryCtx, `SELECT pg_database_size(current_database()), `+postgresBlockCacheRateSQL+`, + + var err error + if db.Backend() == dbconfig.SQLite { + native := &sqliteDatabaseSample{} + err = db.conn.QueryRowContext(queryCtx, `SELECT + (SELECT page_count FROM pragma_page_count),(SELECT page_size FROM pragma_page_size), + (SELECT cache_size FROM pragma_cache_size),(SELECT freelist_count FROM pragma_freelist_count), + (SELECT journal_mode FROM pragma_journal_mode),EXISTS(SELECT 1 FROM sqlite_master WHERE name='sqlite_stat1'), + (SELECT COUNT(*) FROM transmissions),(SELECT COUNT(*) FROM observations),(SELECT COUNT(*) FROM nodes),(SELECT COUNT(*) FROM observers)`).Scan( + &native.PageCount, &native.PageSize, &native.CacheSize, &native.FreelistCount, &native.JournalMode, &native.PlannerStats, + &sample.Rows.Transmissions, &sample.Rows.Observations, &sample.Rows.Nodes, &sample.Rows.Observers) + if err == nil { + var info os.FileInfo + info, err = os.Stat(db.path) + if err == nil { + sample.DatabaseBytes = info.Size() + } + } + if err == nil { + if info, statErr := os.Stat(db.path + "-wal"); statErr == nil { + native.WalSize = info.Size() + } else if !errors.Is(statErr, os.ErrNotExist) { + err = statErr + } + } + sample.SQLite = native + } else { + err = db.conn.QueryRowContext(queryCtx, `SELECT pg_database_size(current_database()), `+postgresBlockCacheRateSQL+`, (SELECT COUNT(*) FROM transmissions), (SELECT COUNT(*) FROM observations), (SELECT COUNT(*) FROM nodes), (SELECT COUNT(*) FROM observers) FROM pg_catalog.pg_stat_database WHERE datname=current_database()`).Scan( - &sample.DatabaseBytes, &sample.CacheHitRate, &sample.Rows.Transmissions, - &sample.Rows.Observations, &sample.Rows.Nodes, &sample.Rows.Observers) + &sample.DatabaseBytes, &sample.CacheHitRate, &sample.Rows.Transmissions, + &sample.Rows.Observations, &sample.Rows.Nodes, &sample.Rows.Observers) + } if err != nil { return nil, err } diff --git a/cmd/server/postgres_stats_test.go b/cmd/server/postgres_stats_test.go index 701b946d9..428df39a0 100644 --- a/cmd/server/postgres_stats_test.go +++ b/cmd/server/postgres_stats_test.go @@ -37,6 +37,7 @@ func TestPostgresInitialDiagnosticFailureIsExplicit(t *testing.T) { } func TestPostgresBlockCacheRatioNullAndZero(t *testing.T) { + postgresOnly(t) db := setupTestDB(t) nineTenths, zero, one := 0.9, 0.0, 1.0 for _, tc := range []struct { diff --git a/cmd/server/postgres_test.go b/cmd/server/postgres_test.go index 7aed7edb5..5d671f783 100644 --- a/cmd/server/postgres_test.go +++ b/cmd/server/postgres_test.go @@ -8,12 +8,14 @@ import ( "strings" "testing" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/dbschema" "github.com/meshcore-analyzer/pgutil" "github.com/meshcore-analyzer/pgutil/pgtest" ) func TestPostgresCursorStopsAtLimitWithoutSorting(t *testing.T) { + postgresOnly(t) db := setupTestDB(t) copyTestRows(t, db.conn, "transmissions", []string{"id", "raw_hex", "hash", "first_seen"}, 20000, func(i int) []any { return []any{i + 1, "AA", fmt.Sprintf("cursor-%d", i), "2026-01-01T00:00:00Z"} @@ -75,6 +77,7 @@ func TestPostgresNegativeOffsetsPreserveFirstPage(t *testing.T) { } func TestPostgresCanonicalSchemaAPIs(t *testing.T) { + postgresOnly(t) dsn := pgtest.NewSchema(t) writer, err := openFixtureSQL(dsn) if err != nil { @@ -84,7 +87,7 @@ func TestPostgresCanonicalSchemaAPIs(t *testing.T) { if err := dbschema.Apply(writer, nil); err != nil { t.Fatal(err) } - seedTestData(t, &DB{conn: writer}) + seedTestData(t, &DB{backend: testBackendValue(), conn: writer}) if _, err := writer.Exec(`UPDATE transmissions SET last_seen=EXTRACT(EPOCH FROM first_seen::timestamptz)::bigint`); err != nil { t.Fatal(err) } @@ -127,9 +130,9 @@ func TestPostgresCanonicalSchemaAPIs(t *testing.T) { } } -func TestPostgresOpenRejectsLegacyFileWithoutLeakingInput(t *testing.T) { +func TestPostgresSelectionCannotOpenLegacyFileWithoutURL(t *testing.T) { legacy := filepath.Join(t.TempDir(), "private-measurements.db") - db, err := OpenDB(legacy) + db, err := OpenStorage(dbconfig.Storage{Backend: dbconfig.Postgres, DBPath: legacy}) if db != nil { db.Close() } @@ -142,6 +145,7 @@ func TestPostgresOpenRejectsLegacyFileWithoutLeakingInput(t *testing.T) { } func TestPostgresReaderRejectsWriterCredential(t *testing.T) { + postgresOnly(t) db, err := OpenDB(pgtest.NewSchema(t)) if db != nil { db.Close() @@ -152,6 +156,7 @@ func TestPostgresReaderRejectsWriterCredential(t *testing.T) { } func TestPostgresScopeAuditSnapshotKeepsConcurrentCommitOut(t *testing.T) { + postgresOnly(t) dsn := pgtest.NewSchema(t) writer, err := openFixtureSQL(dsn) if err != nil { @@ -169,7 +174,7 @@ func TestPostgresScopeAuditSnapshotKeepsConcurrentCommitOut(t *testing.T) { t.Fatal(err) } defer conn.Close() - db := &DB{conn: conn} + db := &DB{backend: testBackendValue(), conn: conn} tx, err := db.beginScopeAuditSnapshot(context.Background()) if err != nil { t.Fatal(err) @@ -216,17 +221,22 @@ func TestPostgresScopeAuditSnapshotKeepsConcurrentCommitOut(t *testing.T) { } func TestPostgresReaderUsesRestrictedRole(t *testing.T) { + postgresOnly(t) dsn := pgtest.NewSchema(t) writer, err := pgutil.Open(dsn, false) if err != nil { t.Fatal(err) } defer writer.Close() - _, err = writer.Exec(`CREATE TABLE transmissions (id BIGINT PRIMARY KEY, hash TEXT); + _, err = writer.Exec(testNativeSQL(`CREATE TABLE transmissions (id BIGINT PRIMARY KEY, hash TEXT); + CREATE TABLE observations (id BIGINT PRIMARY KEY, timestamp BIGINT, observer_idx BIGINT); + CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, last_seen TEXT); + CREATE TABLE observers (id TEXT, inactive INTEGER); + INSERT INTO transmissions VALUES (7, 'stable-hash')`, `CREATE TABLE transmissions (id BIGINT PRIMARY KEY, hash TEXT); CREATE TABLE observations (id BIGINT PRIMARY KEY, timestamp BIGINT, observer_idx BIGINT); CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, last_seen TEXT); CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, inactive INTEGER); - INSERT INTO transmissions VALUES (7, 'stable-hash')`) + INSERT INTO transmissions VALUES (7, 'stable-hash')`)) if err != nil { t.Fatal(err) } diff --git a/cmd/server/postgres_value_parity_test.go b/cmd/server/postgres_value_parity_test.go index c70613539..11d01d04a 100644 --- a/cmd/server/postgres_value_parity_test.go +++ b/cmd/server/postgres_value_parity_test.go @@ -9,7 +9,6 @@ import ( "time" "github.com/gorilla/mux" - "github.com/meshcore-analyzer/dbschema" ) func openPostgresValueFixture(t *testing.T) (*DB, *sql.DB) { @@ -20,7 +19,7 @@ func openPostgresValueFixture(t *testing.T) (*DB, *sql.DB) { t.Fatal(err) } t.Cleanup(func() { writer.Close() }) - if err := dbschema.Apply(writer, nil); err != nil { + if err := applyTestSchema(t, writer); err != nil { t.Fatal(err) } reader, err := openFixtureReader(t, dsn) diff --git a/cmd/server/preparable_schema_test.go b/cmd/server/preparable_schema_test.go index e9b424c94..1f75f09a5 100644 --- a/cmd/server/preparable_schema_test.go +++ b/cmd/server/preparable_schema_test.go @@ -2,6 +2,7 @@ package main import ( "database/sql" + "github.com/meshcore-analyzer/dbschema/legacy" "testing" ) @@ -9,10 +10,10 @@ import ( // ensurePreparable supplies the minimum surface required by OpenDB's prepared // statements. It never creates or modifies a runtime database. var preparableTables = []string{ - `CREATE TABLE IF NOT EXISTS transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`, - `CREATE TABLE IF NOT EXISTS observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, timestamp TEXT)`, + testNativeSQL(`CREATE TABLE IF NOT EXISTS transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT)`, `CREATE TABLE IF NOT EXISTS transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`), + testNativeSQL(`CREATE TABLE IF NOT EXISTS observations (id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp TEXT)`, `CREATE TABLE IF NOT EXISTS observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, timestamp TEXT)`), `CREATE TABLE IF NOT EXISTS nodes (public_key TEXT PRIMARY KEY)`, - `CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT)`, + testNativeSQL(`CREATE TABLE IF NOT EXISTS observers (id TEXT)`, `CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT)`), } // preparableColumns are added when absent. A fixture that already declared the @@ -36,7 +37,18 @@ func ensurePreparable(tb testing.TB, conn *sql.DB) { } } for _, c := range preparableColumns { - _, err := conn.Exec("ALTER TABLE " + c.table + " ADD COLUMN IF NOT EXISTS " + c.column + " " + c.decl) + var err error + if testBackendValue() == "sqlite" { + exists, e := legacy.TableHasColumn(conn, c.table, c.column) + if e != nil { + tb.Fatal(e) + } + if !exists { + _, err = conn.Exec("ALTER TABLE " + c.table + " ADD COLUMN " + c.column + " " + c.decl) + } + } else { + _, err = conn.Exec("ALTER TABLE " + c.table + " ADD COLUMN IF NOT EXISTS " + c.column + " " + c.decl) + } // "duplicate column name" means the fixture already declared it. if err != nil { tb.Fatalf("ensurePreparable: add %s.%s: %v", c.table, c.column, err) @@ -59,7 +71,7 @@ func TestEnsurePreparableMatchesPrepareStatements(t *testing.T) { conn.SetMaxOpenConns(1) ensurePreparable(t, conn) - db := &DB{conn: conn} + db := &DB{backend: testBackendValue(), conn: conn} if err := db.prepareStatements(); err != nil { t.Fatalf("prepareStatements against the minimal preparable schema: %v\n"+ "Add the missing table/column to preparableTables or preparableColumns.", err) diff --git a/cmd/server/proposal_handlers_test.go b/cmd/server/proposal_handlers_test.go index 4e68742af..b4aeee46b 100644 --- a/cmd/server/proposal_handlers_test.go +++ b/cmd/server/proposal_handlers_test.go @@ -10,7 +10,6 @@ import ( "time" "github.com/gorilla/mux" - "github.com/meshcore-analyzer/pgutil/pgtest" "github.com/meshcore-analyzer/users" ) @@ -27,7 +26,7 @@ func newProposalFixture(t *testing.T, ps proposalSettings) *authFixture { func newProposalFixtureWithConfig(t *testing.T, ps proposalSettings, cfg *Config) *authFixture { t.Helper() - a, fake, ownerURL := newTestAuthServiceWithURL(t, pgtest.NewSchema(t), "admin@example.org") + a, fake, ownerURL := newTestAuthServiceWithURL(t, postgresTestDSN(t), "admin@example.org") a.set.proposals = ps srv := &Server{cfg: cfg, perfStats: NewPerfStats(), auth: a} r := mux.NewRouter() diff --git a/cmd/server/readonly_invariant_test.go b/cmd/server/readonly_invariant_test.go index 2df7104f3..3effbc33e 100644 --- a/cmd/server/readonly_invariant_test.go +++ b/cmd/server/readonly_invariant_test.go @@ -254,8 +254,8 @@ func TestUsersOpenIsTheOnlyServerWritePath(t *testing.T) { if err != nil { t.Fatal(err) } - call := regexp.MustCompile(`users\.Open\(`) - want := regexp.MustCompile(`users\.Open\(set\.databaseURL, measurementDatabaseURL\)`) + call := regexp.MustCompile(`users\.OpenStorage\(`) + want := regexp.MustCompile(`users\.OpenStorage\(storage\)`) calls, good := 0, 0 for _, e := range entries { name := e.Name() diff --git a/cmd/server/resolved_index.go b/cmd/server/resolved_index.go index af21be065..98accfc8a 100644 --- a/cmd/server/resolved_index.go +++ b/cmd/server/resolved_index.go @@ -158,7 +158,7 @@ func (s *PacketStore) confirmResolvedPathContains(txID int, pubkey string) bool needle := `"` + strings.ToLower(pubkey) + `"` var count int err := s.db.conn.QueryRow( - `SELECT COUNT(*) FROM observations WHERE transmission_id = $1 AND strpos(LOWER(resolved_path), $2) > 0`, + `SELECT COUNT(*) FROM observations WHERE transmission_id = `+s.db.parameter(1)+` AND `+s.db.nativeSQL("instr", "strpos")+`(LOWER(resolved_path), `+s.db.parameter(2)+`) > 0`, txID, needle, ).Scan(&count) if err != nil { @@ -173,8 +173,9 @@ func (s *PacketStore) fetchResolvedPathsForTx(txID int) map[int][]*string { if s.db == nil || s.db.conn == nil { return nil } + s.beforeCacheLoad("resolvedPath") rows, err := s.db.conn.Query( - `SELECT id, resolved_path FROM observations WHERE transmission_id = $1 AND resolved_path IS NOT NULL`, + `SELECT id, resolved_path FROM observations WHERE transmission_id = `+s.db.parameter(1)+` AND resolved_path IS NOT NULL`, txID, ) if err != nil { @@ -213,9 +214,10 @@ func (s *PacketStore) fetchResolvedPathForObs(obsID int) []*string { } s.lruMu.RUnlock() + s.beforeCacheLoad("resolvedPath") var rpJSON sql.NullString err := s.db.conn.QueryRow( - `SELECT resolved_path FROM observations WHERE id = $1`, obsID, + `SELECT resolved_path FROM observations WHERE id = `+s.db.parameter(1), obsID, ).Scan(&rpJSON) if err != nil || !rpJSON.Valid { return nil @@ -230,7 +232,26 @@ func (s *PacketStore) fetchResolvedPathForObs(obsID int) []*string { return rp } -// fetchResolvedPathForTxBest returns the best observation's resolved_path for a tx. +// rpObs is the part of an observation that the best-resolved-path pick +// needs, copied while s.mu is held. The path is kept as its JSON string and +// parsed by bestResolvedPath, so the parse runs after s.mu is released. +type rpObs struct { + id int + pathJSON string +} + +// snapshotRPObs copies what bestResolvedPath needs from tx. Caller holds s.mu. +func snapshotRPObs(tx *StoreTx) []rpObs { + out := make([]rpObs, len(tx.Observations)) + for i, o := range tx.Observations { + out[i] = rpObs{id: o.ID, pathJSON: o.PathJSON} + } + return out +} + +// bestResolvedPath returns the best observation's resolved_path for a tx, +// given a snapshot of its observations (snapshotRPObs), so it runs without +// s.mu. // // "Best" = the longest path_json among observations that actually have a stored // resolved_path. Earlier versions picked the longest-path obs unconditionally @@ -239,41 +260,43 @@ func (s *PacketStore) fetchResolvedPathForObs(obsID int) []*string { // callers (e.g. /api/nodes/{pk}/health.recentPackets) lost the field. Fixes // #810 by checking all observations and falling back to the longest sibling // that has a stored path. -func (s *PacketStore) fetchResolvedPathForTxBest(tx *StoreTx) []*string { - if tx == nil || len(tx.Observations) == 0 { +func (s *PacketStore) bestResolvedPath(txID int, observations []rpObs) []*string { + if len(observations) == 0 { return nil } + lens := make([]int, len(observations)) + for i, obs := range observations { + lens[i] = pathLen(obs.pathJSON) + } // Fast path: try the longest-path obs first via the LRU/SQL helper. - longest := tx.Observations[0] - longestLen := pathLen(longest.PathJSON) - for _, obs := range tx.Observations[1:] { - if l := pathLen(obs.PathJSON); l > longestLen { - longest = obs - longestLen = l + longest := 0 + for i := 1; i < len(observations); i++ { + if lens[i] > lens[longest] { + longest = i } } - if rp := s.fetchResolvedPathForObs(longest.ID); rp != nil { + if rp := s.fetchResolvedPathForObs(observations[longest].id); rp != nil { return rp } // Fallback: longest-path obs has no stored resolved_path. Query all // observations for this tx and pick the one with the longest path_json // that actually has a stored resolved_path. - rpMap := s.fetchResolvedPathsForTx(tx.ID) + rpMap := s.fetchResolvedPathsForTx(txID) if len(rpMap) == 0 { return nil } var bestRP []*string bestObsID := 0 bestLen := -1 - for _, obs := range tx.Observations { - rp, ok := rpMap[obs.ID] + for i, obs := range observations { + rp, ok := rpMap[obs.id] if !ok || rp == nil { continue } - if l := pathLen(obs.PathJSON); l > bestLen { - bestLen = l + if lens[i] > bestLen { + bestLen = lens[i] bestRP = rp - bestObsID = obs.ID + bestObsID = obs.id } } // Populate LRU so repeat lookups for this tx don't re-issue the multi-row @@ -286,6 +309,43 @@ func (s *PacketStore) fetchResolvedPathForTxBest(tx *StoreTx) []*string { return bestRP } +// rpJob is one resolved_path lookup for a response map. Code holding s.mu +// collects jobs; applyResolvedPaths runs them after s.mu is released, so the +// SQL never waits for a pool connection while the store lock is held (#2146). +type rpJob struct { + m map[string]interface{} + // forTx picks the best observation of transmission txID from txObs; + // otherwise the job looks up observation obsID. + forTx bool + txID int + txObs []rpObs + obsID int +} + +func txRPJob(m map[string]interface{}, tx *StoreTx) rpJob { + return rpJob{m: m, forTx: true, txID: tx.ID, txObs: snapshotRPObs(tx)} +} + +func obsRPJob(m map[string]interface{}, obsID int) rpJob { + return rpJob{m: m, obsID: obsID} +} + +// applyResolvedPaths sets "resolved_path" on each job's map. Must be called +// without s.mu held. +func (s *PacketStore) applyResolvedPaths(jobs []rpJob) { + for _, j := range jobs { + var rp []*string + if j.forTx { + rp = s.bestResolvedPath(j.txID, j.txObs) + } else { + rp = s.fetchResolvedPathForObs(j.obsID) + } + if rp != nil { + j.m["resolved_path"] = rp + } + } +} + // --- Simple LRU cache for resolved paths --- const lruMaxSize = 10000 @@ -357,7 +417,7 @@ func (s *PacketStore) resolvedPubkeysForEvictionBatch(txIDs []int) map[int][]str args[i] = id } query := "SELECT transmission_id, resolved_path FROM observations WHERE transmission_id IN (" + - sqlPlaceholders(len(chunk)) + ") AND resolved_path IS NOT NULL" + s.db.sqlPlaceholders(len(chunk)) + ") AND resolved_path IS NOT NULL" rows, err := s.db.conn.Query(query, args...) if err != nil { diff --git a/cmd/server/resolved_index_test.go b/cmd/server/resolved_index_test.go index dbde326aa..5cf895412 100644 --- a/cmd/server/resolved_index_test.go +++ b/cmd/server/resolved_index_test.go @@ -1099,11 +1099,15 @@ func TestConfirmResolvedPathContains_SpecialChars(t *testing.T) { } defer db.Close() - _, err = db.Exec(`CREATE TABLE observations ( + _, err = db.Exec(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER, + resolved_path TEXT + )`, `CREATE TABLE observations ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, resolved_path TEXT - )`) + )`)) if err != nil { t.Fatal(err) } @@ -1116,7 +1120,7 @@ func TestConfirmResolvedPathContains_SpecialChars(t *testing.T) { } store := &PacketStore{ - db: &DB{conn: db}, + db: &DB{backend: testBackendValue(), conn: db}, } // Normal match should work diff --git a/cmd/server/resolved_path_unlocked_test.go b/cmd/server/resolved_path_unlocked_test.go new file mode 100644 index 000000000..2f332b104 --- /dev/null +++ b/cmd/server/resolved_path_unlocked_test.go @@ -0,0 +1,133 @@ +package main + +import ( + "testing" + "time" +) + +// #2146: resolved_path is fetched from SQLite on demand. The store functions +// that build response maps under s.mu must run that SQL only after releasing +// it, so a slow pool connection never holds the store lock. + +// newResolvedPathStore loads one transmission whose shorter-path observation +// carries a resolved_path (the #810 fixture), so both the single-observation +// lookup and the best-of-transmission fallback run SQL. +func newResolvedPathStore(t *testing.T) *PacketStore { + t.Helper() + db := setupTestDB(t) + t.Cleanup(func() { db.Close() }) + exec := func(query string, args ...any) { + t.Helper() + if _, err := db.conn.Exec(query, args...); err != nil { + t.Fatalf("seed resolved path fixture: %v", err) + } + } + now := time.Now().UTC() + recent := now.Add(-1 * time.Hour).Format(time.RFC3339) + recentEpoch := now.Add(-1 * time.Hour).Unix() + exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs1','O1',`+db.parameter(1)+`, '2026-01-01T00:00:00Z', 100)`, recent) + exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs2','O2',`+db.parameter(1)+`, '2026-01-01T00:00:00Z', 100)`, recent) + exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ('aabbccdd11223344','R','repeater',`+db.parameter(1)+`, '2026-01-01T00:00:00Z', 1)`, recent) + exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES ('AABB','testhash00000001',`+db.parameter(1)+`,1,4,'{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, recent) + exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (1,1,12.5,-90,'["aa","bb","cc"]',`+db.parameter(1)+`)`, recentEpoch) + exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) VALUES (1,2,8.0,-95,'["aa","bb"]',`+db.parameter(1)+`,'["aabbccdd11223344","eeff00112233aabb"]')`, recentEpoch-100) + + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatal(err) + } + return store +} + +// expectNoStoreLockDuringResolvedPathSQL fails the test when a resolved_path +// query runs while the calling goroutine holds s.mu. Background goroutines +// started by Load can hold s.mu briefly, so TryLock is retried: it keeps +// failing for the whole window only when the caller itself holds the lock. +func expectNoStoreLockDuringResolvedPathSQL(t *testing.T, s *PacketStore, caseName *string) (queries *int) { + t.Helper() + queries = new(int) + s.cacheLoadHook = func(kind string) { + if kind != "resolvedPath" { + return + } + *queries++ + for deadline := time.Now().Add(time.Second); time.Now().Before(deadline); time.Sleep(time.Millisecond) { + if s.mu.TryLock() { + s.mu.Unlock() + return + } + } + t.Errorf("%s: resolved_path SQL ran while s.mu was held", *caseName) + } + return queries +} + +func clearResolvedPathLRU(s *PacketStore) { + s.lruMu.Lock() + s.apiResolvedPathLRU = make(map[int][]*string) + s.lruOrder = nil + s.lruMu.Unlock() +} + +func TestResolvedPathSQLRunsWithoutStoreLock(t *testing.T) { + s := newResolvedPathStore(t) + var caseName string + queries := expectNoStoreLockDuringResolvedPathSQL(t, s, &caseName) + + s.mu.RLock() + tx := s.byHash["testhash00000001"] + s.mu.RUnlock() + + hasRP := func(m map[string]interface{}) bool { return m != nil && m["resolved_path"] != nil } + cases := []struct { + name string + run func() bool + }{ + {"QueryPackets", func() bool { + r := s.QueryPackets(PacketQuery{Limit: 10, ExpandObservations: true}) + return len(r.Packets) == 1 && hasRP(r.Packets[0]) + }}, + {"QueryMultiNodePackets", func() bool { + r := s.QueryMultiNodePackets([]string{"aabbccdd11223344"}, 10, 0, "DESC", "", "") + return len(r.Packets) == 1 && hasRP(r.Packets[0]) + }}, + {"GetTransmissionByID", func() bool { return hasRP(s.GetTransmissionByID(tx.ID)) }}, + {"GetPacketByHash", func() bool { return hasRP(s.GetPacketByHash("testhash00000001")) }}, + {"GetPacketByID", func() bool { + // the observation carrying the resolved_path + for _, o := range tx.Observations { + if m := s.GetPacketByID(o.ID); hasRP(m) { + return true + } + } + return false + }}, + {"GetObservationsForHash", func() bool { + for _, m := range s.GetObservationsForHash("testhash00000001") { + if hasRP(m) { + return true + } + } + return false + }}, + {"GetNodeHealth", func() bool { + h, err := s.GetNodeHealth("aabbccdd11223344") + if err != nil || h == nil { + return false + } + rp, _ := h["recentPackets"].([]map[string]interface{}) + return len(rp) == 1 && hasRP(rp[0]) + }}, + } + for _, c := range cases { + caseName = c.name + clearResolvedPathLRU(s) + before := *queries + if !c.run() { + t.Errorf("%s: want resolved_path in the response", c.name) + } + if *queries == before { + t.Errorf("%s: no resolved_path SQL ran, so the lock check proved nothing", c.name) + } + } +} diff --git a/cmd/server/rf_noise.go b/cmd/server/rf_noise.go index f166dbd0c..ac91a7c14 100644 --- a/cmd/server/rf_noise.go +++ b/cmd/server/rf_noise.go @@ -146,11 +146,11 @@ func (s *Server) requireClientRfSamples(w http.ResponseWriter, r *http.Request) // queryRfNoiseRows returns raw RF-environment samples within a bbox, over a // time window (days; 0 = all time). Read-only (server RO connection). func (s *Server) queryRfNoiseRows(days int, b bbox) ([]rfNoiseRow, error) { - where := []string{"lat BETWEEN $1 AND $2", "lon BETWEEN $3 AND $4", "noise_floor IS NOT NULL"} + where := []string{"lat BETWEEN " + s.db.parameter(1) + " AND " + s.db.parameter(2), "lon BETWEEN " + s.db.parameter(3) + " AND " + s.db.parameter(4), "noise_floor IS NOT NULL"} args := []interface{}{b.MinLat, b.MaxLat, b.MinLon, b.MaxLon} if days > 0 { since := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339) - where = append(where, "sampled_at >= $5") + where = append(where, "sampled_at >= "+s.db.parameter(5)) args = append(args, since) } rows, err := s.db.conn.Query(` diff --git a/cmd/server/rf_noise_test.go b/cmd/server/rf_noise_test.go index fc20f625b..6b1e5e486 100644 --- a/cmd/server/rf_noise_test.go +++ b/cmd/server/rf_noise_test.go @@ -187,13 +187,19 @@ func TestRfNoiseRouteGatedOn(t *testing.T) { func seedRfNoiseDB(t *testing.T) *DB { db := setupTestDBv2(t) - mustExecDB(t, db, `CREATE TABLE client_rf_samples ( + mustExecDB(t, db, testNativeSQL(`CREATE TABLE client_rf_samples ( + id INTEGER PRIMARY KEY AUTOINCREMENT, rx_pubkey TEXT, sampled_at TEXT, ingested_at TEXT, + lat DOUBLE PRECISION, lon DOUBLE PRECISION, pos_acc_m DOUBLE PRECISION, stationary INTEGER NOT NULL DEFAULT 0, + uptime_secs INTEGER, battery_mv INTEGER, queue_len INTEGER, errors INTEGER, + noise_floor INTEGER, last_rssi INTEGER, last_snr DOUBLE PRECISION, tx_air_secs INTEGER, + rx_air_secs INTEGER, recv INTEGER, sent INTEGER, flood_rx INTEGER, direct_rx INTEGER, + flood_tx INTEGER, direct_tx INTEGER, recv_errors INTEGER)`, `CREATE TABLE client_rf_samples ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, rx_pubkey TEXT, sampled_at TEXT, ingested_at TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, pos_acc_m DOUBLE PRECISION, stationary INTEGER NOT NULL DEFAULT 0, uptime_secs INTEGER, battery_mv INTEGER, queue_len INTEGER, errors INTEGER, noise_floor INTEGER, last_rssi INTEGER, last_snr DOUBLE PRECISION, tx_air_secs INTEGER, rx_air_secs INTEGER, recv INTEGER, sent INTEGER, flood_rx INTEGER, direct_rx INTEGER, - flood_tx INTEGER, direct_tx INTEGER, recv_errors INTEGER)`) + flood_tx INTEGER, direct_tx INTEGER, recv_errors INTEGER)`)) return db } diff --git a/cmd/server/routes.go b/cmd/server/routes.go index a86b94a50..289924fff 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -273,6 +273,7 @@ func (s *Server) RegisterRoutes(r *mux.Router) { r.HandleFunc("/api/scope-audit", s.handleScopeAudit).Methods("GET") // #1975 r.HandleFunc("/api/perf", s.handlePerf).Methods("GET") r.HandleFunc("/api/perf/io", s.handlePerfIO).Methods("GET") + r.HandleFunc("/api/perf/database", s.handlePerfPostgres).Methods("GET") r.HandleFunc("/api/perf/postgres", s.handlePerfPostgres).Methods("GET") r.HandleFunc("/api/perf/sqlite", s.handlePerfPostgres).Methods("GET") // legacy diagnostic route alias r.HandleFunc("/api/perf/write-sources", s.handlePerfWriteSources).Methods("GET") @@ -405,7 +406,7 @@ func (s *Server) perfMiddleware(next http.Handler) http.Handler { key := r.URL.Path if route := mux.CurrentRoute(r); route != nil { if tmpl, err := route.GetPathTemplate(); err == nil { - key = muxBraceParam.ReplaceAllString(tmpl, ":$1") + key = muxBraceParam.ReplaceAllString(tmpl, ":"+s.db.parameter(1)) } } if key == r.URL.Path { @@ -1031,21 +1032,27 @@ func (s *Server) handlePerf(w http.ResponseWriter, r *http.Request) { } // PostgreSQL diagnostics reuse the bounded database sample. - var postgresStats *PostgresStats + var postgresStats, sqliteStats, databaseStats *PostgresStats if s.db != nil { ss := s.db.GetDBSizeStatsTyped() - postgresStats = &ss + databaseStats = &ss + if ss.Engine == "sqlite" { + sqliteStats = &ss + } else { + postgresStats = &ss + } } writeJSON(w, PerfResponse{ - Uptime: uptimeSec, - TotalRequests: totalRequests, - AvgMs: safeAvg(totalMs, float64(totalRequests)), - Endpoints: summary, - SlowQueries: slowQueries, - Cache: perfCS, - PacketStore: pktStoreStats, - Postgres: postgresStats, + Uptime: uptimeSec, + TotalRequests: totalRequests, + AvgMs: safeAvg(totalMs, float64(totalRequests)), + Endpoints: summary, + SlowQueries: slowQueries, + Cache: perfCS, + PacketStore: pktStoreStats, + Postgres: postgresStats, + Sqlite: sqliteStats, Database: databaseStats, MemoryBreakdown: memBreakdown, MemoryBreakdownNote: breakdownNote, GoRuntime: func() *GoRuntimeStats { @@ -1949,12 +1956,23 @@ func (s *Server) handleNodePaths(w http.ResponseWriter, r *http.Request) { } candidates = filtered + // The canonical resolved_path lookup below runs without s.mu, so it reads + // a snapshot of each surviving candidate's observations instead of + // tx.Observations, which ingest appends to under the write lock. Taken in + // a short read lock of its own so only the survivors are copied. + s.store.mu.RLock() + rpSnapshots := make(map[int][]rpObs, len(candidates)) + for _, tx := range candidates { + rpSnapshots[tx.ID] = snapshotRPObs(tx) + } + s.store.mu.RUnlock() + // #1278: Read the CANONICAL persisted resolved_path for each surviving - // candidate OUTSIDE s.mu (fetchResolvedPathForTxBest takes lruMu; the + // candidate OUTSIDE s.mu (bestResolvedPath takes lruMu; the // lock-ordering contract forbids acquiring lruMu under s.mu). // // Option A from the issue: the packets page renders each tx via - // fetchResolvedPathForTxBest. For /api/nodes/{pk}/paths to stay + // bestResolvedPath. For /api/nodes/{pk}/paths to stay // CONSISTENT with the packets page, BOTH the containsTarget membership // decision AND the displayed hop names must come from that same // canonical resolved_path — not a re-resolution biased by passing the @@ -1965,7 +1983,7 @@ func (s *Server) handleNodePaths(w http.ResponseWriter, r *http.Request) { // there's no canonical answer to be consistent with. canonicalRP := make(map[int][]*string, len(candidates)) for _, tx := range candidates { - if rp := s.store.fetchResolvedPathForTxBest(tx); rp != nil { + if rp := s.store.bestResolvedPath(tx.ID, rpSnapshots[tx.ID]); rp != nil { canonicalRP[tx.ID] = rp } } diff --git a/cmd/server/routes_test.go b/cmd/server/routes_test.go index a89c04205..8c931373f 100644 --- a/cmd/server/routes_test.go +++ b/cmd/server/routes_test.go @@ -552,7 +552,7 @@ func TestPerfEndpoint(t *testing.T) { } // Verify sqlite stats - sqliteStats, ok := body["postgres"].(map[string]interface{}) + sqliteStats, ok := body[testNativeSQL("sqlite", "postgres")].(map[string]interface{}) if !ok { t.Fatal("expected postgres object in perf response") } @@ -2030,7 +2030,7 @@ func TestHandlerErrorPaths(t *testing.T) { srv.RegisterRoutes(router) t.Run("stats error", func(t *testing.T) { - if _, err := db.conn.Exec("DROP TABLE IF EXISTS transmissions CASCADE"); err != nil { + if _, err := db.conn.Exec(testNativeSQL("DROP TABLE IF EXISTS transmissions", "DROP TABLE IF EXISTS transmissions CASCADE")); err != nil { t.Fatal(err) } req := httptest.NewRequest("GET", "/api/stats", nil) @@ -2051,7 +2051,7 @@ func TestHandlerErrorChannels(t *testing.T) { router := mux.NewRouter() srv.RegisterRoutes(router) - if _, err := db.conn.Exec("DROP TABLE IF EXISTS transmissions CASCADE"); err != nil { + if _, err := db.conn.Exec(testNativeSQL("DROP TABLE IF EXISTS transmissions", "DROP TABLE IF EXISTS transmissions CASCADE")); err != nil { t.Fatal(err) } @@ -2149,7 +2149,7 @@ func TestHandlerErrorPackets(t *testing.T) { srv.RegisterRoutes(router) // Drop transmissions table to trigger error in transmission-centric query - if _, err := db.conn.Exec("DROP TABLE IF EXISTS transmissions CASCADE"); err != nil { + if _, err := db.conn.Exec(testNativeSQL("DROP TABLE IF EXISTS transmissions", "DROP TABLE IF EXISTS transmissions CASCADE")); err != nil { t.Fatal(err) } diff --git a/cmd/server/rx_coverage.go b/cmd/server/rx_coverage.go index 6d768fcba..5de325091 100644 --- a/cmd/server/rx_coverage.go +++ b/cmd/server/rx_coverage.go @@ -9,7 +9,6 @@ import ( "strings" "github.com/gorilla/mux" - "github.com/jackc/pgx/v5" ) // coverageRow is one raw reception read from client_receptions. @@ -261,14 +260,14 @@ func prefixOrEmpty(s string, n int) string { // sqlPlaceholders builds native PostgreSQL positional parameters for an IN-list. // The optional first index supports lists following other bound predicates. -func sqlPlaceholders(n int, first ...int) string { +func (db *DB) sqlPlaceholders(n int, first ...int) string { start := 1 if len(first) > 0 { start = first[0] } parts := make([]string, n) for i := range parts { - parts[i] = fmt.Sprintf("$%d", start+i) + parts[i] = fmt.Sprintf("%s", db.parameter(start+i)) } return strings.Join(parts, ",") } @@ -289,8 +288,8 @@ func (s *Server) queryCoverageRows(pubkey string, b bbox) ([]coverageRow, error) rows, err := s.db.conn.Query(` SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions - WHERE heard_key IN (`+sqlPlaceholders(len(cands))+`) - AND lat BETWEEN `+fmt.Sprintf("$%d AND $%d AND lon BETWEEN $%d AND $%d", len(cands)+1, len(cands)+2, len(cands)+3, len(cands)+4), append([]any{pgx.QueryExecModeExec}, args...)...) + WHERE heard_key IN (`+s.db.sqlPlaceholders(len(cands))+`) + AND lat BETWEEN `+fmt.Sprintf("%s AND %s AND lon BETWEEN %s AND %s", s.db.parameter(len(cands)+1), s.db.parameter(len(cands)+2), s.db.parameter(len(cands)+3), s.db.parameter(len(cands)+4)), s.db.planWithValues(args)...) if err != nil { return nil, err } @@ -311,7 +310,7 @@ func (s *Server) mobileRxStats(pubkey string) (count, clients int) { } s.db.conn.QueryRow(` SELECT COUNT(*), COUNT(DISTINCT rx_pubkey) FROM client_receptions - WHERE heard_key IN (`+sqlPlaceholders(len(cands))+`)`, args...).Scan(&count, &clients) + WHERE heard_key IN (`+s.db.sqlPlaceholders(len(cands))+`)`, args...).Scan(&count, &clients) return count, clients } diff --git a/cmd/server/rx_coverage_endpoint_test.go b/cmd/server/rx_coverage_endpoint_test.go index 818af940e..a2a65d2af 100644 --- a/cmd/server/rx_coverage_endpoint_test.go +++ b/cmd/server/rx_coverage_endpoint_test.go @@ -10,9 +10,11 @@ import ( func seedCoverageDB(t *testing.T) *DB { db := setupTestDBv2(t) - mustExecDB(t, db, `CREATE TABLE client_receptions ( + mustExecDB(t, db, testNativeSQL(`CREATE TABLE client_receptions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, rx_pubkey TEXT, heard_key TEXT, heard_keylen INTEGER, + rssi INTEGER, snr DOUBLE PRECISION, lat DOUBLE PRECISION, lon DOUBLE PRECISION, pos_acc_m DOUBLE PRECISION, rx_at TEXT, ingested_at TEXT, src TEXT)`, `CREATE TABLE client_receptions ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, rx_pubkey TEXT, heard_key TEXT, heard_keylen INTEGER, - rssi INTEGER, snr DOUBLE PRECISION, lat DOUBLE PRECISION, lon DOUBLE PRECISION, pos_acc_m DOUBLE PRECISION, rx_at TEXT, ingested_at TEXT, src TEXT)`) + rssi INTEGER, snr DOUBLE PRECISION, lat DOUBLE PRECISION, lon DOUBLE PRECISION, pos_acc_m DOUBLE PRECISION, rx_at TEXT, ingested_at TEXT, src TEXT)`)) mustExecDB(t, db, `CREATE TABLE client_observers (pubkey TEXT PRIMARY KEY, name TEXT, last_seen TEXT)`) return db } diff --git a/cmd/server/rx_coverage_postgres_plan_test.go b/cmd/server/rx_coverage_postgres_plan_test.go index 688c81fb7..3cd0cb8a9 100644 --- a/cmd/server/rx_coverage_postgres_plan_test.go +++ b/cmd/server/rx_coverage_postgres_plan_test.go @@ -10,21 +10,20 @@ import ( "sort" "testing" "time" - - "github.com/meshcore-analyzer/dbschema" ) // A generic plan estimates an unknown bbox as tiny. With a wide viewport it // instead visits all 128K receptions through the geo index, rejecting 127936 // rows after fetching them. The first five custom plans hide this regression. func TestCoverageQueriesKeepValueSpecificPlans(t *testing.T) { + postgresOnly(t) dsn := postgresTestDSN(t) writer, err := openFixtureSQL(dsn) if err != nil { t.Fatal(err) } defer writer.Close() - if err := dbschema.Apply(writer, nil); err != nil { + if err := applyTestSchema(t, writer); err != nil { t.Fatal(err) } keys := make([]string, 2000) diff --git a/cmd/server/rx_dashboard.go b/cmd/server/rx_dashboard.go index 9a1120e68..7da1bbfe4 100644 --- a/cmd/server/rx_dashboard.go +++ b/cmd/server/rx_dashboard.go @@ -11,8 +11,6 @@ import ( "strconv" "strings" "time" - - "github.com/jackc/pgx/v5" ) // scanCoverageRows reads (lat,lon,snr,rssi,heard_key,rx_at) rows into coverageRow values. @@ -102,7 +100,7 @@ func (s *Server) batchResolveHeardKeys(keys []string) map[string][2]string { // so this stays injection-safe regardless of how hexPrefixRe later // evolves. The per-prefix LIMIT 2 lives in a subquery because a bare // LIMIT on a UNION ALL term is a SQLite syntax error. - parts[j] = fmt.Sprintf("SELECT * FROM (SELECT $%d::text AS pfx, public_key, COALESCE(name,'') AS nm FROM nodes WHERE public_key LIKE $%d LIMIT 2) AS matched", len(args)+1, len(args)+2) + parts[j] = fmt.Sprintf(s.db.nativeSQL("SELECT * FROM (SELECT CAST(%s AS TEXT) AS pfx, public_key, COALESCE(name,'') AS nm FROM nodes WHERE public_key LIKE %s LIMIT 2) AS matched", "SELECT * FROM (SELECT %s::text AS pfx, public_key, COALESCE(name,'') AS nm FROM nodes WHERE public_key LIKE %s LIMIT 2) AS matched"), s.db.parameter(len(args)+1), s.db.parameter(len(args)+2)) args = append(args, k, k+"%") } rows, err := s.db.conn.Query(strings.Join(parts, " UNION ALL "), args...) @@ -161,28 +159,28 @@ func (s *Server) resolveHeardKey(heardKey string) (string, string) { // by heard node (prefix/pubkey), contributing client (rx_pubkey), and time window // (days; 0 = all time). Powers the global and per-observer coverage maps. func (s *Server) queryCoverageFiltered(node, rx string, days int, b bbox) ([]coverageRow, error) { - where := []string{"lat BETWEEN $1 AND $2", "lon BETWEEN $3 AND $4"} + where := []string{"lat BETWEEN " + s.db.parameter(1) + " AND " + s.db.parameter(2), "lon BETWEEN " + s.db.parameter(3) + " AND " + s.db.parameter(4)} args := []interface{}{b.MinLat, b.MaxLat, b.MinLon, b.MaxLon} if node != "" { // Sargable heard_key IN-list (see coverageHeardKeyCandidates) so the // (heard_key, …) composite index is used instead of a substr() scan (#5). cands := coverageHeardKeyCandidates(node) - where = append(where, "heard_key IN ("+sqlPlaceholders(len(cands), len(args)+1)+")") + where = append(where, "heard_key IN ("+s.db.sqlPlaceholders(len(cands), len(args)+1)+")") for _, c := range cands { args = append(args, c) } } if rx != "" { - where = append(where, fmt.Sprintf("rx_pubkey = $%d", len(args)+1)) + where = append(where, fmt.Sprintf("rx_pubkey = %s", s.db.parameter(len(args)+1))) args = append(args, strings.ToLower(rx)) } if days > 0 { since := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339) - where = append(where, fmt.Sprintf("rx_at >= $%d", len(args)+1)) + where = append(where, fmt.Sprintf("rx_at >= %s", s.db.parameter(len(args)+1))) args = append(args, since) } // As in queryCoverageRows, choose the plan using the actual bbox and filters. - rows, err := s.db.conn.Query("SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions WHERE "+strings.Join(where, " AND "), append([]any{pgx.QueryExecModeExec}, args...)...) + rows, err := s.db.conn.Query("SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions WHERE "+strings.Join(where, " AND "), s.db.planWithValues(args)...) if err != nil { return nil, err } @@ -279,9 +277,9 @@ func (s *Server) rxLeaderboard(ctx context.Context, days, limit int) ([]LeaderOb FROM client_receptions cr LEFT JOIN nodes n ON n.public_key = cr.rx_pubkey LEFT JOIN client_observers co ON co.pubkey = cr.rx_pubkey - WHERE cr.rx_at >= $1 + WHERE cr.rx_at >= `+s.db.parameter(1)+` ORDER BY cr.rx_at DESC NULLS LAST - LIMIT $2`, since, leaderboardScanCap) + LIMIT `+s.db.parameter(2), since, leaderboardScanCap) if err != nil { return nil, err } diff --git a/cmd/server/scope_audit.go b/cmd/server/scope_audit.go index b1fe61f8b..87ed49ef1 100644 --- a/cmd/server/scope_audit.go +++ b/cmd/server/scope_audit.go @@ -112,7 +112,7 @@ func (db *DB) declaredRegionsTablePresent() bool { return true } var n int - if err := db.conn.QueryRow(`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='node_declared_regions'`).Scan(&n); err != nil || n == 0 { + if err := db.conn.QueryRow(db.nativeSQL("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='node_declared_regions'", `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='node_declared_regions'`)).Scan(&n); err != nil || n == 0 { return false } db.declaredRegionsTableLate.Store(true) @@ -252,7 +252,7 @@ func (db *DB) scopeAuditNodeIdentities(pubkeys []string) map[string]scopeAuditNo placeholders := make([]string, len(pubkeys)) args := make([]interface{}, len(pubkeys)) for i, k := range pubkeys { - placeholders[i] = fmt.Sprintf("$%d", i+1) + placeholders[i] = fmt.Sprintf("%s", db.parameter(i+1)) args[i] = strings.ToLower(k) } rows, err := db.conn.Query( @@ -391,6 +391,16 @@ var scopeAuditForwarderScanQuery = ` AND o.path_json IS NOT NULL AND LENGTH(je.value) >= ` + fmt.Sprint(minForwarderHopHexLen) + ` ` +var sqliteScopeAuditForwarderScanQuery = ` + SELECT t.id, je.value + FROM transmissions t + JOIN observations o ON o.transmission_id = t.id + JOIN json_each(CASE WHEN json_valid(o.path_json) THEN CASE WHEN json_type(o.path_json)='array' THEN o.path_json ELSE '[]' END ELSE '[]' END) je + WHERE t.first_seen >= $1 + AND ` + scopeConformanceForwarderRouteTypesSQL + ` + AND o.path_json IS NOT NULL + AND LENGTH(je.value) >= ` + fmt.Sprint(minForwarderHopHexLen) + ` +` // scopeAuditWindowMetaQuery reads the two per-TRANSMISSION facts the hop scan // used to carry on every hop row: the scope name and the timestamp. It applies @@ -589,7 +599,7 @@ func (s *PacketStore) ScopeAuditForwarding(sinceISO string, targets []string) (m return nil, err } - rows, err := tx.Query(scopeAuditForwarderScanQuery, sinceISO) + rows, err := tx.Query(s.db.nativeSQL(sqliteScopeAuditForwarderScanQuery, scopeAuditForwarderScanQuery), sinceISO) if err != nil { return nil, fmt.Errorf("scope audit forwarder scan: %w", err) } diff --git a/cmd/server/scope_audit_test.go b/cmd/server/scope_audit_test.go index 504c537e6..2156fd0f4 100644 --- a/cmd/server/scope_audit_test.go +++ b/cmd/server/scope_audit_test.go @@ -35,7 +35,25 @@ func setupScopeConformanceDB(t *testing.T) *DB { t.Fatal(err) } conn.SetMaxOpenConns(1) - schema := ` + schema := testNativeSQL(` + CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_hex TEXT NOT NULL, + hash TEXT NOT NULL UNIQUE, + first_seen TEXT NOT NULL, + route_type INTEGER, + payload_type INTEGER, + code1 TEXT, + code2 TEXT, + scope_name TEXT + ); + CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER NOT NULL REFERENCES transmissions(id), + path_json TEXT, + timestamp INTEGER NOT NULL + ); + `, ` CREATE TABLE transmissions ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT NOT NULL, @@ -53,11 +71,11 @@ func setupScopeConformanceDB(t *testing.T) *DB { path_json TEXT, timestamp INTEGER NOT NULL ); - ` + `) if _, err := conn.Exec(schema); err != nil { t.Fatal(err) } - return &DB{conn: conn} + return &DB{backend: testBackendValue(), conn: conn} } // newScopeTestStore wires a *PacketStore to a freshly seeded scope-conformance @@ -727,7 +745,17 @@ func TestHandleScopeAuditConfigStateAllFourShapes(t *testing.T) { // would skip the table and every assertion below would pass vacuously. func createDeclaredRegionsTable(t *testing.T, srv *Server) { t.Helper() - if _, err := srv.db.conn.Exec(` + if _, err := srv.db.conn.Exec(testNativeSQL(` + CREATE TABLE node_declared_regions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + target TEXT NOT NULL, + rx_pubkey TEXT NOT NULL, + observed_at TEXT NOT NULL, + ingested_at TEXT NOT NULL, + regions_csv TEXT NOT NULL, + truncated INTEGER NOT NULL DEFAULT 0, + UNIQUE(target, rx_pubkey, observed_at) + )`, ` CREATE TABLE node_declared_regions ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, target TEXT NOT NULL, @@ -737,7 +765,7 @@ func createDeclaredRegionsTable(t *testing.T, srv *Server) { regions_csv TEXT NOT NULL, truncated INTEGER NOT NULL DEFAULT 0, UNIQUE(target, rx_pubkey, observed_at) - )`); err != nil { + )`)); err != nil { t.Fatal(err) } if err := srv.db.detectSchema(context.Background(), srv.db.conn); err != nil { @@ -759,7 +787,17 @@ func TestDeclaredRegionsTableCreatedAfterStartupIsRead(t *testing.T) { } // Created behind the server's back, with no detectSchema re-probe: exactly // what happens when the ingestor creates it after the server started. - if _, err := srv.db.conn.Exec(` + if _, err := srv.db.conn.Exec(testNativeSQL(` + CREATE TABLE node_declared_regions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + target TEXT NOT NULL, + rx_pubkey TEXT NOT NULL, + observed_at TEXT NOT NULL, + ingested_at TEXT NOT NULL, + regions_csv TEXT NOT NULL, + truncated INTEGER NOT NULL DEFAULT 0, + UNIQUE(target, rx_pubkey, observed_at) + )`, ` CREATE TABLE node_declared_regions ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, target TEXT NOT NULL, @@ -769,7 +807,7 @@ func TestDeclaredRegionsTableCreatedAfterStartupIsRead(t *testing.T) { regions_csv TEXT NOT NULL, truncated INTEGER NOT NULL DEFAULT 0, UNIQUE(target, rx_pubkey, observed_at) - )`); err != nil { + )`)); err != nil { t.Fatal(err) } if srv.db.hasDeclaredRegionsTable { diff --git a/cmd/server/scope_verify.go b/cmd/server/scope_verify.go index 834651573..5c7893353 100644 --- a/cmd/server/scope_verify.go +++ b/cmd/server/scope_verify.go @@ -184,11 +184,11 @@ func (s *PacketStore) unmatchedTransmissionsInWindow(sinceISO string) ([]unmatch rows, err := s.db.conn.Query(` SELECT t.id, t.raw_hex FROM transmissions t - WHERE t.first_seen >= $1 + WHERE t.first_seen >= `+s.db.parameter(1)+` AND t.scope_name = '' AND `+scopeConformanceForwarderRouteTypesSQL+` ORDER BY t.first_seen DESC - LIMIT $2`, sinceISO, scopeVerifyMaxWindowPackets) + LIMIT `+s.db.parameter(2), sinceISO, scopeVerifyMaxWindowPackets) if err != nil { return nil, false, fmt.Errorf("unmatched transmissions scan: %w", err) } diff --git a/cmd/server/state_dir.go b/cmd/server/state_dir.go index 0620a10e0..9e9f2c735 100644 --- a/cmd/server/state_dir.go +++ b/cmd/server/state_dir.go @@ -1,6 +1,9 @@ package main -import "path/filepath" +import ( + "github.com/meshcore-analyzer/dbconfig" + "path/filepath" +) // runtimeStateDir is set once before the server's background workers start. var runtimeStateDir = "data" @@ -11,5 +14,8 @@ func (db *DB) statePath() string { if dir == "" { dir = runtimeStateDir } + if db.Backend() == dbconfig.SQLite { + return filepath.Join(dir, filepath.Base(db.path)) + } return filepath.Join(dir, "meshcore") } diff --git a/cmd/server/storage_backend.go b/cmd/server/storage_backend.go new file mode 100644 index 000000000..20b3ec6a6 --- /dev/null +++ b/cmd/server/storage_backend.go @@ -0,0 +1,117 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "net/url" + "path/filepath" + "strings" + "time" + + "github.com/jackc/pgx/v5" + _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" + "github.com/meshcore-analyzer/pgutil" +) + +// Backend is selected before opening a connection, never after an open fails. +func (db *DB) Backend() dbconfig.Backend { + if db == nil || db.backend == "" { + return dbconfig.SQLite + } + return db.backend +} +func (db *DB) parameter(n int) string { return db.Backend().Parameter(n) } +func (db *DB) nativeSQL(sqlite, postgres string) string { + if db.Backend() == dbconfig.SQLite { + return sqlite + } + return postgres +} + +// OpenStorage opens only the selected reader. The application performs the +// selected schema/import readiness check before starting its HTTP listener. +func OpenStorage(storage dbconfig.Storage) (*DB, error) { + var conn *sql.DB + var err error + path := storage.DBPath + switch storage.Backend { + case dbconfig.SQLite: + if strings.Contains(path, "://") || strings.HasPrefix(path, "postgres:") || strings.HasPrefix(path, "postgresql:") { + return nil, fmt.Errorf("SQLite telemetry requires a filesystem path") + } + var dsn string + dsn, err = dbconfig.SQLiteURI(path, url.Values{"mode": {"ro"}, "_query_only": {"on"}, "_busy_timeout": {"5000"}, "_cache_size": {"-2000"}}) + if err == nil { + conn, err = sql.Open("sqlite3", dsn) + } + case dbconfig.Postgres: + path = storage.ReaderDatabaseURL + conn, err = pgutil.Open(path, true) + default: + return nil, fmt.Errorf("telemetry requires an explicitly selected storage backend") + } + if err != nil { + return nil, err + } + conn.SetMaxOpenConns(4) + conn.SetMaxIdleConns(4) + if storage.Backend == dbconfig.Postgres { + if err = pgutil.AssertReadOnly(conn); err != nil { + conn.Close() + return nil, err + } + } + state := storage.StateDir + if state == "" { + if storage.Backend == dbconfig.SQLite { + state = filepath.Dir(path) + } else { + state = "data" + } + } + d := &DB{conn: conn, path: path, stateDir: state, backend: storage.Backend} + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + sc, err := conn.Conn(ctx) + if err != nil { + conn.Close() + return nil, fmt.Errorf("schema detection: acquire connection: %w", err) + } + err = d.detectSchema(ctx, sc) + _ = sc.Close() + if err != nil { + conn.Close() + return nil, fmt.Errorf("schema detection failed: %w", err) + } + if err = d.prepareStatements(); err != nil { + conn.Close() + return nil, fmt.Errorf("prepare statements: %w", err) + } + return d, nil +} + +func (db *DB) tableHasColumn(table, column string) (bool, error) { + if db.Backend() == dbconfig.SQLite { + return legacy.TableHasColumn(db.conn, table, column) + } + return dbschema.TableHasColumn(db.conn, table, column) +} + +// Only the PostgreSQL driver understands this extended-protocol query option. +func (db *DB) planWithValues(args []any) []any { + if db.Backend() == dbconfig.Postgres { + return append([]any{pgx.QueryExecModeExec}, args...) + } + return args +} + +func (db *DB) AssertReady() error { + if db.Backend() == dbconfig.SQLite { + return dbschema.AssertSQLiteReady(db.conn) + } + return dbschema.AssertPostgresReady(db.conn) +} diff --git a/cmd/server/storage_backend_test.go b/cmd/server/storage_backend_test.go new file mode 100644 index 000000000..645ea18a3 --- /dev/null +++ b/cmd/server/storage_backend_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "database/sql" + "errors" + "github.com/meshcore-analyzer/dbconfig" + "os" + "path/filepath" + "testing" + + _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" +) + +func TestOpenDBNativeSQLiteWithoutPostgres(t *testing.T) { + t.Setenv("CORESCOPE_TEST_POSTGRES_URL", "") + path := filepath.Join(t.TempDir(), "telemetry.db") + dsn, err := legacy.WriterDSN(path) + if err != nil { + t.Fatal(err) + } + owner, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal(err) + } + defer owner.Close() + if err := dbschema.ApplySQLite(owner, nil); err != nil { + t.Fatal(err) + } + reader, err := OpenDB(path) + if err != nil { + t.Fatalf("native SQLite reader must work without PostgreSQL: %v", err) + } + defer reader.Close() + if _, err := reader.conn.Exec("CREATE TABLE forbidden_write(id INTEGER)"); err == nil { + t.Fatal("reader acquired write access") + } + var count int + if err := reader.conn.QueryRow("SELECT count(*) FROM transmissions").Scan(&count); err != nil || count != 0 { + t.Fatalf("reader count=%d err=%v", count, err) + } + if _, err := owner.Exec(`CREATE TABLE ` + dbconfig.SQLiteImportPendingTable + `(phase TEXT)`); err != nil { + t.Fatal(err) + } + if err := reader.AssertReady(); !errors.Is(err, dbconfig.ErrSQLiteImportIncomplete) { + t.Fatal("reader accepted incomplete converted target", err) + } +} + +func testBackend(t testing.TB) dbconfig.Backend { + t.Helper() + v := os.Getenv("CORESCOPE_TEST_BACKEND") + if v != "" && v != "sqlite" && v != "postgres" { + t.Fatalf("invalid CORESCOPE_TEST_BACKEND") + } + if v == "postgres" { + return dbconfig.Postgres + } + return dbconfig.SQLite +} +func testBackendValue() dbconfig.Backend { + if os.Getenv("CORESCOPE_TEST_BACKEND") == "postgres" { + return dbconfig.Postgres + } + return dbconfig.SQLite +} +func testNativeSQL(sqlite, postgres string) string { + if testBackendValue() == dbconfig.Postgres { + return postgres + } + return sqlite +} diff --git a/cmd/server/storage_config.go b/cmd/server/storage_config.go new file mode 100644 index 000000000..b554f062a --- /dev/null +++ b/cmd/server/storage_config.go @@ -0,0 +1,112 @@ +package main + +import ( + "errors" + "io" + "path/filepath" + "strings" + + "github.com/meshcore-analyzer/dbconfig" +) + +type storageFlags struct { + Backend dbconfig.Backend + DBPath, DatabaseURL, StateDir string +} + +// storageInputs gathers bootstrap values without guessing an installation's +// state. The shared persisted selector applies before ResolveStorage runs. +func (c *Config) storageInputs(baseDir string, flags storageFlags, getenv func(string) string) (dbconfig.StorageInputs, error) { + base, err := filepath.Abs(baseDir) + if err != nil { + return dbconfig.StorageInputs{}, errors.New("resolve storage base directory") + } + in := dbconfig.StorageInputs{BaseDir: base, DBPath: c.DBPath, StateDir: envOrValue(getenv, "CORESCOPE_STATE_DIR", c.StateDir), + EnvBackend: dbconfig.Backend(getenv("CORESCOPE_DB_BACKEND")), + DatabaseURL: envOrValue(getenv, "CORESCOPE_DATABASE_URL", c.DatabaseURL), + ReaderDatabaseURL: getenv("CORESCOPE_READER_DATABASE_URL"), WriterDatabaseURL: getenv("CORESCOPE_WRITER_DATABASE_URL"), + UsersDatabaseURL: getenv("CORESCOPE_USERS_DATABASE_URL"), ApprovedChannelsDatabaseURL: getenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL")} + if in.DBPath == "" { + in.DBPath = getenv("DB_PATH") + } + if c.DB != nil { + in.Backend = c.DB.Backend + } + if c.UserManagement != nil { + in.UsersDBPath = strings.TrimSpace(c.UserManagement.DBPath) + in.UsersDatabaseURL = envOrValue(getenv, "CORESCOPE_USERS_DATABASE_URL", c.UserManagement.DatabaseURL) + } + if flags.Backend != "" { + in.Backend = flags.Backend + } + if flags.DBPath != "" { + in.DBPath = flags.DBPath + } + if flags.DatabaseURL != "" { + in.DatabaseURL = flags.DatabaseURL + in.ReaderDatabaseURL = flags.DatabaseURL + } + if flags.StateDir != "" { + in.StateDir = flags.StateDir + } + // Legacy explicit paths are relative to the working directory, whereas the + // server's omitted telemetry path defaults relative to config-dir. + for _, p := range []*string{&in.DBPath, &in.UsersDBPath} { + if *p == "" || strings.Contains(*p, "://") { + continue + } + if filepath.VolumeName(*p) != "" && !filepath.IsAbs(*p) { + return in, errors.New("drive-relative storage paths are ambiguous") + } + *p, err = filepath.Abs(*p) + if err != nil { + return in, errors.New("resolve SQLite storage path") + } + } + return in, nil +} + +// Discovery uses the stable legacy anchor, including after a backend switch. +func storageSelectionPath(raw dbconfig.StorageInputs) (string, error) { + state := raw.StateDir + if state == "" { + if strings.Contains(raw.DBPath, "://") || strings.HasPrefix(raw.DBPath, "postgres:") || strings.HasPrefix(raw.DBPath, "postgresql:") { + return "", errors.New("a database URL cannot be used as the legacy state directory anchor") + } + path := raw.DBPath + if path == "" { + path = filepath.Join(raw.BaseDir, "data", "meshcore.db") + } + state = filepath.Dir(path) + } + if strings.Contains(state, "://") || strings.ContainsRune(state, 0) { + return "", errors.New("invalid storage state directory") + } + if !filepath.IsAbs(state) { + state = filepath.Join(raw.BaseDir, state) + } + return dbconfig.SelectionPath(filepath.Clean(state)), nil +} +func resolveRuntimeStorage(raw dbconfig.StorageInputs) (dbconfig.Storage, io.Closer, error) { + path, err := storageSelectionPath(raw) + if err != nil { + return dbconfig.Storage{}, nil, err + } + selected, lease, err := dbconfig.OpenSelection(path) + if err != nil { + return dbconfig.Storage{}, nil, err + } + fail := func(err error) (dbconfig.Storage, io.Closer, error) { + lease.Close() + return dbconfig.Storage{}, nil, err + } + raw, err = selected.ApplyTo(raw) + if err != nil { + return fail(err) + } + storage, err := dbconfig.ResolveStorage(raw) + if err != nil { + return fail(err) + } + return storage, lease, nil +} diff --git a/cmd/server/storage_config_test.go b/cmd/server/storage_config_test.go new file mode 100644 index 000000000..75c2074c7 --- /dev/null +++ b/cmd/server/storage_config_test.go @@ -0,0 +1,155 @@ +package main + +import ( + "errors" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" +) + +func TestStorageInputsKeepBootstrapRawAndLegacyPrecedence(t *testing.T) { + base := t.TempDir() + cfg := &Config{DBPath: "configured.db", UserManagement: &UserManagementConfig{DBPath: "accounts.db"}} + env := func(k string) string { + if k == "DB_PATH" { + return "environment.db" + } + return "" + } + in, err := cfg.storageInputs(base, storageFlags{}, env) + if err != nil { + t.Fatal(err) + } + want, _ := filepath.Abs("configured.db") + if in.DBPath != want || in.StateDir != "" || in.Backend != "" || in.FreshInstall { + t.Fatalf("raw values/defaults changed: %+v", in) + } + in, err = cfg.storageInputs(base, storageFlags{DBPath: "cli.db"}, env) + if err != nil { + t.Fatal(err) + } + want, _ = filepath.Abs("cli.db") + if in.DBPath != want { + t.Fatal("CLI path did not win") + } + in, err = (&Config{}).storageInputs(base, storageFlags{}, func(string) string { return "" }) + if err != nil { + t.Fatal(err) + } + if _, err = dbconfig.ResolveStorage(in); err == nil { + t.Fatal("unknown existing install was guessed fresh") + } + in.FreshInstall = true + got, err := dbconfig.ResolveStorage(in) + if err != nil { + t.Fatal(err) + } + if got.Backend != dbconfig.SQLite || got.DBPath != filepath.Join(base, "data", "meshcore.db") { + t.Fatalf("fresh default: %+v", got) + } +} + +func TestRecordedPostgresPortCannotDriftThroughEnvironment(t *testing.T) { + t.Setenv("PGPORT", "") + const target = "postgresql://reader@localhost/telemetry?sslmode=disable" + dir := t.TempDir() + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.Postgres, StateDir: dir, ReaderDatabaseURL: target, WriterDatabaseURL: target}) + if err != nil { + t.Fatal(err) + } + t.Setenv("PGPORT", "6543") + raw, err := selected.ApplyTo(dbconfig.StorageInputs{BaseDir: dir, DatabaseURL: target}) + if err != nil { + return + } // Rejecting a changed effective endpoint is safe too. + storage, err := dbconfig.ResolveStorage(raw) + if err != nil { + t.Fatal(err) + } + config, err := pgutil.ParseConfig(storage.ReaderDatabaseURL) + if err != nil { + t.Fatal(err) + } + if config.Port != selected.Telemetry.Postgres.Port { + t.Fatalf("recorded port %d became effective port %d", selected.Telemetry.Postgres.Port, config.Port) + } +} + +func TestStorageInputsReaderURLAndStatePrecedence(t *testing.T) { + base := t.TempDir() + cfg := &Config{DB: &DBConfig{Backend: dbconfig.Postgres}, DatabaseURL: "postgresql://reader@localhost/config", StateDir: "configured"} + env := func(k string) string { + switch k { + case "CORESCOPE_DATABASE_URL": + return "postgresql://reader@localhost/generic" + case "CORESCOPE_READER_DATABASE_URL": + return "postgresql://reader@localhost/reader" + case "CORESCOPE_STATE_DIR": + return "state" + } + return "" + } + in, err := cfg.storageInputs(base, storageFlags{}, env) + if err != nil { + t.Fatal(err) + } + got, err := dbconfig.ResolveStorage(in) + if err != nil { + t.Fatal(err) + } + if got.ReaderDatabaseURL != env("CORESCOPE_READER_DATABASE_URL") || got.StateDir != filepath.Join(base, "state") { + t.Fatal("reader environment precedence changed") + } + in, err = cfg.storageInputs(base, storageFlags{DatabaseURL: "postgresql://reader@localhost/cli", StateDir: "cli-state"}, env) + if err != nil { + t.Fatal(err) + } + got, err = dbconfig.ResolveStorage(in) + if err != nil { + t.Fatal(err) + } + if got.ReaderDatabaseURL != "postgresql://reader@localhost/cli" || got.WriterDatabaseURL != "postgresql://reader@localhost/cli" || got.StateDir != filepath.Join(base, "cli-state") { + t.Fatal("CLI overrides lost") + } +} + +func TestRuntimeUsesInstalledSelectionAndHoldsLease(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "selected.db") + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.SQLite, StateDir: dir, DBPath: path, UsersDBPath: filepath.Join(dir, "users.db")}) + if err != nil { + t.Fatal(err) + } + if _, err = dbconfig.AdoptSelection(dbconfig.SelectionPath(dir), selected); err != nil { + t.Fatal(err) + } + raw := dbconfig.StorageInputs{BaseDir: dir, StateDir: dir, Backend: dbconfig.Postgres, EnvBackend: dbconfig.Postgres, DBPath: filepath.Join(dir, "old.db"), DatabaseURL: "postgresql://unused@localhost/old"} + storage, lease, err := resolveRuntimeStorage(raw) + if err != nil { + t.Fatal(err) + } + defer lease.Close() + if storage.Backend != dbconfig.SQLite || storage.DBPath != path { + t.Fatal("recorded SQLite choice not honored") + } + if _, err := dbconfig.BeginSelectionUpdate(dbconfig.SelectionPath(dir), selected.Generation); !errors.Is(err, dbconfig.ErrSelectionBusy) { + t.Fatal("reader did not retain selection lease", err) + } +} +func TestReaderMissingSelectionDoesNotAdoptOrHoldLease(t *testing.T) { + dir := t.TempDir() + raw := dbconfig.StorageInputs{BaseDir: dir, StateDir: dir, DBPath: filepath.Join(dir, "legacy.db")} + if _, lease, err := resolveRuntimeStorage(raw); !errors.Is(err, dbconfig.ErrSelectionMissing) || lease != nil { + t.Fatal("reader guessed unrecorded storage", err) + } + update, err := dbconfig.BeginSelectionUpdate(dbconfig.SelectionPath(dir), "") + if err != nil { + t.Fatal("reader blocked setup adoption", err) + } + defer update.Close() + if err = update.Abort(); err != nil { + t.Fatal(err) + } +} diff --git a/cmd/server/storage_diagnostics_test.go b/cmd/server/storage_diagnostics_test.go new file mode 100644 index 000000000..d48eb0d2f --- /dev/null +++ b/cmd/server/storage_diagnostics_test.go @@ -0,0 +1,44 @@ +package main + +import ( + "encoding/json" + "net/http/httptest" + "testing" + + "github.com/meshcore-analyzer/dbconfig" +) + +func TestDatabaseDiagnosticsMatchSelectedBackend(t *testing.T) { + fixture := setupTestDB(t) + reader, err := openFixtureReader(t, fixture.path) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + s := &Server{db: reader} + w := httptest.NewRecorder() + s.handlePerfPostgres(w, httptest.NewRequest("GET", "/api/perf/database", nil)) + var response struct { + Engine string `json:"engine"` + PageCount *int64 `json:"pageCount"` + WalSize *int64 `json:"walSize"` + CacheHitRate *float64 `json:"cacheHitRate"` + } + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if reader.Backend() == dbconfig.SQLite { + if response.Engine != "sqlite" || response.PageCount == nil || *response.PageCount <= 0 || response.WalSize == nil { + t.Fatalf("missing native SQLite diagnostics: %s", w.Body.String()) + } + if response.CacheHitRate != nil { + t.Fatal("SQLite must not label PacketStore hits as database cache hits") + } + var stats int + if err := reader.conn.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name='sqlite_stat1'`).Scan(&stats); err != nil || stats != 0 { + t.Fatal("reader diagnostics attempted planner maintenance", stats, err) + } + } else if response.Engine != "postgresql" || response.PageCount != nil || response.WalSize != nil { + t.Fatalf("PostgreSQL must not publish SQLite counters: %s", w.Body.String()) + } +} diff --git a/cmd/server/store.go b/cmd/server/store.go index e94792f59..aef042c79 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -92,8 +92,8 @@ type StoreObs struct { // fallback chain on every read; for /api/observers/{id}/analytics // that fires 60k+ times per request under RLock. ParsedTime returns // the parsed value once, caching for the lifetime of the StoreObs. - tsParseOnce sync.Once tsParsed time.Time + tsParseOnce sync.Once tsParsedOK bool } @@ -262,6 +262,9 @@ type PacketStore struct { regionObsMu sync.Mutex regionObsCache map[string]map[string]bool regionObsCacheTime time.Time + // Regions looked up since the last background refresh; that refresh + // re-queries only these. + regionObsUsed map[string]bool // Cached area key → node pubkey set (30s per-key TTL) areaNodeMu sync.RWMutex areaNodeCache map[string]map[string]bool @@ -276,6 +279,16 @@ type PacketStore struct { nodeCache []nodeInfo nodePM *prefixMap nodeCacheTime time.Time + // nodeCacheGen counts InvalidateNodeCache calls, guarded by cacheMu. A + // background rebuild stores its result only if the generation it started + // under is still current. + nodeCacheGen uint64 + // cacheRefresh limits the node, region-observer and area-node caches to + // one background refresh per key (#2146). + cacheRefresh cacheRefresher + // cacheLoadHook, when set, runs right before one of those caches queries + // SQL. Tests only. + cacheLoadHook func(kind string) // Per-store dedupe set for one-shot schema-degradation warnings. Field // (not package-level) so each test gets a fresh state — see #1199 item 5. schemaDegradationLogged sync.Map @@ -873,10 +886,13 @@ func (s *PacketStore) Load() error { } if s.db.isV3 { - loadSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, + loadSQL = s.db.nativeSQL(`SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, t.payload_type, t.payload_version, t.decoded_json, o.id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, - o.snr, o.rssi, o.score, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')` + obsRawHexCol + rpCol + scopeNameCol + ` + o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')`, `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, + t.payload_type, t.payload_version, t.decoded_json, + o.id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, + o.snr, o.rssi, o.score, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`) + obsRawHexCol + rpCol + scopeNameCol + ` FROM transmissions t LEFT JOIN observations o ON o.transmission_id = t.id LEFT JOIN observers obs ON obs.rowid = o.observer_idx` + filterClause + ` @@ -1186,19 +1202,22 @@ func (s *PacketStore) loadChunk(from, to time.Time) error { var filterClause string var qArgs []interface{} if s.db.hasLastSeen { - filterClause = "\n\t\t\tWHERE t.last_seen >= $1 AND t.last_seen < $2" + filterClause = "\n\t\t\tWHERE t.last_seen >= " + s.db.parameter(1) + " AND t.last_seen < " + s.db.parameter(2) qArgs = []interface{}{fromUnix, toUnix} } else { - filterClause = "\n\t\t\tWHERE t.first_seen >= $1 AND t.first_seen < $2" + filterClause = "\n\t\t\tWHERE t.first_seen >= " + s.db.parameter(1) + " AND t.first_seen < " + s.db.parameter(2) qArgs = []interface{}{fromStr, toStr} } var chunkSQL string if s.db.isV3 { - chunkSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, + chunkSQL = s.db.nativeSQL(`SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, t.payload_type, t.payload_version, t.decoded_json, o.id, obs.id, obs.name, o.direction, - o.snr, o.rssi, o.score, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')` + obsRawHexCol + rpCol + scopeNameCol + ` + o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')`, `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, + t.payload_type, t.payload_version, t.decoded_json, + o.id, obs.id, obs.name, o.direction, + o.snr, o.rssi, o.score, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`) + obsRawHexCol + rpCol + scopeNameCol + ` FROM transmissions t LEFT JOIN observations o ON o.transmission_id = t.id LEFT JOIN observers obs ON obs.rowid = o.observer_idx` + filterClause + ` @@ -1602,10 +1621,10 @@ func (s *PacketStore) loadBackgroundChunks() { var countQuery string var countArg interface{} if s.db.hasLastSeen { - countQuery = `SELECT COUNT(*) FROM transmissions WHERE last_seen >= $1` + countQuery = `SELECT COUNT(*) FROM transmissions WHERE last_seen >= ` + s.db.parameter(1) countArg = retentionFloor } else { - countQuery = `SELECT COUNT(*) FROM transmissions WHERE first_seen >= $1` + countQuery = `SELECT COUNT(*) FROM transmissions WHERE first_seen >= ` + s.db.parameter(1) countArg = retentionFloorStr } if err := s.db.conn.QueryRow(countQuery, countArg).Scan(&totalInDB); err != nil { @@ -1913,9 +1932,17 @@ func (s *PacketStore) QueryPackets(q PacketQuery) *PacketResult { } } atomic.AddInt64(&s.queryCount, 1) + nodePK := s.resolveQueryNode(q.Node) s.mu.RLock() - defer s.mu.RUnlock() + result, jobs := s.queryPacketsLocked(q, nodePK) + s.mu.RUnlock() + s.applyResolvedPaths(jobs) + return result +} +// queryPacketsLocked is the in-memory part of QueryPackets. Caller holds +// s.mu and runs the returned resolved_path jobs after releasing it. +func (s *PacketStore) queryPacketsLocked(q PacketQuery, nodePK string) (*PacketResult, []rpJob) { if q.Limit <= 0 { q.Limit = 50 } @@ -1923,7 +1950,7 @@ func (s *PacketStore) QueryPackets(q PacketQuery) *PacketResult { q.Order = "DESC" } - results := s.filterPackets(q) + results := s.filterPackets(q, nodePK) total := len(results) // #1345: order by ingest id, not insertion-into-s.packets order. After @@ -1944,7 +1971,7 @@ func (s *PacketStore) QueryPackets(q PacketQuery) *PacketResult { // for ASC read forwards. Both are O(page_size) — no sort copy needed. start := q.Offset if start >= total { - return &PacketResult{Packets: []map[string]interface{}{}, Total: total} + return &PacketResult{Packets: []map[string]interface{}{}, Total: total}, nil } pageSize := q.Limit if start+pageSize > total { @@ -1952,9 +1979,12 @@ func (s *PacketStore) QueryPackets(q PacketQuery) *PacketResult { } packets := make([]map[string]interface{}, 0, pageSize) + var jobs []rpJob if q.Order == "ASC" { for _, tx := range results[start : start+pageSize] { - packets = append(packets, s.txToMapWithRP(tx, q.ExpandObservations)) + var m map[string]interface{} + m, jobs = txToMapRPJobs(jobs, tx, q.ExpandObservations) + packets = append(packets, m) } } else { // DESC: newest items are at the tail; page 0 = last pageSize items reversed @@ -1964,10 +1994,12 @@ func (s *PacketStore) QueryPackets(q PacketQuery) *PacketResult { startIdx = 0 } for i := endIdx - 1; i >= startIdx; i-- { - packets = append(packets, s.txToMapWithRP(results[i], q.ExpandObservations)) + var m map[string]interface{} + m, jobs = txToMapRPJobs(jobs, results[i], q.ExpandObservations) + packets = append(packets, m) } } - return &PacketResult{Packets: packets, Total: total} + return &PacketResult{Packets: packets, Total: total}, jobs } // QueryGroupedPackets returns transmissions grouped by hash (already 1:1). @@ -2012,8 +2044,9 @@ func (s *PacketStore) QueryGroupedPackets(q PacketQuery) *PacketResult { s.groupedCacheMu.Unlock() // Collect StoreTx pointers under read lock; sort outside it. + nodePK := s.resolveQueryNode(q.Node) s.mu.RLock() - results := s.filterPackets(q) + results := s.filterPackets(q, nodePK) txs := make([]*StoreTx, len(results)) copy(txs, results) s.mu.RUnlock() @@ -2177,7 +2210,7 @@ func (s *PacketStore) GetStoreStats() (*Stats, error) { defer wg.Done() nodeErr = s.db.conn.QueryRow( `SELECT - (SELECT COUNT(*) FROM nodes WHERE last_seen > $1) AS active_nodes, + (SELECT COUNT(*) FROM nodes WHERE last_seen > `+s.db.parameter(1)+`) AS active_nodes, (SELECT COUNT(*) FROM nodes) AS all_nodes, (SELECT COUNT(*) FROM observers WHERE inactive IS NULL OR inactive = 0) AS observers`, sevenDaysAgo, @@ -2221,9 +2254,9 @@ func (s *PacketStore) refreshObsCounts(oneHourAgo, oneDayAgo int64) (int, int, e var lastHour, last24h int if qErr := s.db.conn.QueryRow( `SELECT - COALESCE(SUM(CASE WHEN timestamp > $1 THEN 1 ELSE 0 END), 0), - COALESCE(SUM(CASE WHEN timestamp > $2 THEN 1 ELSE 0 END), 0) - FROM observations WHERE timestamp > $3`, + COALESCE(SUM(CASE WHEN timestamp > `+s.db.parameter(1)+` THEN 1 ELSE 0 END), 0), + COALESCE(SUM(CASE WHEN timestamp > `+s.db.parameter(2)+` THEN 1 ELSE 0 END), 0) + FROM observations WHERE timestamp > `+s.db.parameter(3), oneHourAgo, oneDayAgo, oneDayAgo, ).Scan(&lastHour, &last24h); qErr != nil { return nil, qErr @@ -2567,53 +2600,63 @@ func (s *PacketStore) GetStoreMemoryBreakdown() *StoreMemoryBreakdown { // GetTransmissionByID returns a transmission by its DB ID, formatted as a map. func (s *PacketStore) GetTransmissionByID(id int) map[string]interface{} { s.mu.RLock() - defer s.mu.RUnlock() - tx := s.byTxID[id] if tx == nil { + s.mu.RUnlock() return nil } - return s.txToMapWithRP(tx, true) + m, jobs := txToMapRPJobs(nil, tx, true) + s.mu.RUnlock() + s.applyResolvedPaths(jobs) + return m } // GetPacketByHash returns a transmission by content hash. func (s *PacketStore) GetPacketByHash(hash string) map[string]interface{} { s.mu.RLock() - defer s.mu.RUnlock() - tx := s.byHash[strings.ToLower(hash)] if tx == nil { + s.mu.RUnlock() return nil } - return s.txToMapWithRP(tx, true) + m, jobs := txToMapRPJobs(nil, tx, true) + s.mu.RUnlock() + s.applyResolvedPaths(jobs) + return m } // GetPacketByID returns an observation (enriched with transmission fields) by observation ID. func (s *PacketStore) GetPacketByID(id int) map[string]interface{} { s.mu.RLock() - defer s.mu.RUnlock() - obs := s.byObsID[id] if obs == nil { + s.mu.RUnlock() return nil } - return s.enrichObs(obs) + m, job := s.enrichObsRPJob(obs) + s.mu.RUnlock() + s.applyResolvedPaths([]rpJob{job}) + return m } // GetObservationsForHash returns all observations for a hash, enriched with transmission fields. func (s *PacketStore) GetObservationsForHash(hash string) []map[string]interface{} { s.mu.RLock() - defer s.mu.RUnlock() - tx := s.byHash[strings.ToLower(hash)] if tx == nil { + s.mu.RUnlock() return []map[string]interface{}{} } result := make([]map[string]interface{}, 0, len(tx.Observations)) + jobs := make([]rpJob, 0, len(tx.Observations)) for _, obs := range tx.Observations { - result = append(result, s.enrichObs(obs)) + m, job := s.enrichObsRPJob(obs) + result = append(result, m) + jobs = append(jobs, job) } + s.mu.RUnlock() + s.applyResolvedPaths(jobs) return result } @@ -2640,21 +2683,28 @@ func (s *PacketStore) GetTimestamps(since string) []string { // QueryMultiNodePackets filters packets matching any of the given pubkeys. func (s *PacketStore) QueryMultiNodePackets(pubkeys []string, limit, offset int, order, since, until string) *PacketResult { + resolved := make([]string, len(pubkeys)) + for i, pk := range pubkeys { + resolved[i] = s.resolveQueryNode(pk) + } s.mu.RLock() - defer s.mu.RUnlock() + result, jobs := s.queryMultiNodePacketsLocked(resolved, limit, offset, order, since, until) + s.mu.RUnlock() + s.applyResolvedPaths(jobs) + return result +} - if len(pubkeys) == 0 { - return &PacketResult{Packets: []map[string]interface{}{}, Total: 0} +// queryMultiNodePacketsLocked is the in-memory part of QueryMultiNodePackets, +// for pubkeys already resolved by resolveQueryNode. Caller holds s.mu and runs +// the returned resolved_path jobs after releasing it. +func (s *PacketStore) queryMultiNodePacketsLocked(resolved []string, limit, offset int, order, since, until string) (*PacketResult, []rpJob) { + if len(resolved) == 0 { + return &PacketResult{Packets: []map[string]interface{}{}, Total: 0}, nil } if limit <= 0 { limit = 50 } - resolved := make([]string, len(pubkeys)) - for i, pk := range pubkeys { - resolved[i] = s.db.resolveNodePubkey(pk) - } - // Use byNode index instead of scanning all packets (O(indexed) vs O(all×pubkeys×json)). hashSet := make(map[string]bool) var filtered []*StoreTx @@ -2683,7 +2733,7 @@ func (s *PacketStore) QueryMultiNodePackets(pubkeys []string, limit, offset int, // filtered is oldest-first (built by iterating s.packets forward). // Apply same DESC/ASC pagination logic as QueryPackets. if offset >= total { - return &PacketResult{Packets: []map[string]interface{}{}, Total: total} + return &PacketResult{Packets: []map[string]interface{}{}, Total: total}, nil } pageSize := limit if offset+pageSize > total { @@ -2691,9 +2741,12 @@ func (s *PacketStore) QueryMultiNodePackets(pubkeys []string, limit, offset int, } packets := make([]map[string]interface{}, 0, pageSize) + var jobs []rpJob if order == "ASC" { for _, tx := range filtered[offset : offset+pageSize] { - packets = append(packets, s.txToMapWithRP(tx)) + var m map[string]interface{} + m, jobs = txToMapRPJobs(jobs, tx) + packets = append(packets, m) } } else { endIdx := total - offset @@ -2702,10 +2755,12 @@ func (s *PacketStore) QueryMultiNodePackets(pubkeys []string, limit, offset int, startIdx = 0 } for i := endIdx - 1; i >= startIdx; i-- { - packets = append(packets, s.txToMapWithRP(filtered[i])) + var m map[string]interface{} + m, jobs = txToMapRPJobs(jobs, filtered[i]) + packets = append(packets, m) } } - return &PacketResult{Packets: packets, Total: total} + return &PacketResult{Packets: packets, Total: total}, jobs } // IngestNewFromDB loads new transmissions from PostgreSQL into memory and returns @@ -2729,14 +2784,17 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac scopeNameCol = ", t.scope_name" } if s.db.isV3 { - querySQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, + querySQL = s.db.nativeSQL(`SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, t.payload_type, t.payload_version, t.decoded_json, o.id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, - o.snr, o.rssi, o.score, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')` + obsRHCol + scopeNameCol + ` + o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')`, `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, + t.payload_type, t.payload_version, t.decoded_json, + o.id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, + o.snr, o.rssi, o.score, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`) + obsRHCol + scopeNameCol + ` FROM transmissions t LEFT JOIN observations o ON o.transmission_id = t.id LEFT JOIN observers obs ON obs.rowid = o.observer_idx - WHERE t.id > $1 + WHERE t.id > ` + s.db.parameter(1) + ` ORDER BY t.id ASC, o.timestamp DESC` } else { querySQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, @@ -2746,7 +2804,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac FROM transmissions t LEFT JOIN observations o ON o.transmission_id = t.id LEFT JOIN observers obs ON obs.id = o.observer_id - WHERE t.id > $1 + WHERE t.id > ` + s.db.parameter(1) + ` ORDER BY t.id ASC, o.timestamp DESC` } @@ -3154,21 +3212,22 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string] obsRHCol2 = ", o.raw_hex" } if s.db.isV3 { - querySQL = `SELECT o.id, o.transmission_id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, - o.snr, o.rssi, o.score, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')` + obsRHCol2 + ` + querySQL = s.db.nativeSQL(`SELECT o.id, o.transmission_id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, + o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')`, `SELECT o.id, o.transmission_id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, + o.snr, o.rssi, o.score, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`) + obsRHCol2 + ` FROM observations o LEFT JOIN observers obs ON obs.rowid = o.observer_idx - WHERE o.id > $1 + WHERE o.id > ` + s.db.parameter(1) + ` ORDER BY o.id ASC - LIMIT $2` + LIMIT ` + s.db.parameter(2) } else { querySQL = `SELECT o.id, o.transmission_id, o.observer_id, o.observer_name, COALESCE(obs.iata, ''), o.direction, o.snr, o.rssi, o.score, o.path_json, o.timestamp` + obsRHCol2 + ` FROM observations o LEFT JOIN observers obs ON obs.id = o.observer_id - WHERE o.id > $1 + WHERE o.id > ` + s.db.parameter(1) + ` ORDER BY o.id ASC - LIMIT $2` + LIMIT ` + s.db.parameter(2) } rows, err := s.db.conn.Query(querySQL, sinceObsID, limit) @@ -3478,8 +3537,18 @@ func (s *PacketStore) MaxObservationID() int { // --- Internal filter/query helpers --- +// resolveQueryNode maps a node filter (pubkey or name) to a pubkey. It runs +// SQL, so callers resolve before taking s.mu (#2146). Empty stays empty. +func (s *PacketStore) resolveQueryNode(node string) string { + if node == "" { + return "" + } + return s.db.resolveNodePubkey(node) +} + // filterPackets applies PacketQuery filters to the in-memory packet list. -func (s *PacketStore) filterPackets(q PacketQuery) []*StoreTx { +// nodePK is q.Node resolved by resolveQueryNode. Caller holds s.mu. +func (s *PacketStore) filterPackets(q PacketQuery, nodePK string) []*StoreTx { // Fast path: single-key index lookups if q.Hash != "" && q.Type == nil && q.Route == nil && q.Observer == "" && q.Region == "" && q.Area == "" && q.Node == "" && q.Channel == "" && q.Since == "" && q.Until == "" { @@ -3544,11 +3613,9 @@ func (s *PacketStore) filterPackets(q PacketQuery) []*StoreTx { } // Pre-compute node filter parameters. - var nodePK string var nodeHashSet map[string]bool hasNode := q.Node != "" if hasNode { - nodePK = s.db.resolveNodePubkey(q.Node) indexed := s.byNode[nodePK] nodeHashSet = make(map[string]bool, len(indexed)) for _, tx := range indexed { @@ -3705,50 +3772,103 @@ func (s *PacketStore) transmissionsForObserver(observerIDs string, from []*Store return result } +// regionObsTTL is how long the region → observer cache is served before one +// background refresh re-queries it. +const regionObsTTL = 30 * time.Second + // resolveRegionObservers returns a set of observer IDs for a given IATA region. -// Results are cached for 30 seconds to avoid repeated DB queries. -// Uses its own mutex (regionObsMu) so callers holding s.mu won't deadlock. +// Callers often hold s.mu, so a cache past its TTL is served as-is while one +// background refresh re-queries it (#2146). A region seen for the first time +// is queried inline, without holding regionObsMu. func (s *PacketStore) resolveRegionObservers(region string) map[string]bool { s.regionObsMu.Lock() - defer s.regionObsMu.Unlock() + m, cached := s.regionObsCache[region] + stale := time.Since(s.regionObsCacheTime) >= regionObsTTL + s.markRegionObsUsed(region) + s.regionObsMu.Unlock() - if s.regionObsCache != nil && time.Since(s.regionObsCacheTime) < 30*time.Second { - if m, ok := s.regionObsCache[region]; ok { - return m + if cached { + if stale { + s.cacheRefresh.start("regionObs", s.refreshRegionObs) } - return s.fetchAndCacheRegionObs(region) + return m } - // Cache expired — rebuild. - s.regionObsCache = make(map[string]map[string]bool) - s.regionObsCacheTime = time.Now() - // Fetch for the requested region and cache it. - return s.fetchAndCacheRegionObs(region) + m = s.queryRegionObs(region) + s.regionObsMu.Lock() + if s.regionObsCache == nil { + s.regionObsCache = make(map[string]map[string]bool) + s.regionObsCacheTime = time.Now() + } + s.regionObsCache[region] = m + s.regionObsMu.Unlock() + return m } -// fetchAndCacheRegionObs fetches observer IDs for a region from the DB and stores in cache. -// Caller must hold regionObsMu. -func (s *PacketStore) fetchAndCacheRegionObs(region string) map[string]bool { - if m, ok := s.regionObsCache[region]; ok { - return m +// markRegionObsUsed records a lookup so the next refresh re-queries the +// region. Caller must hold regionObsMu. +func (s *PacketStore) markRegionObsUsed(region string) { + if s.regionObsUsed == nil { + s.regionObsUsed = make(map[string]bool) } + s.regionObsUsed[region] = true +} + +// refreshRegionObs re-queries the regions looked up since the previous +// refresh and replaces the cache with them. Regions nobody asked for are +// dropped, which bounds the cache: the region comes from the query string. +func (s *PacketStore) refreshRegionObs() { + s.regionObsMu.Lock() + regions := make([]string, 0, len(s.regionObsUsed)) + for r := range s.regionObsUsed { + regions = append(regions, r) + } + s.regionObsUsed = nil + s.regionObsMu.Unlock() + + fresh := make(map[string]map[string]bool, len(regions)) + for _, r := range regions { + fresh[r] = s.queryRegionObs(r) + } + + s.regionObsMu.Lock() + // Regions first looked up while this refresh ran keep their entry. + for r := range s.regionObsUsed { + if _, ok := fresh[r]; !ok { + if m, ok := s.regionObsCache[r]; ok { + fresh[r] = m + } + } + } + s.regionObsCache = fresh + s.regionObsCacheTime = time.Now() + s.regionObsMu.Unlock() +} + +// queryRegionObs loads the observer IDs for a region from SQLite. It returns +// nil when the region has no observers or the query fails. +func (s *PacketStore) queryRegionObs(region string) map[string]bool { + s.beforeCacheLoad("regionObs") ids, err := s.db.GetObserverIdsForRegion(region) if err != nil || len(ids) == 0 { - s.regionObsCache[region] = nil return nil } m := make(map[string]bool, len(ids)) for _, id := range ids { m[id] = true } - s.regionObsCache[region] = m return m } +// areaNodeTTL is how long an area's node set is served before one background +// refresh re-queries it. +const areaNodeTTL = 30 * time.Second + // resolveAreaNodes returns a set of node pubkeys whose GPS coordinates fall // inside the named area polygon. Returns nil if the area key is not in config. -// Results are cached per-key for 30 seconds. Uses its own RWMutex so callers -// holding s.mu won't deadlock. +// Results are cached per key. Callers often hold s.mu, so an entry past its +// TTL is served as-is while one background refresh re-queries it (#2146). +// Uses its own RWMutex so callers holding s.mu won't deadlock. func (s *PacketStore) resolveAreaNodes(areaKey string) map[string]bool { if s.config == nil || s.config.Areas == nil { return nil @@ -3758,16 +3878,23 @@ func (s *PacketStore) resolveAreaNodes(areaKey string) map[string]bool { return nil } - // Fast path: serve from cache if the per-key TTL is still valid. s.areaNodeMu.RLock() - if t, ok := s.areaNodeCacheTimes[areaKey]; ok && time.Since(t) < 30*time.Second { - m := s.areaNodeCache[areaKey] - s.areaNodeMu.RUnlock() + t, cached := s.areaNodeCacheTimes[areaKey] + m := s.areaNodeCache[areaKey] + s.areaNodeMu.RUnlock() + if cached { + if time.Since(t) >= areaNodeTTL { + s.cacheRefresh.start("areaNodes:"+areaKey, func() { s.loadAreaNodes(areaKey, entry) }) + } return m } - s.areaNodeMu.RUnlock() + return s.loadAreaNodes(areaKey, entry) +} - // Slow path: query the DB outside any lock, then write back under Lock. +// loadAreaNodes queries the node set for an area outside any lock and stores +// it under areaNodeMu. +func (s *PacketStore) loadAreaNodes(areaKey string, entry AreaEntry) map[string]bool { + s.beforeCacheLoad("areaNodes") pks, err := s.db.GetNodePubkeysInArea(entry) var m map[string]bool if err == nil && len(pks) > 0 { @@ -3779,7 +3906,7 @@ func (s *PacketStore) resolveAreaNodes(areaKey string) map[string]bool { s.areaNodeMu.Lock() // Re-check in case another goroutine already refreshed while we queried. - if t, ok := s.areaNodeCacheTimes[areaKey]; !ok || time.Since(t) >= 30*time.Second { + if t, ok := s.areaNodeCacheTimes[areaKey]; !ok || time.Since(t) >= areaNodeTTL { s.areaNodeCache[areaKey] = m s.areaNodeCacheTimes[areaKey] = time.Now() } else { @@ -3816,6 +3943,23 @@ func iataMatchesRegion(iata, regionParam string) bool { return false } +// observerIATAs returns observer ID → normalised IATA code. It reads the +// observers table, so callers load it before taking s.mu (#2146). +func (s *PacketStore) observerIATAs() map[string]string { + obsIATA := make(map[string]string, 64) + if s.db == nil { + return obsIATA + } + if observers, err := s.db.GetObservers(); err == nil { + for _, o := range observers { + if o.IATA != nil && *o.IATA != "" { + obsIATA[o.ID] = strings.TrimSpace(strings.ToUpper(*o.IATA)) + } + } + } + return obsIATA +} + // computeNodeHomeRegions returns a pubkey → IATA map deriving each node's // HOME region from zero-hop DIRECT adverts. A zero-hop direct advert is the // most authoritative location signal because the path byte is set locally on @@ -3825,24 +3969,13 @@ func iataMatchesRegion(iata, regionParam string) bool { // When a node has zero-hop direct adverts heard by observers from multiple // regions, the most-frequently-observed region wins (geographic plurality). // +// obsIATA is the observer → IATA map from observerIATAs, loaded before s.mu. // Caller must hold s.mu (read or write). Returns empty map (not nil) if no // observers are loaded or no zero-hop direct adverts have been seen. // // #804: feeds analytics region-attribution so a multi-byte repeater whose // flood adverts get relayed across regions is still attributed to its home. -func (s *PacketStore) computeNodeHomeRegions() map[string]string { - // Build observer → IATA map. observers table is small (≪ packets), so a - // single DB read here is acceptable; resolveRegionObservers does similar. - obsIATA := make(map[string]string, 64) - if s.db != nil { - if observers, err := s.db.GetObservers(); err == nil { - for _, o := range observers { - if o.IATA != nil && *o.IATA != "" { - obsIATA[o.ID] = strings.TrimSpace(strings.ToUpper(*o.IATA)) - } - } - } - } +func (s *PacketStore) computeNodeHomeRegions(obsIATA map[string]string) map[string]string { if len(obsIATA) == 0 { return map[string]string{} } @@ -3921,26 +4054,39 @@ func (s *PacketStore) computeNodeHomeRegions() map[string]string { return out } -// enrichObs returns a map with observation fields + transmission fields. -// Looks up the transmission in s.byTxID itself — safe only when the caller -// already holds s.mu (directly, or via a defer'd RLock spanning the call). +// enrichObsRPJob returns a map with observation fields + transmission fields, +// plus the resolved_path lookup as a job for applyResolvedPaths. Looks up the +// transmission in s.byTxID itself, so the caller must hold s.mu, and must run +// the job after releasing it. // Callers that snapshot observations under RLock and then release it before // iterating (e.g. handleObserverAnalytics, #1830) must use enrichObsWithTx // with a tx pointer resolved during that same snapshot instead. -func (s *PacketStore) enrichObs(obs *StoreObs) map[string]interface{} { - return s.enrichObsWithTx(obs, s.byTxID[obs.TransmissionID]) +func (s *PacketStore) enrichObsRPJob(obs *StoreObs) (map[string]interface{}, rpJob) { + m := enrichObsFields(obs, s.byTxID[obs.TransmissionID]) + return m, obsRPJob(m, obs.ID) } -// enrichObsWithTx is enrichObs with the transmission pointer already -// resolved by the caller, instead of looking it up in s.byTxID here. #1830: +// enrichObsWithTx is enrichObsRPJob with the transmission pointer already +// resolved by the caller, instead of looking it up in s.byTxID here, and with +// the resolved_path fetched inline. #1830: // s.byTxID is guarded by s.mu (writes from ingest/eviction); reading it // without holding at least RLock races with those writers — Go maps can // panic with "concurrent map read and map write" during a rehash, not just // fail under -race. Callers that need to read byTxID after releasing their // RLock (to keep JSON decode / enrichment off the hot lock, per #1481) // should resolve the *StoreTx for each observation during their RLock-held -// snapshot and pass it in here. +// snapshot and pass it in here. Must be called without s.mu held. func (s *PacketStore) enrichObsWithTx(obs *StoreObs, tx *StoreTx) map[string]interface{} { + m := enrichObsFields(obs, tx) + if rp := s.fetchResolvedPathForObs(obs.ID); rp != nil { + m["resolved_path"] = rp + } + return m +} + +// enrichObsFields returns the observation fields plus, when tx is non-nil, +// the transmission fields. It runs no SQL. +func enrichObsFields(obs *StoreObs, tx *StoreTx) map[string]interface{} { m := map[string]interface{}{ "id": obs.ID, "timestamp": strOrNil(obs.Timestamp), @@ -3953,11 +4099,6 @@ func (s *PacketStore) enrichObsWithTx(obs *StoreObs, tx *StoreTx) map[string]int "score": floatPtrOrNil(obs.Score), "path_json": strOrNil(obs.PathJSON), } - // On-demand SQL fetch for resolved_path - rp := s.fetchResolvedPathForObs(obs.ID) - if rp != nil { - m["resolved_path"] = rp - } if tx != nil { m["hash"] = strOrNil(tx.Hash) @@ -4030,28 +4171,23 @@ func txToMap(tx *StoreTx, includeObservations ...bool) map[string]interface{} { return m } -// txToMapWithRP is like txToMap but also fetches resolved_path on demand from the store. -func (s *PacketStore) txToMapWithRP(tx *StoreTx, includeObservations ...bool) map[string]interface{} { +// txToMapRPJobs is txToMap plus the resolved_path lookups for the +// transmission and, when included, its observation sub-maps, appended to +// jobs. Caller holds s.mu; run the jobs with applyResolvedPaths after +// releasing it. +func txToMapRPJobs(jobs []rpJob, tx *StoreTx, includeObservations ...bool) (map[string]interface{}, []rpJob) { m := txToMap(tx, includeObservations...) - // On-demand SQL fetch for resolved_path - rp := s.fetchResolvedPathForTxBest(tx) - if rp != nil { - m["resolved_path"] = rp - } - // Also add resolved_path to observation sub-maps if present + jobs = append(jobs, txRPJob(m, tx)) if len(includeObservations) > 0 && includeObservations[0] { if obsList, ok := m["observations"].([]map[string]interface{}); ok { for i, o := range tx.Observations { if i < len(obsList) { - obsRP := s.fetchResolvedPathForObs(o.ID) - if obsRP != nil { - obsList[i]["resolved_path"] = obsRP - } + jobs = append(jobs, obsRPJob(obsList[i], o.ID)) } } } } - return m + return m, jobs } func strOrNil(s string) interface{} { @@ -6978,17 +7114,36 @@ func buildPrefixMap(nodes []nodeInfo) *prefixMap { return pm } -// getCachedNodesAndPM returns cached node list and prefix map, rebuilding if stale. -// Must be called with s.mu held (RLock or Lock). +// nodeCacheTTL is how long getCachedNodesAndPM serves its node list before it +// starts a background rebuild. +const nodeCacheTTL = 30 * time.Second + +// getCachedNodesAndPM returns the cached node list and prefix map. Callers +// often hold s.mu, so a cache past its TTL is served as-is while one +// background rebuild runs the SQL (#2146). Only a cache that was never built +// or was invalidated is rebuilt inline. func (s *PacketStore) getCachedNodesAndPM() ([]nodeInfo, *prefixMap) { s.cacheMu.Lock() - if s.nodeCache != nil && time.Since(s.nodeCacheTime) < 30*time.Second { - nodes, pm := s.nodeCache, s.nodePM - s.cacheMu.Unlock() - return nodes, pm + nodes, pm, builtAt := s.nodeCache, s.nodePM, s.nodeCacheTime + s.cacheMu.Unlock() + + if builtAt.IsZero() { + return s.rebuildNodeCache() + } + if time.Since(builtAt) >= nodeCacheTTL { + s.cacheRefresh.start("nodes", func() { s.rebuildNodeCache() }) } + return nodes, pm +} + +// rebuildNodeCache loads the node list and prefix map from SQLite and stores +// them, unless InvalidateNodeCache ran while it was loading. +func (s *PacketStore) rebuildNodeCache() ([]nodeInfo, *prefixMap) { + s.cacheMu.Lock() + gen := s.nodeCacheGen s.cacheMu.Unlock() + s.beforeCacheLoad("nodes") nodes := s.getAllNodes() pm := buildPrefixMap(nodes) // Issue #1290: exclude observers that advertised `repeat:off` from @@ -7004,20 +7159,24 @@ func (s *PacketStore) getCachedNodesAndPM() ([]nodeInfo, *prefixMap) { } s.cacheMu.Lock() - s.nodeCache = nodes - s.nodePM = pm - s.nodeCacheTime = time.Now() + if s.nodeCacheGen == gen { + s.nodeCache = nodes + s.nodePM = pm + s.nodeCacheTime = time.Now() + } s.cacheMu.Unlock() return nodes, pm } -// InvalidateNodeCache forces the next getCachedNodesAndPM call to rebuild. +// InvalidateNodeCache forces the next getCachedNodesAndPM call to rebuild +// inline. func (s *PacketStore) InvalidateNodeCache() { s.cacheMu.Lock() s.nodeCache = nil s.nodePM = nil s.nodeCacheTime = time.Time{} + s.nodeCacheGen++ s.cacheMu.Unlock() } @@ -7368,18 +7527,26 @@ func (s *PacketStore) GetAnalyticsTopologyWithWindow(region, area string, window } func (s *PacketStore) computeAnalyticsTopology(region, area string, window TimeWindow) map[string]interface{} { + // These may query SQLite, so they run before s.mu (#2146). var areaNodes map[string]bool if area != "" { areaNodes = s.resolveAreaNodes(area) } - - s.mu.RLock() - defer s.mu.RUnlock() - var regionObs map[string]bool if region != "" { regionObs = s.resolveRegionObservers(region) } + // Use DB 7-day active node count (matches /api/stats totalNodes). Only + // that count: GetStats also scans transmissions and observations. + uniqueNodes := 0 + if s.db != nil { + if n, err := s.db.CountActiveNodes(); err == nil { + uniqueNodes = n + } + } + + s.mu.RLock() + defer s.mu.RUnlock() allNodes, pm := s.getCachedNodesAndPM() _ = allNodes // only pm is needed for topology @@ -7859,14 +8026,6 @@ func (s *PacketStore) computeAnalyticsTopology(region, area string, window TimeW bestPathList = bestPathList[:50] } - // Use DB 7-day active node count (matches /api/stats totalNodes) - uniqueNodes := 0 - if s.db != nil { - if stats, err := s.db.GetStats(); err == nil { - uniqueNodes = stats.TotalNodes - } - } - return map[string]interface{}{ "uniqueNodes": uniqueNodes, "avgHops": avgHops, @@ -8331,9 +8490,7 @@ func (s *PacketStore) computeAnalyticsHashSizesWithCapability(region, area strin } func (s *PacketStore) computeAnalyticsHashSizes(region, area string) map[string]interface{} { - s.mu.RLock() - defer s.mu.RUnlock() - + // These may query SQLite, so they run before s.mu (#2146). var regionObs map[string]bool if region != "" { regionObs = s.resolveRegionObservers(region) @@ -8342,12 +8499,16 @@ func (s *PacketStore) computeAnalyticsHashSizes(region, area string) map[string] if area != "" { areaNodes = s.resolveAreaNodes(area) } + obsIATA := s.observerIATAs() + + s.mu.RLock() + defer s.mu.RUnlock() // #804: derive each node's HOME region from zero-hop direct adverts (the // most authoritative location signal — those packets cannot have been // relayed). When non-empty, multi-byte node attribution prefers this // over observer-region. Falls back to observer-region when unknown. - nodeHomeRegion := s.computeNodeHomeRegions() + nodeHomeRegion := s.computeNodeHomeRegions(obsIATA) attributionMethod := "observer" if region != "" && len(nodeHomeRegion) > 0 { attributionMethod = "repeater" @@ -9504,6 +9665,48 @@ func (s *PacketStore) computeMultiByteCapability(adopterHashSizes map[string]int // --- Bulk Health (in-memory) --- +// bulkHealthNode is one nodes-table row for GetBulkHealth. +type bulkHealthNode struct { + pk, name, role string + lat, lon interface{} +} + +// bulkHealthNodes reads the most recently seen nodes for GetBulkHealth, +// keeping only areaNodes when set. With a region or area filter it reads up +// to 10000 rows so the caller does not under-fill after its exclusions; +// otherwise it stops at limit. +func (s *PacketStore) bulkHealthNodes(limit int, regionFilter bool, areaNodes map[string]bool) ([]bulkHealthNode, error) { + filtered := regionFilter || areaNodes != nil + queryLimit := limit + if filtered { + queryLimit = 10000 + } + rows, err := s.db.conn.Query("SELECT public_key, name, role, lat, lon FROM nodes ORDER BY last_seen DESC NULLS LAST LIMIT "+s.db.parameter(1), queryLimit) + if err != nil { + return nil, err + } + defer rows.Close() + + var nodes []bulkHealthNode + for rows.Next() { + var pk string + var name, role sql.NullString + var lat, lon sql.NullFloat64 + rows.Scan(&pk, &name, &role, &lat, &lon) + if areaNodes != nil && !areaNodes[pk] { + continue + } + nodes = append(nodes, bulkHealthNode{ + pk: pk, name: nullStrVal(name), role: nullStrVal(role), + lat: nullFloat(lat), lon: nullFloat(lon), + }) + if !filtered && len(nodes) >= limit { + break + } + } + return nodes, nil +} + func (s *PacketStore) GetBulkHealth(limit int, region, area string) []map[string]interface{} { var areaNodes map[string]bool if area != "" { @@ -9514,13 +9717,23 @@ func (s *PacketStore) GetBulkHealth(limit int, region, area string) []map[string directHeard := s.loadDirectHeard() nonRelaySet, seenSet := s.canRelaySets() + // SQL, so before s.mu (#2146). The region filter needs the store, so it + // is applied to these rows under the lock below. + var regionObs map[string]bool + if region != "" { + regionObs = s.resolveRegionObservers(region) + } + nodes, err := s.bulkHealthNodes(limit, regionObs != nil, areaNodes) + if err != nil { + return []map[string]interface{}{} + } + s.mu.RLock() defer s.mu.RUnlock() // Region filtering var regionNodeKeys map[string]bool if region != "" { - regionObs := s.resolveRegionObservers(region) if regionObs != nil { regionalHashes := make(map[string]bool) for obsID := range regionObs { @@ -9544,40 +9757,14 @@ func (s *PacketStore) GetBulkHealth(limit int, region, area string) []map[string } } - // Get nodes from DB — fetch more when filtering so we don't under-fill after exclusions - queryLimit := limit - if regionNodeKeys != nil || areaNodes != nil { - queryLimit = 10000 - } - rows, err := s.db.conn.Query("SELECT public_key, name, role, lat, lon FROM nodes ORDER BY last_seen DESC NULLS LAST LIMIT $1", queryLimit) - if err != nil { - return []map[string]interface{}{} - } - defer rows.Close() - - type dbNode struct { - pk, name, role string - lat, lon interface{} - } - var nodes []dbNode - for rows.Next() { - var pk string - var name, role sql.NullString - var lat, lon sql.NullFloat64 - rows.Scan(&pk, &name, &role, &lat, &lon) - if regionNodeKeys != nil && !regionNodeKeys[pk] { - continue - } - if areaNodes != nil && !areaNodes[pk] { - continue - } - nodes = append(nodes, dbNode{ - pk: pk, name: nullStrVal(name), role: nullStrVal(role), - lat: nullFloat(lat), lon: nullFloat(lon), - }) - if regionNodeKeys == nil && areaNodes == nil && len(nodes) >= limit { - break + if regionNodeKeys != nil { + kept := nodes[:0] + for _, n := range nodes { + if regionNodeKeys[n.pk] { + kept = append(kept, n) + } } + nodes = kept } // Only cap to limit in the global (no-filter) case; area/region returns full filtered set if regionNodeKeys != nil && areaNodes == nil && len(nodes) > limit { @@ -9683,10 +9870,10 @@ func (s *PacketStore) GetNodeHealth(pubkey string) (map[string]interface{}, erro // Loaded before taking s.mu so the lock order stays s.mu → // analyticsRecomputerMu everywhere (computeDirectHeard takes s.mu). directHeard := s.loadDirectHeard() + // SQL, so also before s.mu (#2146). + nonRelaySet, seenSet := s.canRelaySets() s.mu.RLock() - defer s.mu.RUnlock() - packets := s.byNode[pubkey] todayStart := time.Now().UTC().Truncate(24 * time.Hour).Format(time.RFC3339) @@ -9727,7 +9914,6 @@ func (s *PacketStore) GetNodeHealth(pubkey string) (map[string]interface{}, erro } } - nonRelaySet, seenSet := s.canRelaySets() directByObs := directHeard[strings.ToLower(pubkey)] observerRows := buildDirectObserverRows(directByObs, nonRelaySet, seenSet) relayObserverCount := relayOnlyObserverCount(relayObservers, directByObs) @@ -9751,11 +9937,15 @@ func (s *PacketStore) GetNodeHealth(pubkey string) (map[string]interface{}, erro recentLimit = len(packets) } recentPackets := make([]map[string]interface{}, 0, recentLimit) + var rpJobs []rpJob for i := len(packets) - 1; i >= len(packets)-recentLimit; i-- { - p := s.txToMapWithRP(packets[i]) + var p map[string]interface{} + p, rpJobs = txToMapRPJobs(rpJobs, packets[i]) delete(p, "observations") recentPackets = append(recentPackets, p) } + s.mu.RUnlock() + s.applyResolvedPaths(rpJobs) return map[string]interface{}{ "node": node, diff --git a/cmd/server/store_lock_convoy_test.go b/cmd/server/store_lock_convoy_test.go new file mode 100644 index 000000000..05b595aea --- /dev/null +++ b/cmd/server/store_lock_convoy_test.go @@ -0,0 +1,46 @@ +package main + +import ( + "testing" + "time" +) + +// #2146: the lock convoy. A request that waits on SQL while holding +// s.mu.RLock blocks the ingest writer's Lock, and a pending writer blocks +// every new reader, even ones that do no SQL (/api/healthz). +// +// Here the resolved_path query takes slowSQL. While /api/packets is in that +// query, the ingest poller asks for the write lock and then a reader asks +// for RLock. The reader must not wait for the slow query. +func TestSlowResolvedPathSQLDoesNotStallReaders(t *testing.T) { + const slowSQL = 300 * time.Millisecond + s := newResolvedPathStore(t) + clearResolvedPathLRU(s) + + inSQL := make(chan struct{}, 8) + s.cacheLoadHook = func(kind string) { + if kind == "resolvedPath" { + inSQL <- struct{}{} + time.Sleep(slowSQL) + } + } + + go s.QueryPackets(PacketQuery{Limit: 10}) + <-inSQL + + go func() { // the ingest poller + s.mu.Lock() + s.mu.Unlock() + }() + time.Sleep(10 * time.Millisecond) // let the writer queue up + + start := time.Now() + s.mu.RLock() + waited := time.Since(start) + s.mu.RUnlock() + + t.Logf("reader waited %v behind a %v resolved_path query", waited, slowSQL) + if waited > slowSQL/3 { + t.Fatalf("reader waited %v: the slow query held s.mu", waited) + } +} diff --git a/cmd/server/store_lock_sql_invariant_test.go b/cmd/server/store_lock_sql_invariant_test.go new file mode 100644 index 000000000..03f38e887 --- /dev/null +++ b/cmd/server/store_lock_sql_invariant_test.go @@ -0,0 +1,313 @@ +package main + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "sort" + "strings" + "testing" +) + +// #2146: no SQL while PacketStore.mu is held. The server's read pool has 4 +// connections; a store-lock holder that waits for one stalls the ingest +// writer, and behind a pending writer every other reader of the store. +// +// This test parses the package and fails when code inside a store.mu +// section calls, directly or through other package functions, a function +// that queries SQLite. Calls are resolved by name only (no type checking), +// which is enough for this package's receiver conventions. + +// sqlUnderLockAllowedCallees query SQLite only on a cold or invalidated +// cache; past their TTL they serve the cached value and refresh in the +// background (cache_refresh.go). +var sqlUnderLockAllowedCallees = map[string]bool{ + "PacketStore.getCachedNodesAndPM": true, + "PacketStore.resolveRegionObservers": true, + "PacketStore.resolveAreaNodes": true, +} + +// sqlUnderLockAllowedHolders run at startup, before or while the store is +// first filled. Load and scanAndMergeChunk already release the lock around +// their evidence query; the scan cannot see an Unlock in a nested block. +var sqlUnderLockAllowedHolders = map[string]bool{ + "PacketStore.Load": true, + "PacketStore.LoadChunked": true, + "PacketStore.loadChunk": true, + "PacketStore.scanAndMergeChunk": true, +} + +func TestNoSQLUnderStoreLock(t *testing.T) { + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + srcs := map[string]string{} + for _, f := range files { + if strings.HasSuffix(f, "_test.go") { + continue + } + b, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + srcs[f] = string(b) + } + hits, regions := findSQLUnderStoreLock(t, srcs) + // Guard against a scan that silently stops matching anything. + if regions < 40 { + t.Fatalf("found only %d store.mu sections; the scan no longer recognises the locking code", regions) + } + for _, h := range hits { + t.Errorf("SQL under store.mu: %s", h) + } +} + +// The scan itself must catch a direct and an indirect query under the lock, +// and ignore one after the unlock. +func TestNoSQLUnderStoreLockDetects(t *testing.T) { + src := `package main +type PacketStore struct{} +type DB struct{} +func (db *DB) GetThing() {} +func (s *PacketStore) helper() { s.db.GetThing() } +func (s *PacketStore) Direct() { + s.mu.RLock() + defer s.mu.RUnlock() + s.db.conn.Query("SELECT 1") +} +func (s *PacketStore) Indirect() { + s.mu.RLock() + s.helper() + s.mu.RUnlock() +} +func (s *PacketStore) After() { + s.mu.RLock() + x := 1 + s.mu.RUnlock() + s.helper() + _ = x +} +` + hits, _ := findSQLUnderStoreLock(t, map[string]string{"x.go": src}) + got := strings.Join(hits, "\n") + for _, want := range []string{"PacketStore.Direct", "PacketStore.Indirect calls PacketStore.helper"} { + if !strings.Contains(got, want) { + t.Errorf("scan missed %q; hits:\n%s", want, got) + } + } + if strings.Contains(got, "PacketStore.After") { + t.Errorf("scan flagged a query after the unlock; hits:\n%s", got) + } +} + +var sqlMethodNames = map[string]bool{ + "Query": true, "QueryRow": true, "QueryContext": true, "QueryRowContext": true, + "Exec": true, "ExecContext": true, "Prepare": true, "Begin": true, "BeginTx": true, +} + +func lockScanExpr(e ast.Expr) string { + switch x := e.(type) { + case *ast.Ident: + return x.Name + case *ast.SelectorExpr: + return lockScanExpr(x.X) + "." + x.Sel.Name + case *ast.CallExpr: + return lockScanExpr(x.Fun) + "()" + } + return "?" +} + +func lockScanRecv(d *ast.FuncDecl) string { + if d.Recv == nil || len(d.Recv.List) == 0 { + return "" + } + t := d.Recv.List[0].Type + if s, ok := t.(*ast.StarExpr); ok { + t = s.X + } + if id, ok := t.(*ast.Ident); ok { + return id.Name + } + return "" +} + +// findSQLUnderStoreLock returns one line per call made inside a store.mu +// section that reaches SQL, and the number of sections it found. +func findSQLUnderStoreLock(t *testing.T, srcs map[string]string) (hits []string, regions int) { + t.Helper() + fset := token.NewFileSet() + type fn struct { + decl *ast.FuncDecl + recv string + } + funcs := map[string]fn{} + for name, src := range srcs { + f, err := parser.ParseFile(fset, name, src, 0) + if err != nil { + t.Fatal(err) + } + for _, d := range f.Decls { + fd, ok := d.(*ast.FuncDecl) + if !ok || fd.Body == nil { + continue + } + recv := lockScanRecv(fd) + funcs[recv+"."+fd.Name.Name] = fn{fd, recv} + } + } + resolve := func(c *ast.CallExpr, recv string) string { + switch f := c.Fun.(type) { + case *ast.Ident: + if _, ok := funcs["."+f.Name]; ok { + return "." + f.Name + } + case *ast.SelectorExpr: + x := lockScanExpr(f.X) + var typ string + switch { + case x == "s" && recv != "": + typ = recv + case strings.HasSuffix(x, "store"): + typ = "PacketStore" + case strings.HasSuffix(x, "db"): + typ = "DB" + } + if _, ok := funcs[typ+"."+f.Sel.Name]; ok { + return typ + "." + f.Sel.Name + } + } + return "" + } + + // via[k] is the next step from k towards a function that runs SQL. + via := map[string]string{} + edges := map[string][]string{} + for k, f := range funcs { + if f.recv == "DB" { + via[k] = k + } + ast.Inspect(f.decl.Body, func(n ast.Node) bool { + c, ok := n.(*ast.CallExpr) + if !ok { + return true + } + if se, ok := c.Fun.(*ast.SelectorExpr); ok && sqlMethodNames[se.Sel.Name] && strings.HasSuffix(lockScanExpr(se.X), "conn") { + via[k] = k + } + if callee := resolve(c, f.recv); callee != "" { + edges[k] = append(edges[k], callee) + } + return true + }) + } + for k := range sqlUnderLockAllowedCallees { + delete(via, k) + } + for changed := true; changed; { + changed = false + for k, es := range edges { + if _, ok := via[k]; ok || sqlUnderLockAllowedCallees[k] { + continue + } + for _, e := range es { + if _, ok := via[e]; ok { + via[k] = e + changed = true + break + } + } + } + } + chain := func(k string) string { + out := []string{k} + for i := 0; i < 10 && via[k] != k; i++ { + k = via[k] + out = append(out, k) + } + return strings.Join(out, " -> ") + } + + isStoreMu := func(x ast.Expr, recv string) bool { + s := lockScanExpr(x) + return s == "s.store.mu" || s == "store.mu" || (s == "s.mu" && recv == "PacketStore") + } + isUnlock := func(st ast.Stmt) bool { + es, ok := st.(*ast.ExprStmt) + if !ok { + return false + } + c, ok := es.X.(*ast.CallExpr) + if !ok { + return false + } + s := lockScanExpr(c.Fun) + return strings.HasSuffix(s, "mu.RUnlock") || strings.HasSuffix(s, "mu.Unlock") + } + + for k, f := range funcs { + if sqlUnderLockAllowedHolders[k] { + continue + } + var spans [][2]token.Pos + ast.Inspect(f.decl.Body, func(n ast.Node) bool { + b, ok := n.(*ast.BlockStmt) + if !ok { + return true + } + for i, st := range b.List { + es, ok := st.(*ast.ExprStmt) + if !ok { + continue + } + c, ok := es.X.(*ast.CallExpr) + if !ok { + continue + } + se, ok := c.Fun.(*ast.SelectorExpr) + if !ok || (se.Sel.Name != "RLock" && se.Sel.Name != "Lock") || !isStoreMu(se.X, f.recv) { + continue + } + end := b.End() // a deferred unlock holds it to the end of the block + for _, st2 := range b.List[i+1:] { + if isUnlock(st2) { + end = st2.Pos() + break + } + } + spans = append(spans, [2]token.Pos{c.Pos(), end}) + } + return true + }) + regions += len(spans) + ast.Inspect(f.decl.Body, func(n ast.Node) bool { + c, ok := n.(*ast.CallExpr) + if !ok { + return true + } + inside := false + for _, sp := range spans { + if c.Pos() > sp[0] && c.Pos() < sp[1] { + inside = true + } + } + if !inside { + return true + } + if se, ok := c.Fun.(*ast.SelectorExpr); ok && sqlMethodNames[se.Sel.Name] && strings.HasSuffix(lockScanExpr(se.X), "conn") { + hits = append(hits, fset.Position(c.Pos()).String()+" "+k+" queries directly") + return true + } + if callee := resolve(c, f.recv); callee != "" { + if _, ok := via[callee]; ok { + hits = append(hits, fset.Position(c.Pos()).String()+" "+k+" calls "+chain(callee)) + } + } + return true + }) + } + sort.Strings(hits) + return hits, regions +} diff --git a/cmd/server/store_memory_diagnostics_test.go b/cmd/server/store_memory_diagnostics_test.go index 9edf9719f..f027a48e4 100644 --- a/cmd/server/store_memory_diagnostics_test.go +++ b/cmd/server/store_memory_diagnostics_test.go @@ -97,16 +97,23 @@ func TestObsRawHexNotRetainedOnLoad(t *testing.T) { t.Fatal(err) } stmts := []string{ - `CREATE TABLE transmissions ( + testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT + )`, `CREATE TABLE transmissions ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`, - `CREATE TABLE observations ( + )`), + testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, + observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp BIGINT, raw_hex TEXT + )`, `CREATE TABLE observations ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp BIGINT, raw_hex TEXT - )`, - `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, + )`), + testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`), `CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION @@ -162,7 +169,7 @@ func TestObsRawHexNotRetainedOnLoad(t *testing.T) { // The read path must still serve raw_hex via the parent-tx fallback. store.mu.RLock() - m := store.enrichObs(obs) + m, _ := store.enrichObsRPJob(obs) store.mu.RUnlock() rh, _ := m["raw_hex"].(string) if rh != txHex { diff --git a/cmd/server/store_obs_memory_test.go b/cmd/server/store_obs_memory_test.go new file mode 100644 index 000000000..effa63523 --- /dev/null +++ b/cmd/server/store_obs_memory_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "runtime" + "testing" + "unsafe" +) + +// Observations outnumber transmissions at high-fanout receivers. Keep the +// timestamp cache packed so each retained observation fits the 192-byte +// allocation class on the supported 64-bit deployment targets. +func TestStoreObsMemoryFootprint(t *testing.T) { + if unsafe.Sizeof(uintptr(0)) != 8 { + t.Skip("allocation budget applies to 64-bit targets") + } + if size := unsafe.Sizeof(StoreObs{}); size > 192 { + t.Fatalf("StoreObs is %d bytes; want <=192 to avoid the 208-byte allocation class", size) + } +} + +// Measure live objects after GC; a heap profile taken after this benchmark +// returns would otherwise lose the observation set being compared. +func BenchmarkStoreObsRetainedMemory(b *testing.B) { + const count = 100000 + for i := 0; i < b.N; i++ { + b.StopTimer() + observations := make([]*StoreObs, count) + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + b.StartTimer() + for j := range observations { + observations[j] = &StoreObs{ID: j + 1, TransmissionID: j/16 + 1, Timestamp: "2026-01-01T00:00:00Z"} + observations[j].ParsedTime() + } + b.StopTimer() + runtime.GC() + runtime.ReadMemStats(&after) + b.ReportMetric(float64(int64(after.HeapAlloc)-int64(before.HeapAlloc))/count, "retained-B/obs") + runtime.KeepAlive(observations) + b.StartTimer() + } +} diff --git a/cmd/server/topology_dedup_test.go b/cmd/server/topology_dedup_test.go index b8e2980b0..17bfa3b9d 100644 --- a/cmd/server/topology_dedup_test.go +++ b/cmd/server/topology_dedup_test.go @@ -21,15 +21,21 @@ func TestTopologyDedup_RepeatersMergeByPubkey(t *testing.T) { t.Fatalf("SQL exec failed: %v\nSQL: %s", err, s) } } - exec(`CREATE TABLE transmissions ( + exec(testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT + )`, `CREATE TABLE transmissions ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) - exec(`CREATE TABLE observations ( + )`)) + exec(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT + )`, `CREATE TABLE observations ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT - )`) - exec(`CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + )`)) + exec(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) exec(`CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, frequency DOUBLE PRECISION @@ -144,15 +150,21 @@ func TestTopologyDedup_AmbiguousPrefixNotMerged(t *testing.T) { t.Fatalf("SQL exec failed: %v\nSQL: %s", err, s) } } - exec(`CREATE TABLE transmissions ( + exec(testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT + )`, `CREATE TABLE transmissions ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) - exec(`CREATE TABLE observations ( + )`)) + exec(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT + )`, `CREATE TABLE observations ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT - )`) - exec(`CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + )`)) + exec(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) exec(`CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, frequency DOUBLE PRECISION @@ -249,15 +261,21 @@ func TestTopologyDedup_PairsMergeByPubkey(t *testing.T) { t.Fatalf("SQL exec failed: %v\nSQL: %s", err, s) } } - exec(`CREATE TABLE transmissions ( + exec(testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT + )`, `CREATE TABLE transmissions ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) - exec(`CREATE TABLE observations ( + )`)) + exec(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT + )`, `CREATE TABLE observations ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT - )`) - exec(`CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + )`)) + exec(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) exec(`CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, frequency DOUBLE PRECISION diff --git a/cmd/server/types.go b/cmd/server/types.go index b6fa5fedd..014a3bef7 100644 --- a/cmd/server/types.go +++ b/cmd/server/types.go @@ -269,6 +269,8 @@ type DatabaseRowCounts struct { } type PostgresStats struct { + WalSizeMB *float64 `json:"walSizeMB,omitempty"` + FreelistMB *float64 `json:"freelistMB,omitempty"` Engine string `json:"engine"` DbSizeMB *float64 `json:"dbSizeMB"` Rows *DatabaseRowCounts `json:"rows"` @@ -287,6 +289,8 @@ type PerfResponse struct { Cache PerfCacheStats `json:"cache"` PacketStore *PerfPacketStoreStats `json:"packetStore"` Postgres *PostgresStats `json:"postgres"` + Sqlite *PostgresStats `json:"sqlite,omitempty"` + Database *PostgresStats `json:"database,omitempty"` GoRuntime *GoRuntimeStats `json:"goRuntime,omitempty"` // MemoryBreakdown is populated only for /api/perf?mem=1 (an O(tx+obs) // walk, opt-in so the normal hot endpoint stays cheap). It sizes the diff --git a/cmd/server/user_mgmt_config.go b/cmd/server/user_mgmt_config.go index 2a899a384..a8d10fd6b 100644 --- a/cmd/server/user_mgmt_config.go +++ b/cmd/server/user_mgmt_config.go @@ -193,13 +193,14 @@ var fakeMailerAllowed bool func resolveUserManagement(u *UserManagementConfig, stateDir string, getenv func(string) string) (*userMgmtSettings, error) { set := &userMgmtSettings{adminEmails: map[string]bool{}} - if u.DBPath != "" { - return nil, errors.New("userManagement.dbPath is no longer supported; import the accounts database and configure userManagement.databaseURL") - } set.databaseURL = envOrValue(getenv, "CORESCOPE_USERS_DATABASE_URL", u.DatabaseURL) if set.databaseURL == "" { - return nil, errors.New("userManagement.databaseURL or CORESCOPE_USERS_DATABASE_URL is required for the separate PostgreSQL accounts database") + set.databaseURL = strings.TrimSpace(u.DBPath) + } + if set.databaseURL == "" { + set.databaseURL = filepath.Join(stateDir, "users.db") } + for _, raw := range u.AdminEmails { e, err := users.NormalizeEmail(raw) if err != nil { diff --git a/cmd/server/user_mgmt_config_test.go b/cmd/server/user_mgmt_config_test.go index a06cef3f8..425c60350 100644 --- a/cmd/server/user_mgmt_config_test.go +++ b/cmd/server/user_mgmt_config_test.go @@ -157,8 +157,9 @@ func TestResolveUsersBackup(t *testing.T) { t.Fatalf("absent block = %+v; want %+v", set.backup, defaults) } u.DBPath = "private-users.db" - if _, err := resolveUserManagement(u, meas, noEnv); err == nil || strings.Contains(err.Error(), u.DBPath) { - t.Fatalf("legacy account path must be refused: %v", err) + u.DatabaseURL = "" + if set, err := resolveUserManagement(u, meas, noEnv); err != nil || set.databaseURL != u.DBPath { + t.Fatalf("legacy account path lost: %v", err) } u.DBPath = "" off, on := false, true diff --git a/cmd/server/user_mgmt_off_test.go b/cmd/server/user_mgmt_off_test.go index adcbc8887..48a9c352d 100644 --- a/cmd/server/user_mgmt_off_test.go +++ b/cmd/server/user_mgmt_off_test.go @@ -8,7 +8,6 @@ import ( "testing" "github.com/gorilla/mux" - "github.com/meshcore-analyzer/pgutil/pgtest" ) func TestUserManagementOffIsUnchanged(t *testing.T) { @@ -84,10 +83,11 @@ func TestInitUserManagementRefusesMeasurementDB(t *testing.T) { } func TestInitUserManagementRequiresBootstrappedDatabase(t *testing.T) { + postgresOnly(t) u := validUM() - u.DatabaseURL = pgtest.NewDatabase(t) + u.DatabaseURL = testDatabaseDSN(t) srv := &Server{cfg: &Config{UserManagement: u}} - if err := srv.initUserManagement(pgtest.NewDatabase(t)); err == nil { + if err := srv.initUserManagement(testDatabaseDSN(t)); err == nil { t.Fatal("runtime created missing account schema") } db, err := openFixtureSQL(u.DatabaseURL) @@ -96,7 +96,7 @@ func TestInitUserManagementRequiresBootstrappedDatabase(t *testing.T) { } defer db.Close() var tables int - if err := db.QueryRow(`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='public'`).Scan(&tables); err != nil || tables != 0 { + if err := db.QueryRow(testNativeSQL(`SELECT COUNT(*) FROM sqlite_master WHERE type='table'`, `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='public'`)).Scan(&tables); err != nil || tables != 0 { t.Fatalf("runtime created tables: %d, %v", tables, err) } } diff --git a/cmd/server/users_backup.go b/cmd/server/users_backup.go index f920e7793..182577554 100644 --- a/cmd/server/users_backup.go +++ b/cmd/server/users_backup.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/users" ) @@ -26,18 +27,22 @@ const ( // usersBackupName matches the snapshot files this code writes and rotates. // Apart from orphaned temporary files (usersBackupTemp), nothing else in // the directory is ever touched. -var usersBackupName = regexp.MustCompile(`^users-\d{8}-\d{6}\.dump$`) +var usersBackupName = regexp.MustCompile(`^users-\d{8}-\d{6}\.(?:db|dump)$`) // usersBackupTemp matches the temporary name a snapshot is written under. // One is only left behind when the process died mid-write. -var usersBackupTemp = regexp.MustCompile(`^users-\d{8}-\d{6}\.dump\.tmp$`) +var usersBackupTemp = regexp.MustCompile(`^users-\d{8}-\d{6}\.(?:db|dump)\.tmp$`) // usersBackupTempMaxAge is how old an orphaned temporary snapshot must be // before the sweep removes it; no snapshot write takes this long. const usersBackupTempMaxAge = 24 * time.Hour -func usersBackupFile(t time.Time) string { - return "users-" + t.UTC().Format(usersBackupLayout) + ".dump" +func usersBackupFile(t time.Time, backend ...dbconfig.Backend) string { + extension := ".dump" + if len(backend) > 0 && backend[0] == dbconfig.SQLite { + extension = ".db" + } + return "users-" + t.UTC().Format(usersBackupLayout) + extension } // listUsersBackups returns the snapshot file names in dir, oldest first @@ -66,7 +71,7 @@ func listUsersBackups(dir string) ([]string, error) { // back) are skipped so they cannot stop the backups. func usersBackupDue(names []string, now time.Time) bool { for i := len(names) - 1; i >= 0; i-- { - t, err := time.Parse(usersBackupLayout, strings.TrimSuffix(strings.TrimPrefix(names[i], "users-"), ".dump")) + t, err := time.Parse(usersBackupLayout, strings.TrimSuffix(strings.TrimSuffix(strings.TrimPrefix(names[i], "users-"), ".dump"), ".db")) if err != nil || t.After(now) { continue } @@ -81,7 +86,7 @@ func writeUsersBackup(st *users.Store, dir string, now time.Time) (string, int64 if err := os.MkdirAll(dir, 0o700); err != nil { return "", 0, err } - path := filepath.Join(dir, usersBackupFile(now)) + path := filepath.Join(dir, usersBackupFile(now, st.Backend())) tmp := path + ".tmp" if err := st.Snapshot(tmp); err != nil { return "", 0, err @@ -202,7 +207,7 @@ func (s *Server) handleAdminUsersBackup(w http.ResponseWriter, r *http.Request, log.Printf("[users] backup download cleanup: %v", err) } }() - name := "corescope-users-" + time.Now().UTC().Format(usersBackupLayout) + ".dump" + name := "corescope-" + usersBackupFile(time.Now(), a.st.Backend()) path := filepath.Join(tmpDir, name) if err := a.st.SnapshotContext(r.Context(), path); err != nil { log.Printf("[users] backup download: %v", err) diff --git a/cmd/server/users_backup_test.go b/cmd/server/users_backup_test.go index 72b841fd6..456d6011b 100644 --- a/cmd/server/users_backup_test.go +++ b/cmd/server/users_backup_test.go @@ -11,7 +11,6 @@ import ( "testing" "time" - "github.com/meshcore-analyzer/pgutil/pgtest" "github.com/meshcore-analyzer/users" ) @@ -53,7 +52,7 @@ func TestUsersBackupWrittenWhenNoneExists(t *testing.T) { } a.maybeBackup(backupNow) names := snapshotNames(t, dir) - if len(names) != 1 || names[0] != usersBackupFile(backupNow) || names[0] != "users-20261008-120000.dump" { + if len(names) != 1 || names[0] != testUsersBackupFile(backupNow) || names[0] != testNativeSQL("users-20261008-120000.db", "users-20261008-120000.dump") { t.Fatalf("snapshots = %v", names) } db := restorePostgresTestBackup(t, filepath.Join(dir, names[0])) @@ -73,7 +72,7 @@ func TestUsersBackupFileMode(t *testing.T) { } a, dir := backupService(t, 7) a.maybeBackup(backupNow) - fi, err := os.Stat(filepath.Join(dir, usersBackupFile(backupNow))) + fi, err := os.Stat(filepath.Join(dir, testUsersBackupFile(backupNow))) if err != nil { t.Fatal(err) } @@ -91,7 +90,7 @@ func TestUsersBackupFileMode(t *testing.T) { func TestUsersBackupSkippedWhenFresh(t *testing.T) { a, dir := backupService(t, 7) - fresh := usersBackupFile(backupNow.Add(-time.Hour)) + fresh := testUsersBackupFile(backupNow.Add(-time.Hour)) writeBackupFile(t, dir, fresh) a.maybeBackup(backupNow) a.maybeBackup(backupNow.Add(22 * time.Hour)) @@ -102,20 +101,20 @@ func TestUsersBackupSkippedWhenFresh(t *testing.T) { func TestUsersBackupWrittenWhenStale(t *testing.T) { a, dir := backupService(t, 7) - stale := usersBackupFile(backupNow.Add(-25 * time.Hour)) + stale := testUsersBackupFile(backupNow.Add(-25 * time.Hour)) writeBackupFile(t, dir, stale) a.maybeBackup(backupNow) - if names := snapshotNames(t, dir); !reflect.DeepEqual(names, []string{stale, usersBackupFile(backupNow)}) { + if names := snapshotNames(t, dir); !reflect.DeepEqual(names, []string{stale, testUsersBackupFile(backupNow)}) { t.Fatalf("snapshots = %v", names) } } func TestUsersBackupFutureSnapshotDoesNotBlock(t *testing.T) { a, dir := backupService(t, 7) - future := usersBackupFile(backupNow.Add(48 * time.Hour)) + future := testUsersBackupFile(backupNow.Add(48 * time.Hour)) writeBackupFile(t, dir, future) a.maybeBackup(backupNow) - want := []string{usersBackupFile(backupNow), future} + want := []string{testUsersBackupFile(backupNow), future} if names := snapshotNames(t, dir); !reflect.DeepEqual(names, want) { t.Fatalf("snapshots = %v; want %v", names, want) } @@ -128,10 +127,10 @@ func TestUsersBackupFutureSnapshotDoesNotBlock(t *testing.T) { func TestUsersBackupRotationNeverDeletesTheNewSnapshot(t *testing.T) { a, dir := backupService(t, 1) - future := usersBackupFile(backupNow.Add(48 * time.Hour)) + future := testUsersBackupFile(backupNow.Add(48 * time.Hour)) writeBackupFile(t, dir, future) a.maybeBackup(backupNow) - want := []string{usersBackupFile(backupNow)} + want := []string{testUsersBackupFile(backupNow)} if names := snapshotNames(t, dir); !reflect.DeepEqual(names, want) { t.Fatalf("snapshots = %v; want %v", names, want) } @@ -141,16 +140,16 @@ func TestUsersBackupRotationKeepsNewestAndForeignFiles(t *testing.T) { a, dir := backupService(t, 3) var old []string // oldest first for i := 5; i >= 1; i-- { - n := usersBackupFile(backupNow.Add(-time.Duration(i) * 25 * time.Hour)) + n := testUsersBackupFile(backupNow.Add(-time.Duration(i) * 25 * time.Hour)) writeBackupFile(t, dir, n) old = append(old, n) } - foreign := []string{"notes.txt", "users-before-upgrade.db", "users.db", usersBackupFile(backupNow.Add(-300*time.Hour)) + ".tmp"} + foreign := []string{"notes.txt", "users-before-upgrade.db", "users.db", testUsersBackupFile(backupNow.Add(-300*time.Hour)) + ".tmp"} for _, n := range foreign { writeBackupFile(t, dir, n) } a.maybeBackup(backupNow) - want := []string{old[3], old[4], usersBackupFile(backupNow)} + want := []string{old[3], old[4], testUsersBackupFile(backupNow)} if names := snapshotNames(t, dir); !reflect.DeepEqual(names, want) { t.Fatalf("after rotation = %v; want %v", names, want) } @@ -170,21 +169,21 @@ func TestUsersBackupSweepsOldTempFiles(t *testing.T) { t.Fatal(err) } } - orphan := usersBackupFile(backupNow.Add(-30*time.Hour)) + ".tmp" - recent := usersBackupFile(backupNow.Add(-time.Hour)) + ".tmp" + orphan := testUsersBackupFile(backupNow.Add(-30*time.Hour)) + ".tmp" + recent := testUsersBackupFile(backupNow.Add(-time.Hour)) + ".tmp" foreign := []string{"other.db.tmp", "users-20261001-000000.db.tmp.bak", "users-before-upgrade.db.tmp"} for _, n := range append([]string{orphan, recent}, foreign...) { writeBackupFile(t, dir, n) setAge(n, 25*time.Hour) } setAge(recent, time.Hour) - dirLike := usersBackupFile(backupNow.Add(-40*time.Hour)) + ".tmp" + dirLike := testUsersBackupFile(backupNow.Add(-40*time.Hour)) + ".tmp" if err := os.Mkdir(filepath.Join(dir, dirLike), 0o700); err != nil { t.Fatal(err) } setAge(dirLike, 25*time.Hour) // A fresh snapshot: no new one is due, the sweep still runs. - writeBackupFile(t, dir, usersBackupFile(backupNow.Add(-time.Hour))) + writeBackupFile(t, dir, testUsersBackupFile(backupNow.Add(-time.Hour))) a.maybeBackup(backupNow) if _, err := os.Stat(filepath.Join(dir, orphan)); !os.IsNotExist(err) { @@ -199,7 +198,7 @@ func TestUsersBackupSweepsOldTempFiles(t *testing.T) { func TestUsersBackupFailureKeepsOldSnapshots(t *testing.T) { a, dir := backupService(t, 1) - stale := usersBackupFile(backupNow.Add(-48 * time.Hour)) + stale := testUsersBackupFile(backupNow.Add(-48 * time.Hour)) writeBackupFile(t, dir, stale) a.st.Close() // every store call fails from here; the fixture's cleanup closes again, which is a no-op a.maybeBackup(backupNow) @@ -232,7 +231,7 @@ func TestInitUserManagementBacksUpAtStartup(t *testing.T) { u.DatabaseURL = a.set.databaseURL u.Backup = &UsersBackupConfig{Dir: dir} srv := &Server{cfg: &Config{UserManagement: u}} - if err := srv.initUserManagement(pgtest.NewDatabase(t)); err != nil { + if err := srv.initUserManagement(testDatabaseDSN(t)); err != nil { t.Fatal(err) } srv.closeUserManagement() @@ -241,7 +240,7 @@ func TestInitUserManagementBacksUpAtStartup(t *testing.T) { } } -var usersBackupFilenameRE = regexp.MustCompile(`^attachment; filename="corescope-users-\d{8}-\d{6}\.dump"$`) +var usersBackupFilenameRE = regexp.MustCompile(`^attachment; filename="corescope-users-\d{8}-\d{6}\.(?:db|dump)"$`) func TestAdminUsersBackupDownload(t *testing.T) { f := newBackupAuthFixture(t, "boss@example.org") @@ -266,7 +265,7 @@ func TestAdminUsersBackupDownload(t *testing.T) { t.Errorf("Content-Disposition = %q", cd) } body := w.Body.Bytes() - if !bytes.HasPrefix(body, []byte("PGDMP")) { + if !bytes.HasPrefix(body, []byte(postgresMagic)) { t.Fatalf("body is not a PostgreSQL custom archive (%d bytes)", len(body)) } if cl := w.Header().Get("Content-Length"); cl != strconv.Itoa(len(body)) { @@ -293,3 +292,5 @@ func TestAdminUsersBackupDownload(t *testing.T) { t.Fatalf("user.backup row = %+v", e) } } + +func testUsersBackupFile(stamp time.Time) string { return usersBackupFile(stamp, testBackendValue()) } diff --git a/config.example.json b/config.example.json index 0b372d260..7e0eac3d4 100644 --- a/config.example.json +++ b/config.example.json @@ -1,9 +1,10 @@ { "port": 3000, "apiKey": "your-secret-api-key-here", + "dbPath": "", "databaseURL": "", "stateDir": "data", - "_comment_databaseURL": "PostgreSQL telemetry URL. Prefer environment credentials: CORESCOPE_READER_DATABASE_URL for the server, CORESCOPE_WRITER_DATABASE_URL for the ingestor, or CORESCOPE_DATABASE_URL when running a single process. Runtime roles are separate from the bootstrap owner. SQLite dbPath is no longer a runtime setting; upgrade existing databases with the offline importer described in docs/postgresql-upgrade.md.", + "_comment_databaseURL": "PostgreSQL telemetry URL. Prefer environment credentials: CORESCOPE_READER_DATABASE_URL for the server, CORESCOPE_WRITER_DATABASE_URL for the ingestor, or CORESCOPE_DATABASE_URL when running a single process. Runtime roles are separate from the bootstrap owner. Used only when PostgreSQL is selected. SQLite uses dbPath/DB_PATH. After installation the private storage-selection.json record wins over bootstrap paths/backend settings; change engines only through the offline operator workflow in docs/storage.md.", "_comment_stateDir": "Directory shared by the server and ingestor for file queues, statistics and account backups; CORESCOPE_STATE_DIR overrides it. This is a filesystem path, never a database URL. The default is data; containers use /app/data.", "nodeBlacklist": [], "_comment_nodeBlacklist": "Public keys of nodes to hide from all API responses. Use for trolls, offensive names, or nodes reporting false data that operators refuse to fix.", @@ -23,11 +24,14 @@ "_comment": "nodeDays: nodes not seen in N days moved to inactive_nodes (default 7). observerDays: observers not sending data in N days are removed (-1 = keep forever, default 14). observerPurgeDays: observers already marked inactive AND not seen in N days are hard-deleted, but only when no observation, observer_metrics or dropped_packets row still references them (0 = disabled, the default). Set it above both observerDays and packetDays — below those the reference guards keep every row anyway. packetDays: transmissions older than N days are deleted (0 = disabled). clientRxDays: mobile client-RX coverage rows (client_receptions/client_observers) older than N days are deleted (0 = disabled) — bounds the opt-in coverage tables (#1727). clientRxObsDays: diagnostic client_rx_observations rows (by rx_at) older than N days are deleted (0 = disabled) — bounds the opt-in clientRxObservations table. clientRfDays: RF environment sample rows (client_rf_samples, by sampled_at) older than N days are deleted (0 = disabled) — bounds the opt-in clientRfSamples table; unset means unbounded growth at roughly 240 rows/hour per driver. clientRegionsDays: region-discovery answer rows (node_declared_regions, by observed_at) older than N days are deleted (0 = disabled) — bounds the opt-in clientRegions table. NOTE (#1283): all retention fields are consumed by the INGESTOR process. The server is read-only and never prunes." }, "db": { + "backend": "sqlite", + "vacuumOnStartup": false, + "incrementalVacuumPages": 1024, "load": { "chunkSize": 10000, "_comment": "chunkSize: rows fetched per chunk by PacketStore.LoadChunked during startup (#1009). Default 10000. Lower values surface the early-HTTP-readiness signal sooner (the listener binds after the first chunk) at the cost of more SQL round-trips. Higher values reduce per-chunk overhead but delay first-chunk readiness. The X-CoreScope-Load-Status response header reports loading|ready and progress= until the load completes." }, - "_comment": "PostgreSQL manages VACUUM, ANALYZE and checkpoints. The former SQLite vacuumOnStartup, incrementalVacuumPages and analysisLimit controls do not apply. Keep autovacuum enabled in the database service. load.chunkSize controls application startup loading; see the nested comment.", + "_comment": "backend selects a fresh install: sqlite (default) or postgres; CORESCOPE_DB_BACKEND supplies a bootstrap environment choice. Installed storage-selection.json is authoritative, including when older config still says sqlite after a verified PostgreSQL conversion. Backend changes are offline operator actions, not live customizer toggles. SQLite vacuumOnStartup is opt-in, can block startup for minutes and needs working disk space; incrementalVacuumPages defaults to 1024. These SQLite maintenance controls are ignored by PostgreSQL, which manages VACUUM/ANALYZE/checkpoints. Keep PostgreSQL autovacuum enabled. load.chunkSize applies to both engines.", "_comment_slowWriterMs": "Writer-lock log threshold (default 500 ms). Set CORESCOPE_DB_SLOW_WRITER_MS on the ingestor to adjust logging. Per-component wait/hold time and contention counters are available at /api/perf/write-sources. These counters describe the application writer, not PostgreSQL cluster-wide commits or WAL flushes." }, "listLimits": { @@ -433,6 +437,7 @@ "_comment_analytics": "Issue #1240 + #1256 + #1265. Each analytics endpoint (topology, rf, distance, channels, hashCollisions, hashSizes, roles, observersClockSkew, nodesClockSkew) is recomputed in the background on the configured interval and served from an atomic-pointer cache. Reads never block on compute. Default 300s (5 min) per endpoint reflects the operator principle: serving slightly stale data quickly beats real-time data slowly. Lower values = fresher data at higher CPU cost. Only the default query (no region/window) is precomputed; region- and window-filtered requests fall back to the legacy on-request compute + 60s TTL cache.", "userManagement": { "enabled": false, + "dbPath": "", "databaseURL": "", "approvedChannelsDatabaseURL": "", "adminEmails": ["operator@example.org"], @@ -465,10 +470,10 @@ "keep": 7 } }, - "_comment_userManagement": "Optional accounts (off by default; see docs/user-guide/accounts.md). When enabled: visitors can register with email + password and activate through a mailed link; addresses in adminEmails become admins; admins manage users at #/admin/users and can use the operator endpoints without the apiKey. The dashboard stays public. databaseURL (or CORESCOPE_USERS_DATABASE_URL) selects a separate PostgreSQL account database with its own restricted writer role. The server never writes telemetry. Initialize schemas with the migration owner before starting either service. publicBaseUrl is required and is used for every link in every mail. Mail goes through Brevo's transactional API (a free account is enough); brevoApiKey and webhookSecret may instead come from CORESCOPE_BREVO_API_KEY and CORESCOPE_BREVO_WEBHOOK_SECRET. Set webhookSecret (16+ chars) and point a Brevo transactional webhook at /api/mail/brevo/webhook with bearer auth to see delivery status per user. trustedProxies (CIDRs) lets login rate limits use X-Forwarded-For behind a reverse proxy. Startup fails if enabled with an incomplete mail setup.", + "_comment_userManagement": "Optional accounts (off by default; see docs/user-guide/accounts.md). When enabled: visitors can register with email + password and activate through a mailed link; addresses in adminEmails become admins; admins manage users at #/admin/users and can use the operator endpoints without the apiKey. The dashboard stays public. SQLite dbPath defaults to users.db beside telemetry; PostgreSQL databaseURL (or CORESCOPE_USERS_DATABASE_URL) uses a separate account database with a restricted writer role. The recorded account target persists while this feature is disabled and participates in an offline backend conversion. The server never writes telemetry. Packaged setup validates/initializes the selected stores before startup. publicBaseUrl is required and is used for every link in every mail. Mail goes through Brevo's transactional API (a free account is enough); brevoApiKey and webhookSecret may instead come from CORESCOPE_BREVO_API_KEY and CORESCOPE_BREVO_WEBHOOK_SECRET. Set webhookSecret (16+ chars) and point a Brevo transactional webhook at /api/mail/brevo/webhook with bearer auth to see delivery status per user. trustedProxies (CIDRs) lets login rate limits use X-Forwarded-For behind a reverse proxy. Startup fails if enabled with an incomplete mail setup.", "_comment_channelProposals": "Optional, needs userManagement.enabled (docs/user-guide/accounts.md, Channel proposals). Logged-in users propose hashtag channels; admins approve, reject or revoke them on the Proposals tab. An approved channel is decrypted by the ingestor from its next refresh (once a minute, no restart) and listed for every visitor on the Channels page. maxApproved bounds the decryption work (every approved key is tried on every group message); perUserPerDay and maxPending limit proposals. The ingestor reads only the approved_channels view using userManagement.approvedChannelsDatabaseURL or CORESCOPE_APPROVED_CHANNELS_DATABASE_URL. This reader must have no access to account tables or write privileges.", "_comment_notifications": "Optional, needs userManagement.enabled (docs/user-guide/accounts.md, Node notifications). Logged-in users watch nodes and get one mail per check when a watched node goes offline, comes back or reports a low battery; admins can also watch for new foreign nodes and observers going offline. Thresholds are healthThresholds and batteryThresholds. A check runs every intervalMinutes; at most perUserPerDay mails per user and maxMailsPerDay in total go out per 24 hours (over a limit the change is recorded and dropped, never queued). maxMailsPerDay counts notification mail only: keep it below the mail provider's daily quota so activation, password-reset and address-change mail still go out (the default 100 leaves 200 of Brevo's free 300 a day); maxWatchesPerUser bounds each watch list. Every mail has a one-click unsubscribe.", - "_comment_usersBackup": "With accounts enabled, the server creates a consistent PostgreSQL custom-format archive at startup when the newest snapshot is older than 24 hours, then daily. Files are users-.dump under backup.dir (default: backups under stateDir). keep limits retained snapshots (default 7). pg_dump 18 must be available. Archives contain authentication state: protect the directory and follow the account restore procedure before restarting, because restoring also restores old passwords, sessions, used tokens and deleted accounts.", + "_comment_usersBackup": "With accounts enabled, the server creates a consistent native snapshot at startup when the newest snapshot is older than 24 hours, then daily. SQLite snapshots use .db; PostgreSQL custom-format archives use .dump. Files are users- under backup.dir (default: backups under stateDir). keep limits retained snapshots (default 7). PostgreSQL requires pg_dump18; the supplied image includes it. Archives contain authentication state: protect the directory and follow the account restore procedure before restarting, because restoring also restores old passwords, sessions, used tokens and deleted accounts.", "customizer": { "disabledTabs": [], "_comment_disabledTabs": "Issue #1508. List of customizer-modal tab ids to hide from end users. Useful when operators want to expose only daily-use viewer controls and keep one-shot admin chrome out of the way. Recognized ids: branding, theme, nodes, home, display, geofilter, export. Example: [\"branding\", \"geofilter\", \"export\"] hides the admin tabs and leaves theme/colors/home/display visible. Default [] keeps the legacy behavior (all tabs visible). Operators edit this in config.json directly — there is no in-app UI for this list (by design)." diff --git a/docker-compose.example.postgres.yml b/docker-compose.example.postgres.yml new file mode 100644 index 000000000..1d7392eb9 --- /dev/null +++ b/docker-compose.example.postgres.yml @@ -0,0 +1,28 @@ +# Optional PostgreSQL storage. Combine with the matching SQLite-default base +# using -f base.yml -f this-file.yml. No profiles: unused PG credentials and +# services are absent from the base model. Changing files is not a conversion; +# use the documented offline storage switch before changing an installed backend. +services: + postgres: + extends: + file: docker/postgres.compose.yml + service: postgres + volumes: + - ${POSTGRES_DATA_DIR:-./postgres-data}:/var/lib/postgresql + bootstrap: + extends: + file: docker/postgres.compose.yml + service: bootstrap + image: ${CORESCOPE_IMAGE:-ghcr.io/kpa-clawbot/corescope:latest} + volumes: + - ${DATA_DIR:-./data}:/app/data + corescope: + depends_on: + bootstrap: + condition: service_completed_successfully + environment: + CORESCOPE_DB_BACKEND: postgres + CORESCOPE_READER_DATABASE_URL: postgresql://corescope_reader:${CORESCOPE_READER_PASSWORD:?Set CORESCOPE_READER_PASSWORD}@postgres:5432/corescope_telemetry?sslmode=disable&search_path=public + CORESCOPE_WRITER_DATABASE_URL: postgresql://corescope_writer:${CORESCOPE_WRITER_PASSWORD:?Set CORESCOPE_WRITER_PASSWORD}@postgres:5432/corescope_telemetry?sslmode=disable&search_path=public + CORESCOPE_USERS_DATABASE_URL: postgresql://corescope_accounts:${CORESCOPE_ACCOUNTS_PASSWORD:?Set CORESCOPE_ACCOUNTS_PASSWORD}@postgres:5432/corescope_accounts?sslmode=disable&search_path=public + CORESCOPE_APPROVED_CHANNELS_DATABASE_URL: postgresql://corescope_channels:${CORESCOPE_CHANNELS_PASSWORD:?Set CORESCOPE_CHANNELS_PASSWORD}@postgres:5432/corescope_accounts?sslmode=disable&search_path=public diff --git a/docker-compose.example.yml b/docker-compose.example.yml index fba94a6fa..3eec371f0 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -1,4 +1,4 @@ -# CoreScope — application and PostgreSQL deployment using the pre-built +# CoreScope — SQLite by default, using the pre-built # application image from GHCR. # # AUDIENCE: third-party operators running a self-hosted CoreScope on @@ -11,39 +11,21 @@ # `meshcore-net`), see `docker-compose.staging.yml`. # # Usage: docker compose -f docker-compose.example.yml up -d +# Optional PostgreSQL: add -f docker-compose.example.postgres.yml after setup. # Docs: https://github.com/Kpa-clawbot/CoreScope/blob/master/DEPLOY.md name: corescope services: - postgres: - extends: - file: docker/postgres.compose.yml - service: postgres - volumes: - - ${POSTGRES_DATA_DIR:-./postgres-data}:/var/lib/postgresql - bootstrap: - extends: - file: docker/postgres.compose.yml - service: bootstrap - image: ${CORESCOPE_IMAGE:-ghcr.io/kpa-clawbot/corescope:latest} - volumes: - - ${DATA_DIR:-./data}:/app/data:ro corescope: - depends_on: - bootstrap: - condition: service_completed_successfully image: ${CORESCOPE_IMAGE:-ghcr.io/kpa-clawbot/corescope:latest} ports: - "${HTTP_PORT:-80}:80" volumes: - ${DATA_DIR:-./data}:/app/data environment: - - CORESCOPE_STATE_DIR=/app/data - - CORESCOPE_READER_DATABASE_URL=postgresql://corescope_reader:${CORESCOPE_READER_PASSWORD:?Set CORESCOPE_READER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable - - CORESCOPE_WRITER_DATABASE_URL=postgresql://corescope_writer:${CORESCOPE_WRITER_PASSWORD:?Set CORESCOPE_WRITER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable - - CORESCOPE_USERS_DATABASE_URL=postgresql://corescope_accounts:${CORESCOPE_ACCOUNTS_PASSWORD:?Set CORESCOPE_ACCOUNTS_PASSWORD}@postgres/corescope_accounts?sslmode=disable - - CORESCOPE_APPROVED_CHANNELS_DATABASE_URL=postgresql://corescope_channels:${CORESCOPE_CHANNELS_PASSWORD:?Set CORESCOPE_CHANNELS_PASSWORD}@postgres/corescope_accounts?sslmode=disable + - CORESCOPE_DB_BACKEND=sqlite + - CORESCOPE_STATE_DIR=${CORESCOPE_STATE_DIR:-/app/data} - DISABLE_CADDY=${DISABLE_CADDY:-false} - DISABLE_MOSQUITTO=${DISABLE_MOSQUITTO:-false} restart: unless-stopped diff --git a/docker-compose.no-mosquitto.yml b/docker-compose.no-mosquitto.yml index ebc0f055f..39c04a640 100644 --- a/docker-compose.no-mosquitto.yml +++ b/docker-compose.no-mosquitto.yml @@ -16,23 +16,7 @@ name: corescope services: - postgres: - extends: - file: docker/postgres.compose.yml - service: postgres - volumes: - - ${PROD_POSTGRES_DATA_DIR:-~/corescope-postgres}:/var/lib/postgresql - bootstrap: - extends: - file: docker/postgres.compose.yml - service: bootstrap - image: corescope:latest - volumes: - - ${PROD_DATA_DIR:-~/meshcore-data}:/app/data:ro prod: - depends_on: - bootstrap: - condition: service_completed_successfully build: context: . args: @@ -54,11 +38,8 @@ services: - ${PROD_DATA_DIR:-~/meshcore-data}:/app/data - caddy-data:/data/caddy environment: - - CORESCOPE_STATE_DIR=/app/data - - CORESCOPE_READER_DATABASE_URL=postgresql://corescope_reader:${CORESCOPE_READER_PASSWORD:?Set CORESCOPE_READER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable - - CORESCOPE_WRITER_DATABASE_URL=postgresql://corescope_writer:${CORESCOPE_WRITER_PASSWORD:?Set CORESCOPE_WRITER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable - - CORESCOPE_USERS_DATABASE_URL=postgresql://corescope_accounts:${CORESCOPE_ACCOUNTS_PASSWORD:?Set CORESCOPE_ACCOUNTS_PASSWORD}@postgres/corescope_accounts?sslmode=disable - - CORESCOPE_APPROVED_CHANNELS_DATABASE_URL=postgresql://corescope_channels:${CORESCOPE_CHANNELS_PASSWORD:?Set CORESCOPE_CHANNELS_PASSWORD}@postgres/corescope_accounts?sslmode=disable + - CORESCOPE_DB_BACKEND=sqlite + - CORESCOPE_STATE_DIR=${CORESCOPE_STATE_DIR:-/app/data} - NODE_ENV=production - DISABLE_MOSQUITTO=true healthcheck: diff --git a/docker-compose.postgres.yml b/docker-compose.postgres.yml new file mode 100644 index 000000000..132310170 --- /dev/null +++ b/docker-compose.postgres.yml @@ -0,0 +1,28 @@ +# Optional PostgreSQL storage. Combine with the matching SQLite-default base +# using -f base.yml -f this-file.yml. No profiles: unused PG credentials and +# services are absent from the base model. Changing files is not a conversion; +# use the documented offline storage switch before changing an installed backend. +services: + postgres: + extends: + file: docker/postgres.compose.yml + service: postgres + volumes: + - ${PROD_POSTGRES_DATA_DIR:-~/corescope-postgres}:/var/lib/postgresql + bootstrap: + extends: + file: docker/postgres.compose.yml + service: bootstrap + image: corescope:latest + volumes: + - ${PROD_DATA_DIR:-~/meshcore-data}:/app/data + prod: + depends_on: + bootstrap: + condition: service_completed_successfully + environment: + CORESCOPE_DB_BACKEND: postgres + CORESCOPE_READER_DATABASE_URL: postgresql://corescope_reader:${CORESCOPE_READER_PASSWORD:?Set CORESCOPE_READER_PASSWORD}@postgres:5432/corescope_telemetry?sslmode=disable&search_path=public + CORESCOPE_WRITER_DATABASE_URL: postgresql://corescope_writer:${CORESCOPE_WRITER_PASSWORD:?Set CORESCOPE_WRITER_PASSWORD}@postgres:5432/corescope_telemetry?sslmode=disable&search_path=public + CORESCOPE_USERS_DATABASE_URL: postgresql://corescope_accounts:${CORESCOPE_ACCOUNTS_PASSWORD:?Set CORESCOPE_ACCOUNTS_PASSWORD}@postgres:5432/corescope_accounts?sslmode=disable&search_path=public + CORESCOPE_APPROVED_CHANNELS_DATABASE_URL: postgresql://corescope_channels:${CORESCOPE_CHANNELS_PASSWORD:?Set CORESCOPE_CHANNELS_PASSWORD}@postgres:5432/corescope_accounts?sslmode=disable&search_path=public diff --git a/docker-compose.staging.no-mosquitto.yml b/docker-compose.staging.no-mosquitto.yml index a1eb0e6ca..e58dd11c5 100644 --- a/docker-compose.staging.no-mosquitto.yml +++ b/docker-compose.staging.no-mosquitto.yml @@ -15,23 +15,7 @@ name: corescope-staging services: - postgres: - extends: - file: docker/postgres.compose.yml - service: postgres - volumes: - - ${STAGING_POSTGRES_DATA_DIR:-~/corescope-staging-postgres}:/var/lib/postgresql - bootstrap: - extends: - file: docker/postgres.compose.yml - service: bootstrap - image: corescope-go:latest - volumes: - - ${STAGING_DATA_DIR:-~/meshcore-staging-data}:/app/data:ro staging-go: - depends_on: - bootstrap: - condition: service_completed_successfully build: context: . dockerfile: Dockerfile @@ -58,11 +42,8 @@ services: - ${STAGING_DATA_DIR:-~/meshcore-staging-data}:/app/data - caddy-data-staging-go:/data/caddy environment: - - CORESCOPE_STATE_DIR=/app/data - - CORESCOPE_READER_DATABASE_URL=postgresql://corescope_reader:${CORESCOPE_READER_PASSWORD:?Set CORESCOPE_READER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable - - CORESCOPE_WRITER_DATABASE_URL=postgresql://corescope_writer:${CORESCOPE_WRITER_PASSWORD:?Set CORESCOPE_WRITER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable - - CORESCOPE_USERS_DATABASE_URL=postgresql://corescope_accounts:${CORESCOPE_ACCOUNTS_PASSWORD:?Set CORESCOPE_ACCOUNTS_PASSWORD}@postgres/corescope_accounts?sslmode=disable - - CORESCOPE_APPROVED_CHANNELS_DATABASE_URL=postgresql://corescope_channels:${CORESCOPE_CHANNELS_PASSWORD:?Set CORESCOPE_CHANNELS_PASSWORD}@postgres/corescope_accounts?sslmode=disable + - CORESCOPE_DB_BACKEND=sqlite + - CORESCOPE_STATE_DIR=${CORESCOPE_STATE_DIR:-/app/data} - NODE_ENV=staging - ENABLE_PPROF=true - DISABLE_MOSQUITTO=true diff --git a/docker-compose.staging.postgres.yml b/docker-compose.staging.postgres.yml new file mode 100644 index 000000000..a716170f2 --- /dev/null +++ b/docker-compose.staging.postgres.yml @@ -0,0 +1,28 @@ +# Optional PostgreSQL storage. Combine with the matching SQLite-default base +# using -f base.yml -f this-file.yml. No profiles: unused PG credentials and +# services are absent from the base model. Changing files is not a conversion; +# use the documented offline storage switch before changing an installed backend. +services: + postgres: + extends: + file: docker/postgres.compose.yml + service: postgres + volumes: + - ${STAGING_POSTGRES_DATA_DIR:-~/corescope-staging-postgres}:/var/lib/postgresql + bootstrap: + extends: + file: docker/postgres.compose.yml + service: bootstrap + image: corescope-go:latest + volumes: + - ${STAGING_DATA_DIR:-~/meshcore-staging-data}:/app/data + staging-go: + depends_on: + bootstrap: + condition: service_completed_successfully + environment: + CORESCOPE_DB_BACKEND: postgres + CORESCOPE_READER_DATABASE_URL: postgresql://corescope_reader:${CORESCOPE_READER_PASSWORD:?Set CORESCOPE_READER_PASSWORD}@postgres:5432/corescope_telemetry?sslmode=disable&search_path=public + CORESCOPE_WRITER_DATABASE_URL: postgresql://corescope_writer:${CORESCOPE_WRITER_PASSWORD:?Set CORESCOPE_WRITER_PASSWORD}@postgres:5432/corescope_telemetry?sslmode=disable&search_path=public + CORESCOPE_USERS_DATABASE_URL: postgresql://corescope_accounts:${CORESCOPE_ACCOUNTS_PASSWORD:?Set CORESCOPE_ACCOUNTS_PASSWORD}@postgres:5432/corescope_accounts?sslmode=disable&search_path=public + CORESCOPE_APPROVED_CHANNELS_DATABASE_URL: postgresql://corescope_channels:${CORESCOPE_CHANNELS_PASSWORD:?Set CORESCOPE_CHANNELS_PASSWORD}@postgres:5432/corescope_accounts?sslmode=disable&search_path=public diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml index 7bb91ea1c..30e4f5af5 100644 --- a/docker-compose.staging.yml +++ b/docker-compose.staging.yml @@ -23,23 +23,7 @@ name: corescope-staging services: - postgres: - extends: - file: docker/postgres.compose.yml - service: postgres - volumes: - - ${STAGING_POSTGRES_DATA_DIR:-~/corescope-staging-postgres}:/var/lib/postgresql - bootstrap: - extends: - file: docker/postgres.compose.yml - service: bootstrap - image: corescope-go:latest - volumes: - - ${STAGING_DATA_DIR:-~/meshcore-staging-data}:/app/data:ro staging-go: - depends_on: - bootstrap: - condition: service_completed_successfully build: context: . dockerfile: Dockerfile @@ -73,11 +57,8 @@ services: - ${STAGING_DATA_DIR:-~/meshcore-staging-data}:/app/data - caddy-data-staging-go:/data/caddy environment: - - CORESCOPE_STATE_DIR=/app/data - - CORESCOPE_READER_DATABASE_URL=postgresql://corescope_reader:${CORESCOPE_READER_PASSWORD:?Set CORESCOPE_READER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable - - CORESCOPE_WRITER_DATABASE_URL=postgresql://corescope_writer:${CORESCOPE_WRITER_PASSWORD:?Set CORESCOPE_WRITER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable - - CORESCOPE_USERS_DATABASE_URL=postgresql://corescope_accounts:${CORESCOPE_ACCOUNTS_PASSWORD:?Set CORESCOPE_ACCOUNTS_PASSWORD}@postgres/corescope_accounts?sslmode=disable - - CORESCOPE_APPROVED_CHANNELS_DATABASE_URL=postgresql://corescope_channels:${CORESCOPE_CHANNELS_PASSWORD:?Set CORESCOPE_CHANNELS_PASSWORD}@postgres/corescope_accounts?sslmode=disable + - CORESCOPE_DB_BACKEND=sqlite + - CORESCOPE_STATE_DIR=${CORESCOPE_STATE_DIR:-/app/data} - NODE_ENV=staging - ENABLE_PPROF=true # Default TRUE: the standalone mqtt-broker container owns MQTT on diff --git a/docker-compose.yml b/docker-compose.yml index a229d18d7..ed00d9860 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,23 +6,7 @@ name: corescope services: - postgres: - extends: - file: docker/postgres.compose.yml - service: postgres - volumes: - - ${PROD_POSTGRES_DATA_DIR:-~/corescope-postgres}:/var/lib/postgresql - bootstrap: - extends: - file: docker/postgres.compose.yml - service: bootstrap - image: corescope:latest - volumes: - - ${PROD_DATA_DIR:-~/meshcore-data}:/app/data:ro prod: - depends_on: - bootstrap: - condition: service_completed_successfully build: context: . args: @@ -45,11 +29,8 @@ services: - ${PROD_DATA_DIR:-~/meshcore-data}:/app/data - caddy-data:/data/caddy environment: - - CORESCOPE_STATE_DIR=/app/data - - CORESCOPE_READER_DATABASE_URL=postgresql://corescope_reader:${CORESCOPE_READER_PASSWORD:?Set CORESCOPE_READER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable - - CORESCOPE_WRITER_DATABASE_URL=postgresql://corescope_writer:${CORESCOPE_WRITER_PASSWORD:?Set CORESCOPE_WRITER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable - - CORESCOPE_USERS_DATABASE_URL=postgresql://corescope_accounts:${CORESCOPE_ACCOUNTS_PASSWORD:?Set CORESCOPE_ACCOUNTS_PASSWORD}@postgres/corescope_accounts?sslmode=disable - - CORESCOPE_APPROVED_CHANNELS_DATABASE_URL=postgresql://corescope_channels:${CORESCOPE_CHANNELS_PASSWORD:?Set CORESCOPE_CHANNELS_PASSWORD}@postgres/corescope_accounts?sslmode=disable + - CORESCOPE_DB_BACKEND=sqlite + - CORESCOPE_STATE_DIR=${CORESCOPE_STATE_DIR:-/app/data} - NODE_ENV=production - DISABLE_MOSQUITTO=${DISABLE_MOSQUITTO:-false} - DISABLE_CADDY=${DISABLE_CADDY:-false} diff --git a/docker/entrypoint-go.sh b/docker/entrypoint-go.sh index 9f47efc15..5f229cfd5 100644 --- a/docker/entrypoint-go.sh +++ b/docker/entrypoint-go.sh @@ -1,4 +1,5 @@ #!/bin/sh +set -eu # Config lives in the data directory (bind-mounted from host) # The Go server already searches /app/data/config.json via LoadConfig @@ -21,6 +22,8 @@ if [ -f /app/data/.env ]; then set +a fi +/app/storage.sh start + SUPERVISORD_CONF="/etc/supervisor/conf.d/supervisord.conf" if [ "${DISABLE_MOSQUITTO:-false}" = "true" ] && [ "${DISABLE_CADDY:-false}" = "true" ]; then echo "[config] internal MQTT broker disabled (DISABLE_MOSQUITTO=true)" diff --git a/docker/postgres-bootstrap.sh b/docker/postgres-bootstrap.sh index 8f5386bae..73d5579f1 100755 --- a/docker/postgres-bootstrap.sh +++ b/docker/postgres-bootstrap.sh @@ -1,26 +1,27 @@ #!/bin/sh set -eu -# A legacy instance must go through the explicit offline importer. Retained -# SQLite files are harmless once both PostgreSQL stores have verified readiness. -if [ -f /app/data/meshcore.db ]; then - /app/corescope-migrate -check-import-kind=telemetry -from-sqlite=/app/data/meshcore.db || { - echo 'SQLite telemetry exists. Follow docs/postgresql-upgrade.md before starting this instance.' >&2 - exit 1 - } -fi -if [ -f /app/data/users.db ]; then - /app/corescope-migrate -check-import-kind=accounts -users-from-sqlite=/app/data/users.db || { - echo 'SQLite account data exists. Follow docs/postgresql-upgrade.md before starting this instance.' >&2 - exit 1 - } -fi -/app/corescope-migrate +# Only the owner service may initialize/adopt PostgreSQL. Existing SQLite data +# requires the explicit verified switch; a missing record is never a fallback. +/app/storage.sh postgres-setup +/app/storage.sh managed-postgres +# Legacy check-ready remains a read-only schema check; use owner-only environment +# for this child without changing the runtime account role URL or leaking argv. +databases=corescope_telemetry +account_owner= +case $(/app/storage.sh field has_accounts) in + true) + databases="$databases corescope_accounts" + account_owner=${CORESCOPE_USERS_OWNER_DATABASE_URL:?set account owner URL} + ;; + false) ;; + *) echo 'Cannot read the recorded account target.' >&2; exit 1 ;; +esac +CORESCOPE_USERS_DATABASE_URL="$account_owner" /app/corescope-migrate -check-ready -# libpq uses these environment settings; URLs/passwords never appear in argv. -export PGHOST="${PGHOST:-postgres}" PGPORT="${PGPORT:-5432}" PGSSLMODE=disable +export PGHOST=postgres PGPORT=5432 PGSSLMODE=disable export PGUSER=corescope_owner PGPASSWORD="${CORESCOPE_OWNER_PASSWORD:?set CORESCOPE_OWNER_PASSWORD}" -export PGDATABASE=corescope_telemetry -psql -X -v ON_ERROR_STOP=1 -f /app/postgres-grants.sql -export PGDATABASE=corescope_accounts -psql -X -v ON_ERROR_STOP=1 -f /app/postgres-grants.sql +for PGDATABASE in $databases; do + export PGDATABASE + psql -X -v ON_ERROR_STOP=1 -f /app/postgres-grants.sql +done diff --git a/docker/postgres.compose.yml b/docker/postgres.compose.yml index 32304de06..9d8d508d3 100644 --- a/docker/postgres.compose.yml +++ b/docker/postgres.compose.yml @@ -29,6 +29,12 @@ services: postgres: condition: service_healthy environment: - CORESCOPE_DATABASE_URL: postgresql://corescope_owner:${CORESCOPE_OWNER_PASSWORD:?Set CORESCOPE_OWNER_PASSWORD}@postgres/corescope_telemetry?sslmode=disable - CORESCOPE_USERS_DATABASE_URL: postgresql://corescope_owner:${CORESCOPE_OWNER_PASSWORD:?Set CORESCOPE_OWNER_PASSWORD}@postgres/corescope_accounts?sslmode=disable + CORESCOPE_DB_BACKEND: postgres + CORESCOPE_STATE_DIR: ${CORESCOPE_STATE_DIR:-/app/data} + CORESCOPE_DATABASE_URL: postgresql://corescope_owner:${CORESCOPE_OWNER_PASSWORD:?Set CORESCOPE_OWNER_PASSWORD}@postgres:5432/corescope_telemetry?sslmode=disable&search_path=public + CORESCOPE_USERS_OWNER_DATABASE_URL: postgresql://corescope_owner:${CORESCOPE_OWNER_PASSWORD:?Set CORESCOPE_OWNER_PASSWORD}@postgres:5432/corescope_accounts?sslmode=disable&search_path=public + CORESCOPE_READER_DATABASE_URL: postgresql://corescope_reader:${CORESCOPE_READER_PASSWORD:?Set CORESCOPE_READER_PASSWORD}@postgres:5432/corescope_telemetry?sslmode=disable&search_path=public + CORESCOPE_WRITER_DATABASE_URL: postgresql://corescope_writer:${CORESCOPE_WRITER_PASSWORD:?Set CORESCOPE_WRITER_PASSWORD}@postgres:5432/corescope_telemetry?sslmode=disable&search_path=public + CORESCOPE_USERS_DATABASE_URL: postgresql://corescope_accounts:${CORESCOPE_ACCOUNTS_PASSWORD:?Set CORESCOPE_ACCOUNTS_PASSWORD}@postgres:5432/corescope_accounts?sslmode=disable&search_path=public + CORESCOPE_APPROVED_CHANNELS_DATABASE_URL: postgresql://corescope_channels:${CORESCOPE_CHANNELS_PASSWORD:?Set CORESCOPE_CHANNELS_PASSWORD}@postgres:5432/corescope_accounts?sslmode=disable&search_path=public CORESCOPE_OWNER_PASSWORD: ${CORESCOPE_OWNER_PASSWORD:?Set CORESCOPE_OWNER_PASSWORD} diff --git a/docker/storage.sh b/docker/storage.sh new file mode 100755 index 000000000..40f2426f0 --- /dev/null +++ b/docker/storage.sh @@ -0,0 +1,87 @@ +#!/bin/sh +# Packaged selection boundary. Go owns discovery, validation and conversion. +set -eu +state_dir=${CORESCOPE_STATE_DIR:-/app/data} +case "$state_dir" in /app/data|/app/data/*) ;; *) + echo 'CORESCOPE_STATE_DIR must be /app/data or a persistent subdirectory of that mounted data directory.' >&2; exit 1 ;; +esac +case "/$state_dir/" in */../*|*/./*) echo 'Use a normalized persistent state directory without dot segments.' >&2; exit 1 ;; esac +# Refuse a plain image directory: losing the selection on recreation is unsafe. +awk '$5 == "/app/data" { found=1 } END { exit !found }' /proc/self/mountinfo || { + echo '/app/data must be a persistent data mount before storage setup.' >&2; exit 1 +} +persistent_root=$(cd /app/data && pwd -P) +existing=$state_dir +while [ ! -e "$existing" ] && [ ! -L "$existing" ]; do existing=${existing%/*}; done +resolved=$(readlink -f "$existing") || { echo 'Cannot resolve the selected state directory.' >&2; exit 1; } +case "$resolved" in "$persistent_root"|"$persistent_root"/*) ;; *) + echo 'The state directory resolves outside the persistent data mount.' >&2; exit 1 ;; +esac +selection_file=$state_dir/storage-selection.json +action=${1:-start} +[ "$#" -eq 0 ] || shift +status() { /app/corescope-migrate -storage-action=status -selection-file "$selection_file"; } +mutate() { + operation=$1; shift + /app/corescope-migrate -storage-action="$operation" -selection-file "$selection_file" \ + -config-dir /app -state-dir "$state_dir/storage-jobs" -offline "$@" +} +blocked() { + job=$(printf '%s\n' "$report" | jq -r '.job_id') + echo "Storage is not ready; keep services stopped and use storage status, then resume or abort job $job." >&2 + exit 1 +} +case "$action" in + status) status ;; + field) + case "${1:-}" in has_accounts|state|backend|generation|job_id|state_dir|source_backend|target_backend|telemetry.sqlite_path|accounts.sqlite_path|telemetry.postgres.host|telemetry.postgres.port|telemetry.postgres.database|telemetry.postgres.schema|accounts.postgres.host|accounts.postgres.port|accounts.postgres.database|accounts.postgres.schema) ;; + *) echo 'Unsupported storage status field.' >&2; exit 1 ;; + esac + report=$(status) || exit 1 + case "$1" in + has_accounts) printf '%s\n' "$report" | jq -r '.accounts != null' ;; + source_backend|target_backend) printf '%s\n' "$report" | jq -r ".$1" ;; + *) printf '%s\n' "$report" | jq -er ".$1 // empty" ;; + esac + ;; + managed-postgres) + report=$(status) || exit 1 + printf '%s\n' "$report" | jq -e ' + .state == "ready" and .backend == "postgres" + and .telemetry.postgres == {host:"postgres",port:5432,database:"corescope_telemetry",schema:"public"} + and (.accounts == null or .accounts.postgres == {host:"postgres",port:5432,database:"corescope_accounts",schema:"public"}) + ' >/dev/null || { + echo 'Managed operations require the recorded local PostgreSQL telemetry/accounts targets. Use explicit owner tooling for custom endpoints; no local substitute was used.' >&2 + exit 1 + } + ;; + start) + report=$(status) || exit 1 + case $(printf '%s\n' "$report" | jq -er '.state') in + ready) ;; # Runtime opens the recorded backend; bootstrap hints cannot replace it. + unrecorded) + [ "${CORESCOPE_DB_BACKEND:-sqlite}" = sqlite ] || { + echo 'PostgreSQL needs its matching bootstrap override before application startup.' >&2; exit 1 + } + # This explicit packaged setup intent still validates all existing data. + # Neither missing selection nor a missing database is a fallback signal. + mutate setup -backend=sqlite + ;; + *) blocked ;; + esac + ;; + postgres-setup) + report=$(status) || exit 1 + case $(printf '%s\n' "$report" | jq -er '.state') in + unrecorded) mutate setup -backend=postgres ;; + ready) + [ "$(printf '%s\n' "$report" | jq -er '.backend')" = postgres ] || { + echo 'SQLite is recorded. Use the explicit offline storage switch before adding PostgreSQL.' >&2; exit 1 + } + ;; + *) blocked ;; + esac + ;; + setup|adopt|init|switch|resume|abort) mutate "$action" "$@" ;; + *) echo 'Unsupported packaged storage action.' >&2; exit 1 ;; +esac diff --git a/docker/supervisord-go-no-caddy.conf b/docker/supervisord-go-no-caddy.conf index a89179a1a..8c1c88fae 100644 --- a/docker/supervisord-go-no-caddy.conf +++ b/docker/supervisord-go-no-caddy.conf @@ -29,7 +29,7 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 [program:corescope-server] -command=/app/corescope-server -config-dir /app -state-dir /app/data -public /app/public -port 3000 +command=/app/corescope-server -config-dir /app -public /app/public -port 3000 directory=/app autostart=true autorestart=true diff --git a/docker/supervisord-go-no-mosquitto-no-caddy.conf b/docker/supervisord-go-no-mosquitto-no-caddy.conf index 3d2386740..5683d625e 100644 --- a/docker/supervisord-go-no-mosquitto-no-caddy.conf +++ b/docker/supervisord-go-no-mosquitto-no-caddy.conf @@ -20,7 +20,7 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 [program:corescope-server] -command=/app/corescope-server -config-dir /app -state-dir /app/data -public /app/public -port 3000 +command=/app/corescope-server -config-dir /app -public /app/public -port 3000 directory=/app autostart=true autorestart=true diff --git a/docker/supervisord-go-no-mosquitto.conf b/docker/supervisord-go-no-mosquitto.conf index 1acd90427..5d12ece36 100644 --- a/docker/supervisord-go-no-mosquitto.conf +++ b/docker/supervisord-go-no-mosquitto.conf @@ -20,7 +20,7 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 [program:corescope-server] -command=/app/corescope-server -config-dir /app -state-dir /app/data -public /app/public -port 3000 +command=/app/corescope-server -config-dir /app -public /app/public -port 3000 directory=/app autostart=true autorestart=true diff --git a/docker/supervisord-go.conf b/docker/supervisord-go.conf index 697405cad..4354f2d4f 100644 --- a/docker/supervisord-go.conf +++ b/docker/supervisord-go.conf @@ -29,7 +29,7 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 [program:corescope-server] -command=/app/corescope-server -config-dir /app -state-dir /app/data -public /app/public -port 3000 +command=/app/corescope-server -config-dir /app -public /app/public -port 3000 directory=/app autostart=true autorestart=true diff --git a/docs/backend-switching.md b/docs/backend-switching.md new file mode 100644 index 000000000..d2c701640 --- /dev/null +++ b/docs/backend-switching.md @@ -0,0 +1,105 @@ +# Choosing and switching storage + +A fresh installation defaults to SQLite. PostgreSQL is an explicit installation choice. An update retains the installed backend and both selected database targets, including an account store whose feature is disabled. + +The stable state directory contains one `storage-selection.json`. After setup, that record is authoritative for the backend and SQLite file paths. Config/env backend selectors and file paths are bootstrap inputs, so an old `db.backend: sqlite` does not undo a completed PostgreSQL switch. PostgreSQL credentials and TLS settings remain outside the record; they may rotate but must still identify the recorded host, port, database and schema. + +`accounts: null` means no account store has been initialized. A non-null target remains recorded even when account features are disabled. Runtime opens never create a missing account database. Fresh SQLite setup initializes its account store upfront; legacy adoption preserves any valid existing account file but records null when it is absent. Backend switches preserve that absence. Explicit setup can later initialize the first account target under an exclusive installation lease and atomically record it. PostgreSQL initialization requires supplied account owner and runtime credentials; ordinary setup with neither preserves null. A previously recorded missing target always fails instead of being replaced. + +Keep the runtime state-directory setting stable across updates and switches. The migration tool's `-selection-file` identifies that record. Its separate `-state-dir` chooses private recovery/job storage; it defaults to `migration` beside the selection record. + +## Inspect and initialize + +The packaged startup runs guarded setup before either service starts. Direct installations can use the same command while every service is stopped: + +```sh +corescope-migrate -storage-action=status \ + -selection-file /app/data/storage-selection.json + +corescope-migrate -storage-action=setup -offline \ + -selection-file /app/data/storage-selection.json \ + -config-dir /app +``` + +Status reads only local metadata. It needs no PostgreSQL credentials or connection and returns one JSON object. `ready`, `unrecorded`, and `pending` return exit status 0; `corrupt` returns 1. **Unrecorded is not proof of a fresh installation.** Pending or corrupt state must never trigger an empty SQLite fallback or a new installation. Status includes credential-free selected targets and the stable state directory for native backup tools. + +Setup validates an existing recorded choice, adopts a supported legacy SQLite installation, or initializes provably empty targets. `-storage-action=init` refuses existing schemas; `adopt` requires an existing supported telemetry store. A legacy SQLite writer/setup may apply supported native schema upgrades before recording its choice. Conversion copies, in contrast, leave their sources unchanged. + +Use `-config FILE` for an explicit configuration file; missing or invalid explicit files fail. Otherwise `-config-dir DIR` checks `config.json`, then `data/config.json`, continuing only when a candidate is absent. An existing unreadable or malformed primary configuration stops setup. Explicit relative telemetry/account paths resolve from the process working directory. An omitted telemetry path defaults to `config-dir/data/meshcore.db`; accounts default to `users.db` beside telemetry. On unrecorded adoption, conflicting config `dbPath` and `DB_PATH` fail unless `-sqlite-path` explicitly resolves the conflict. `-users-sqlite-path` is the corresponding account override. + +For a fresh PostgreSQL install, add `-backend=postgres`. This explicit choice overrides bootstrap config/env defaults. Setup refuses to initialize an empty PostgreSQL target while known primary SQLite data exists. A ready PostgreSQL installation with retained SQLite files needs a matching completed-import record or explicit adoption; switching live data uses the verified procedure below. + +## Prepare an offline switch + +Stop **every** writer and reader, including other hosts, scheduled jobs, and administrative database clients. Keep both original databases and their native backups. The tool takes local installation leases and native source locks; a local lease is not a distributed deployment coordinator. + +Prepare fresh, unselected destination targets. Existing SQLite files are never overwritten, and existing PostgreSQL schema/data is not an import destination. Telemetry and accounts remain separate databases/files. Conversion includes every recorded account store even when account features are disabled. + +Supply PostgreSQL credentials through the private environment, not process arguments or logs: + +| Variable | New storage actions | +|---|---| +| `CORESCOPE_DATABASE_URL` | Telemetry migration owner for the PostgreSQL side | +| `CORESCOPE_USERS_OWNER_DATABASE_URL` | Account migration owner for the PostgreSQL side | +| `CORESCOPE_READER_DATABASE_URL` | Restricted telemetry reader | +| `CORESCOPE_WRITER_DATABASE_URL` | Restricted telemetry writer | +| `CORESCOPE_USERS_DATABASE_URL` | Restricted account runtime writer | +| `CORESCOPE_APPROVED_CHANNELS_DATABASE_URL` | Optional restricted approved-channel projection reader | + +The owner creates/imports destination schemas, applies the existing narrow runtime grant policy, then verifies actual runtime access before selection changes. Runtime roles never gain schema ownership or DML on readiness/import metadata; the account writer cannot modify `schema_version`. Credentials are not stored in selection records, job manifests or command output. Role URL target overrides through query parameters or service settings are not supported. + +SQLite to PostgreSQL: + +```sh +corescope-migrate -storage-action=switch -backend=postgres -offline \ + -selection-file /app/data/storage-selection.json \ + -state-dir /app/data/migration +``` + +PostgreSQL to new SQLite files: + +```sh +corescope-migrate -storage-action=switch -backend=sqlite -offline \ + -selection-file /app/data/storage-selection.json \ + -sqlite-path /app/data/returned/meshcore.db \ + -users-sqlite-path /app/data/returned/users.db \ + -state-dir /app/data/migration +``` + +For either direction, the owner URLs address the PostgreSQL side: the destination for the first command, the source for the second. Do not pre-bootstrap an import destination. Original invocation without `-storage-action` retains its prior PostgreSQL bootstrap/forward-import behavior, including its legacy account-owner environment convention. + +## What verification covers + +Native SQLite recovery snapshots retain WAL contents and hidden rowids; PostgreSQL recovery is a private `pg_dump` custom archive. Copying uses bounded batches and a 64 MiB per-row safety limit. Every application table is compared by complete row count and ordered canonical digest, including NULL/empty distinctions, raw text/JSON/timestamps and representable fractional values. Unsupported values or custom source schemas fail before selection instead of being coerced or discarded. + +Identity verification includes retained IDs, deleted high-water marks, historical references and backfill cursors. PostgreSQL `last_value` and `is_called` are retained in the private manifest; SQLite non-reusing allocators are seeded above every retained/reference maximum. No dummy rows are inserted. Account SQLite schema 6 uses explicit `mail_events.id`; PostgreSQL keeps its native account schema version 5. Those schema version values describe the engine, not copied account data. + +Reverse conversion adds required SQLite physical migration markers while preserving every source ledger entry. The private reverse manifest lists those additions. It builds the native observer/time index offline even if the retained async ledger already says the task is done. PostgreSQL-specific readiness/import bookkeeping is retained in its native recovery archive, not copied into application SQLite tables. + +SQLite reverse targets contain `corescope_reverse_progress` until complete data verification. Native open/setup/adoption paths reject that marker, including through another state directory. Runtime schema readiness alone is never treated as proof that an import completed. The PostgreSQL source's managed table/column/constraint/index/view layout is checked against a versioned native manifest; custom extensions require explicit review. + +Only after both stores verify and destination runtime access succeeds does one atomic replacement select the new backend and both targets together. Original files/databases and recovery copies remain available. This is a planned-downtime switch, not live replication or an automatic rollback service. + +## Interruption and recovery + +Inspect status to obtain the pending `job_id`. Keep services stopped and retain the same selection path, recovery directory and destination targets. Supply credentials again, then resume: + +```sh +corescope-migrate -storage-action=resume -offline \ + -selection-file /app/data/storage-selection.json \ + -state-dir /app/data/migration -job-id JOB_ID +``` + +Resume rechecks the exact source, sequences, native recovery copy, destination and committed copy progress. A changed source or target is refused. An interruption before a complete preparation manifest may require preserving the failed job and starting a new job with fresh targets after abort; partial artifacts are not silently reused. + +First account initialization uses the same journal, with identical source and target backends. Resume can finish an existing ready account target, without PostgreSQL credentials for SQLite. It never recreates a missing target or guesses that an empty target is fresh. If initialization stopped before readiness, preserve/inspect it, abort the uncommitted metadata update, then rerun explicit setup. A journal with no staged destination permits only metadata-only abort; no database connection is needed. + +Before selection replacement, abort can reopen the original selected installation: + +```sh +corescope-migrate -storage-action=abort -offline \ + -selection-file /app/data/storage-selection.json \ + -state-dir /app/data/migration -job-id JOB_ID +``` + +Abort preserves staged databases and recovery files. After selection replacement, abort refuses to restore an old selection; resume verifies and finishes that job instead. Inspect status after a cleanup error, since the verified new selection may already be committed. Returning later to the previous backend is another explicit verified conversion into fresh targets, preserving data written since the first switch. diff --git a/docs/deployment.md b/docs/deployment.md index 557cd2894..6fd7a362f 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -18,7 +18,7 @@ Comprehensive guide to deploying and operating CoreScope. For a quick start, see ## System Requirements -The supplied images support `linux/amd64` and `linux/arm64`. Use Docker with the Compose plugin and PostgreSQL 18.6. Budget for the application and database together: earlier SQLite memory figures do not size a PostgreSQL installation. Measure the retained dataset, packet cache, database indexes/WAL and backup storage before choosing capacity. +The supplied images support `linux/amd64` and `linux/arm64`. Use Docker with the Compose plugin. SQLite is the default; PostgreSQL18.6 is an optional separate service. Budget for the application and database together: earlier SQLite memory figures do not size a PostgreSQL installation. Measure the retained dataset, packet cache, database indexes/WAL and backup storage before choosing capacity. For migration, retain the original data plus recovery and normalized snapshots while PostgreSQL builds its own data and indexes. Required disk space and downtime must be measured on a representative copy. See [offline upgrade and recovery](postgresql-upgrade.md). @@ -26,16 +26,16 @@ For migration, retain the original data plus recovery and normalized snapshots w ### Complete Compose checkout -Use [DEPLOY.md](../DEPLOY.md#quick-start) to clone and pin the complete repository, populate the private `.env` and select a matching reviewed image. A downloaded YAML alone is insufficient: it extends `docker/postgres.compose.yml` and mounts the committed initialization script. +Use [DEPLOY.md](../DEPLOY.md#quick-start) to clone and pin the complete repository, populate the private `.env` and select a matching reviewed image. Keep the matching checkout, including optional PostgreSQL overrides and their initialization scripts. ```sh docker compose -f docker-compose.example.yml up -d docker compose -f docker-compose.example.yml ps ``` -The PostgreSQL service creates separate telemetry and account databases. The one-shot bootstrap installs schemas and grants before CoreScope starts. Existing SQLite files require a completed offline import; startup refuses a manually initialized empty replacement. +The default app validates/adopts existing SQLite storage or initializes a provably fresh install before starting. The optional `docker-compose.example.postgres.yml` override adds PostgreSQL and owner bootstrap. An installed selection is authoritative; a missing connection or an interrupted conversion never permits SQLite fallback. -The application container runs the server and ingestor, optional Mosquitto, and optional Caddy. PostgreSQL runs separately. Runtime services receive restricted roles; the schema owner belongs only to bootstrap/import. +The application container runs the server and ingestor, optional Mosquitto, and optional Caddy. When selected, PostgreSQL runs separately. Its runtime services receive restricted roles; the owner belongs only to setup/conversion. | Setting | Purpose | |---|---| @@ -45,7 +45,7 @@ The application container runs the server and ingestor, optional Mosquitto, and | `CORESCOPE_IMAGE` | Reviewed application image tag or digest matching the checkout | | `DISABLE_MOSQUITTO` | Use an external broker when true | | `DISABLE_CADDY` | Use an external reverse proxy when true | -| `CORESCOPE_*_PASSWORD` | Distinct owner and runtime credentials; populate every field in `.env.example` | +| `CORESCOPE_*_PASSWORD` | Distinct owner and runtime credentials; only for PostgreSQL, merge `.env.postgres.example` | The production and staging variants use their own directory/port variables listed in `.env.example`. `manage.sh` operates those variants; `./manage.sh start --with-staging` clones telemetry only into an empty staging database and keeps staging accounts independent. @@ -58,7 +58,7 @@ docker compose -f docker-compose.example.yml pull docker compose -f docker-compose.example.yml up -d ``` -A SQLite-to-PostgreSQL upgrade requires the [offline cutover](postgresql-upgrade.md), not only an image update. Keep the previous application and recovery files until validation is complete. +A backend change in either direction requires the [offline storage switch](storage.md), not only an image update. Ordinary updates keep the recorded choice. Keep the previous application and recovery files until validation is complete. --- @@ -68,7 +68,9 @@ CoreScope uses a layered configuration system (highest priority wins): 1. **Environment variables** — `MQTT_BROKER`, `CORESCOPE_READER_DATABASE_URL`, etc. 2. **`/app/data/config.json`** — full config file (volume-mounted) -3. **Built-in defaults** — database URLs and credentials are still required +3. **Built-in defaults** — SQLite for a provably fresh installation; PostgreSQL requires its role URLs + +After setup, `storage-selection.json` overrides bootstrap backend/path hints. Credentials and TLS remain private config/environment settings. Keep the same shared state directory on restart and update. ### Environment variable overrides @@ -76,8 +78,8 @@ CoreScope uses a layered configuration system (highest priority wins): |----------|---------|-------------| | `MQTT_BROKER` | `mqtt://localhost:1883` | MQTT broker URL (overrides config file) | | `MQTT_TOPIC` | `meshcore/#` | MQTT topic subscription pattern | -| `CORESCOPE_READER_DATABASE_URL` | Required by server | Restricted telemetry reader URL | -| `CORESCOPE_WRITER_DATABASE_URL` | Required by ingestor | Restricted telemetry writer URL | +| `CORESCOPE_READER_DATABASE_URL` | Required for PostgreSQL | Restricted telemetry reader URL | +| `CORESCOPE_WRITER_DATABASE_URL` | Required for PostgreSQL | Restricted telemetry writer URL | | `CORESCOPE_USERS_DATABASE_URL` | Required when accounts are enabled | Separate account database writer URL | | `CORESCOPE_APPROVED_CHANNELS_DATABASE_URL` | Required when proposals are enabled | Restricted approved-channel view reader URL | | `CORESCOPE_STATE_DIR` | `data` for native binaries | Shared filesystem state, separate from database URLs | @@ -86,7 +88,7 @@ CoreScope uses a layered configuration system (highest priority wins): ### config.json -Place `config.json` in the mounted state directory (`DATA_DIR` in the example deployment). Use private environment configuration for database passwords; do not publish rendered Compose configuration or URLs. Supported native keys are `databaseURL`, `userManagement.databaseURL` and `stateDir`; legacy `dbPath`/`DB_PATH` settings are refused. +Place `config.json` in the mounted state directory (`DATA_DIR` in the example deployment). Use private environment configuration for database passwords; do not publish rendered Compose configuration or URLs. Supported keys include `db.backend`, SQLite `dbPath`/`DB_PATH` and `userManagement.dbPath`, optional PostgreSQL role URLs, and `stateDir`. Explicit relative SQLite paths use the process working directory; containers use `/app`. Keep container databases/state inside `/app/data` so the bind mount persists them. Set `CORESCOPE_STATE_DIR=/app/data/` in the private `.env` when using a custom container state directory; every service shares it. See `config.example.json` in the repository for all available options including: - MQTT sources (multiple brokers) @@ -444,7 +446,7 @@ docker stats corescope ## Backup & Restore -Use native custom-format PostgreSQL archives. The authenticated `GET /api/backup` and admin `GET /api/admin/users-backup` endpoints download `.dump` files. The server needs PostgreSQL 18 `pg_dump` on `PATH`; the supplied image includes it. +Use native SQLite snapshots for SQLite or custom-format PostgreSQL archives for PostgreSQL. The authenticated `GET /api/backup` and admin `GET /api/admin/users-backup` endpoints download `.dump` files. The server needs PostgreSQL 18 `pg_dump` on `PATH`; the supplied image includes it. For the production variant managed by `manage.sh`: @@ -454,7 +456,7 @@ For the production variant managed by `manage.sh`: This saves `telemetry.dump`, `accounts.dump` and available config/theme/Caddy files. Each archive is consistent, but two sequential dumps are not a shared cross-database snapshot; stop both writers for a coordinated pair. Preserve credentials, queues/state and TLS material separately. Encrypt private backups off-host. -`./manage.sh restore ` requires the application stopped and both target databases empty. It reapplies runtime grants and leaves the application stopped for validation. It never replaces a nonempty PostgreSQL database or accepts a SQLite file. Use a fresh PostgreSQL storage location while retaining the previous cluster for recovery. +`./manage.sh restore ` requires the application stopped and fresh/empty native targets. SQLite restores stage and validate the full state bundle in a new empty state directory. PostgreSQL restores require both databases empty. It reapplies runtime grants and leaves the application stopped for validation. It never replaces a nonempty PostgreSQL database or overwrites an existing SQLite state directory. Use a fresh PostgreSQL storage location while retaining the previous cluster for recovery. Account backups contain password hashes, addresses, sessions and tokens. Restoration can revive deleted accounts or revoked credentials. Review the [account recovery procedure](user-guide/accounts.md#backups) before reopening the service. See [native backup and cutover boundaries](postgresql-upgrade.md#native-backups-and-restores) for downloaded archives and rollback. diff --git a/docs/postgresql-performance.md b/docs/postgresql-performance.md index 8f5afbb40..e601fb432 100644 --- a/docs/postgresql-performance.md +++ b/docs/postgresql-performance.md @@ -1,5 +1,10 @@ # PostgreSQL conversion performance +> Historical measurement of candidate `ce023e0`, before optional backend selection +> and upstream's #2147 concurrency fix. It does not measure the current dual-backend +> implementation. Keep these results separate from the new comparison against +> upstream `3e21b17`. + The PostgreSQL conversion reduced retention-related ingestion delays in this test. Median writes and several reads were slower, and CPU, memory and storage costs increased. These results describe the implemented conversion under one diff --git a/docs/postgresql-upgrade.md b/docs/postgresql-upgrade.md index a82c2495a..fc82c1a71 100644 --- a/docs/postgresql-upgrade.md +++ b/docs/postgresql-upgrade.md @@ -1,8 +1,10 @@ +> PostgreSQL is optional. Normal installs and updates keep SQLite by default. Use [storage operations](storage.md) for the current setup and verified bidirectional switch. This page retains the lower-level forward-import and PostgreSQL-native recovery details; adding a Compose override alone is not a conversion. + # PostgreSQL installation and offline upgrade -CoreScope uses PostgreSQL for telemetry and optional accounts. The tested database version is PostgreSQL 18.6. Keep the application checkout, image, shared Compose file, bootstrap scripts and configuration example from the same reviewed revision. +When selected, PostgreSQL stores telemetry and optional accounts. The tested database version is PostgreSQL 18.6. Keep the application checkout, image, shared Compose file, bootstrap scripts and configuration example from the same reviewed revision. -Existing SQLite instances need a planned outage. Stop **both** the ingestor and server: the account janitor, sessions, proposals and notification evaluator also write data. MQTT messages arriving during this window may be lost unless the broker and publishers provide a separately verified replay mechanism. +Switching an existing SQLite instance to PostgreSQL needs a planned outage. Stop **both** the ingestor and server: the account janitor, sessions, proposals and notification evaluator also write data. MQTT messages arriving during this window may be lost unless the broker and publishers provide a separately verified replay mechanism. For an existing instance, the upgrade path is: @@ -24,9 +26,9 @@ The supplied deployment creates `corescope_telemetry` and `corescope_accounts` o | `corescope_accounts` | Account data writes; no schema or readiness-marker writes. | | `corescope_channels` | Ingestor reads of the approved-channel view; no access to passwords or sessions. | -Preserve an existing private `.env` and merge only missing settings from `.env.example`; do not replace its paths, ports, broker settings or passwords. For a new PostgreSQL data directory, fill each missing password field with a different value from `openssl rand -hex 32` and keep `.env` permissions at `0600`. PostgreSQL creates roles only on first initialization: editing `.env` later does **not** rotate their passwords. If an interrupted installation has lost its matching `.env`, recover that file or perform a coordinated administrator password rotation with all application services stopped; do not delete the PostgreSQL volume to fix authentication. The Compose files supply runtime URLs; credentials do not belong in Git, screenshots, issue reports or command arguments. On native installations, use private environment configuration with `CORESCOPE_READER_DATABASE_URL`, `CORESCOPE_WRITER_DATABASE_URL`, `CORESCOPE_USERS_DATABASE_URL` and `CORESCOPE_APPROVED_CHANNELS_DATABASE_URL`. +Preserve an existing private `.env` and merge only missing PostgreSQL settings from `.env.postgres.example`; do not replace its paths, ports, broker settings or passwords. For a new PostgreSQL data directory, fill each missing password field with a different value from `openssl rand -hex 32` and keep `.env` permissions at `0600`. PostgreSQL creates roles only on first initialization: editing `.env` later does **not** rotate their passwords. If an interrupted installation has lost its matching `.env`, recover that file or perform a coordinated administrator password rotation with all application services stopped; do not delete the PostgreSQL volume to fix authentication. The Compose files supply runtime URLs; credentials do not belong in Git, screenshots, issue reports or command arguments. On native installations, use private environment configuration with `CORESCOPE_READER_DATABASE_URL`, `CORESCOPE_WRITER_DATABASE_URL`, `CORESCOPE_USERS_DATABASE_URL` and `CORESCOPE_APPROVED_CHANNELS_DATABASE_URL`. -`databaseURL` replaces `dbPath`; `CORESCOPE_DATABASE_URL` is the per-process fallback. Remove legacy `dbPath`, `userManagement.dbPath` and `DB_PATH` settings after preserving their old values for recovery. `stateDir` / `CORESCOPE_STATE_DIR` is a separate filesystem directory for queues, statistics and account backups. A database URL is never a directory name. +Before initial adoption, configure the intended PostgreSQL URLs explicitly; `CORESCOPE_DATABASE_URL` is the per-process fallback. Once recorded, the selected targets override stale `dbPath`, `userManagement.dbPath` and `DB_PATH` bootstrap hints, so removing those hints is unnecessary. `stateDir` / `CORESCOPE_STATE_DIR` is a separate, stable filesystem directory for the selection, queues, statistics and account backups. A database URL is never a directory name. Main runtime pool limits are four server telemetry connections, one ingestor connection and one account connection. Bootstrap, identity checks and backups open additional connections. Connection attempts default to a ten-second upper bound; a shorter URL `connect_timeout` is retained. Native application dumps also have a ten-minute deadline. Backend credentials and pool limits are not theme/customizer controls. @@ -36,7 +38,7 @@ Main runtime pool limits are four server telemetry connections, one ingestor con |---|---| | Telemetry v3 with `observations.observer_idx` and the known optional columns | Supported. Missing supported columns/tables are added only to a working copy. | | Telemetry v2 with `observer_id`, or an older Node-era layout | Refused. Upgrade a recovery copy using the last SQLite release, review the result, then import it. | -| Account `schema_version` 0–5 | Supported, including empty version 0. Each declared version must have its complete historical shape. | +| Account `schema_version` 0–6 | Supported, including empty version 0. Each declared version must have its complete historical shape. | | No account database has ever existed | Omit `-users-from-sqlite`; the configured account destination is initialized empty. | | Accounts currently disabled, but a users database exists | Import it with `-users-from-sqlite` so accounts, audit records and tokens remain available if re-enabled. | | Unknown/newer columns, tables, views, triggers or virtual tables | Refused, with no silent field or record loss. | @@ -50,7 +52,7 @@ Identity sequences are reseeded above imported IDs, available `sqlite_sequence` 1. While still using the **old checkout and its original `.env`**, record the Compose file, project name, application service/container, image tag **and image ID**, and the actual telemetry/account SQLite paths. Read the old `dbPath`, `userManagement.dbPath` and `DB_PATH` values before removing them. A custom `stateDir` does not prove where either database lives. 2. Stop the old deployment with its own command, before switching checkout or configuration. For example, from that checkout, use `docker compose -f "$OLD_COMPOSE" -p "$OLD_PROJECT" stop "$OLD_SERVICE"` with the recorded values; old `manage.sh` installations can use their existing `./manage.sh stop`. Verify the old application container is stopped, and stop any separately launched server, ingestor or maintenance writer. The new Compose project may have a different name and may require settings the old one never used. -3. Make a private, quiescent recovery bundle containing the old Compose files/scripts, `.env`, image identity, configuration/theme, keys and **whole state/source directories**, including SQLite `-wal` and `-shm` siblings. Keep an untouched original. Do not use the new PostgreSQL-only `manage.sh backup` to back up SQLite. Restore the preserved old configuration and image together if rolling back. +3. Make a private, quiescent recovery bundle containing the old Compose files/scripts, `.env`, image identity, configuration/theme, keys and **whole state/source directories**, including SQLite `-wal` and `-shm` siblings. Keep an untouched original. Before first adoption, use the old deployment's backup procedure or those quiescent copies; the new `manage.sh backup` requires a validated selection and will not guess an unrecorded backend. Restore the preserved old configuration and image together if rolling back. 4. Select new, empty PostgreSQL storage and allocate a private migration-state directory. It will hold a WAL-aware recovery snapshot, normalized working snapshot and source/destination manifest for each store. Snapshot creation uses SQLite's backup API so hidden observer/mail-event row IDs do not change. 5. Measure free space against the actual source, both snapshot copies, PostgreSQL data/indexes, WAL and temporary work. The required space and duration depend on the data; no fixed multiplier or downtime estimate is guaranteed. @@ -64,13 +66,13 @@ The common path below uses `docker-compose.example.yml`. Keep the complete pinne Preserve/merge `.env`; do not regenerate passwords after PostgreSQL has initialized. In the example variant, set `DATA_DIR` to the **existing absolute host state directory**, `POSTGRES_DATA_DIR` to the selected PostgreSQL storage and `CORESCOPE_IMAGE` to the matching reviewed image. `PROD_DATA_DIR` and `STAGING_DATA_DIR` do not control this variant. Set these before running bootstrap. Keep PostgreSQL storage separate from state/recovery directories. -Inside the supplied container, the selected host `DATA_DIR` is always `/app/data` and the runtime state directory is `/app/data`. Keep config/theme/queues there; remove legacy database-path settings from the active config and any `/app/data/.env` only after saving their original values in the recovery bundle. Preserve unrelated configuration and broker/mail credentials. +Inside the supplied container, the selected host `DATA_DIR` is always `/app/data` and the runtime state directory defaults to `/app/data`. Keep any configured `CORESCOPE_STATE_DIR` subdirectory stable and preserve config/theme/queues and the original database-path settings in the recovery bundle. Preserve unrelated configuration and broker/mail credentials. Confirm the exact source mounts before initializing schemas. This check does not start the application or PostgreSQL: ```sh set -eu -docker compose -f docker-compose.example.yml run --rm --no-deps \ +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml run --rm --no-deps \ --entrypoint /bin/sh bootstrap -eu -c \ 'test -f /app/data/config.json; test -f /app/data/meshcore.db; test -f /app/data/users.db' ``` @@ -81,17 +83,17 @@ Remove the `users.db` check and import flag only when no account source exists, ```sh set -eu -docker compose -f docker-compose.example.yml up -d --wait postgres +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml up -d --wait postgres install -d -m 700 migration-state -docker compose -f docker-compose.example.yml run --rm \ - --entrypoint /app/corescope-migrate \ - -v "$PWD/migration-state:/migration" bootstrap \ +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml run --rm \ + --entrypoint /bin/sh \ + -v "$PWD/migration-state:/migration" bootstrap -eu -c 'export CORESCOPE_USERS_DATABASE_URL="$CORESCOPE_USERS_OWNER_DATABASE_URL"; exec /app/corescope-migrate "$@"' sh \ -offline -from-sqlite /app/data/meshcore.db \ -users-from-sqlite /app/data/users.db -state-dir /migration ``` -The bootstrap service supplies owner URLs through its environment. Its PostgreSQL health check only proves the server is accepting connections; successful importer authentication is still required. The importer copies in bounded batches, commits progress with each batch, verifies counts and logical digests in primary-key order, reseeds identities and analyzes the imported tables. Both requested imports must verify before readiness finalization begins. +The current bootstrap service supplies the account owner as `CORESCOPE_USERS_OWNER_DATABASE_URL` and keeps `CORESCOPE_USERS_DATABASE_URL` for runtime. For the legacy commands on this page that omit `-storage-action`, run through `/bin/sh -eu -c 'export CORESCOPE_USERS_DATABASE_URL="$CORESCOPE_USERS_OWNER_DATABASE_URL"; exec /app/corescope-migrate "$@"' sh ...` in the bootstrap container, replacing `...` with the listed non-secret flags. Prefer the current storage-switch workflow above for routine operations. Its PostgreSQL health check only proves the server is accepting connections; successful importer authentication is still required. The importer copies in bounded batches, commits progress with each batch, verifies counts and logical digests in primary-key order, reseeds identities and analyzes the imported tables. Both requested imports must verify before readiness finalization begins. Success is exit status zero, a JSON report with `"verified":true` for each imported store, followed by: @@ -114,22 +116,22 @@ If an old source needed an explicit repair/upgrade, the verified import belongs ```sh set -eu -docker compose -f docker-compose.example.yml run --rm \ - --entrypoint /app/corescope-migrate bootstrap \ +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml run --rm \ + --entrypoint /bin/sh bootstrap -eu -c 'export CORESCOPE_USERS_DATABASE_URL="$CORESCOPE_USERS_OWNER_DATABASE_URL"; exec /app/corescope-migrate "$@"' sh \ -check-import-kind=telemetry -from-sqlite /app/data/meshcore.db -docker compose -f docker-compose.example.yml run --rm \ - --entrypoint /app/corescope-migrate bootstrap \ +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml run --rm \ + --entrypoint /bin/sh bootstrap -eu -c 'export CORESCOPE_USERS_DATABASE_URL="$CORESCOPE_USERS_OWNER_DATABASE_URL"; exec /app/corescope-migrate "$@"' sh \ -check-import-kind=accounts -users-from-sqlite /app/data/users.db -docker compose -f docker-compose.example.yml run --rm \ - --entrypoint /app/corescope-migrate bootstrap -check-ready +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml run --rm \ + --entrypoint /bin/sh bootstrap -eu -c 'export CORESCOPE_USERS_DATABASE_URL="$CORESCOPE_USERS_OWNER_DATABASE_URL"; exec /app/corescope-migrate "$@"' sh -check-ready -docker compose -f docker-compose.example.yml run --rm bootstrap -docker compose -f docker-compose.example.yml up -d corescope +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml run --rm bootstrap +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml up -d corescope ``` -Skip the account-source check only when there was no account source. The source checks print ` verified import matches the retained SQLite source`; readiness prints ` PostgreSQL schema is ready` for both configured destinations. Normal bootstrap checks retained `/app/data/meshcore.db` and `/app/data/users.db`, then grants restricted runtime privileges. A manually initialized empty database does not satisfy this upgrade guard. Custom source paths require the explicit checks above because automatic bootstrap only checks those two default paths. Absent and zero-length WAL sidecars share a fingerprint; every nonempty WAL byte remains significant. +Skip the account-source check only when there was no account source. The source checks print ` verified import matches the retained SQLite source`; readiness prints ` PostgreSQL schema is ready` for both configured destinations. These legacy source checks verify the retained files explicitly. Current packaged startup uses the authoritative installation selection and the new guarded setup action; publish a validated PostgreSQL selection with the current storage workflow before starting the application. Do not manually substitute an empty PostgreSQL target for a SQLite installation. Absent and zero-length WAL sidecars share a fingerprint; every nonempty WAL byte remains significant. The example publishes HTTP port 80 through bundled Caddy. Open the configured host port and verify `/api/healthz` plus actual packet ingestion. If you explicitly disable Caddy, your proxy or port mapping must instead reach container port 3000. @@ -139,7 +141,7 @@ For native installations, invoke `corescope-migrate` with owner URLs in `CORESCO Verify packet and observation counts, representative raw/null values, observers and paths, live ingestion/WebSocket visibility, retention, account login, existing sessions/CSRF, proposals, notifications and account export. Test accounts enabled and disabled as applicable. Restore a native backup into fresh isolated databases and exercise the API/login there before declaring recovery proven. -Rollback **before PostgreSQL accepts new writes**: stop the new application with its new Compose project, leave its PostgreSQL storage and migration evidence intact, then return to the preserved old checkout/Compose project, original `.env`, pinned SQLite image and untouched source/state directories. Restore the old database-path settings from that bundle; the PostgreSQL configuration cannot be passed to the SQLite release. Confirm the new writers are stopped before starting the old application. Once PostgreSQL has accepted new telemetry or account changes, the old SQLite copy is stale. There is no automatic lossless reverse converter; stop and decide explicitly how to handle the new data before reverting. +Rollback **before PostgreSQL accepts new writes**: stop the new application with its new Compose project, leave its PostgreSQL storage and migration evidence intact, then return to the preserved old checkout/Compose project, original `.env`, pinned SQLite image and untouched source/state directories. Restore the old database-path settings from that bundle; the PostgreSQL configuration cannot be passed to the SQLite release. Confirm the new writers are stopped before starting the old application. Once PostgreSQL has accepted new telemetry or account changes, the old SQLite copy is stale. Use the verified PostgreSQL-to-SQLite storage switch to retain new data; returning to an old pre-cutover copy would discard those writes. Keep at least one verified recovery copy. If you later archive the old SQLite files outside the active state directory, move their WAL siblings with them and preserve the import report and original fingerprint evidence. Do not remove the only rollback copy to reclaim space. @@ -155,7 +157,7 @@ For the **example Compose variant**, use its PostgreSQL container directly. This ```sh pg_owner() { - docker compose -f docker-compose.example.yml exec -T postgres sh -eu -c ' + docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml exec -T postgres sh -eu -c ' export PGHOST=127.0.0.1 PGUSER=corescope_owner PGSSLMODE=disable export PGPASSWORD="$CORESCOPE_OWNER_PASSWORD" exec "$@" @@ -168,7 +170,7 @@ For a coordinated backup pair, stop the application, create a new private backup ```sh set -eu umask 077 -docker compose -f docker-compose.example.yml stop corescope +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml stop corescope BACKUP_DIR=$(mktemp -d "$PWD/corescope-backup.XXXXXX") for store in telemetry accounts; do pg_owner pg_dump --no-password --format=custom --no-owner --no-privileges \ @@ -176,14 +178,14 @@ for store in telemetry accounts; do mv "$BACKUP_DIR/$store.dump.partial" "$BACKUP_DIR/$store.dump" done # Preserve matching .env, config/theme, keys and remaining state privately too. -docker compose -f docker-compose.example.yml up -d corescope +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml up -d corescope ``` Restore into **new empty PostgreSQL storage**, keeping the previous storage intact. With the original Compose file/project and `.env`, stop `corescope` and then `postgres` before changing any storage path. Record the old `POSTGRES_DATA_DIR`, select a new empty path in `.env`, use matching recovered credentials, and keep `corescope` stopped. Restore the matching config/theme and remaining state from the private bundle before bootstrap; preserve their previous copies. Set `BACKUP_DIR` to the verified backup bundle. Before either restore, check that both destinations are empty: ```sh set -eu -docker compose -f docker-compose.example.yml up -d --wait postgres +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml up -d --wait postgres for store in telemetry accounts; do test -f "$BACKUP_DIR/$store.dump" objects=$(pg_owner psql -X -v ON_ERROR_STOP=1 -At --dbname="corescope_$store" \ @@ -195,9 +197,9 @@ for store in telemetry accounts; do --exit-on-error --single-transaction --dbname="corescope_$store" \ < "$BACKUP_DIR/$store.dump" done -docker compose -f docker-compose.example.yml run --rm bootstrap -docker compose -f docker-compose.example.yml run --rm \ - --entrypoint /app/corescope-migrate bootstrap -check-ready +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml run --rm bootstrap +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml run --rm \ + --entrypoint /bin/sh bootstrap -eu -c 'export CORESCOPE_USERS_DATABASE_URL="$CORESCOPE_USERS_OWNER_DATABASE_URL"; exec /app/corescope-migrate "$@"' sh -check-ready ``` If one restore fails, the other database may already have restored successfully: keep services stopped and preserve both the failure evidence and old storage. Do not retry into an occupied database. After successful grants/readiness and the account checks below, validate the restored instance in isolation before starting it for normal traffic. diff --git a/docs/storage.md b/docs/storage.md new file mode 100644 index 000000000..185a906a3 --- /dev/null +++ b/docs/storage.md @@ -0,0 +1,97 @@ +# Choose, update and recover storage + +CoreScope uses **SQLite by default**. PostgreSQL18.6 is optional. Both keep telemetry and accounts separate; the ingestor writes telemetry and the server reads it. Accounts remain at their recorded target while user management is disabled. If a legacy installation never initialized accounts, enabling them is an explicit setup operation: stop writers and run `./manage.sh setup` after editing the account configuration. Restart alone does not invent a missing account database. + +## Install + +Keep a complete checkout pinned to the same release as the image. For the usual single-host installation, preserve an existing private `.env` or copy `.env.example`, set `DATA_DIR` to the intended persistent directory, then run: + +```sh +docker compose -f docker-compose.example.yml up -d +``` + +No PostgreSQL variables or service are required. Before supervisor starts either process, packaged setup validates an existing supported SQLite installation or initializes only a provably fresh one. A missing selection alone is not permission to discard data. If setup reports an error, keep the application stopped and preserve the files. + +For a **new PostgreSQL installation**, merge `.env.postgres.example` into the private `.env`. Generate a distinct `openssl rand -hex 32` password for each blank field only before the first initialization of an empty PostgreSQL directory. Then add the matching override: + +```sh +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml up -d +``` + +The override starts a separate PostgreSQL service and a short-lived owner bootstrap. Runtime containers receive reader, writer, account and approved-channel roles, never owner credentials. There is no published database port. Existing SQLite data is not converted by adding this file. + +For source installations, `./manage.sh setup` offers SQLite (default) or PostgreSQL. It preserves existing private environment settings and keeps an installed backend. Its production files are `docker-compose.yml` / `docker-compose.no-mosquitto.yml`; PostgreSQL adds `docker-compose.postgres.yml`. Staging uses its own base and `docker-compose.staging.postgres.yml`, persistent directories and project. The example deployment is separate from the variants managed by `manage.sh`. + +Success means setup exits successfully, `/api/healthz` reports ready, and actual packets appear through the API/UI. A running container or PostgreSQL health check alone is not evidence of completed setup or authentication. + +## Keep an installed backend on update + +`storage-selection.json` in the shared state directory records the backend and both targets atomically. It contains no passwords. Old `db.backend`, `CORESCOPE_DB_BACKEND`, `dbPath` and account-path bootstrap hints cannot replace a recorded target. Credentials and TLS settings stay in private config/environment and can rotate without changing the endpoint identity. + +Use the same state mount and Compose files on restart/update. `./manage.sh update` keeps the recorded backend. For a pre-built image, pull the matching image and run `up -d` with the same files, including the PostgreSQL override when selected. Removing an override does not perform a reverse conversion. + +Container state must stay at `/app/data` or a persistent subdirectory, selected with `CORESCOPE_STATE_DIR`. All supplied services share that value. The entrypoint refuses an ephemeral directory or a symlink escaping the data mount. Native relative SQLite paths resolve from the process working directory; container processes run in `/app`. A `config.json` that exists but cannot be read or parsed stops startup; it is not ignored in favour of defaults. + +Before the first upgrade from an older deployment, stop that old stack using its actual Compose project/service and checkout, before replacing configuration files. Preserve its checkout/Compose files, image identity, private `.env`, config/theme and a quiescent copy of the complete data directory including WAL/SHM siblings. Do not use a raw copy of a live SQLite main file as a backup. + +## Explicit offline backend change + +Plan a maintenance window and measure disk headroom on a representative copy. Keep source data, recovery snapshots and the new target together until validation is complete. Data/index/WAL and process memory costs differ between engines; one synthetic benchmark does not size every installation. + +For the managed production deployment: + +```sh +./manage.sh storage status +./manage.sh backup ./backups/before-storage-change +./manage.sh storage switch postgres # or: sqlite +``` + +The switch asks for confirmation, stops the owned application processes, starts only the database service when needed, and invokes the verified converter. It does not bootstrap the destination schema before conversion. Existing accounts participate even when their UI is disabled. A PostgreSQL destination must be empty; preserve an old cluster and choose a new PostgreSQL directory instead of overwriting it. Switching to SQLite creates new destination paths under the existing persistent state directory, preserving the old source files. + +The converter snapshots/normalizes supported source data, copies bounded batches, verifies table counts, typed logical digests and reference/identity high-water marks, applies narrow destination grants, and checks runtime connections. Both requested stores must verify before the single selection replacement. The state directory itself does not move. + +On success, inspect the credential-free report and recorded targets. The management command leaves writers stopped: + +```sh +./manage.sh storage status +./manage.sh start +``` + +Validate packet/observation counts, raw bytes, paths, live ingestion, accounts/login, existing sessions, settings, proposals and notifications as applicable. Preserve source and recovery data after success. A pre-cutover copy becomes stale once the new backend accepts writes; it is not a lossless reverse migration. To return while retaining new writes, perform a verified switch in the other direction. + +### Interrupted conversion + +A pending journal blocks startup. Do not delete it, rewrite the selection by hand, or manually mark a target ready. Keep the same source, configured PostgreSQL host/port/schema, target paths, recovery directory and matching converter version. Status supplies the job ID: + +```sh +./manage.sh storage status +./manage.sh storage resume JOB_ID +# Or, only while the old selection is still authoritative: +./manage.sh storage abort JOB_ID +``` + +Resume rechecks saved source/target identity, snapshots, progress, data and sequences. Changed data or mismatched evidence is refused. Abort is allowed only before selection publication and preserves staged/recovery files. If preparation failed before a complete manifest existed, preserve the incomplete directory and original files; follow the reported recovery instructions instead of inventing progress or deleting the only snapshots. + +The native CLI has the same actions through `-storage-action`, always with an absolute `-selection-file`, and `-offline` for mutation. Its `-state-dir` is private recovery/job storage, not the stable application state directory. Use `-config` / `-config-dir` for the same fail-closed configuration rules as runtime. Owner connections belong in `CORESCOPE_DATABASE_URL` and, for new storage actions, `CORESCOPE_USERS_OWNER_DATABASE_URL`; `CORESCOPE_USERS_DATABASE_URL` remains the runtime account writer. Never put credential URLs in command arguments. Custom/external PostgreSQL endpoints require explicit native owner tooling; managed backup/restore refuses to substitute the local default database. + +## Native backups and restore + +`./manage.sh backup DIRECTORY` creates native SQLite `telemetry.db` / `accounts.db` snapshots or PostgreSQL `telemetry.dump` / `accounts.dump`, plus a private `state.tar`, matching `compose.env`, and available config/theme/Caddy files. SQLite uses a read-only native backup that includes committed WAL pages. The state archive excludes the selected live SQLite files/sidecars and PostgreSQL storage. An absent, never-created SQLite account store is recorded explicitly. Use a new empty backup directory outside live data. + +Each database snapshot is consistent; sequential snapshots plus copied state are not one cross-database transaction. Stop both writers when a coordinated pair/queue state is required. Keep the bundle and matching private `.env` encrypted off-host; do not commit or upload real databases, account state or credentials. + +For SQLite restore, stop the application, retain the current data directory, and select a **new empty** `PROD_DATA_DIR`. `./manage.sh restore DIRECTORY` stages the state archive and native files, validates schemas and integrity, then publishes the restored directory. It refuses occupied targets, archive links/escaping paths and targets outside the persistent mount. Failure leaves source backups and staged recovery material intact. The services remain stopped for review. Older standalone snapshots or custom layouts use the native recovery procedure instead of guessing a selection record. + +For PostgreSQL restore, keep the application stopped and preserve its matching state/selection and private settings. Select a new PostgreSQL storage directory, start only the database service, and restore into two empty databases with `./manage.sh restore DIRECTORY`. Do not run schema bootstrap first. The helper refuses occupied targets, restores transactionally per database, reapplies grants and analyzes data. If one restore fails, keep everything stopped; do not rerun into the already-restored database. For a lost host, restore and review the matching private state/selection before database restoration, using the same container paths. See [native PostgreSQL recovery](postgresql-upgrade.md#native-backups-and-restores). + +Restoring accounts can revive deleted users, old passwords, revoked sessions and used links. Preserve relevant current audit entries before replacing state; disable affected accounts and clear restored sessions/tokens before serving traffic, then repeat deletion through the application to anonymize mail history and audit it. See [account backups](user-guide/accounts.md#backups). + +## Supported source layouts + +| Store | Supported current/upgrade boundary | +|---|---| +| SQLite telemetry | Known v3 layout and supported append-only migrations; older/unknown layouts require the prior version's explicit upgrade/repair | +| SQLite accounts | Original versions1–5 upgrade to v6; v6 adds non-reusing identities and explicit mail-event IDs | +| PostgreSQL telemetry/accounts | Readiness marker version1; account logical schema version5 | + +Unknown/future columns, hidden generated columns, unsupported objects or conflicting data refuse conversion. Preserve a raw recovery snapshot before an explicit repair/upgrade of a disposable working copy. Deleted legacy integer IDs without allocator or retained-reference evidence have no recoverable historical high-water mark. Do not infer one or discard references to make validation pass. diff --git a/docs/user-guide/accounts.md b/docs/user-guide/accounts.md index 9514198c5..4cf551a2d 100644 --- a/docs/user-guide/accounts.md +++ b/docs/user-guide/accounts.md @@ -45,7 +45,8 @@ incomplete, and the log says what is missing. |---|---| | `adminEmails` | Addresses that become admin on activation. They cannot be demoted, disabled or deleted from the UI. Remove an address here first. | | `publicBaseUrl` | The address visitors use. Every mail link is built from it, never from the request. It must match the browser origin, because state-changing requests from another origin are refused. | -| `databaseURL` | Required separate PostgreSQL account database, using the restricted account writer. Prefer `CORESCOPE_USERS_DATABASE_URL` in private environment configuration. | +| `dbPath` | SQLite account file, default `users.db` beside telemetry. The installed target remains recorded while accounts are disabled. | +| `databaseURL` | When PostgreSQL is selected, use a separate account database and restricted writer via `CORESCOPE_USERS_DATABASE_URL`. | | `sessionDays` | Login lifetime, extended while in use. Default 30, maximum 365. | | `trustedProxies` | CIDRs of your reverse proxy, so the per-IP login limits see real client IPs. Without it, behind a proxy every client shares the proxy's IP for the per-IP limits (they are switched off when that IP is loopback or private). Per-address and per-account limits apply either way. | | `mail.webhookSecret` | Enables delivery status (below). At least 16 characters. | @@ -187,12 +188,12 @@ watched node changes state. Admins can also watch the instance. ### Backups -The separate PostgreSQL account database holds password hashes and addresses. The server keeps its own snapshots of +The separate SQLite account file or PostgreSQL account database holds password hashes and addresses. The server keeps its own snapshots of it: at startup when the newest snapshot is older than 24 hours (or there is none), then -about every 24 hours (the check runs hourly). A snapshot is a native PostgreSQL custom-format archive named `users-.dump` (UTC), +about every 24 hours (the check runs hourly). A native snapshot is named `users-.db` for SQLite or `.dump` for PostgreSQL (UTC), readable by the server's user only, in `backups/` under the configured `stateDir`. After each new snapshot the oldest ones beyond `keep` are deleted, never the one just written. -Temporary files of an interrupted snapshot (`users-.dump.tmp`) are +Temporary files of an interrupted snapshot (`users-.db.tmp` or `.dump.tmp`) are deleted once they are older than 24 hours; other files in that directory are never touched. A failed snapshot is logged (`[users] backup failed: ...`) and the next run tries again. @@ -219,7 +220,7 @@ delete that copy. Snapshots on the same disk are lost with that disk. To keep a copy elsewhere, log in as an admin and open `/api/admin/users-backup` in the same browser: it downloads a fresh -snapshot (`corescope-users-.dump`) and records it in the audit log +snapshot (`corescope-users-.db` or `.dump`, matching the backend) and records it in the audit log (`user.backup`). Store the download encrypted: it holds every password hash and address. The analyzer database has its own backup route, `GET /api/backup`. @@ -233,7 +234,9 @@ The analyzer database has its own backup route, `GET /api/backup`. The audit log is replaced too, so note the deletions before you overwrite it. -**Restore** into a fresh account database while the server and every account writer are stopped. Keep the old database intact for recovery and audit review. Supply PostgreSQL connection credentials through private libpq environment settings (`PGHOST`, `PGUSER`, `PGDATABASE` and a protected password source), never command arguments. +For SQLite, restore the native `.db` snapshot only while all account writers are stopped, into a new account target; keep the current file and WAL siblings for audit/recovery. Use the matching installation state and [native storage restore procedure](../storage.md#native-backups-and-restore). Do not replace a live main file or treat an old snapshot as a reverse migration after new writes. The deletion query below can use `datetime(at, 'unixepoch')` and an integer Unix cutoff on SQLite; the session/token cleanup applies to both engines. + +**PostgreSQL restore:** use a fresh account database while the server and every account writer are stopped. Keep the old database intact for recovery and audit review. Supply PostgreSQL connection credentials through private libpq environment settings (`PGHOST`, `PGUSER`, `PGDATABASE` and a protected password source), never command arguments. 1. On the current database, record accounts deleted since the snapshot. Replace the timestamp with the snapshot's UTC time: @@ -262,7 +265,7 @@ The audit log is replaced too, so note the deletions before you overwrite it. COMMIT; ``` -4. Check schema readiness with the matching `corescope-migrate -check-ready` binary and the owner account URL. Update the restricted runtime URLs to the restored destination and start the server. +4. Check schema readiness with the matching `corescope-migrate -check-ready` binary and the owner account URL. Keep the matching selection and restricted runtime target identities, validate the restored stores, then start the server. A different target must be explicitly adopted/selected through the offline operator workflow, not silently substituted by editing a URL. 5. Delete those accounts again through the admin area. Application deletion also replaces addresses in mail history with a hash and records the deletion in the audit log. Keep notification delivery paused until its restored state has been reviewed; old notification preferences and unsubscribe links also return. A newer schema is refused by older binaries. Never delete or overwrite the current database just to make restoration succeed. See [native backups and restores](../postgresql-upgrade.md#native-backups-and-restores) for the two-database recovery procedure. diff --git a/internal/dbconfig/selection.go b/internal/dbconfig/selection.go new file mode 100644 index 000000000..e58130c69 --- /dev/null +++ b/internal/dbconfig/selection.go @@ -0,0 +1,331 @@ +package dbconfig + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "net" + "net/url" + "os" + "path/filepath" + "reflect" + "regexp" + "strconv" + "strings" +) + +const selectionVersion = 1 + +var ( + ErrSelectionMissing = errors.New("installation selection is absent; setup or the writer must validate and adopt existing storage before reader startup") + ErrSelectionBusy = errors.New("installation is in use; stop its services before backend adoption, switching or recovery") + ErrSelectionInProgress = errors.New("an incomplete backend switch blocks startup; keep writers stopped and explicitly resume or abort that job") + ErrSelectionChanged = errors.New("recorded storage target or generation changed; use a verified offline selection update") +) + +type PostgresTarget struct { + Host string `json:"host"` + Port uint16 `json:"port"` + Database string `json:"database"` + Schema string `json:"schema"` +} + +type Target struct { + SQLitePath string `json:"sqlite_path,omitempty"` + Postgres *PostgresTarget `json:"postgres,omitempty"` +} + +// Selection records endpoints, never authentication or TLS settings. Accounts +// describe the configured store independently of whether its feature is enabled. +type Selection struct { + Version int `json:"version"` + Generation string `json:"generation"` + Backend Backend `json:"backend"` + StateDir string `json:"state_dir"` + Telemetry Target `json:"telemetry"` + Accounts *Target `json:"accounts"` +} + +func SelectionPath(stateDir string) string { return filepath.Join(stateDir, "storage-selection.json") } + +func newSelectionID() (string, error) { + var value [16]byte + if _, err := rand.Read(value[:]); err != nil { + return "", err + } + return hex.EncodeToString(value[:]), nil +} + +func validSelectionID(value string) bool { + decoded, err := hex.DecodeString(value) + return err == nil && len(decoded) == 16 && value == strings.ToLower(value) +} + +// NewSelection constructs a proposal. Callers validate the same native targets +// before adopting/committing it. It never connects or infers freshness. +func NewSelection(storage Storage) (Selection, error) { + s := Selection{Version: selectionVersion, Backend: storage.Backend, StateDir: storage.StateDir} + var err error + if s.Generation, err = newSelectionID(); err != nil { + return Selection{}, err + } + switch s.Backend { + case SQLite: + s.Telemetry = Target{SQLitePath: storage.DBPath} + if storage.UsersDBPath != "" { + s.Accounts = &Target{SQLitePath: storage.UsersDBPath} + } + case Postgres: + if s.Telemetry, err = postgresRoleTarget(storage.ReaderDatabaseURL, storage.WriterDatabaseURL); err != nil { + return Selection{}, err + } + if storage.UsersDatabaseURL != "" || storage.ApprovedChannelsDatabaseURL != "" { + target, err := postgresRoleTarget(storage.UsersDatabaseURL, storage.ApprovedChannelsDatabaseURL) + if err != nil { + return Selection{}, err + } + s.Accounts = &target + } + default: + return Selection{}, errors.New("selection requires a validated backend") + } + return s, s.validate() +} + +func (s Selection) validate() error { + if s.Version != selectionVersion || !validSelectionID(s.Generation) || !filepath.IsAbs(s.StateDir) || filepath.Clean(s.StateDir) != s.StateDir { + return errors.New("invalid or unsupported installation selection") + } + check := func(target Target) bool { + if s.Backend == SQLite { + return target.Postgres == nil && filepath.IsAbs(target.SQLitePath) && filepath.Clean(target.SQLitePath) == target.SQLitePath && !strings.ContainsRune(target.SQLitePath, 0) + } + p := target.Postgres + return s.Backend == Postgres && target.SQLitePath == "" && p != nil && p.Host != "" && p.Port != 0 && p.Database != "" && p.Schema != "" && !strings.ContainsRune(p.Host+p.Database+p.Schema, 0) + } + if !check(s.Telemetry) || (s.Accounts != nil && !check(*s.Accounts)) { + return errors.New("installation selection has invalid storage targets") + } + if s.Accounts != nil { + if s.Backend == SQLite && s.Telemetry.SQLitePath == s.Accounts.SQLitePath { + return errors.New("telemetry and account targets must remain separate") + } + if s.Backend == Postgres { + a, b := s.Telemetry.Postgres, s.Accounts.Postgres + if a.Host == b.Host && a.Port == b.Port && a.Database == b.Database { + return errors.New("telemetry and account databases must remain separate") + } + } + } + return nil +} + +func postgresRoleTarget(first, second string) (Target, error) { + if first == "" { + first = second + } + if first == "" { + return Target{}, errors.New("PostgreSQL target requires a role URL") + } + value, err := postgresTarget(first) + if err != nil { + return Target{}, err + } + if err := bootstrapPostgresEnvironment(first); err != nil { + return Target{}, err + } + if second != "" { + other, err := postgresTarget(second) + if err != nil { + return Target{}, err + } + if value != other { + return Target{}, ErrSelectionChanged + } + if err := bootstrapPostgresEnvironment(second); err != nil { + return Target{}, err + } + } + return Target{Postgres: &value}, nil +} + +func bootstrapPostgresEnvironment(value string) error { + if os.Getenv("PGOPTIONS") != "" || os.Getenv("PGSERVICE") != "" { + return errors.New("initial PostgreSQL selection requires explicit URL settings instead of PGSERVICE or PGOPTIONS") + } + u, err := url.Parse(value) + if err != nil { + return errors.New("invalid PostgreSQL target URL") + } + if u.Port() == "" && os.Getenv("PGPORT") != "" { + port, err := strconv.ParseUint(os.Getenv("PGPORT"), 10, 16) + if err != nil || port != 5432 { + return errors.New("initial PostgreSQL selection has an ambiguous ambient PGPORT; supply an explicit URL port") + } + } + return nil +} + +var unquotedSchema = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_$]*$`) + +func selectionSchema(value string) (string, error) { + value = strings.TrimSpace(value) + if value == "" { + return "public", nil + } + if strings.HasPrefix(value, `"`) && strings.HasSuffix(value, `"`) && len(value) > 2 { + inner := value[1 : len(value)-1] + if strings.Contains(strings.ReplaceAll(inner, `""`, ""), `"`) || strings.ContainsRune(inner, 0) { + return "", errors.New("selection requires one explicit PostgreSQL schema") + } + return strings.ReplaceAll(inner, `""`, `"`), nil + } + if !unquotedSchema.MatchString(value) { + return "", errors.New("selection requires one explicit PostgreSQL schema") + } + return strings.ToLower(value), nil +} + +func postgresTarget(value string) (PostgresTarget, error) { + u, err := url.Parse(value) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") || u.Hostname() == "" || strings.Contains(u.Host, ",") || strings.TrimPrefix(u.Path, "/") == "" { + return PostgresTarget{}, errors.New("invalid PostgreSQL target URL") + } + port := uint64(5432) + if u.Port() != "" { + port, err = strconv.ParseUint(u.Port(), 10, 16) + if err != nil || port == 0 { + return PostgresTarget{}, errors.New("invalid PostgreSQL target port") + } + } + q, err := url.ParseQuery(u.RawQuery) + if err != nil || len(q["search_path"]) > 1 { + return PostgresTarget{}, errors.New("invalid PostgreSQL target schema setting") + } + // Drivers accept connection fields in the query as overrides of the URL. + // Keep the recorded endpoint unambiguous, including service-file settings + // and startup options that can replace search_path outside this parser. + for _, key := range []string{"host", "hostaddr", "port", "dbname", "database", "service", "servicefile", "options"} { + if _, present := q[key]; present { + return PostgresTarget{}, errors.New("PostgreSQL target overrides must be expressed in the URL authority, database path and search_path") + } + } + schema, err := selectionSchema(q.Get("search_path")) + if err != nil { + return PostgresTarget{}, err + } + host := strings.ToLower(u.Hostname()) + if ip := net.ParseIP(host); ip != nil { + host = ip.String() + } + return PostgresTarget{Host: host, Port: uint16(port), Database: strings.TrimPrefix(u.Path, "/"), Schema: schema}, nil +} + +func bindSelectedURL(value string, target *PostgresTarget) (string, error) { + if value == "" { + return "", nil + } + if target == nil { + return "", ErrSelectionChanged + } + u, err := url.Parse(value) + if err != nil { + return "", errors.New("invalid PostgreSQL target URL") + } + // Missing port/schema are bootstrap defaults, not permission to change an + // installed target through PGPORT or a rotated role's default search_path. + if u.Port() == "" { + u.Host = net.JoinHostPort(u.Hostname(), strconv.Itoa(int(target.Port))) + } + q, err := url.ParseQuery(u.RawQuery) + if err != nil { + return "", errors.New("invalid PostgreSQL target URL query") + } + if !q.Has("search_path") { + q.Set("search_path", `"`+strings.ReplaceAll(target.Schema, `"`, `""`)+`"`) + u.RawQuery = q.Encode() + } + actual, err := postgresTarget(u.String()) + if err != nil { + return "", err + } + if actual != *target { + return "", ErrSelectionChanged + } + u.Host = net.JoinHostPort(target.Host, strconv.Itoa(int(target.Port))) + u.Path = "/" + target.Database + u.RawPath = "" + // Pin the selected schema even when rotated role defaults differ. Host and + // database are never rebound; credentials and TLS options remain untouched. + q.Set("search_path", `"`+strings.ReplaceAll(target.Schema, `"`, `""`)+`"`) + u.RawQuery = q.Encode() + return u.String(), nil +} + +// ApplyTo makes the installed record authoritative over bootstrap selectors +// and paths. Only active PostgreSQL role credentials/TLS remain caller inputs. +func (s Selection) ApplyTo(raw StorageInputs) (StorageInputs, error) { + if err := s.validate(); err != nil { + return StorageInputs{}, err + } + raw.Backend, raw.EnvBackend, raw.ExistingBackend = s.Backend, s.Backend, s.Backend + raw.FreshInstall = false + raw.StateDir = s.StateDir + raw.DBPath, raw.UsersDBPath = "", "" + if s.Backend == SQLite { + raw.DBPath = s.Telemetry.SQLitePath + if s.Accounts != nil { + raw.UsersDBPath = s.Accounts.SQLitePath + } + return raw, nil + } + reader, writer := raw.ReaderDatabaseURL, raw.WriterDatabaseURL + if reader == "" { + reader = raw.DatabaseURL + } + if writer == "" { + writer = raw.DatabaseURL + } + var err error + if raw.ReaderDatabaseURL, err = bindSelectedURL(reader, s.Telemetry.Postgres); err != nil { + return StorageInputs{}, err + } + if raw.WriterDatabaseURL, err = bindSelectedURL(writer, s.Telemetry.Postgres); err != nil { + return StorageInputs{}, err + } + var account *PostgresTarget + if s.Accounts != nil { + account = s.Accounts.Postgres + } else { + raw.UsersDatabaseURL, raw.ApprovedChannelsDatabaseURL = "", "" + } + if raw.UsersDatabaseURL, err = bindSelectedURL(raw.UsersDatabaseURL, account); err != nil { + return StorageInputs{}, err + } + if raw.ApprovedChannelsDatabaseURL, err = bindSelectedURL(raw.ApprovedChannelsDatabaseURL, account); err != nil { + return StorageInputs{}, err + } + raw.DatabaseURL = "" // Role URLs are now bound; an obsolete fallback is inactive. + return raw, nil +} + +func sameSelectionTargets(a, b Selection) bool { + a.Generation, b.Generation = "", "" + return reflect.DeepEqual(a, b) +} + +func cloneSelection(s Selection) Selection { + if s.Telemetry.Postgres != nil { + target := *s.Telemetry.Postgres + s.Telemetry.Postgres = &target + } + if s.Accounts != nil { + target := *s.Accounts + s.Accounts = &target + if target.Postgres != nil { + postgres := *target.Postgres + target.Postgres = &postgres + } + } + return s +} diff --git a/internal/dbconfig/selection_file.go b/internal/dbconfig/selection_file.go new file mode 100644 index 000000000..804fd9900 --- /dev/null +++ b/internal/dbconfig/selection_file.go @@ -0,0 +1,500 @@ +package dbconfig + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "sync" +) + +const maxSelectionBytes = 64 * 1024 + +type selectionLease struct { + file *os.File + once sync.Once + err error +} + +func (l *selectionLease) Close() error { + l.once.Do(func() { l.err = errors.Join(unlockSelectionFile(l.file), l.file.Close()) }) + return l.err +} + +func acquireSelectionLease(path string, exclusive bool) (*selectionLease, error) { + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return nil, errors.New("selection path must be absolute and normalized") + } + f, err := os.OpenFile(path+".lock", os.O_CREATE|os.O_RDWR, 0600) + if errors.Is(err, os.ErrNotExist) { + return nil, ErrSelectionMissing + } + if err != nil { + return nil, err + } + if err = lockSelectionFile(f, exclusive); err != nil { + f.Close() + return nil, err + } + return &selectionLease{file: f}, nil +} + +func readSelectionJSON(path string, out any) error { + info, err := os.Lstat(path) + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Size() > maxSelectionBytes { + return errors.New("invalid selection metadata file") + } + f, err := openSelectionRead(path) + if err != nil { + return err + } + data, err := io.ReadAll(io.LimitReader(f, maxSelectionBytes+1)) + closeErr := f.Close() + if err != nil { + return err + } + if closeErr != nil { + return closeErr + } + if len(data) > maxSelectionBytes { + return errors.New("selection metadata exceeds its bound") + } + d := json.NewDecoder(bytes.NewReader(data)) + d.DisallowUnknownFields() + if err := d.Decode(out); err != nil { + return errors.New("invalid selection metadata JSON") + } + if err := d.Decode(new(any)); err != io.EOF { + return errors.New("unexpected trailing selection metadata") + } + return nil +} + +func readSelected(path string) (Selection, error) { + var s Selection + if err := readSelectionJSON(path, &s); err != nil { + if errors.Is(err, os.ErrNotExist) { + return s, ErrSelectionMissing + } + return s, err + } + if err := s.validate(); err != nil { + return Selection{}, err + } + if SelectionPath(s.StateDir) != path { + return Selection{}, errors.New("selection state directory does not match its discovery path") + } + return s, nil +} + +func selectionJournalError(path string) error { + _, err := os.Lstat(path + ".switch.json") + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + var journal selectionJournal + if readSelectionJSON(path+".switch.json", &journal) == nil && validSelectionID(journal.ID) { + return fmt.Errorf("%w (job %s)", ErrSelectionInProgress, journal.ID) + } + return ErrSelectionInProgress +} + +// ReadSelection is for inspection. A runtime must retain OpenSelection's lease. +func ReadSelection(path string) (Selection, error) { + if err := selectionJournalError(path); err != nil { + return Selection{}, err + } + s, err := readSelected(path) + if err != nil { + return Selection{}, err + } + if err := selectionJournalError(path); err != nil { + return Selection{}, err + } + return s, nil +} + +// OpenSelection returns a shared local lease held until service shutdown. On +// any error, including an absent record, no lease remains to deadlock adoption. +func OpenSelection(path string) (Selection, io.Closer, error) { + lease, err := acquireSelectionLease(path, false) + if err != nil { + return Selection{}, nil, err + } + s, err := ReadSelection(path) + if err != nil { + lease.Close() + return Selection{}, nil, err + } + return s, lease, nil +} + +// AdoptSelection follows caller validation of an existing or newly initialized +// installation. It never silently changes an existing selection's targets. +func AdoptSelection(path string, validated Selection) (Selection, error) { + if err := validated.validate(); err != nil { + return Selection{}, err + } + if SelectionPath(validated.StateDir) != path { + return Selection{}, errors.New("adoption state directory differs from discovery path") + } + if current, lease, err := OpenSelection(path); err == nil { + defer lease.Close() + if !sameSelectionTargets(current, validated) { + return Selection{}, ErrSelectionChanged + } + return current, nil + } else if !errors.Is(err, ErrSelectionMissing) { + return Selection{}, err + } + if err := os.MkdirAll(validated.StateDir, 0700); err != nil { + return Selection{}, err + } + u, err := BeginSelectionUpdate(path, "") + if err != nil { + return Selection{}, err + } + defer u.Close() + if _, err := u.Commit(validated); err != nil { + return Selection{}, err + } + return validated, nil +} + +type selectionJournal struct { + Version int `json:"version"` + ID string `json:"id"` + Source *Selection `json:"source"` + Next *Selection `json:"next,omitempty"` +} + +// SelectionStatus is safe to inspect before choosing a database service or +// loading credentials. Unrecorded means setup validation is still required. +type SelectionStatus struct { + Version int `json:"version"` + State string `json:"state"` + Backend Backend `json:"backend"` + Generation string `json:"generation"` + JobID string `json:"job_id"` + SourceBackend Backend `json:"source_backend"` + TargetBackend Backend `json:"target_backend"` + StateDir string `json:"state_dir"` + Telemetry *Target `json:"telemetry"` + Accounts *Target `json:"accounts"` +} + +func (s *SelectionStatus) selectedTargets(selected Selection) { + s.Backend, s.Generation, s.StateDir = selected.Backend, selected.Generation, selected.StateDir + s.Telemetry, s.Accounts = &selected.Telemetry, selected.Accounts +} + +func InspectSelection(path string) (SelectionStatus, error) { + var status SelectionStatus + var err error + for attempt := 0; attempt < 3; attempt++ { + status, err = inspectSelection(path) + if !errors.Is(err, ErrSelectionInProgress) && !errors.Is(err, ErrSelectionChanged) { + return status, err + } + } + if errors.Is(err, ErrSelectionInProgress) { + status.State = "pending" + return status, nil + } + return status, err +} + +func inspectSelection(path string) (SelectionStatus, error) { + status := SelectionStatus{Version: selectionVersion, State: "corrupt"} + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return status, errors.New("selection path must be absolute and normalized") + } + status.StateDir = filepath.Dir(path) + var journal selectionJournal + err := readSelectionJSON(path+".switch.json", &journal) + if err == nil { + if err = journal.validate(path); err != nil { + return status, err + } + u := SelectionUpdate{path: path, journal: journal} + replaced, err := u.currentState() + if err != nil { + return status, err + } + status.State, status.JobID = "pending", journal.ID + if journal.Source != nil { + status.SourceBackend = journal.Source.Backend + status.selectedTargets(*journal.Source) + } + if journal.Next != nil { + status.TargetBackend = journal.Next.Backend + if replaced { + status.selectedTargets(*journal.Next) + } + } + return status, nil + } + if !errors.Is(err, os.ErrNotExist) { + return status, err + } + s, err := ReadSelection(path) + if errors.Is(err, ErrSelectionMissing) { + status.State = "unrecorded" + return status, nil + } + if err != nil { + return status, err + } + status.State = "ready" + status.selectedTargets(s) + return status, nil +} + +func (j selectionJournal) validate(path string) error { + if j.Version != 1 || !validSelectionID(j.ID) { + return errors.New("invalid selection recovery journal") + } + for _, s := range []*Selection{j.Source, j.Next} { + if s == nil { + continue + } + if err := s.validate(); err != nil { + return err + } + if SelectionPath(s.StateDir) != path { + return errors.New("journal state directory differs from discovery path") + } + } + if j.Source != nil && j.Next != nil && j.Source.Generation == j.Next.Generation { + return ErrSelectionChanged + } + return nil +} + +// SelectionUpdate is a single local selection-file transaction, not a database +// converter. Its caller verifies both stores and holds native DB writer fences. +type SelectionUpdate struct { + path string + lease *selectionLease + journal selectionJournal + afterReplace func() error // fault injection in package tests +} + +func (u *SelectionUpdate) ID() string { return u.journal.ID } +func (u *SelectionUpdate) Current() Selection { + if u.journal.Source == nil { + return Selection{} + } + return cloneSelection(*u.journal.Source) +} + +func (u *SelectionUpdate) Target() Selection { + if u.journal.Next == nil { + return Selection{} + } + return cloneSelection(*u.journal.Next) +} + +// Stage binds the exact unselected destination before any copy begins. It does +// not authorize startup or claim data verification; only Commit selects it. +func (u *SelectionUpdate) Stage(next Selection) error { + if u.lease == nil { + return errors.New("selection update is closed") + } + if _, err := u.currentState(); err != nil { + return err + } + if err := next.validate(); err != nil { + return err + } + if SelectionPath(next.StateDir) != u.path || (u.journal.Source != nil && next.StateDir != u.journal.Source.StateDir) { + return errors.New("backend switching cannot relocate the installation state directory") + } + if u.journal.Source != nil && next.Generation == u.journal.Source.Generation { + return errors.New("selection update requires a new generation") + } + if u.journal.Next != nil { + if !reflect.DeepEqual(next, *u.journal.Next) { + return ErrSelectionChanged + } + return nil + } + staged := u.journal + next = cloneSelection(next) + staged.Next = &next + if _, err := writeSelectionJSON(u.path+".switch.json", staged); err != nil { + return err + } + u.journal = staged + return nil +} +func (u *SelectionUpdate) Close() error { + if u.lease == nil { + return nil + } + err := u.lease.Close() + u.lease = nil + return err +} + +func BeginSelectionUpdate(path, expectedGeneration string) (*SelectionUpdate, error) { + lease, err := acquireSelectionLease(path, true) + if err != nil { + return nil, err + } + fail := func(err error) (*SelectionUpdate, error) { lease.Close(); return nil, err } + if err := selectionJournalError(path); err != nil { + return fail(err) + } + current, err := readSelected(path) + if err != nil && !errors.Is(err, ErrSelectionMissing) { + return fail(err) + } + if (err == nil && current.Generation != expectedGeneration) || (err != nil && expectedGeneration != "") { + return fail(ErrSelectionChanged) + } + id, err := newSelectionID() + if err != nil { + return fail(err) + } + u := &SelectionUpdate{path: path, lease: lease, journal: selectionJournal{Version: 1, ID: id}} + if current.Generation != "" { + u.journal.Source = ¤t + } + if _, err = writeSelectionJSON(path+".switch.json", u.journal); err != nil { + return fail(err) + } + return u, nil +} + +func ResumeSelectionUpdate(path, jobID string) (*SelectionUpdate, error) { + lease, err := acquireSelectionLease(path, true) + if err != nil { + return nil, err + } + fail := func(err error) (*SelectionUpdate, error) { lease.Close(); return nil, err } + var journal selectionJournal + if err := readSelectionJSON(path+".switch.json", &journal); err != nil { + return fail(err) + } + if journal.ID != jobID { + return fail(errors.New("selection recovery requires the exact recorded job")) + } + if err := journal.validate(path); err != nil { + return fail(err) + } + u := &SelectionUpdate{path: path, lease: lease, journal: journal} + if _, err := u.currentState(); err != nil { + return fail(err) + } + return u, nil +} + +func (u *SelectionUpdate) currentState() (bool, error) { + current, err := readSelected(u.path) + if errors.Is(err, ErrSelectionMissing) && u.journal.Source == nil { + return false, nil + } + if err != nil { + return false, err + } + if u.journal.Next != nil && reflect.DeepEqual(current, *u.journal.Next) { + return true, nil + } + if u.journal.Source != nil && reflect.DeepEqual(current, *u.journal.Source) { + return false, nil + } + return false, ErrSelectionChanged +} + +// Commit is called after complete source/target verification. A true result +// with an error means replacement occurred but recovery/cleanup is unfinished. +// Inspect status after cleanup errors; Abort never restores the old record. +func (u *SelectionUpdate) Commit(next Selection) (bool, error) { + if u.lease == nil { + return false, errors.New("selection update is closed") + } + replaced, err := u.currentState() + if err != nil { + return false, err + } + if err := u.Stage(next); err != nil { + return replaced, err + } + if !replaced { + replaced, err = writeSelectionJSON(u.path, next) + if err != nil { + // Native replacement errors can be ambiguous. Classify from the actual + // record, keeping the journal regardless; never claim an old selection. + if actual, readErr := readSelected(u.path); readErr == nil && reflect.DeepEqual(actual, next) { + replaced = true + } + return replaced, err + } + if u.afterReplace != nil { + if err := u.afterReplace(); err != nil { + return true, err + } + } + } + if err := os.Remove(u.path + ".switch.json"); err != nil { + return true, err + } + return true, syncSelectionDirectory(filepath.Dir(u.path)) +} + +func (u *SelectionUpdate) Abort() error { + if u.lease == nil { + return errors.New("selection update is closed") + } + replaced, err := u.currentState() + if err != nil { + return err + } + if replaced { + return errors.New("selected generation already changed; resume verified completion instead of aborting") + } + if err := os.Remove(u.path + ".switch.json"); err != nil { + return err + } + return syncSelectionDirectory(filepath.Dir(u.path)) +} + +func writeSelectionJSON(path string, value any) (replaced bool, err error) { + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return false, err + } + if len(data) > maxSelectionBytes { + return false, errors.New("selection metadata exceeds its bound") + } + f, err := os.CreateTemp(filepath.Dir(path), ".storage-selection-*") + if err != nil { + return false, err + } + temp := f.Name() + defer func() { f.Close(); os.Remove(temp) }() + if _, err = f.Write(append(data, '\n')); err != nil { + return false, err + } + if err = f.Sync(); err != nil { + return false, err + } + if err = f.Close(); err != nil { + return false, err + } + if err = replaceSelectionFile(temp, path); err != nil { + return false, err + } + return true, syncSelectionDirectory(filepath.Dir(path)) +} diff --git a/internal/dbconfig/selection_test.go b/internal/dbconfig/selection_test.go new file mode 100644 index 000000000..3238f43ee --- /dev/null +++ b/internal/dbconfig/selection_test.go @@ -0,0 +1,464 @@ +package dbconfig + +import ( + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +func sqliteSelection(t *testing.T) Selection { + t.Helper() + s, err := ResolveStorage(StorageInputs{FreshInstall: true, BaseDir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + selection, err := NewSelection(s) + if err != nil { + t.Fatal(err) + } + return selection +} + +func TestSelectionMissingAndAuthoritativeAdoption(t *testing.T) { + want := sqliteSelection(t) + path := SelectionPath(want.StateDir) + if _, _, err := OpenSelection(path); !errors.Is(err, ErrSelectionMissing) { + t.Fatalf("missing record=%v", err) + } + got, err := AdoptSelection(path, want) + if err != nil { + t.Fatal(err) + } + if got.Generation != want.Generation { + t.Fatal("initial generation changed") + } + raw := StorageInputs{BaseDir: t.TempDir(), Backend: Postgres, EnvBackend: Postgres, FreshInstall: true, DBPath: "obsolete.db", UsersDBPath: "obsolete-users.db", StateDir: "old-state", DatabaseURL: "inactive-private-url"} + raw, err = got.ApplyTo(raw) + if err != nil { + t.Fatal(err) + } + resolved, err := ResolveStorage(raw) + if err != nil { + t.Fatal(err) + } + if raw.FreshInstall || raw.ExistingBackend != SQLite || resolved.Backend != SQLite || resolved.DBPath != want.Telemetry.SQLitePath || resolved.UsersDBPath != want.Accounts.SQLitePath || resolved.StateDir != want.StateDir { + t.Fatal("bootstrap settings overrode installed selection") + } + duplicate := want + duplicate.Generation = strings.Repeat("a", 32) + got, err = AdoptSelection(path, duplicate) + if err != nil || got.Generation != want.Generation { + t.Fatalf("idempotent adoption: %v", err) + } + duplicate.Telemetry.SQLitePath = filepath.Join(want.StateDir, "other.db") + if _, err := AdoptSelection(path, duplicate); !errors.Is(err, ErrSelectionChanged) { + t.Fatalf("adoption changed targets: %v", err) + } +} + +func TestSelectionPostgresCredentialsAreNotPersisted(t *testing.T) { + t.Setenv("PGPORT", "") + t.Setenv("PGOPTIONS", "") + t.Setenv("PGSERVICE", "") + base := t.TempDir() + storage, err := ResolveStorage(StorageInputs{BaseDir: base, Backend: Postgres, DatabaseURL: "postgres://first:never-record-password@localhost/telemetry?sslmode=require&search_path=public", UsersDatabaseURL: "postgres://accounts:never-record-password@localhost/accounts"}) + if err != nil { + t.Fatal(err) + } + selected, err := NewSelection(storage) + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(selected) + if err != nil { + t.Fatal(err) + } + for _, private := range []string{"never-record-password", "sslmode", "postgres://", "first:"} { + if strings.Contains(string(data), private) { + t.Fatal("credentials or TLS settings persisted") + } + } + raw := StorageInputs{BaseDir: base, Backend: SQLite, DBPath: "inactive.db", DatabaseURL: "postgres://rotated:replacement-secret@localhost:5432/telemetry?sslmode=verify-full", UsersDatabaseURL: "postgres://rotated:replacement-secret@localhost/accounts?sslmode=require"} + raw, err = selected.ApplyTo(raw) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(raw.ReaderDatabaseURL, "replacement-secret") || !strings.Contains(raw.ReaderDatabaseURL, "sslmode=verify-full") || !strings.Contains(raw.ReaderDatabaseURL, "search_path=") { + t.Fatal("rotation lost credentials/TLS or left schema unbound") + } + raw.ReaderDatabaseURL = "postgres://reader:secret@localhost/another" + if _, err := selected.ApplyTo(raw); !errors.Is(err, ErrSelectionChanged) { + t.Fatalf("active PG retarget accepted: %v", err) + } +} + +func TestSelectionPinsRecordedPortAndRefusesAmbiguousBootstrap(t *testing.T) { + t.Setenv("PGPORT", "") + t.Setenv("PGOPTIONS", "") + t.Setenv("PGSERVICE", "") + base := t.TempDir() + selected, err := NewSelection(Storage{Backend: Postgres, StateDir: base, WriterDatabaseURL: "postgres://owner:secret@localhost/data"}) + if err != nil { + t.Fatal(err) + } + t.Setenv("PGPORT", "6543") + bound, err := selected.ApplyTo(StorageInputs{BaseDir: base, DatabaseURL: "postgres://rotated:secret@localhost/data"}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(bound.WriterDatabaseURL, "localhost:5432/") { + t.Fatal("recorded port was left to ambient PGPORT") + } + if _, err := NewSelection(Storage{Backend: Postgres, StateDir: base, WriterDatabaseURL: "postgres://owner:secret@localhost/data"}); err == nil { + t.Fatal("ambiguous implicit bootstrap port accepted") + } + if _, err := NewSelection(Storage{Backend: Postgres, StateDir: base, WriterDatabaseURL: "postgres://owner:secret@localhost:5432/data"}); err != nil { + t.Fatal("explicit port refused", err) + } + for _, key := range []string{"PGOPTIONS", "PGSERVICE"} { + t.Setenv(key, "ambiguous") + if _, err := NewSelection(Storage{Backend: Postgres, StateDir: base, WriterDatabaseURL: "postgres://owner:secret@localhost:5432/data"}); err == nil { + t.Fatal("ambient target options accepted", key) + } + t.Setenv(key, "") + } +} + +func TestRecordedAccountAbsenceDoesNotDefaultOrActivateRawSettings(t *testing.T) { + base := t.TempDir() + selected, err := NewSelection(Storage{Backend: SQLite, StateDir: base, DBPath: filepath.Join(base, "telemetry.sqlite")}) + if err != nil { + t.Fatal(err) + } + if selected.Accounts != nil { + t.Fatal("absent accounts recorded as an initialized target") + } + raw, err := selected.ApplyTo(StorageInputs{BaseDir: base, UsersDBPath: "must-not-create.db", UsersDatabaseURL: "inactive"}) + if err != nil { + t.Fatal(err) + } + resolved, err := ResolveStorage(raw) + if err != nil { + t.Fatal(err) + } + if resolved.UsersDBPath != "" { + t.Fatal("recorded account absence defaulted to a new file") + } + bootstrap, err := ResolveStorage(StorageInputs{BaseDir: base, Backend: SQLite, DBPath: filepath.Join(base, "telemetry.sqlite")}) + if err != nil || bootstrap.UsersDBPath == "" { + t.Fatal("bootstrap account default lost", err) + } + pg, err := NewSelection(Storage{Backend: Postgres, StateDir: base, WriterDatabaseURL: "postgres://owner:secret@localhost:5432/data"}) + if err != nil { + t.Fatal(err) + } + raw, err = pg.ApplyTo(StorageInputs{BaseDir: base, DatabaseURL: "postgres://writer:secret@localhost:5432/data", UsersDatabaseURL: "inactive-not-selected", ApprovedChannelsDatabaseURL: "inactive-not-selected"}) + if err != nil { + t.Fatal(err) + } + resolved, err = ResolveStorage(raw) + if err != nil || resolved.UsersDatabaseURL != "" || resolved.ApprovedChannelsDatabaseURL != "" { + t.Fatal("raw credentials activated uninitialized accounts", err) + } +} + +func TestSelectionRuntimeLeasesAndCrashRecovery(t *testing.T) { + current := sqliteSelection(t) + path := SelectionPath(current.StateDir) + if _, err := AdoptSelection(path, current); err != nil { + t.Fatal(err) + } + _, a, err := OpenSelection(path) + if err != nil { + t.Fatal(err) + } + _, b, err := OpenSelection(path) + if err != nil { + t.Fatal(err) + } + if _, err := BeginSelectionUpdate(path, current.Generation); !errors.Is(err, ErrSelectionBusy) { + t.Fatalf("runtime lease did not fence switch: %v", err) + } + a.Close() + b.Close() + u, err := BeginSelectionUpdate(path, current.Generation) + if err != nil { + t.Fatal(err) + } + id := u.ID() + if _, _, err := OpenSelection(path); err == nil { + t.Fatal("runtime entered active switch") + } + if _, err := ResumeSelectionUpdate(path, id); !errors.Is(err, ErrSelectionBusy) { + t.Fatalf("concurrent resume=%v", err) + } + if err := u.Close(); err != nil { + t.Fatal(err) + } // process-exit behavior: journal remains + if _, _, err := OpenSelection(path); !errors.Is(err, ErrSelectionInProgress) { + t.Fatalf("crash journal ignored: %v", err) + } + if _, err := ResumeSelectionUpdate(path, "wrong-job"); err == nil { + t.Fatal("wrong resume accepted") + } + u, err = ResumeSelectionUpdate(path, id) + if err != nil { + t.Fatal(err) + } + if u.Current().Generation != current.Generation { + t.Fatal("resume changed source generation") + } + if err := u.Abort(); err != nil { + t.Fatal(err) + } + u.Close() + got, lease, err := OpenSelection(path) + if err != nil { + t.Fatal(err) + } + defer lease.Close() + if got.Generation != current.Generation { + t.Fatal("abort changed selection") + } +} + +func TestSelectionCommitAndFailedCommitRemainRecoverable(t *testing.T) { + old := sqliteSelection(t) + path := SelectionPath(old.StateDir) + if _, err := AdoptSelection(path, old); err != nil { + t.Fatal(err) + } + u, err := BeginSelectionUpdate(path, old.Generation) + if err != nil { + t.Fatal(err) + } + next := old + next.Generation = strings.Repeat("b", 32) + next.Telemetry.SQLitePath = filepath.Join(old.StateDir, "staged.db") + bad := next + bad.StateDir = t.TempDir() + if replaced, err := u.Commit(bad); err == nil || replaced { + t.Fatal("switch moved state directory") + } + if err := u.Stage(next); err != nil { + t.Fatal(err) + } + if u.Target().Generation != next.Generation { + t.Fatal("staged target absent") + } + view := u.Target() + view.Accounts.SQLitePath = filepath.Join(old.StateDir, "mutated-view.db") + if u.Target().Accounts.SQLitePath != next.Accounts.SQLitePath { + t.Fatal("caller mutated staged target through a shared pointer") + } + changed := next + changed.Telemetry.SQLitePath = filepath.Join(old.StateDir, "substituted.db") + if err := u.Stage(changed); !errors.Is(err, ErrSelectionChanged) { + t.Fatal("staged target changed", err) + } + u.afterReplace = func() error { return errors.New("injected crash after replacement") } + if replaced, err := u.Commit(next); err == nil || !replaced { + t.Fatalf("post-replace result=%v %v", replaced, err) + } + id := u.ID() + u.Close() + if _, _, err := OpenSelection(path); !errors.Is(err, ErrSelectionInProgress) { + t.Fatal("post-replace failure allowed startup") + } + u, err = ResumeSelectionUpdate(path, id) + if err != nil { + t.Fatal(err) + } + if err := u.Abort(); err == nil { + t.Fatal("abort rolled back a changed selection") + } + if replaced, err := u.Commit(Selection{}); err == nil || !replaced { + t.Fatal("failed retry concealed an already replaced selection") + } + if replaced, err := u.Commit(next); err != nil || !replaced { + t.Fatalf("verified retry failed: %v %v", replaced, err) + } + u.Close() + got, lease, err := OpenSelection(path) + if err != nil { + t.Fatal(err) + } + defer lease.Close() + if got.Generation != next.Generation || got.Telemetry.SQLitePath != next.Telemetry.SQLitePath { + t.Fatal("retry did not publish exact verified target") + } +} + +func TestSelectionStatusAndNativeProcessCrash(t *testing.T) { + s := sqliteSelection(t) + path := SelectionPath(s.StateDir) + status, err := InspectSelection(path) + if err != nil || status.State != "unrecorded" || status.Backend != "" { + t.Fatalf("missing status: %+v %v", status, err) + } + if _, err := AdoptSelection(path, s); err != nil { + t.Fatal(err) + } + status, err = InspectSelection(path) + if err != nil || status.State != "ready" || status.Generation != s.Generation || status.Backend != SQLite { + t.Fatalf("ready status: %+v %v", status, err) + } + if status.StateDir != s.StateDir || status.Telemetry == nil || status.Telemetry.SQLitePath != s.Telemetry.SQLitePath || status.Accounts == nil || status.Accounts.SQLitePath != s.Accounts.SQLitePath { + t.Fatal("status omitted selected targets needed for native backups") + } + child := exec.Command(os.Args[0], "-test.run=^TestSelectionCrashChild$") + ready := filepath.Join(t.TempDir(), "ready") + child.Env = append(os.Environ(), "CORESCOPE_SELECTION_CRASH_TEST="+path, "CORESCOPE_SELECTION_CRASH_READY="+ready) + if err := child.Start(); err != nil { + t.Fatal(err) + } + defer func() { child.Process.Kill(); child.Wait() }() + deadline := time.Now().Add(10 * time.Second) + for { + if _, err := os.Stat(ready); err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("child did not acquire native lease") + } + time.Sleep(10 * time.Millisecond) + } + if _, _, err := OpenSelection(path); !errors.Is(err, ErrSelectionBusy) { + t.Fatalf("another process entered switch lease: %v", err) + } + if err := child.Process.Kill(); err != nil { + t.Fatal(err) + } + child.Wait() + status, err = InspectSelection(path) + if err != nil || status.State != "pending" || status.JobID == "" || status.SourceBackend != SQLite { + t.Fatalf("crashed process status: %+v %v", status, err) + } + if _, _, err := OpenSelection(path); !errors.Is(err, ErrSelectionInProgress) { + t.Fatalf("crash did not retain startup gate: %v", err) + } + u, err := ResumeSelectionUpdate(path, status.JobID) + if err != nil { + t.Fatal(err) + } + if err := u.Abort(); err != nil { + t.Fatal(err) + } + u.Close() + if err := os.WriteFile(path+".switch.json", []byte(`{"version":99}`), 0600); err != nil { + t.Fatal(err) + } + status, err = InspectSelection(path) + if err == nil || status.State != "corrupt" { + t.Fatalf("corrupt status: %+v %v", status, err) + } +} + +func TestSelectionCrashChild(t *testing.T) { + path := os.Getenv("CORESCOPE_SELECTION_CRASH_TEST") + if path == "" { + return + } + s, err := ReadSelection(path) + if err != nil { + t.Fatal(err) + } + u, err := BeginSelectionUpdate(path, s.Generation) + if err != nil { + t.Fatal(err) + } + defer u.Close() + if err := os.WriteFile(os.Getenv("CORESCOPE_SELECTION_CRASH_READY"), []byte("ready"), 0600); err != nil { + t.Fatal(err) + } + time.Sleep(time.Minute) + t.Fatal("parent did not terminate child") +} + +func TestSelectionRefusesHiddenPostgresRetargeting(t *testing.T) { + for _, query := range []string{"host=other", "hostaddr=127.0.0.2", "port=5000", "dbname=other", "database=other", "service=other", "servicefile=other", "options=-csearch_path%3Dother"} { + if _, err := postgresTarget("postgres://role:secret@localhost/telemetry?" + query); err == nil { + t.Errorf("accepted target override %s", query) + } + } +} + +func TestSelectionStatusDoesNotBlockAtomicReplacement(t *testing.T) { + selected := sqliteSelection(t) + path := SelectionPath(selected.StateDir) + if _, err := AdoptSelection(path, selected); err != nil { + t.Fatal(err) + } + stop := make(chan struct{}) + fail := make(chan error, 1) + var reader sync.WaitGroup + reader.Add(1) + go func() { + defer reader.Done() + for { + select { + case <-stop: + return + default: + } + status, err := InspectSelection(path) + if err != nil || (status.State != "ready" && status.State != "pending") { + if err == nil { + err = errors.New("unexpected status during update") + } + select { + case fail <- err: + default: + } + return + } + } + }() + defer func() { + close(stop) + reader.Wait() + select { + case err := <-fail: + t.Error(err) + default: + } + }() + for i := 0; i < 30; i++ { + update, err := BeginSelectionUpdate(path, selected.Generation) + if err != nil { + t.Fatal(err) + } + next := cloneSelection(selected) + next.Generation, err = newSelectionID() + if err != nil { + update.Close() + t.Fatal(err) + } + if _, err := update.Commit(next); err != nil { + update.Close() + t.Fatal(err) + } + update.Close() + selected = next + } +} + +func TestSelectionCorruptionDoesNotDefault(t *testing.T) { + s := sqliteSelection(t) + path := SelectionPath(s.StateDir) + if _, err := AdoptSelection(path, s); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(`{"version":99,"backend":"sqlite"}`), 0600); err != nil { + t.Fatal(err) + } + if _, _, err := OpenSelection(path); err == nil || errors.Is(err, ErrSelectionMissing) { + t.Fatal("invalid record treated as fresh") + } +} diff --git a/internal/dbconfig/selection_unix.go b/internal/dbconfig/selection_unix.go new file mode 100644 index 000000000..c7ed64c73 --- /dev/null +++ b/internal/dbconfig/selection_unix.go @@ -0,0 +1,33 @@ +//go:build linux || darwin || freebsd || openbsd || netbsd || dragonfly + +package dbconfig + +import ( + "errors" + "os" + "syscall" +) + +func openSelectionRead(path string) (*os.File, error) { return os.Open(path) } + +func lockSelectionFile(f *os.File, exclusive bool) error { + flag := syscall.LOCK_SH | syscall.LOCK_NB + if exclusive { + flag = syscall.LOCK_EX | syscall.LOCK_NB + } + err := syscall.Flock(int(f.Fd()), flag) + if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) { + return ErrSelectionBusy + } + return err +} +func unlockSelectionFile(f *os.File) error { return syscall.Flock(int(f.Fd()), syscall.LOCK_UN) } +func replaceSelectionFile(source, target string) error { return os.Rename(source, target) } +func syncSelectionDirectory(path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + return f.Sync() +} diff --git a/internal/dbconfig/selection_windows.go b/internal/dbconfig/selection_windows.go new file mode 100644 index 000000000..0626c2975 --- /dev/null +++ b/internal/dbconfig/selection_windows.go @@ -0,0 +1,102 @@ +//go:build windows + +package dbconfig + +import ( + "os" + "strings" + "syscall" + "time" + "unsafe" +) + +var selectionKernel = syscall.NewLazyDLL("kernel32.dll") +var selectionLock = selectionKernel.NewProc("LockFileEx") +var selectionUnlock = selectionKernel.NewProc("UnlockFileEx") +var selectionMove = selectionKernel.NewProc("MoveFileExW") + +func openSelectionRead(path string) (*os.File, error) { + name, err := syscall.UTF16PtrFromString(selectionLongPath(path)) + if err != nil { + return nil, err + } + // Inspection must not prevent another process replacing the complete + // metadata file. The open handle still reads the old complete generation. + var handle syscall.Handle + for attempt := 0; attempt < 32; attempt++ { + handle, err = syscall.CreateFile(name, syscall.GENERIC_READ, syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, nil, syscall.OPEN_EXISTING, syscall.FILE_ATTRIBUTE_NORMAL, 0) + if err != syscall.Errno(5) && err != syscall.Errno(32) { + break + } + time.Sleep(2 * time.Millisecond) + } + if err != nil { + return nil, &os.PathError{Op: "open", Path: path, Err: err} + } + return os.NewFile(uintptr(handle), path), nil +} + +func lockSelectionFile(f *os.File, exclusive bool) error { + flags := uintptr(1) // LOCKFILE_FAIL_IMMEDIATELY + if exclusive { + flags |= 2 + } + var overlap syscall.Overlapped + ok, _, err := selectionLock.Call(f.Fd(), flags, 0, 1, 0, uintptr(unsafe.Pointer(&overlap))) + if ok != 0 { + return nil + } + if err == syscall.Errno(33) || err == syscall.Errno(997) { + return ErrSelectionBusy + } + return err +} + +func unlockSelectionFile(f *os.File) error { + var overlap syscall.Overlapped + ok, _, err := selectionUnlock.Call(f.Fd(), 0, 1, 0, uintptr(unsafe.Pointer(&overlap))) + if ok != 0 { + return nil + } + return err +} + +func selectionLongPath(path string) string { + if strings.HasPrefix(path, `\\?\`) { + return path + } + if strings.HasPrefix(path, `\\`) { + return `\\?\UNC\` + strings.TrimPrefix(path, `\\`) + } + return `\\?\` + path +} + +func replaceSelectionFile(source, target string) error { + from, err := syscall.UTF16PtrFromString(selectionLongPath(source)) + if err != nil { + return err + } + to, err := syscall.UTF16PtrFromString(selectionLongPath(target)) + if err != nil { + return err + } + // Same-directory rename only: replacement + write-through, never copy/delete. + // A just-deleted journal name can remain delete-pending until an inspection + // handle closes. Retry only Windows' transient sharing/access responses; + // never replace via a copy/delete fallback or wait on a long-lived reader. + var moveErr error + for attempt := 0; attempt < 32; attempt++ { + ok, _, err := selectionMove.Call(uintptr(unsafe.Pointer(from)), uintptr(unsafe.Pointer(to)), 9) + if ok != 0 { + return nil + } + moveErr = err + if err != syscall.Errno(5) && err != syscall.Errno(32) { + break + } + time.Sleep(2 * time.Millisecond) + } + return moveErr +} + +func syncSelectionDirectory(string) error { return nil } // MoveFileExW uses WRITE_THROUGH. diff --git a/internal/dbconfig/sqlite_import.go b/internal/dbconfig/sqlite_import.go new file mode 100644 index 000000000..500a97bf8 --- /dev/null +++ b/internal/dbconfig/sqlite_import.go @@ -0,0 +1,23 @@ +package dbconfig + +import ( + "database/sql" + "errors" +) + +// The native resume ledger also fences adoption through a different state +// directory. Conversion removes it only after complete data verification. +const SQLiteImportPendingTable = "corescope_reverse_progress" + +var ErrSQLiteImportIncomplete = errors.New("SQLite target is a staged incomplete conversion; keep it unselected and resume or abort its original offline job") + +func AssertSQLiteImportComplete(db *sql.DB) error { + var count int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name=?1`, SQLiteImportPendingTable).Scan(&count); err != nil { + return err + } + if count != 0 { + return ErrSQLiteImportIncomplete + } + return nil +} diff --git a/internal/dbconfig/sqlite_uri.go b/internal/dbconfig/sqlite_uri.go new file mode 100644 index 000000000..25ba8a63c --- /dev/null +++ b/internal/dbconfig/sqlite_uri.go @@ -0,0 +1,29 @@ +package dbconfig + +import ( + "errors" + "fmt" + "net/url" + "path/filepath" + "runtime" + "strings" +) + +// SQLiteURI encodes a filesystem path as a native file: URI. It does not open +// the file or choose access/durability options; callers supply those explicitly. +// A path is never interpreted as an existing URI or an in-memory database name. +func SQLiteURI(path string, options url.Values) (string, error) { + if path == "" || strings.ContainsRune(path, 0) { + return "", errors.New("SQLite requires a nonempty filesystem path without NUL") + } + abs, err := filepath.Abs(path) + if err != nil { + return "", fmt.Errorf("resolve SQLite filesystem path: %w", err) + } + abs = filepath.ToSlash(abs) + if runtime.GOOS == "windows" && !strings.HasPrefix(abs, "//") { + abs = "/" + abs + } + u := url.URL{Scheme: "file", Path: abs, RawQuery: options.Encode()} + return u.String(), nil +} diff --git a/internal/dbconfig/sqlite_uri_test.go b/internal/dbconfig/sqlite_uri_test.go new file mode 100644 index 000000000..cbd2a6d10 --- /dev/null +++ b/internal/dbconfig/sqlite_uri_test.go @@ -0,0 +1,66 @@ +package dbconfig + +import ( + "net/url" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" +) + +func TestSQLiteURILiteralPathsAndOptions(t *testing.T) { + for _, name := range []string{"plain.db", "literal # ? % space µ.db"} { + path := filepath.Join(t.TempDir(), name) + opts := url.Values{"mode": {"ro"}, "_pragma": {"busy_timeout(5000)", "foreign_keys(1)"}} + before := opts.Encode() + text, err := SQLiteURI(path, opts) + if err != nil { + t.Fatal(err) + } + u, err := url.Parse(text) + if err != nil { + t.Fatal(err) + } + if u.Scheme != "file" || u.Host != "" || u.Fragment != "" || !reflect.DeepEqual(u.Query(), opts) || opts.Encode() != before { + t.Fatal("URI changed path boundaries/options") + } + want := filepath.ToSlash(path) + if runtime.GOOS == "windows" && !strings.HasPrefix(want, "//") { + want = "/" + want + } + if u.Path != want { + t.Fatalf("path=%q want=%q", u.Path, want) + } + if strings.Contains(text, "#") || strings.Contains(text, " µ") { + t.Fatal("literal path was not escaped") + } + } +} + +func TestSQLiteURIInvalidAndRelativePaths(t *testing.T) { + for _, path := range []string{"", "bad\x00name"} { + if _, err := SQLiteURI(path, nil); err == nil { + t.Fatal("invalid path accepted") + } + } + text, err := SQLiteURI("relative.sqlite", nil) + if err != nil { + t.Fatal(err) + } + u, err := url.Parse(text) + if err != nil { + t.Fatal(err) + } + want, err := filepath.Abs("relative.sqlite") + if err != nil { + t.Fatal(err) + } + want = filepath.ToSlash(want) + if runtime.GOOS == "windows" && !strings.HasPrefix(want, "//") { + want = "/" + want + } + if u.Path != want { + t.Fatalf("relative path=%q want=%q", u.Path, want) + } +} diff --git a/internal/dbconfig/storage.go b/internal/dbconfig/storage.go index c99043113..092cbb3d0 100644 --- a/internal/dbconfig/storage.go +++ b/internal/dbconfig/storage.go @@ -109,11 +109,13 @@ func ResolveStorage(in StorageInputs) (Storage, error) { return Storage{}, err } path = in.UsersDBPath - if path == "" { + if path == "" && in.ExistingBackend == "" { path = filepath.Join(filepath.Dir(out.DBPath), "users.db") } - if out.UsersDBPath, err = storagePath(in.BaseDir, path); err != nil { - return Storage{}, err + if path != "" { + if out.UsersDBPath, err = storagePath(in.BaseDir, path); err != nil { + return Storage{}, err + } } same := out.DBPath == out.UsersDBPath if runtime.GOOS == "windows" { diff --git a/internal/dbconfig/storage_config.go b/internal/dbconfig/storage_config.go new file mode 100644 index 000000000..22620eba0 --- /dev/null +++ b/internal/dbconfig/storage_config.go @@ -0,0 +1,40 @@ +package dbconfig + +import ( + "encoding/json" + "errors" + "io" + "os" +) + +// ReadStorageConfig reads just the shared storage fields, including disabled +// account stores. It preserves raw paths; callers apply their explicit config +// discovery, environment precedence and ResolveStorage exactly once. +func ReadStorageConfig(path string) (StorageInputs, error) { + f, err := os.Open(path) + if err != nil { + return StorageInputs{}, err + } + defer f.Close() + var config *struct { + DB DBConfig `json:"db"` + DBPath string `json:"dbPath"` + DatabaseURL string `json:"databaseURL"` + StateDir string `json:"stateDir"` + Users struct { + DBPath string `json:"dbPath"` + DatabaseURL string `json:"databaseURL"` + ApprovedChannelsDatabaseURL string `json:"approvedChannelsDatabaseURL"` + } `json:"userManagement"` + } + const maxConfigBytes = 4 * 1024 * 1024 + limit := &io.LimitedReader{R: f, N: maxConfigBytes + 1} + decoder := json.NewDecoder(limit) + if err := decoder.Decode(&config); err != nil || config == nil { + return StorageInputs{}, errors.New("invalid storage configuration JSON") + } + if err := decoder.Decode(new(any)); err != io.EOF || limit.N <= 0 { + return StorageInputs{}, errors.New("storage configuration exceeds its bound or has trailing JSON") + } + return StorageInputs{Backend: config.DB.Backend, DBPath: config.DBPath, DatabaseURL: config.DatabaseURL, StateDir: config.StateDir, UsersDBPath: config.Users.DBPath, UsersDatabaseURL: config.Users.DatabaseURL, ApprovedChannelsDatabaseURL: config.Users.ApprovedChannelsDatabaseURL}, nil +} diff --git a/internal/dbconfig/storage_config_test.go b/internal/dbconfig/storage_config_test.go new file mode 100644 index 000000000..5d53869a2 --- /dev/null +++ b/internal/dbconfig/storage_config_test.go @@ -0,0 +1,32 @@ +package dbconfig + +import ( + "os" + "path/filepath" + "testing" +) + +func TestReadStorageConfigKeepsRawPathsAndDisabledAccounts(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(path, []byte(`{"db":{"backend":"sqlite"},"dbPath":"relative.db","databaseURL":"inactive","stateDir":"stable","userManagement":{"enabled":false,"dbPath":"accounts.db","databaseURL":"runtime","approvedChannelsDatabaseURL":"reader"},"port":3000}`), 0600); err != nil { + t.Fatal(err) + } + got, err := ReadStorageConfig(path) + if err != nil { + t.Fatal(err) + } + if got.Backend != SQLite || got.DBPath != "relative.db" || got.UsersDBPath != "accounts.db" || got.StateDir != "stable" || got.UsersDatabaseURL != "runtime" || got.ApprovedChannelsDatabaseURL != "reader" || got.DatabaseURL != "inactive" { + t.Fatal("storage inputs were defaulted or dropped") + } + for _, invalid := range []string{`{"dbPath":8}`, `{} {}`, `null`, `{"db":{"backend":42}}`} { + if err := os.WriteFile(path, []byte(invalid), 0600); err != nil { + t.Fatal(err) + } + if _, err := ReadStorageConfig(path); err == nil { + t.Fatal("invalid explicit config accepted", invalid) + } + } + if _, err := ReadStorageConfig(filepath.Join(t.TempDir(), "missing")); err == nil { + t.Fatal("missing explicit file accepted") + } +} diff --git a/internal/dbschema/legacy/writer_dsn.go b/internal/dbschema/legacy/writer_dsn.go index 829f6b9b5..4a8d53bf4 100644 --- a/internal/dbschema/legacy/writer_dsn.go +++ b/internal/dbschema/legacy/writer_dsn.go @@ -1,5 +1,11 @@ package legacy +import ( + "net/url" + + "github.com/meshcore-analyzer/dbconfig" +) + // WriterDSN builds the DSN every writer must open the database with. // // It lives here because this package is the one thing cmd/ingestor and @@ -24,12 +30,13 @@ package legacy // - _cache_size=-2000 — 2000 KiB of page cache per connection. This is // SQLite's own default, pinned so it cannot drift: the page cache is a C // allocation and therefore sits outside GOMEMLIMIT. -func WriterDSN(path string) string { - return path + - "?_journal_mode=WAL" + - "&_synchronous=FULL" + - "&_auto_vacuum=incremental" + - "&_foreign_keys=on" + - "&_busy_timeout=5000" + - "&_cache_size=-2000" +func WriterDSN(path string) (string, error) { + return dbconfig.SQLiteURI(path, url.Values{ + "_journal_mode": {"WAL"}, + "_synchronous": {"FULL"}, + "_auto_vacuum": {"incremental"}, + "_foreign_keys": {"on"}, + "_busy_timeout": {"5000"}, + "_cache_size": {"-2000"}, + }) } diff --git a/internal/dbschema/legacy/writer_dsn_test.go b/internal/dbschema/legacy/writer_dsn_test.go index 79f2d157f..12766fa54 100644 --- a/internal/dbschema/legacy/writer_dsn_test.go +++ b/internal/dbschema/legacy/writer_dsn_test.go @@ -2,10 +2,14 @@ package legacy import ( "database/sql" + "net/url" + "os" "path/filepath" + "runtime" "testing" _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" ) // TestWriterDSNPragmas pins what every writer connection actually gets. @@ -16,7 +20,11 @@ import ( // pragma unconditionally, so if it ever drops out of the DSN the durability // change is silent. func TestWriterDSNPragmas(t *testing.T) { - db, err := sql.Open("sqlite3", WriterDSN(filepath.Join(t.TempDir(), "w.db"))) + dsn, err := WriterDSN(filepath.Join(t.TempDir(), "w.db")) + if err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite3", dsn) if err != nil { t.Fatal(err) } @@ -41,3 +49,52 @@ func TestWriterDSNPragmas(t *testing.T) { } } } + +func TestWriterDSNLiteralPathAndReadOnlyURI(t *testing.T) { + names := []string{"literal #100%20 space µ.db"} + if runtime.GOOS != "windows" { + names = append(names, "literal ? query.db") + } + for _, name := range names { + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), name) + dsn, err := WriterDSN(path) + if err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal(err) + } + if _, err = db.Exec(`CREATE TABLE kept(value TEXT);INSERT INTO kept VALUES('same file')`); err != nil { + db.Close() + t.Fatal(err) + } + if err = db.Close(); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(path); err != nil { + t.Fatal("literal filename was not created", err) + } + dsn, err = dbconfig.SQLiteURI(path, url.Values{"mode": {"ro"}}) + if err != nil { + t.Fatal(err) + } + ro, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal(err) + } + defer ro.Close() + var value string + if err = ro.QueryRow(`SELECT value FROM kept`).Scan(&value); err != nil || value != "same file" { + t.Fatalf("read-only URI opened another file: %q %v", value, err) + } + if _, err = ro.Exec(`INSERT INTO kept VALUES('forbidden')`); err == nil { + t.Fatal("read-only URI allowed writes") + } + }) + } + if _, err := WriterDSN(""); err == nil { + t.Fatal("empty path accepted") + } +} diff --git a/internal/dbschema/sqlite.go b/internal/dbschema/sqlite.go index 704dd981d..8b2f2044d 100644 --- a/internal/dbschema/sqlite.go +++ b/internal/dbschema/sqlite.go @@ -8,6 +8,7 @@ import ( "regexp" "strings" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/dbschema/legacy" ) @@ -24,6 +25,9 @@ func AssertPostgresReady(db *sql.DB) error { return AssertReady(db) } // are checked before historical migrations can perform cleanup. Older v2 or // unknown layouts require an explicit supported offline upgrade first. func ApplySQLite(db *sql.DB, logf Logger) error { + if err := dbconfig.AssertSQLiteImportComplete(db); err != nil { + return err + } migrated, cols, err := sqliteObserverLayout(db) if err != nil { return err @@ -65,6 +69,9 @@ func ApplySQLite(db *sql.DB, logf Logger) error { if err := legacy.Apply(db, legacy.Logger(logf)); err != nil { return err } + if err := EnsureSQLiteAsyncMigrations(db); err != nil { + return err + } // The identity migration's corpus/FK scans run once. Ordinary restarts // retain existing schema maintenance without rescanning all observations. if !needsIdentity { @@ -73,9 +80,26 @@ func ApplySQLite(db *sql.DB, logf Logger) error { return migrateSQLiteObserverIdentity(db) } +// EnsureSQLiteAsyncMigrations retains the upstream ingestor's exact ledger +// layout. Setup and reverse conversion need the complete schema before the +// ingestor starts; that process also calls this same idempotent entrypoint. +func EnsureSQLiteAsyncMigrations(db *sql.DB) error { + _, err := db.Exec(`CREATE TABLE IF NOT EXISTS _async_migrations ( + name TEXT PRIMARY KEY, + status TEXT NOT NULL, + started_at TEXT NOT NULL DEFAULT (datetime('now')), + ended_at TEXT, + error TEXT + )`) + return err +} + // AssertSQLiteReady only reads schema metadata. It is safe on a mode=ro handle; // it never applies migrations or marks a partially initialized target ready. func AssertSQLiteReady(db *sql.DB) error { + if err := dbconfig.AssertSQLiteImportComplete(db); err != nil { + return err + } if err := legacy.AssertReady(db); err != nil { return err } diff --git a/internal/dbschema/sqlite_indexes.go b/internal/dbschema/sqlite_indexes.go new file mode 100644 index 000000000..459508043 --- /dev/null +++ b/internal/dbschema/sqlite_indexes.go @@ -0,0 +1,16 @@ +package dbschema + +import ( + "context" + "database/sql" +) + +// EnsureSQLiteObserverTimeIndex is shared by the online asynchronous migration +// and offline conversion. PostgreSQL creates its own index during owner setup. +// A transaction or reserved connection may be used during offline finalization. +func EnsureSQLiteObserverTimeIndex(ctx context.Context, db interface { + ExecContext(context.Context, string, ...any) (sql.Result, error) +}) error { + _, err := db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_observations_observer_idx_timestamp ON observations(observer_idx, timestamp)`) + return err +} diff --git a/internal/dbschema/sqlite_test.go b/internal/dbschema/sqlite_test.go index b38bc17c8..bd55113e8 100644 --- a/internal/dbschema/sqlite_test.go +++ b/internal/dbschema/sqlite_test.go @@ -3,19 +3,25 @@ package dbschema import ( "context" "database/sql" + "errors" "fmt" "path/filepath" "strings" "testing" "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/dbschema/legacy" ) func sqliteTestDB(t *testing.T) (*sql.DB, string) { t.Helper() path := filepath.Join(t.TempDir(), "telemetry.db") - db, err := sql.Open("sqlite3", legacy.WriterDSN(path)) + dsn, err := legacy.WriterDSN(path) + if err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite3", dsn) if err != nil { t.Fatal(err) } @@ -31,6 +37,40 @@ func sqliteExec(t *testing.T, db *sql.DB, query string, args ...any) { } } +func TestSQLiteSchemaOwnsAsyncMigrationLedger(t *testing.T) { + db, _ := sqliteTestDB(t) + if err := ApplySQLite(db, nil); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO _async_migrations(name,status,error) VALUES('retained','failed','exact error')`); err != nil { + t.Fatal(err) + } + if err := EnsureSQLiteAsyncMigrations(db); err != nil { + t.Fatal(err) + } + var status, started, detail string + if err := db.QueryRow(`SELECT status,started_at,error FROM _async_migrations WHERE name='retained'`).Scan(&status, &started, &detail); err != nil || status != "failed" || started == "" || detail != "exact error" { + t.Fatal("async ledger changed", err) + } +} + +func TestSQLitePendingConversionCannotBeAdoptedOrOpened(t *testing.T) { + db, _ := sqliteTestDB(t) + if err := ApplySQLite(db, nil); err != nil { + t.Fatal(err) + } + sqliteExec(t, db, `CREATE TABLE corescope_reverse_progress(table_name TEXT PRIMARY KEY,rows_copied INTEGER NOT NULL,sha256 TEXT NOT NULL,complete INTEGER NOT NULL)`) + for _, check := range []func() error{func() error { return ApplySQLite(db, nil) }, func() error { return AssertSQLiteReady(db) }, func() error { return dbconfig.AssertSQLiteImportComplete(db) }} { + if err := check(); !errors.Is(err, dbconfig.ErrSQLiteImportIncomplete) { + t.Fatal("incomplete target accepted", err) + } + } + var count int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name='corescope_reverse_progress'`).Scan(&count); err != nil || count != 1 { + t.Fatal("pending marker changed", err) + } +} + func TestSQLiteFreshAndReadOnlyReadiness(t *testing.T) { db, path := sqliteTestDB(t) if err := AssertSQLiteReady(db); err == nil { diff --git a/internal/sqliteutil/go.mod b/internal/sqliteutil/go.mod new file mode 100644 index 000000000..c8085b762 --- /dev/null +++ b/internal/sqliteutil/go.mod @@ -0,0 +1,10 @@ +module github.com/meshcore-analyzer/sqliteutil + +go 1.25.0 + +require ( + github.com/mattn/go-sqlite3 v1.14.52 + github.com/meshcore-analyzer/dbconfig v0.0.0 +) + +replace github.com/meshcore-analyzer/dbconfig => ../dbconfig diff --git a/internal/sqliteutil/go.sum b/internal/sqliteutil/go.sum new file mode 100644 index 000000000..75bf6eb53 --- /dev/null +++ b/internal/sqliteutil/go.sum @@ -0,0 +1,2 @@ +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= diff --git a/internal/sqliteutil/snapshot.go b/internal/sqliteutil/snapshot.go new file mode 100644 index 000000000..f8c9c330c --- /dev/null +++ b/internal/sqliteutil/snapshot.go @@ -0,0 +1,138 @@ +// Package sqliteutil provides native snapshots without a writable source handle. +package sqliteutil + +import ( + "context" + "database/sql" + "errors" + "fmt" + "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" + "net/url" + "os" + "strings" + "time" +) + +// Snapshot copies one consistent SQLite image, including committed WAL pages, +// hidden rowids and AUTOINCREMENT sequences, to a new private file. It never +// writes the source. The caller may impose a shorter deadline; ten minutes is +// the upper bound. Failure removes only the new destination and its sidecars. +// Offline importers must separately enforce their source-immutability policy. +func Snapshot(ctx context.Context, source, destination string) (err error) { + if err := ctx.Err(); err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + if source == "" || destination == "" || strings.HasPrefix(source, "file:") || strings.Contains(source, "://") || strings.HasPrefix(destination, "file:") || strings.Contains(destination, "://") { + return errors.New("SQLite snapshot requires filesystem paths") + } + sourceURI, err := dbconfig.SQLiteURI(source, url.Values{"mode": {"ro"}, "_query_only": {"on"}, "_busy_timeout": {"100"}}) + if err != nil { + return err + } + destinationURI, err := dbconfig.SQLiteURI(destination, url.Values{"mode": {"rw"}, "_busy_timeout": {"100"}}) + if err != nil { + return err + } + sourceDB, err := sql.Open("sqlite3", sourceURI) + if err != nil { + return err + } + defer sourceDB.Close() + sourceDB.SetMaxOpenConns(1) + if _, err := sourceDB.ExecContext(ctx, `PRAGMA trusted_schema=OFF`); err != nil { + return err + } + var schemaVersion int + if err := sourceDB.QueryRowContext(ctx, `PRAGMA schema_version`).Scan(&schemaVersion); err != nil { + return fmt.Errorf("read SQLite source: %w", err) + } + reserved, err := os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return fmt.Errorf("create new recovery snapshot: %w", err) + } + if err := reserved.Close(); err != nil { + _ = os.Remove(destination) + return err + } + defer func() { + if err != nil { + for _, suffix := range []string{"", "-wal", "-shm", "-journal"} { + _ = os.Remove(destination + suffix) + } + } + }() + destinationDB, err := sql.Open("sqlite3", destinationURI) + if err != nil { + return err + } + defer destinationDB.Close() + destinationDB.SetMaxOpenConns(1) + src, err := sourceDB.Conn(ctx) + if err != nil { + return err + } + defer src.Close() + dst, err := destinationDB.Conn(ctx) + if err != nil { + return err + } + defer dst.Close() + err = src.Raw(func(sourceDriver any) error { + return dst.Raw(func(destinationDriver any) (err error) { + backup, err := destinationDriver.(*sqlite3.SQLiteConn).Backup("main", sourceDriver.(*sqlite3.SQLiteConn), "main") + if err != nil { + return err + } + defer func() { + if finishErr := backup.Finish(); err == nil { + err = finishErr + } + }() + remaining, progressAt := -1, time.Now() + for { + if err := ctx.Err(); err != nil { + return err + } + done, err := backup.Step(256) + if err != nil { + return err + } + if done { + return nil + } + if current := backup.Remaining(); current != remaining { + remaining, progressAt = current, time.Now() + continue + } + if time.Since(progressAt) > 10*time.Second { + return errors.New("SQLite snapshot made no progress") + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(10 * time.Millisecond): + } + } + }) + }) + if err != nil { + return fmt.Errorf("copy SQLite recovery snapshot: %w", err) + } + var check string + if err := dst.QueryRowContext(ctx, `PRAGMA quick_check`).Scan(&check); err != nil { + return err + } + if check != "ok" { + return errors.New("SQLite recovery snapshot failed integrity verification") + } + if err := dst.Close(); err != nil { + return err + } + if err := destinationDB.Close(); err != nil { + return err + } + return nil +} diff --git a/internal/sqliteutil/snapshot_test.go b/internal/sqliteutil/snapshot_test.go new file mode 100644 index 000000000..9f850b155 --- /dev/null +++ b/internal/sqliteutil/snapshot_test.go @@ -0,0 +1,124 @@ +package sqliteutil + +import ( + "context" + "database/sql" + "errors" + "net/url" + "os" + "path/filepath" + "testing" + "time" + + _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" +) + +func open(t *testing.T, path, mode string) *sql.DB { + t.Helper() + dsn, err := dbconfig.SQLiteURI(path, url.Values{"mode": {mode}, "_busy_timeout": {"50"}}) + if err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + return db +} + +func TestSnapshotPreservesWALRowIDsAndSequence(t *testing.T) { + dir := t.TempDir() + source := filepath.Join(dir, "live # %.db") + destination := filepath.Join(dir, "snapshot # %.db") + db := open(t, source, "rwc") + if _, err := db.Exec(`PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0; + CREATE TABLE legacy(id TEXT); INSERT INTO legacy(rowid,id) VALUES(3,'a'),(101,'b'); + CREATE TABLE identities(id INTEGER PRIMARY KEY AUTOINCREMENT,value TEXT); + INSERT INTO identities(id,value) VALUES(900,'deleted'); DELETE FROM identities; + INSERT INTO identities(value) VALUES('last');`); err != nil { + t.Fatal(err) + } + if info, err := os.Stat(source + "-wal"); err != nil || info.Size() == 0 { + t.Fatal("fixture must contain uncheckpointed WAL") + } + if err := Snapshot(context.Background(), source, destination); err != nil { + t.Fatal(err) + } + copy := open(t, destination, "ro") + var rowid, id int + if err := copy.QueryRow(`SELECT rowid FROM legacy WHERE id='b'`).Scan(&rowid); err != nil || rowid != 101 { + t.Fatalf("rowid=%d: %v", rowid, err) + } + if err := copy.QueryRow(`SELECT id FROM identities`).Scan(&id); err != nil || id != 901 { + t.Fatalf("identity=%d: %v", id, err) + } + if err := copy.QueryRow(`SELECT seq FROM sqlite_sequence WHERE name='identities'`).Scan(&id); err != nil || id != 901 { + t.Fatalf("sequence=%d: %v", id, err) + } + if _, err := copy.Exec(`CREATE TABLE forbidden(id)`); err == nil { + t.Fatal("readonly snapshot handle allowed write") + } + if _, err := open(t, source, "ro").Exec(`DELETE FROM legacy`); err == nil { + t.Fatal("readonly source allowed write") + } +} + +func TestSnapshotRefusesExistingAndMissingSource(t *testing.T) { + dir := t.TempDir() + source := filepath.Join(dir, "source.db") + if _, err := open(t, source, "rwc").Exec(`CREATE TABLE t(id);INSERT INTO t VALUES(1)`); err != nil { + t.Fatal(err) + } + for _, destination := range []string{source, filepath.Join(dir, "existing.db")} { + if destination != source { + if err := os.WriteFile(destination, []byte("keep"), 0600); err != nil { + t.Fatal(err) + } + } + before, err := os.ReadFile(destination) + if err != nil { + t.Fatal(err) + } + if err := Snapshot(context.Background(), source, destination); err == nil { + t.Fatal("overwrote existing target") + } + after, err := os.ReadFile(destination) + if err != nil || string(after) != string(before) { + t.Fatal("existing target changed") + } + } + missing := filepath.Join(dir, "missing.db") + destination := filepath.Join(dir, "new.db") + if err := Snapshot(context.Background(), missing, destination); err == nil { + t.Fatal("missing source accepted") + } + for _, p := range []string{missing, destination} { + if _, err := os.Stat(p); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("unexpected file %q: %v", p, err) + } + } +} + +func TestSnapshotCancellationRemovesPartialOutput(t *testing.T) { + dir := t.TempDir() + source := filepath.Join(dir, "busy.db") + destination := filepath.Join(dir, "cancelled.db") + db := open(t, source, "rwc") + db.SetMaxOpenConns(1) + if _, err := db.Exec(`CREATE TABLE t(id);INSERT INTO t VALUES(1);BEGIN EXCLUSIVE`); err != nil { + t.Fatal(err) + } + defer db.Exec(`ROLLBACK`) + ctx, cancel := context.WithTimeout(context.Background(), 75*time.Millisecond) + defer cancel() + if err := Snapshot(ctx, source, destination); err == nil { + t.Fatal("snapshot ignored locked source/cancellation") + } + for _, suffix := range []string{"", "-wal", "-shm", "-journal"} { + if _, err := os.Stat(destination + suffix); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("partial snapshot remains: %v", err) + } + } +} diff --git a/internal/users/audit.go b/internal/users/audit.go index 9a1091417..e88fcb23d 100644 --- a/internal/users/audit.go +++ b/internal/users/audit.go @@ -45,7 +45,7 @@ func (s *Store) Audit(actor *int64, action string, target *int64, detail map[str if err != nil { return err } - _, err = s.db.Exec(`INSERT INTO audit_log (at, actor_user_id, action, target_user_id, detail) VALUES ($1, $2, $3, $4, $5)`, + _, err = s.db.Exec(`INSERT INTO audit_log (at, actor_user_id, action, target_user_id, detail) VALUES (`+s.p(1)+`, `+s.p(2)+`, `+s.p(3)+`, `+s.p(4)+`, `+s.p(5)+`)`, unix(s.now()), nullInt(actor), action, nullInt(target), string(b)) return err } @@ -61,12 +61,12 @@ func (s *Store) AuditFor(userID int64, limit int) ([]AuditEntry, error) { // AuditAllFor is AuditFor without a cap (the account export). func (s *Store) AuditAllFor(userID int64) ([]AuditEntry, error) { - return s.auditFor(userID, noLimit) + return s.auditFor(userID, s.noLimit()) } func (s *Store) auditFor(userID int64, limit any) ([]AuditEntry, error) { rows, err := s.db.Query(`SELECT id, at, actor_user_id, action, target_user_id, detail FROM audit_log - WHERE target_user_id = $1 OR actor_user_id = $2 ORDER BY at DESC, id DESC LIMIT $3`, userID, userID, limit) + WHERE target_user_id = `+s.p(1)+` OR actor_user_id = `+s.p(2)+` ORDER BY at DESC, id DESC LIMIT `+s.p(3), userID, userID, limit) if err != nil { return nil, err } @@ -81,29 +81,29 @@ func (s *Store) AuditList(f AuditFilter) ([]AuditEntry, error) { var ors []string for _, a := range f.Actions { if group, ok := strings.CutSuffix(a, ".*"); ok { - ors = append(ors, fmt.Sprintf(`action = $%d OR action LIKE $%d ESCAPE '\'`, len(args)+1, len(args)+2)) + ors = append(ors, fmt.Sprintf(`action = %s OR action LIKE %s ESCAPE '\'`, s.p(len(args)+1), s.p(len(args)+2))) args = append(args, group, escapeLike(group)+".%") } else { - ors = append(ors, fmt.Sprintf(`action = $%d`, len(args)+1)) + ors = append(ors, fmt.Sprintf(`action = %s`, s.p(len(args)+1))) args = append(args, a) } } q += ` AND (` + strings.Join(ors, ` OR `) + `)` } if f.UserID != nil { - q += fmt.Sprintf(` AND (actor_user_id = $%d OR target_user_id = $%d)`, len(args)+1, len(args)+2) + q += fmt.Sprintf(` AND (actor_user_id = %s OR target_user_id = %s)`, s.p(len(args)+1), s.p(len(args)+2)) args = append(args, *f.UserID, *f.UserID) } if f.From != nil { - q += fmt.Sprintf(` AND at >= $%d`, len(args)+1) + q += fmt.Sprintf(` AND at >= %s`, s.p(len(args)+1)) args = append(args, unix(*f.From)) } if f.To != nil { - q += fmt.Sprintf(` AND at <= $%d`, len(args)+1) + q += fmt.Sprintf(` AND at <= %s`, s.p(len(args)+1)) args = append(args, unix(*f.To)) } if f.BeforeID > 0 { - q += fmt.Sprintf(` AND id < $%d`, len(args)+1) + q += fmt.Sprintf(` AND id < %s`, s.p(len(args)+1)) args = append(args, f.BeforeID) } limit := f.Limit @@ -113,7 +113,7 @@ func (s *Store) AuditList(f AuditFilter) ([]AuditEntry, error) { if limit > AuditListMax { limit = AuditListMax } - q += fmt.Sprintf(` ORDER BY id DESC LIMIT $%d`, len(args)+1) + q += fmt.Sprintf(` ORDER BY id DESC LIMIT %s`, s.p(len(args)+1)) args = append(args, limit) rows, err := s.db.Query(q, args...) if err != nil { @@ -133,8 +133,8 @@ func (s *Store) PruneAudit(actions []string, maxAge time.Duration) (int64, error args = append(args, a) } args = append(args, unix(s.now())-int64(maxAge/time.Second)) - ph := placeholders(len(actions)) - res, err := s.db.Exec(`DELETE FROM audit_log WHERE action IN (`+ph+`) AND at < $`+fmt.Sprint(len(args)), args...) + ph := s.placeholders(len(actions)) + res, err := s.db.Exec(`DELETE FROM audit_log WHERE action IN (`+ph+`) AND at < `+s.p(len(args)), args...) if err != nil { return 0, err } diff --git a/internal/users/audit_list_test.go b/internal/users/audit_list_test.go index a438f9ae9..407479a29 100644 --- a/internal/users/audit_list_test.go +++ b/internal/users/audit_list_test.go @@ -2,6 +2,7 @@ package users import ( "fmt" + "github.com/meshcore-analyzer/dbconfig" "strings" "testing" "time" @@ -145,7 +146,11 @@ func TestUsersByID(t *testing.T) { func hasIndex(t *testing.T, st *Store, name string) bool { t.Helper() var n int - if err := st.db.QueryRow(`SELECT COUNT(*) FROM pg_indexes WHERE schemaname=current_schema() AND indexname=$1`, name).Scan(&n); err != nil { + q := `SELECT COUNT(*) FROM pg_indexes WHERE schemaname=current_schema() AND indexname=$1` + if st.backend == dbconfig.SQLite { + q = `SELECT COUNT(*) FROM sqlite_schema WHERE type='index' AND name=?1` + } + if err := st.db.QueryRow(q, name).Scan(&n); err != nil { t.Fatal(err) } return n == 1 @@ -161,8 +166,8 @@ func TestFreshDatabaseHasAuditTimeIndex(t *testing.T) { // A native account schema at v2 binary gains the audit_at index and keeps its rows. func TestMigrateV2DatabaseToV3(t *testing.T) { st := migratedTestStore(t, 2, `INSERT INTO audit_log (at,action,detail) VALUES (1,'user.register','{}')`) - if v, err := st.SchemaVersion(); err != nil || v != len(migrations) { - t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, len(migrations)) + if v, err := st.SchemaVersion(); err != nil || v != schemaVersion(t) { + t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, schemaVersion(t)) } if !hasIndex(t, st, "audit_at") { t.Fatal("audit_at index missing after migration") diff --git a/internal/users/go.mod b/internal/users/go.mod index 537ea9d9d..2756be8a4 100644 --- a/internal/users/go.mod +++ b/internal/users/go.mod @@ -5,19 +5,31 @@ go 1.25.0 require ( github.com/jackc/pgx/v5 v5.7.6 golang.org/x/crypto v0.37.0 + modernc.org/sqlite v1.34.5 ) require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect golang.org/x/sync v0.13.0 // indirect golang.org/x/text v0.24.0 // indirect + modernc.org/libc v1.55.3 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect ) require ( + github.com/meshcore-analyzer/dbconfig v0.0.0 github.com/meshcore-analyzer/pgutil v0.0.0 golang.org/x/sys v0.32.0 // indirect ) replace github.com/meshcore-analyzer/pgutil => ../pgutil + +replace github.com/meshcore-analyzer/dbconfig => ../dbconfig diff --git a/internal/users/go.sum b/internal/users/go.sum index fa18bbda0..5cf6e6372 100644 --- a/internal/users/go.sum +++ b/internal/users/go.sum @@ -1,6 +1,12 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -9,8 +15,14 @@ github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -18,13 +30,42 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= +modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= +modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= +modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= +modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= +modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= +modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= +modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/users/helpers_test.go b/internal/users/helpers_test.go index 698a73ad5..67373ee06 100644 --- a/internal/users/helpers_test.go +++ b/internal/users/helpers_test.go @@ -2,9 +2,12 @@ package users import ( "database/sql" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/pgutil" "github.com/meshcore-analyzer/pgutil/pgtest" "net/url" + "os" + "path/filepath" "testing" "time" ) @@ -14,23 +17,64 @@ type fakeClock struct{ t time.Time } func (c *fakeClock) Now() time.Time { return c.t } func (c *fakeClock) Advance(d time.Duration) { c.t = c.t.Add(d) } -// newTestStore bootstraps an isolated PostgreSQL schema with a controllable clock. +// Ordinary tests always exercise a real SQLite database. The explicit PostgreSQL +// matrix requires its service; it never silently substitutes SQLite or skips it. +func testBackend(t *testing.T) dbconfig.Backend { + t.Helper() + switch os.Getenv("CORESCOPE_TEST_BACKEND") { + case "", "sqlite": + return dbconfig.SQLite + case "postgres": + return dbconfig.Postgres + default: + t.Fatal("CORESCOPE_TEST_BACKEND must be sqlite or postgres") + return "" + } +} +func requirePostgres(t *testing.T) { + t.Helper() + if testBackend(t) != dbconfig.Postgres { + t.Skip("PostgreSQL-specific contract; run CORESCOPE_TEST_BACKEND=postgres") + } +} +func testTarget(t *testing.T, wholeDatabase bool) string { + t.Helper() + if testBackend(t) == dbconfig.SQLite { + return filepath.Join(t.TempDir(), "accounts.db") + } + if wholeDatabase { + return pgtest.NewDatabase(t) + } + return pgtest.NewSchema(t) +} +func schemaVersion(t *testing.T) int { + if testBackend(t) == dbconfig.SQLite { + return SQLiteSchemaVersion + } + return CurrentSchemaVersion +} +func applyTestSchema(t *testing.T, db *sql.DB) error { + if testBackend(t) == dbconfig.SQLite { + return ApplySQLite(db) + } + return ApplyPostgres(db) +} + +// newTestStore bootstraps an isolated database with a controllable clock. func newTestStore(t *testing.T) (*Store, *fakeClock) { t.Helper() - return newTestStoreAt(t, pgtest.NewSchema(t)) + return newTestStoreAt(t, testTarget(t, false)) } func newTestStoreAt(t *testing.T, dsn string) (*Store, *fakeClock) { t.Helper() - db, err := pgutil.Open(dsn, false) - if err != nil { - t.Fatal(err) - } - if err := Apply(db); err != nil { + if testBackend(t) == dbconfig.Postgres { + db := testOwner(t, dsn) + if err := Apply(db); err != nil { + t.Fatal(err) + } db.Close() - t.Fatal(err) } - db.Close() st, err := Open(testRuntimeURL(t, dsn)) if err != nil { t.Fatalf("Open: %v", err) @@ -43,6 +87,9 @@ func newTestStoreAt(t *testing.T, dsn string) (*Store, *fakeClock) { func testRuntimeURL(t *testing.T, ownerURL string) string { t.Helper() + if testBackend(t) == dbconfig.SQLite { + return ownerURL + } dsn := pgtest.Writer(t, ownerURL) u, _ := url.Parse(dsn) db, err := pgutil.Open(ownerURL, false) @@ -60,11 +107,8 @@ func testRuntimeURL(t *testing.T, ownerURL string) string { // SQLite source normalization is covered by the offline importer's fixtures. func migratedTestStore(t *testing.T, version int, seed string) *Store { t.Helper() - dsn := pgtest.NewSchema(t) - db, err := pgutil.Open(dsn, false) - if err != nil { - t.Fatal(err) - } + dsn := testTarget(t, false) + db := testOwner(t, dsn) defer db.Close() if _, err := db.Exec(`CREATE TABLE schema_version(version INTEGER NOT NULL)`); err != nil { t.Fatal(err) @@ -72,7 +116,11 @@ func migratedTestStore(t *testing.T, version int, seed string) *Store { if _, err := db.Exec(`INSERT INTO schema_version VALUES ($1)`, version); err != nil { t.Fatal(err) } - for _, migration := range migrations[:version] { + history := migrations + if testBackend(t) == dbconfig.SQLite { + history = legacyMigrations + } + for _, migration := range history[:version] { for _, stmt := range migration { if _, err := db.Exec(stmt); err != nil { t.Fatal(err) @@ -84,7 +132,7 @@ func migratedTestStore(t *testing.T, version int, seed string) *Store { t.Fatal(err) } } - if err := Apply(db); err != nil { + if err := applyTestSchema(t, db); err != nil { t.Fatal(err) } st, err := Open(testRuntimeURL(t, dsn)) @@ -97,7 +145,13 @@ func migratedTestStore(t *testing.T, version int, seed string) *Store { func testOwner(t *testing.T, dsn string) *sql.DB { t.Helper() - db, err := pgutil.Open(dsn, false) + var db *sql.DB + var err error + if testBackend(t) == dbconfig.SQLite { + db, err = sql.Open("sqlite", dsn) + } else { + db, err = pgutil.Open(dsn, false) + } if err != nil { t.Fatal(err) } diff --git a/internal/users/legacy_schema.go b/internal/users/legacy_schema.go index 85ca25f4b..bb8d2ae24 100644 --- a/internal/users/legacy_schema.go +++ b/internal/users/legacy_schema.go @@ -1,7 +1,7 @@ package users -// LegacyMigrations contains only immutable SQLite upgrade metadata for the offline -// importer. Runtime account connections never execute these statements. +// legacyMigrations is the original append-only SQLite upgrade history shared by +// native account upgrades and the offline importer. Keep these five definitions unchanged. var legacyMigrations = [][]string{ { // v1: sub-project A `CREATE TABLE users ( diff --git a/internal/users/mail.go b/internal/users/mail.go index 6b48a4c8d..65a272f07 100644 --- a/internal/users/mail.go +++ b/internal/users/mail.go @@ -58,7 +58,7 @@ func (s *Store) LogMail(userID *int64, to, purpose, messageID string) (int64, er } var id int64 err := s.db.QueryRow(`INSERT INTO mail_log (user_id, to_email, purpose, provider_message_id, sent_at, last_event, last_event_at) - VALUES ($1, $2, $3, $4, $5, 'sent', $6) RETURNING id`, nullInt(userID), to, purpose, mid, now, now).Scan(&id) + VALUES (`+s.p(1)+`, `+s.p(2)+`, `+s.p(3)+`, `+s.p(4)+`, `+s.p(5)+`, 'sent', `+s.p(6)+`) RETURNING id`, nullInt(userID), to, purpose, mid, now, now).Scan(&id) if err != nil { return 0, err } @@ -76,7 +76,7 @@ func (s *Store) RecordMailEvent(messageID, event string, at time.Time, reason st defer tx.Rollback() var id, lastAt int64 var uid sql.NullInt64 - err = tx.QueryRow(`SELECT id, user_id, last_event_at FROM mail_log WHERE provider_message_id = $1 FOR UPDATE`, messageID). + err = tx.QueryRow(`SELECT id, user_id, last_event_at FROM mail_log WHERE provider_message_id = `+s.p(1)+s.forUpdate(), messageID). Scan(&id, &uid, &lastAt) if errors.Is(err, sql.ErrNoRows) { return nil, false, nil @@ -84,12 +84,12 @@ func (s *Store) RecordMailEvent(messageID, event string, at time.Time, reason st if err != nil { return nil, false, err } - if _, err := tx.Exec(`INSERT INTO mail_events (mail_id, event, at, reason) VALUES ($1, $2, $3, $4) ON CONFLICT (mail_id,event,at) DO NOTHING`, + if _, err := tx.Exec(`INSERT INTO mail_events (mail_id, event, at, reason) VALUES (`+s.p(1)+`, `+s.p(2)+`, `+s.p(3)+`, `+s.p(4)+`) ON CONFLICT (mail_id,event,at) DO NOTHING`, id, event, unix(at), reason); err != nil { return nil, false, err } if unix(at) >= lastAt { - if _, err := tx.Exec(`UPDATE mail_log SET last_event = $1, last_event_at = $2, last_reason = $3 WHERE id = $4`, + if _, err := tx.Exec(`UPDATE mail_log SET last_event = `+s.p(1)+`, last_event_at = `+s.p(2)+`, last_reason = `+s.p(3)+` WHERE id = `+s.p(4), event, unix(at), reason, id); err != nil { return nil, false, err } @@ -105,7 +105,7 @@ func (s *Store) RecordMailEvent(messageID, event string, at time.Time, reason st } func (s *Store) eventsFor(mailID int64) ([]MailEvent, error) { - rows, err := s.db.Query(`SELECT event, at, reason FROM mail_events WHERE mail_id = $1 ORDER BY at, id`, mailID) + rows, err := s.db.Query(`SELECT event, at, reason FROM mail_events WHERE mail_id = `+s.p(1)+` ORDER BY at, id`, mailID) if err != nil { return nil, err } @@ -125,7 +125,7 @@ func (s *Store) eventsFor(mailID int64) ([]MailEvent, error) { // MailByID returns one record with its events. func (s *Store) MailByID(id int64) (*MailRecord, error) { - m, err := scanMail(s.db.QueryRow(`SELECT `+mailCols+` FROM mail_log WHERE id = $1`, id)) + m, err := scanMail(s.db.QueryRow(`SELECT `+mailCols+` FROM mail_log WHERE id = `+s.p(1), id)) if err != nil { return nil, err } @@ -146,11 +146,11 @@ func (s *Store) MailForUser(userID int64, limit int) ([]MailRecord, error) { // MailAllForUser is MailForUser without a cap (the account export). func (s *Store) MailAllForUser(userID int64) ([]MailRecord, error) { - return s.mailForUser(userID, noLimit) + return s.mailForUser(userID, s.noLimit()) } func (s *Store) mailForUser(userID int64, limit any) ([]MailRecord, error) { - rows, err := s.db.Query(`SELECT `+mailCols+` FROM mail_log WHERE user_id = $1 ORDER BY sent_at DESC, id DESC LIMIT $2`, userID, limit) + rows, err := s.db.Query(`SELECT `+mailCols+` FROM mail_log WHERE user_id = `+s.p(1)+` ORDER BY sent_at DESC, id DESC LIMIT `+s.p(2), userID, limit) if err != nil { return nil, err } @@ -196,7 +196,7 @@ func (s *Store) LatestMailByUser() (map[int64]MailRecord, error) { // PruneMail deletes mail records (and their events) older than maxAge. func (s *Store) PruneMail(maxAge time.Duration) (int64, error) { - res, err := s.db.Exec(`DELETE FROM mail_log WHERE sent_at < $1`, unix(s.now())-int64(maxAge/time.Second)) + res, err := s.db.Exec(`DELETE FROM mail_log WHERE sent_at < `+s.p(1), unix(s.now())-int64(maxAge/time.Second)) if err != nil { return 0, err } diff --git a/internal/users/notify.go b/internal/users/notify.go index 9c4b6858f..17d9bdcd9 100644 --- a/internal/users/notify.go +++ b/internal/users/notify.go @@ -3,7 +3,7 @@ package users import ( "database/sql" "errors" - "fmt" + "github.com/meshcore-analyzer/dbconfig" "sort" "strings" "time" @@ -114,14 +114,14 @@ func canonicalEvents(list []string) []string { return out } -func placeholders(n int, offset ...int) string { +func (s *Store) placeholders(n int, offset ...int) string { start := 1 if len(offset) > 0 { start += offset[0] } out := make([]string, n) for i := range out { - out[i] = fmt.Sprintf("$%d", start+i) + out[i] = s.p(start + i) } return strings.Join(out, ",") } @@ -146,7 +146,7 @@ func scanPrefs(row rowScanner) (*NotifyPrefs, error) { } func (s *Store) getPrefs(userID int64) (*NotifyPrefs, error) { - return scanPrefs(s.db.QueryRow(`SELECT `+prefsCols+` FROM notification_prefs WHERE user_id = $1`, userID)) + return scanPrefs(s.db.QueryRow(`SELECT `+prefsCols+` FROM notification_prefs WHERE user_id = `+s.p(1), userID)) } // NotifyPrefsFor returns the user's preferences, creating the default row @@ -164,7 +164,7 @@ func (s *Store) NotifyPrefsFor(userID int64) (NotifyPrefs, error) { } d := DefaultNotifyPrefs(userID) if _, err := s.db.Exec(`INSERT INTO notification_prefs (user_id, enabled, events, unsub_token, updated_at) - VALUES ($1, $2, $3, $4, $5) ON CONFLICT(user_id) DO NOTHING`, userID, 1, strings.Join(d.Events, ","), tok, unix(s.now())); err != nil { + VALUES (`+s.p(1)+`, `+s.p(2)+`, `+s.p(3)+`, `+s.p(4)+`, `+s.p(5)+`) ON CONFLICT(user_id) DO NOTHING`, userID, 1, strings.Join(d.Events, ","), tok, unix(s.now())); err != nil { return NotifyPrefs{}, err } p, err := s.getPrefs(userID) @@ -202,14 +202,14 @@ func (s *Store) SetNotifyPrefs(userID int64, enabled bool, events []string) (Not return NotifyPrefs{}, err } defer tx.Rollback() - if _, err := tx.Exec(`UPDATE notification_prefs SET enabled = $1, events = $2, updated_at = $3 WHERE user_id = $4`, + if _, err := tx.Exec(`UPDATE notification_prefs SET enabled = `+s.p(1)+`, events = `+s.p(2)+`, updated_at = `+s.p(3)+` WHERE user_id = `+s.p(4), en, strings.Join(ev, ","), unix(s.now()), userID); err != nil { return NotifyPrefs{}, err } - q := `DELETE FROM notification_state WHERE user_id = $1` + q := `DELETE FROM notification_state WHERE user_id = ` + s.p(1) args := []any{userID} if len(ev) > 0 { - q += ` AND event NOT IN (` + placeholders(len(ev), 1) + `)` + q += ` AND event NOT IN (` + s.placeholders(len(ev), 1) + `)` for _, e := range ev { args = append(args, e) } @@ -236,7 +236,7 @@ func (s *Store) DisableNotifyByToken(token string) (userID int64, wasEnabled boo } defer tx.Rollback() var en int - err = tx.QueryRow(`SELECT user_id,enabled FROM notification_prefs WHERE unsub_token=$1 FOR UPDATE`, token).Scan(&userID, &en) + err = tx.QueryRow(`SELECT user_id,enabled FROM notification_prefs WHERE unsub_token=`+s.p(1)+s.forUpdate(), token).Scan(&userID, &en) if errors.Is(err, sql.ErrNoRows) { return 0, false, ErrTokenInvalid } @@ -244,7 +244,7 @@ func (s *Store) DisableNotifyByToken(token string) (userID int64, wasEnabled boo return 0, false, err } if en != 0 { - if _, err = tx.Exec(`UPDATE notification_prefs SET enabled=0,updated_at=$1 WHERE user_id=$2`, unix(s.now()), userID); err != nil { + if _, err = tx.Exec(`UPDATE notification_prefs SET enabled=0,updated_at=`+s.p(1)+` WHERE user_id=`+s.p(2), unix(s.now()), userID); err != nil { return 0, false, err } } @@ -279,11 +279,11 @@ func (s *Store) AddWatches(userID int64, pubkeys []string, max int) (added, alre return 0, 0, 0, err } defer tx.Rollback() - if err := lockUser(tx, userID); err != nil { + if err := s.lockUser(tx, userID); err != nil { return 0, 0, 0, err } have := map[string]bool{} - rows, err := tx.Query(`SELECT pubkey FROM notification_watches WHERE user_id = $1`, userID) + rows, err := tx.Query(`SELECT pubkey FROM notification_watches WHERE user_id = `+s.p(1), userID) if err != nil { return 0, 0, 0, err } @@ -307,7 +307,7 @@ func (s *Store) AddWatches(userID int64, pubkeys []string, max int) (added, alre case max > 0 && len(have) >= max: overLimit++ default: - if _, err := tx.Exec(`INSERT INTO notification_watches (user_id, pubkey, created_at) VALUES ($1, $2, $3)`, userID, pk, now); err != nil { + if _, err := tx.Exec(`INSERT INTO notification_watches (user_id, pubkey, created_at) VALUES (`+s.p(1)+`, `+s.p(2)+`, `+s.p(3)+`)`, userID, pk, now); err != nil { return 0, 0, 0, err } have[pk] = true @@ -341,15 +341,15 @@ func (s *Store) RemoveWatch(userID int64, pubkey string) error { return err } defer tx.Rollback() - if err := lockUser(tx, userID); errors.Is(err, ErrNotFound) { + if err := s.lockUser(tx, userID); errors.Is(err, ErrNotFound) { return nil } else if err != nil { return err } - if _, err := tx.Exec(`DELETE FROM notification_watches WHERE user_id = $1 AND pubkey = $2`, userID, pubkey); err != nil { + if _, err := tx.Exec(`DELETE FROM notification_watches WHERE user_id = `+s.p(1)+` AND pubkey = `+s.p(2), userID, pubkey); err != nil { return err } - if _, err := tx.Exec(`DELETE FROM notification_state WHERE user_id = $1 AND subject = $2 AND event IN ($3, $4)`, + if _, err := tx.Exec(`DELETE FROM notification_state WHERE user_id = `+s.p(1)+` AND subject = `+s.p(2)+` AND event IN (`+s.p(3)+`, `+s.p(4)+`)`, userID, pubkey, NotifyNodeOffline, NotifyNodeBattery); err != nil { return err } @@ -377,7 +377,7 @@ func (s *Store) queryWatches(q string, args ...any) ([]NotifyWatch, error) { // WatchesFor returns the user's watches, oldest first. func (s *Store) WatchesFor(userID int64) ([]NotifyWatch, error) { - return s.queryWatches(`SELECT user_id, pubkey, created_at FROM notification_watches WHERE user_id = $1 ORDER BY created_at, pubkey`, userID) + return s.queryWatches(`SELECT user_id, pubkey, created_at FROM notification_watches WHERE user_id = `+s.p(1)+` ORDER BY created_at, pubkey`, userID) } // AllWatches returns every watch, by user and pubkey. @@ -414,8 +414,12 @@ func (s *Store) WriteNotifyStates(list []NotifyState) error { return err } defer tx.Rollback() + lock := "" + if s.backend == dbconfig.Postgres { + lock = " FOR KEY SHARE" + } stmt, err := tx.Prepare(`INSERT INTO notification_state (user_id, event, subject, state, changed_at) - SELECT id, $2, $3, $4, $5 FROM users WHERE id=$1 FOR KEY SHARE + SELECT id, ` + s.p(2) + `, ` + s.p(3) + `, ` + s.p(4) + `, ` + s.p(5) + ` FROM users WHERE id=` + s.p(1) + lock + ` ON CONFLICT (user_id, event, subject) DO UPDATE SET state = excluded.state, changed_at = excluded.changed_at`) if err != nil { return err @@ -443,7 +447,7 @@ func (s *Store) DeleteNotifyStates(keys []NotifyKey) error { return err } defer tx.Rollback() - stmt, err := tx.Prepare(`DELETE FROM notification_state WHERE user_id = $1 AND event = $2 AND subject = $3`) + stmt, err := tx.Prepare(`DELETE FROM notification_state WHERE user_id = ` + s.p(1) + ` AND event = ` + s.p(2) + ` AND subject = ` + s.p(3)) if err != nil { return err } @@ -460,7 +464,7 @@ func (s *Store) DeleteNotifyStates(keys []NotifyKey) error { // total (deleted accounts included) and per existing user. func (s *Store) NotifyMailCounts(since time.Time) (total int, perUser map[int64]int, err error) { perUser = map[int64]int{} - rows, err := s.db.Query(`SELECT user_id, COUNT(*) FROM mail_log WHERE purpose = $1 AND sent_at >= $2 GROUP BY user_id`, + rows, err := s.db.Query(`SELECT user_id, COUNT(*) FROM mail_log WHERE purpose = `+s.p(1)+` AND sent_at >= `+s.p(2)+` GROUP BY user_id`, NotifyMailPurpose, unix(since)) if err != nil { return 0, nil, err diff --git a/internal/users/notify_test.go b/internal/users/notify_test.go index 0973457fd..56062ae09 100644 --- a/internal/users/notify_test.go +++ b/internal/users/notify_test.go @@ -43,8 +43,8 @@ func statesByKey(t *testing.T, st *Store) map[NotifyKey]NotifyState { // A native account schema at v4 binary gains the notification tables and keeps its rows. func TestMigrateV4DatabaseToV5(t *testing.T) { st := migratedTestStore(t, 4, `INSERT INTO users (email,display_name,password_hash,created_at) VALUES ('old@example.org','Old','x',1)`) - if v, err := st.SchemaVersion(); err != nil || v != len(migrations) { - t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, len(migrations)) + if v, err := st.SchemaVersion(); err != nil || v != schemaVersion(t) { + t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, schemaVersion(t)) } if len(migrations) < 5 { t.Fatal("migration v5 missing") diff --git a/internal/users/onetime.go b/internal/users/onetime.go index aad937161..2e0f06c22 100644 --- a/internal/users/onetime.go +++ b/internal/users/onetime.go @@ -29,10 +29,10 @@ func (s *Store) IssueToken(userID int64, p Purpose, ttl time.Duration, newEmail return "", err } defer tx.Rollback() - if err := lockUser(tx, userID); err != nil { + if err := s.lockUser(tx, userID); err != nil { return "", err } - if _, err := tx.Exec(`UPDATE tokens SET used_at = $1 WHERE user_id = $2 AND purpose = $3 AND used_at IS NULL`, + if _, err := tx.Exec(`UPDATE tokens SET used_at = `+s.p(1)+` WHERE user_id = `+s.p(2)+` AND purpose = `+s.p(3)+` AND used_at IS NULL`, unix(now), userID, string(p)); err != nil { return "", err } @@ -40,7 +40,7 @@ func (s *Store) IssueToken(userID int64, p Purpose, ttl time.Duration, newEmail if newEmail != "" { ne = newEmail } - if _, err := tx.Exec(`INSERT INTO tokens (token_hash, user_id, purpose, new_email, expires_at) VALUES ($1, $2, $3, $4, $5)`, + if _, err := tx.Exec(`INSERT INTO tokens (token_hash, user_id, purpose, new_email, expires_at) VALUES (`+s.p(1)+`, `+s.p(2)+`, `+s.p(3)+`, `+s.p(4)+`, `+s.p(5)+`)`, hash, userID, string(p), ne, unix(now.Add(ttl))); err != nil { return "", err } @@ -59,7 +59,7 @@ func (s *Store) checkToken(q tokenQuerier, hash string, p Purpose) (userID int64 var expires int64 var ne sql.NullString var used sql.NullInt64 - err = q.QueryRow(`SELECT user_id, purpose, new_email, expires_at, used_at FROM tokens WHERE token_hash = $1`, hash). + err = q.QueryRow(`SELECT user_id, purpose, new_email, expires_at, used_at FROM tokens WHERE token_hash = `+s.p(1), hash). Scan(&userID, &purpose, &ne, &expires, &used) if errors.Is(err, sql.ErrNoRows) { return 0, "", ErrTokenInvalid @@ -96,14 +96,14 @@ func (s *Store) ConsumeToken(raw string, p Purpose) (userID int64, newEmail stri if err != nil { return 0, "", err } - if err := lockUser(tx, userID); err != nil { + if err := s.lockUser(tx, userID); err != nil { return 0, "", ErrTokenInvalid } userID, newEmail, err = s.checkToken(tx, hash, p) if err != nil { return 0, "", err } - if err := expectOne(tx.Exec(`UPDATE tokens SET used_at = $1 WHERE token_hash = $2 AND used_at IS NULL`, unix(s.now()), hash)); err != nil { + if err := expectOne(tx.Exec(`UPDATE tokens SET used_at = `+s.p(1)+` WHERE token_hash = `+s.p(2)+` AND used_at IS NULL`, unix(s.now()), hash)); err != nil { if errors.Is(err, ErrNotFound) { return 0, "", ErrTokenInvalid } @@ -127,7 +127,7 @@ func (s *Store) ActivateWithToken(raw string, id int64, role Role, verifiedHash return err } defer tx.Rollback() - if err := lockUser(tx, id); err != nil { + if err := s.lockUser(tx, id); err != nil { return ErrAccountChanged } uid, _, err := s.checkToken(tx, hash, PurposeActivate) @@ -138,14 +138,14 @@ func (s *Store) ActivateWithToken(raw string, id int64, role Role, verifiedHash return ErrTokenInvalid } now := unix(s.now()) - if err := expectOne(tx.Exec(`UPDATE tokens SET used_at = $1 WHERE token_hash = $2 AND used_at IS NULL`, now, hash)); err != nil { + if err := expectOne(tx.Exec(`UPDATE tokens SET used_at = `+s.p(1)+` WHERE token_hash = `+s.p(2)+` AND used_at IS NULL`, now, hash)); err != nil { if errors.Is(err, ErrNotFound) { return ErrTokenInvalid } return err } - err = expectOne(tx.Exec(`UPDATE users SET status = 'active', role = $1, activated_at = $2, activated_by = NULL - WHERE id = $3 AND status = 'pending' AND password_hash = $4`, string(role), now, id, verifiedHash)) + err = expectOne(tx.Exec(`UPDATE users SET status = 'active', role = `+s.p(1)+`, activated_at = `+s.p(2)+`, activated_by = NULL + WHERE id = `+s.p(3)+` AND status = 'pending' AND password_hash = `+s.p(4), string(role), now, id, verifiedHash)) if errors.Is(err, ErrNotFound) { return ErrAccountChanged } @@ -162,12 +162,12 @@ func (s *Store) InvalidateTokens(userID int64, p Purpose) error { return err } defer tx.Rollback() - if err := lockUser(tx, userID); errors.Is(err, ErrNotFound) { + if err := s.lockUser(tx, userID); errors.Is(err, ErrNotFound) { return nil } else if err != nil { return err } - if _, err = tx.Exec(`UPDATE tokens SET used_at=$1 WHERE user_id=$2 AND purpose=$3 AND used_at IS NULL`, unix(s.now()), userID, string(p)); err != nil { + if _, err = tx.Exec(`UPDATE tokens SET used_at=`+s.p(1)+` WHERE user_id=`+s.p(2)+` AND purpose=`+s.p(3)+` AND used_at IS NULL`, unix(s.now()), userID, string(p)); err != nil { return err } return tx.Commit() @@ -179,7 +179,7 @@ func (s *Store) InvalidateTokens(userID int64, p Purpose) error { func (s *Store) PendingEmailChange(userID int64) (string, error) { var ne sql.NullString err := s.db.QueryRow(`SELECT new_email FROM tokens - WHERE user_id = $1 AND purpose = $2 AND used_at IS NULL AND expires_at > $3 + WHERE user_id = `+s.p(1)+` AND purpose = `+s.p(2)+` AND used_at IS NULL AND expires_at > `+s.p(3)+` ORDER BY expires_at DESC LIMIT 1`, userID, string(PurposeEmailChange), unix(s.now())).Scan(&ne) if errors.Is(err, sql.ErrNoRows) { return "", nil @@ -189,7 +189,7 @@ func (s *Store) PendingEmailChange(userID int64) (string, error) { // PruneTokens deletes tokens that expired more than keep ago. func (s *Store) PruneTokens(keep time.Duration) (int64, error) { - res, err := s.db.Exec(`DELETE FROM tokens WHERE expires_at < $1`, unix(s.now())-int64(keep/time.Second)) + res, err := s.db.Exec(`DELETE FROM tokens WHERE expires_at < `+s.p(1), unix(s.now())-int64(keep/time.Second)) if err != nil { return 0, err } diff --git a/internal/users/postgres_test.go b/internal/users/postgres_test.go index 7cb73370e..82c6499ac 100644 --- a/internal/users/postgres_test.go +++ b/internal/users/postgres_test.go @@ -2,6 +2,7 @@ package users import ( "errors" + "github.com/meshcore-analyzer/dbconfig" "strings" "sync" "sync/atomic" @@ -25,6 +26,7 @@ func TestOpenRejectsCredentialsInErrors(t *testing.T) { } func TestOpenRefusesSchemaOwner(t *testing.T) { + requirePostgres(t) dsn := pgtest.NewSchema(t) db, err := pgutil.Open(dsn, false) if err != nil { @@ -43,9 +45,9 @@ func TestOpenRefusesSchemaOwner(t *testing.T) { } } -func TestPostgresSettingsCASAcrossPools(t *testing.T) { +func TestSettingsCASAcrossPools(t *testing.T) { st, _ := newTestStore(t) - other, err := Open(st.databaseURL) + other, err := Open(st.target) if err != nil { t.Fatal(err) } @@ -78,9 +80,9 @@ func TestPostgresSettingsCASAcrossPools(t *testing.T) { } } -func TestPostgresProposalQuotaAcrossPools(t *testing.T) { +func TestProposalQuotaAcrossPools(t *testing.T) { st, _ := newTestStore(t) - other, err := Open(st.databaseURL) + other, err := Open(st.target) if err != nil { t.Fatal(err) } @@ -114,6 +116,7 @@ func TestPostgresProposalQuotaAcrossPools(t *testing.T) { } func TestOpenRequiresBootstrap(t *testing.T) { + requirePostgres(t) dsn := pgtest.NewSchema(t) st, err := Open(dsn) if st != nil { @@ -124,9 +127,9 @@ func TestOpenRequiresBootstrap(t *testing.T) { } } -func TestPostgresSingleUseAndLatestTokenAcrossPools(t *testing.T) { +func TestSingleUseAndLatestTokenAcrossPools(t *testing.T) { st, _ := newTestStore(t) - other, err := Open(st.databaseURL) + other, err := Open(st.target) if err != nil { t.Fatal(err) } @@ -170,9 +173,9 @@ func TestPostgresSingleUseAndLatestTokenAcrossPools(t *testing.T) { } } -func TestPostgresWatchAndApprovalQuotasAcrossPools(t *testing.T) { +func TestWatchAndApprovalQuotasAcrossPools(t *testing.T) { st, _ := newTestStore(t) - other, err := Open(st.databaseURL) + other, err := Open(st.target) if err != nil { t.Fatal(err) } @@ -233,6 +236,7 @@ func raceStores(first, second *Store, fn func(*Store, int)) { } func TestApplyForImportCannotAppearReady(t *testing.T) { + requirePostgres(t) dsn := pgtest.NewSchema(t) db := testOwner(t, dsn) if err := ApplyForImport(db); err != nil { @@ -256,6 +260,7 @@ func TestApplyForImportCannotAppearReady(t *testing.T) { } func TestAccountRuntimeCannotAlterSchemaOrReadiness(t *testing.T) { + requirePostgres(t) st, _ := newTestStore(t) for _, stmt := range []string{`CREATE TABLE forbidden(id int)`, `ALTER TABLE users ADD COLUMN forbidden int`, `UPDATE corescope_schema SET ready=false`, `DELETE FROM schema_version`} { if _, err := st.db.Exec(stmt); err == nil { @@ -266,12 +271,14 @@ func TestAccountRuntimeCannotAlterSchemaOrReadiness(t *testing.T) { func TestNativeAccountTextUsesBinaryCollation(t *testing.T) { st, _ := newTestStore(t) - var nonBinary int - if err := st.db.QueryRow(`SELECT count(*) FROM information_schema.columns WHERE table_schema=current_schema() AND data_type='text' AND collation_name IS DISTINCT FROM 'C'`).Scan(&nonBinary); err != nil { - t.Fatal(err) - } - if nonBinary != 0 { - t.Fatalf("%d account text columns inherit locale-dependent collation", nonBinary) + if testBackend(t) == dbconfig.Postgres { + var nonBinary int + if err := st.db.QueryRow(`SELECT count(*) FROM information_schema.columns WHERE table_schema=current_schema() AND data_type='text' AND collation_name IS DISTINCT FROM 'C'`).Scan(&nonBinary); err != nil { + t.Fatal(err) + } + if nonBinary != 0 { + t.Fatalf("%d account text columns inherit locale-dependent collation", nonBinary) + } } for _, email := range []string{"a@example.org", "B@example.org", "ä@example.org"} { mustCreate(t, st, email, "Case") @@ -295,10 +302,11 @@ func TestNativeAccountTextUsesBinaryCollation(t *testing.T) { } func TestPendingPruneRechecksTokenAfterConcurrentIssue(t *testing.T) { + requirePostgres(t) st, clk := newTestStore(t) u := mustCreate(t, st, "pending-race@example.org", "Pending") clk.Advance(8 * 24 * time.Hour) - other, err := pgutil.Open(st.databaseURL, false) + other, err := pgutil.Open(st.target, false) if err != nil { t.Fatal(err) } @@ -308,7 +316,7 @@ func TestPendingPruneRechecksTokenAfterConcurrentIssue(t *testing.T) { t.Fatal(err) } defer tx.Rollback() - if err := lockUser(tx, u.ID); err != nil { + if err := st.lockUser(tx, u.ID); err != nil { t.Fatal(err) } type result struct { @@ -347,9 +355,10 @@ func TestPendingPruneRechecksTokenAfterConcurrentIssue(t *testing.T) { } func TestNotifyStatesSkipConcurrentAccountDeletion(t *testing.T) { + requirePostgres(t) st, clk := newTestStore(t) u := mustCreate(t, st, "delete-race@example.org", "Deleted") - other, err := pgutil.Open(st.databaseURL, false) + other, err := pgutil.Open(st.target, false) if err != nil { t.Fatal(err) } diff --git a/internal/users/proposals.go b/internal/users/proposals.go index ebe851522..8a10bcff8 100644 --- a/internal/users/proposals.go +++ b/internal/users/proposals.go @@ -4,6 +4,7 @@ import ( "database/sql" "errors" "fmt" + "github.com/meshcore-analyzer/dbconfig" "time" ) @@ -119,11 +120,11 @@ func (s *Store) Propose(kind, subject string, userID int64, lim ProposalLimits) defer tx.Rollback() // ponytail: serialize the small proposal queue across processes; replace // with per-kind locks only if measured proposal throughput requires it. - if _, err := tx.Exec(`SELECT pg_advisory_xact_lock(hashtextextended(current_schema(),7289041403))`); err != nil { + if err := s.lockProposals(tx); err != nil { return nil, err } now := unix(s.now()) - cur, err := scanProposal(tx.QueryRow(`SELECT `+proposalCols+` FROM proposals WHERE kind = $1 AND subject = $2`, kind, subject)) + cur, err := scanProposal(tx.QueryRow(`SELECT `+proposalCols+` FROM proposals WHERE kind = `+s.p(1)+` AND subject = `+s.p(2), kind, subject)) switch { case errors.Is(err, ErrNotFound): cur = nil @@ -138,7 +139,7 @@ func (s *Store) Propose(kind, subject string, userID int64, lim ProposalLimits) } if lim.PerUserPerDay > 0 { var n int - if err := tx.QueryRow(`SELECT COUNT(*) FROM proposals WHERE proposer_id = $1 AND created_at > $2`, + if err := tx.QueryRow(`SELECT COUNT(*) FROM proposals WHERE proposer_id = `+s.p(1)+` AND created_at > `+s.p(2), userID, now-secs(24*time.Hour)).Scan(&n); err != nil { return nil, err } @@ -157,19 +158,19 @@ func (s *Store) Propose(kind, subject string, userID int64, lim ProposalLimits) } var id int64 if cur == nil { - err := tx.QueryRow(`INSERT INTO proposals (kind, subject, status, proposer_id, created_at) VALUES ($1, $2, 'pending', $3, $4) RETURNING id`, + err := tx.QueryRow(`INSERT INTO proposals (kind, subject, status, proposer_id, created_at) VALUES (`+s.p(1)+`, `+s.p(2)+`, 'pending', `+s.p(3)+`, `+s.p(4)+`) RETURNING id`, kind, subject, userID, now).Scan(&id) if err != nil { return nil, err } } else { id = cur.ID - if _, err := tx.Exec(`UPDATE proposals SET status = 'pending', proposer_id = $1, reviewer_id = NULL, note = '', - created_at = $2, decided_at = NULL WHERE id = $3`, userID, now, id); err != nil { + if _, err := tx.Exec(`UPDATE proposals SET status = 'pending', proposer_id = `+s.p(1)+`, reviewer_id = NULL, note = '', + created_at = `+s.p(2)+`, decided_at = NULL WHERE id = `+s.p(3), userID, now, id); err != nil { return nil, err } } - p, err := scanProposal(tx.QueryRow(`SELECT `+proposalCols+` FROM proposals WHERE id = $1`, id)) + p, err := scanProposal(tx.QueryRow(`SELECT `+proposalCols+` FROM proposals WHERE id = `+s.p(1), id)) if err != nil { return nil, err } @@ -188,10 +189,10 @@ func (s *Store) Decide(id int64, action ProposalAction, reviewerID int64, note s defer tx.Rollback() // ponytail: serialize the small proposal queue across processes; replace // with per-kind locks only if measured proposal throughput requires it. - if _, err := tx.Exec(`SELECT pg_advisory_xact_lock(hashtextextended(current_schema(),7289041403))`); err != nil { + if err := s.lockProposals(tx); err != nil { return nil, err } - p, err := scanProposal(tx.QueryRow(`SELECT `+proposalCols+` FROM proposals WHERE id = $1`, id)) + p, err := scanProposal(tx.QueryRow(`SELECT `+proposalCols+` FROM proposals WHERE id = `+s.p(1), id)) if err != nil { return nil, err } @@ -208,7 +209,7 @@ func (s *Store) Decide(id int64, action ProposalAction, reviewerID int64, note s } if to == ProposalApproved && maxApproved > 0 { var n int - if err := tx.QueryRow(`SELECT COUNT(*) FROM proposals WHERE kind = $1 AND status = 'approved'`, p.Kind).Scan(&n); err != nil { + if err := tx.QueryRow(`SELECT COUNT(*) FROM proposals WHERE kind = `+s.p(1)+` AND status = 'approved'`, p.Kind).Scan(&n); err != nil { return nil, err } if n >= maxApproved { @@ -216,7 +217,7 @@ func (s *Store) Decide(id int64, action ProposalAction, reviewerID int64, note s } } now := s.now() - if _, err := tx.Exec(`UPDATE proposals SET status = $1, reviewer_id = $2, note = $3, decided_at = $4 WHERE id = $5`, + if _, err := tx.Exec(`UPDATE proposals SET status = `+s.p(1)+`, reviewer_id = `+s.p(2)+`, note = `+s.p(3)+`, decided_at = `+s.p(4)+` WHERE id = `+s.p(5), string(to), reviewerID, note, unix(now), id); err != nil { return nil, err } @@ -233,14 +234,14 @@ func (s *Store) ListProposals(f ProposalFilter) ([]Proposal, error) { q := `SELECT ` + proposalCols + ` FROM proposals WHERE 1=1` var args []any if f.Status != "" { - q += fmt.Sprintf(` AND status = $%d`, len(args)+1) + q += fmt.Sprintf(` AND status = %s`, s.p(len(args)+1)) args = append(args, string(f.Status)) } if f.Kind != "" { - q += fmt.Sprintf(` AND kind = $%d`, len(args)+1) + q += fmt.Sprintf(` AND kind = %s`, s.p(len(args)+1)) args = append(args, f.Kind) } - q += fmt.Sprintf(` ORDER BY created_at DESC, id DESC LIMIT $%d`, len(args)+1) + q += fmt.Sprintf(` ORDER BY created_at DESC, id DESC LIMIT %s`, s.p(len(args)+1)) args = append(args, ProposalListMax) return s.queryProposals(q, args...) } @@ -253,12 +254,12 @@ func (s *Store) ProposalsByUser(userID int64) ([]Proposal, error) { // AllProposalsByUser is ProposalsByUser without the cap (the account export). func (s *Store) AllProposalsByUser(userID int64) ([]Proposal, error) { - return s.proposalsByUser(userID, noLimit) + return s.proposalsByUser(userID, s.noLimit()) } func (s *Store) proposalsByUser(userID int64, limit any) ([]Proposal, error) { - return s.queryProposals(`SELECT `+proposalCols+` FROM proposals WHERE proposer_id = $1 - ORDER BY created_at DESC, id DESC LIMIT $2`, userID, limit) + return s.queryProposals(`SELECT `+proposalCols+` FROM proposals WHERE proposer_id = `+s.p(1)+` + ORDER BY created_at DESC, id DESC LIMIT `+s.p(2), userID, limit) } func (s *Store) queryProposals(q string, args ...any) ([]Proposal, error) { @@ -282,10 +283,10 @@ func (s *Store) queryProposals(q string, args ...any) ([]Proposal, error) { // first, at most limit (0 = all). Never nil. cmd/ingestor runs the same // query as raw SQL (see ingestorApprovedQuery in proposals_test.go). func (s *Store) ApprovedSubjects(kind string, limit int) ([]string, error) { - q := `SELECT subject FROM proposals WHERE kind = $1 AND status = 'approved' ORDER BY decided_at, id` + q := `SELECT subject FROM proposals WHERE kind = ` + s.p(1) + ` AND status = 'approved' ORDER BY decided_at, id` args := []any{kind} if limit > 0 { - q += ` LIMIT $2` + q += ` LIMIT ` + s.p(2) args = append(args, limit) } rows, err := s.db.Query(q, args...) @@ -307,10 +308,20 @@ func (s *Store) ApprovedSubjects(kind string, limit int) ([]string, error) { // PruneProposals deletes rejected and revoked proposals decided more than // maxAge ago. Pending and approved rows are kept however old they are. func (s *Store) PruneProposals(maxAge time.Duration) (int64, error) { - res, err := s.db.Exec(`DELETE FROM proposals WHERE status IN ('rejected','revoked') AND decided_at < $1`, + res, err := s.db.Exec(`DELETE FROM proposals WHERE status IN ('rejected','revoked') AND decided_at < `+s.p(1), unix(s.now())-secs(maxAge)) if err != nil { return 0, err } return res.RowsAffected() } + +// SQLite begins write transactions IMMEDIATE. PostgreSQL needs an explicit +// queue lock because several processes can check the same quota concurrently. +func (s *Store) lockProposals(tx *sql.Tx) error { + if s.backend == dbconfig.SQLite { + return nil + } + _, err := tx.Exec(`SELECT pg_advisory_xact_lock(hashtextextended(current_schema(),7289041403))`) + return err +} diff --git a/internal/users/proposals_test.go b/internal/users/proposals_test.go index b635fe007..596a9929b 100644 --- a/internal/users/proposals_test.go +++ b/internal/users/proposals_test.go @@ -2,6 +2,7 @@ package users import ( "errors" + "github.com/meshcore-analyzer/dbconfig" "strings" "testing" "time" @@ -272,7 +273,11 @@ func TestApprovedSubjects(t *testing.T) { if empty, err := st.ApprovedSubjects("nothing", 0); err != nil || empty == nil || len(empty) != 0 { t.Fatalf("no rows = %#v, %v; want a non-nil empty slice", empty, err) } - rows, err := st.db.Query(ingestorApprovedQuery, 2) + ingestorQuery := ingestorApprovedQuery + if st.backend == dbconfig.SQLite { + ingestorQuery = `SELECT subject FROM proposals WHERE kind = 'hashtag_channel' AND status = 'approved' ORDER BY decided_at, id LIMIT ?1` + } + rows, err := st.db.Query(ingestorQuery, 2) if err != nil { t.Fatalf("ingestor query: %v", err) } @@ -334,8 +339,8 @@ func TestProposalSurvivesProposerDeletion(t *testing.T) { // A native account schema at v3 binary gains the proposals table and keeps its rows. func TestMigrateV3DatabaseToV4(t *testing.T) { st := migratedTestStore(t, 3, `INSERT INTO users (email,display_name,password_hash,created_at) VALUES ('old@example.org','Old','x',1)`) - if v, err := st.SchemaVersion(); err != nil || v != len(migrations) { - t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, len(migrations)) + if v, err := st.SchemaVersion(); err != nil || v != schemaVersion(t) { + t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, schemaVersion(t)) } if !hasIndex(t, st, "proposals_kind_subject") || !hasIndex(t, st, "proposals_status") { t.Fatal("proposals indexes missing after migration") diff --git a/internal/users/schema.go b/internal/users/schema.go index 3ac76e6e4..ebe23949b 100644 --- a/internal/users/schema.go +++ b/internal/users/schema.go @@ -135,13 +135,16 @@ func Tables() []string { return []string{"users", "sessions", "tokens", "audit_log", "mail_log", "mail_events", "user_settings", "proposals", "notification_prefs", "notification_watches", "notification_state"} } -// Apply installs account schema using the migration owner's connection. Runtime -// Open never runs DDL. An advisory transaction lock serializes simultaneous +// Apply installs PostgreSQL accounts using the migration owner's connection. +// PostgreSQL runtime Open never runs DDL. An advisory transaction lock serializes simultaneous // bootstraps, and all DDL plus the readiness marker commit atomically. func Apply(db *sql.DB) error { return apply(db, true) } +func ApplyPostgres(db *sql.DB) error { return Apply(db) } +func AssertPostgresReady(db *sql.DB) error { return AssertReady(db) } + // ApplyForImport creates the schema with readiness false in the same commit. // Only the offline importer's verified completion may change it to true. func ApplyForImport(db *sql.DB) error { return apply(db, false) } diff --git a/internal/users/sessions.go b/internal/users/sessions.go index 7caab7882..5715833d5 100644 --- a/internal/users/sessions.go +++ b/internal/users/sessions.go @@ -49,7 +49,7 @@ func (s *Store) CreateSession(userID int64, ttl time.Duration, userAgent string) now := s.now() var id int64 err = s.db.QueryRow(`INSERT INTO sessions (token_hash, user_id, csrf_token, created_at, expires_at, last_seen_at, user_agent) - VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id`, hash, userID, csrf, unix(now), unix(now.Add(ttl)), unix(now), userAgent).Scan(&id) + VALUES (`+s.p(1)+`, `+s.p(2)+`, `+s.p(3)+`, `+s.p(4)+`, `+s.p(5)+`, `+s.p(6)+`, `+s.p(7)+`) RETURNING id`, hash, userID, csrf, unix(now), unix(now.Add(ttl)), unix(now), userAgent).Scan(&id) if err != nil { return "", nil, err } @@ -60,7 +60,7 @@ func (s *Store) CreateSession(userID int64, ttl time.Duration, userAgent string) // LookupSession resolves a raw cookie token. Unknown and expired sessions // return ErrNotFound; expired ones are deleted on the way. func (s *Store) LookupSession(raw string) (*Session, error) { - sess, err := scanSession(s.db.QueryRow(`SELECT `+sessionCols+` FROM sessions WHERE token_hash = $1`, HashToken(raw))) + sess, err := scanSession(s.db.QueryRow(`SELECT `+sessionCols+` FROM sessions WHERE token_hash = `+s.p(1), HashToken(raw))) if errors.Is(err, sql.ErrNoRows) { return nil, ErrNotFound } @@ -68,7 +68,7 @@ func (s *Store) LookupSession(raw string) (*Session, error) { return nil, fmt.Errorf("users: lookup session: %w", err) } if !s.now().Before(sess.ExpiresAt) { - _, _ = s.db.Exec(`DELETE FROM sessions WHERE id = $1`, sess.ID) + _, _ = s.db.Exec(`DELETE FROM sessions WHERE id = `+s.p(1), sess.ID) return nil, ErrNotFound } return sess, nil @@ -77,29 +77,29 @@ func (s *Store) LookupSession(raw string) (*Session, error) { // ExtendSession marks the session seen now and moves its expiry to now+ttl. func (s *Store) ExtendSession(id int64, ttl time.Duration) error { now := s.now() - return expectOne(s.db.Exec(`UPDATE sessions SET last_seen_at = $1, expires_at = $2 WHERE id = $3`, + return expectOne(s.db.Exec(`UPDATE sessions SET last_seen_at = `+s.p(1)+`, expires_at = `+s.p(2)+` WHERE id = `+s.p(3), unix(now), unix(now.Add(ttl)), id)) } func (s *Store) DeleteSessionByToken(raw string) error { - _, err := s.db.Exec(`DELETE FROM sessions WHERE token_hash = $1`, HashToken(raw)) + _, err := s.db.Exec(`DELETE FROM sessions WHERE token_hash = `+s.p(1), HashToken(raw)) return err } // DeleteSession revokes one session, only if it belongs to userID. func (s *Store) DeleteSession(userID, sessionID int64) error { - return expectOne(s.db.Exec(`DELETE FROM sessions WHERE id = $1 AND user_id = $2`, sessionID, userID)) + return expectOne(s.db.Exec(`DELETE FROM sessions WHERE id = `+s.p(1)+` AND user_id = `+s.p(2), sessionID, userID)) } // DeleteUserSessions revokes all of a user's sessions except exceptID (0 = none kept). func (s *Store) DeleteUserSessions(userID, exceptID int64) error { - _, err := s.db.Exec(`DELETE FROM sessions WHERE user_id = $1 AND id != $2`, userID, exceptID) + _, err := s.db.Exec(`DELETE FROM sessions WHERE user_id = `+s.p(1)+` AND id != `+s.p(2), userID, exceptID) return err } // ListSessions returns a user's sessions, most recently seen first. func (s *Store) ListSessions(userID int64) ([]Session, error) { - rows, err := s.db.Query(`SELECT `+sessionCols+` FROM sessions WHERE user_id = $1 ORDER BY last_seen_at DESC, id DESC`, userID) + rows, err := s.db.Query(`SELECT `+sessionCols+` FROM sessions WHERE user_id = `+s.p(1)+` ORDER BY last_seen_at DESC, id DESC`, userID) if err != nil { return nil, err } @@ -116,7 +116,7 @@ func (s *Store) ListSessions(userID int64) ([]Session, error) { } func (s *Store) PruneExpiredSessions() (int64, error) { - res, err := s.db.Exec(`DELETE FROM sessions WHERE expires_at <= $1`, unix(s.now())) + res, err := s.db.Exec(`DELETE FROM sessions WHERE expires_at <= `+s.p(1), unix(s.now())) if err != nil { return 0, err } diff --git a/internal/users/settings.go b/internal/users/settings.go index a89569c7c..27da0c821 100644 --- a/internal/users/settings.go +++ b/internal/users/settings.go @@ -36,7 +36,7 @@ func newGeneration() (string, error) { func (s *Store) GetSettings(userID int64) (string, SettingsVersion, error) { var doc string var v SettingsVersion - err := s.db.QueryRow(`SELECT doc, revision, generation FROM user_settings WHERE user_id = $1`, userID).Scan(&doc, &v.Revision, &v.Generation) + err := s.db.QueryRow(`SELECT doc, revision, generation FROM user_settings WHERE user_id = `+s.p(1), userID).Scan(&doc, &v.Revision, &v.Generation) if errors.Is(err, sql.ErrNoRows) { return "", SettingsVersion{}, nil } @@ -59,7 +59,7 @@ type SettingsRecord struct { func (s *Store) SettingsRecordFor(userID int64) (*SettingsRecord, error) { var r SettingsRecord var updated int64 - err := s.db.QueryRow(`SELECT doc, revision, generation, updated_at FROM user_settings WHERE user_id = $1`, userID). + err := s.db.QueryRow(`SELECT doc, revision, generation, updated_at FROM user_settings WHERE user_id = `+s.p(1), userID). Scan(&r.Doc, &r.Version.Revision, &r.Version.Generation, &updated) if errors.Is(err, sql.ErrNoRows) { return nil, nil @@ -82,11 +82,11 @@ func (s *Store) PutSettings(userID int64, base SettingsVersion, doc string) (Set return SettingsVersion{}, err } defer tx.Rollback() - if err := lockUser(tx, userID); err != nil { + if err := s.lockUser(tx, userID); err != nil { return SettingsVersion{}, err } var cur SettingsVersion - err = tx.QueryRow(`SELECT revision, generation FROM user_settings WHERE user_id = $1`, userID).Scan(&cur.Revision, &cur.Generation) + err = tx.QueryRow(`SELECT revision, generation FROM user_settings WHERE user_id = `+s.p(1), userID).Scan(&cur.Revision, &cur.Generation) if err != nil && !errors.Is(err, sql.ErrNoRows) { return SettingsVersion{}, err } @@ -102,7 +102,7 @@ func (s *Store) PutSettings(userID int64, base SettingsVersion, doc string) (Set return cur, ErrSettingsConflict } next.Revision = cur.Revision + 1 - if _, err := tx.Exec(`INSERT INTO user_settings (user_id, doc, revision, generation, updated_at) VALUES ($1, $2, $3, $4, $5) + if _, err := tx.Exec(`INSERT INTO user_settings (user_id, doc, revision, generation, updated_at) VALUES (`+s.p(1)+`, `+s.p(2)+`, `+s.p(3)+`, `+s.p(4)+`, `+s.p(5)+`) ON CONFLICT(user_id) DO UPDATE SET doc = excluded.doc, revision = excluded.revision, generation = excluded.generation, updated_at = excluded.updated_at`, userID, doc, next.Revision, next.Generation, unix(s.now())); err != nil { return SettingsVersion{}, err @@ -118,12 +118,12 @@ func (s *Store) DeleteSettings(userID int64) error { return err } defer tx.Rollback() - if err = lockUser(tx, userID); errors.Is(err, ErrNotFound) { + if err = s.lockUser(tx, userID); errors.Is(err, ErrNotFound) { return nil } else if err != nil { return err } - if _, err = tx.Exec(`DELETE FROM user_settings WHERE user_id=$1`, userID); err != nil { + if _, err = tx.Exec(`DELETE FROM user_settings WHERE user_id=`+s.p(1), userID); err != nil { return err } return tx.Commit() diff --git a/internal/users/settings_test.go b/internal/users/settings_test.go index 4e8f80474..ff28fa061 100644 --- a/internal/users/settings_test.go +++ b/internal/users/settings_test.go @@ -135,8 +135,8 @@ func TestSettingsSurviveDisable(t *testing.T) { // A native account schema at v1 binary gains user_settings and keeps its rows. func TestMigrateV1DatabaseToV2(t *testing.T) { st := migratedTestStore(t, 1, `INSERT INTO users (email,display_name,password_hash,created_at) VALUES ('old@example.org','Old','x',1)`) - if v, err := st.SchemaVersion(); err != nil || v != len(migrations) { - t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, len(migrations)) + if v, err := st.SchemaVersion(); err != nil || v != schemaVersion(t) { + t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, schemaVersion(t)) } u, err := st.GetByEmail("old@example.org") if err != nil { diff --git a/internal/users/snapshot.go b/internal/users/snapshot.go index 2112f7667..4f059de24 100644 --- a/internal/users/snapshot.go +++ b/internal/users/snapshot.go @@ -3,19 +3,55 @@ package users import ( "context" "errors" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/pgutil" + "os" + "path/filepath" ) -// Snapshot writes a consistent PostgreSQL custom-format archive readable only -// by its owner. Restore with pg_restore; the destination must not exist. +// Snapshot writes a consistent, owner-readable native backup. SQLite produces +// a standalone database; PostgreSQL produces a pg_restore archive. The target +// must not exist, including an existing empty file. func (s *Store) Snapshot(path string) error { return s.SnapshotContext(context.Background(), path) } -// SnapshotContext cancels the dump when the caller disconnects or times out. +// SnapshotContext cancels the native backup when the caller disconnects or times out. func (s *Store) SnapshotContext(ctx context.Context, path string) error { if err := s.db.PingContext(ctx); err != nil { return errors.New("users: account database is not available for snapshot") } - return pgutil.Dump(ctx, s.databaseURL, path) + if s.backend == dbconfig.Postgres { + return pgutil.Dump(ctx, s.target, path) + } + if _, err := os.Lstat(path); err == nil { + return errors.New("users: snapshot target already exists") + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + // VACUUM INTO accepts a pre-created empty file, so private mode applies before + // any account data is written. Publish with a hard link to avoid overwriting a + // target created by another caller while the snapshot was running. + f, err := os.CreateTemp(filepath.Dir(path), ".accounts-snapshot-*") + if err != nil { + return err + } + temporary := f.Name() + defer os.Remove(temporary) + if err := f.Close(); err != nil { + return err + } + if _, err := s.db.ExecContext(ctx, `VACUUM INTO ?1`, temporary); err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + if err := os.Link(temporary, path); err != nil { + if errors.Is(err, os.ErrExist) { + return errors.New("users: snapshot target already exists") + } + return err + } + return nil } diff --git a/internal/users/snapshot_test.go b/internal/users/snapshot_test.go index 89c29c595..5f3fad482 100644 --- a/internal/users/snapshot_test.go +++ b/internal/users/snapshot_test.go @@ -11,12 +11,12 @@ import ( "testing" "time" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/pgutil" - "github.com/meshcore-analyzer/pgutil/pgtest" ) func TestSnapshotCopiesRows(t *testing.T) { - st, clk := newTestStoreAt(t, pgtest.NewDatabase(t)) + st, clk := newTestStoreAt(t, testTarget(t, true)) u := mustCreate(t, st, "a@example.org", "Aaa") if err := st.Activate(u.ID, RoleUser, nil); err != nil { t.Fatal(err) @@ -44,20 +44,23 @@ func TestSnapshotCopiesRows(t *testing.T) { if err := st.Snapshot(path); err != nil { t.Fatal(err) } - target := pgtest.NewDatabase(t) - env, err := pgutil.CommandEnv(target) - if err != nil { - t.Fatal(err) - } - // --dbname is needed for restore mode; use a credential-free database name. - cfg, err := pgutil.ParseConfig(target) - if err != nil { - t.Fatal(err) - } - cmd := exec.Command("pg_restore", "--no-password", "--exit-on-error", "--no-owner", "--no-privileges", "--dbname="+cfg.Database, path) - cmd.Env = env - if err := cmd.Run(); err != nil { - t.Fatal("native account restore failed") + target := path + if st.backend == dbconfig.Postgres { + target = testTarget(t, true) + env, err := pgutil.CommandEnv(target) + if err != nil { + t.Fatal(err) + } + // --dbname is needed for restore mode; use a credential-free database name. + cfg, err := pgutil.ParseConfig(target) + if err != nil { + t.Fatal(err) + } + cmd := exec.Command("pg_restore", "--no-password", "--exit-on-error", "--no-owner", "--no-privileges", "--dbname="+cfg.Database, path) + cmd.Env = env + if err := cmd.Run(); err != nil { + t.Fatal("native account restore failed") + } } cp, err := Open(testRuntimeURL(t, target)) if err != nil { @@ -69,8 +72,8 @@ func TestSnapshotCopiesRows(t *testing.T) { if err != nil || got.ID != u.ID || got.DisplayName != "Aaa" { t.Fatalf("user in snapshot = %+v, %v", got, err) } - if v, err := cp.SchemaVersion(); err != nil || v != len(migrations) { - t.Fatalf("snapshot schema version = %d, %v; want %d", v, err, len(migrations)) + if v, err := cp.SchemaVersion(); err != nil || v != schemaVersion(t) { + t.Fatalf("snapshot schema version = %d, %v; want %d", v, err, schemaVersion(t)) } if e, err := cp.AuditAllFor(u.ID); err != nil || len(e) != 1 { t.Fatalf("audit in snapshot = %d, %v", len(e), err) @@ -93,7 +96,7 @@ func TestSnapshotFileMode(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("Unix file modes") } - st, _ := newTestStoreAt(t, pgtest.NewDatabase(t)) + st, _ := newTestStoreAt(t, testTarget(t, true)) path := filepath.Join(t.TempDir(), "snap.dump") if err := st.Snapshot(path); err != nil { t.Fatal(err) @@ -108,7 +111,7 @@ func TestSnapshotFileMode(t *testing.T) { } func TestSnapshotRefusesExistingTarget(t *testing.T) { - st, _ := newTestStoreAt(t, pgtest.NewDatabase(t)) + st, _ := newTestStoreAt(t, testTarget(t, true)) path := filepath.Join(t.TempDir(), "snap.dump") // Existing empty files must also be preserved. if err := os.WriteFile(path, nil, 0o600); err != nil { @@ -124,7 +127,7 @@ func TestSnapshotRefusesExistingTarget(t *testing.T) { } func TestSnapshotFailureLeavesNoFile(t *testing.T) { - st, _ := newTestStoreAt(t, pgtest.NewDatabase(t)) + st, _ := newTestStoreAt(t, testTarget(t, true)) st.Close() path := filepath.Join(t.TempDir(), "snap.dump") if err := st.Snapshot(path); err == nil { @@ -136,7 +139,7 @@ func TestSnapshotFailureLeavesNoFile(t *testing.T) { } func TestSnapshotCanceledLeavesNoFile(t *testing.T) { - st, _ := newTestStoreAt(t, pgtest.NewDatabase(t)) + st, _ := newTestStoreAt(t, testTarget(t, true)) ctx, cancel := context.WithCancel(context.Background()) cancel() path := filepath.Join(t.TempDir(), "cancel.dump") diff --git a/internal/users/sqlite_backend_test.go b/internal/users/sqlite_backend_test.go new file mode 100644 index 000000000..1de5579e8 --- /dev/null +++ b/internal/users/sqlite_backend_test.go @@ -0,0 +1,124 @@ +package users + +import ( + "database/sql" + "github.com/meshcore-analyzer/dbconfig" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + _ "modernc.org/sqlite" +) + +func TestSQLitePathOpensWithoutPostgres(t *testing.T) { + t.Setenv("CORESCOPE_TEST_POSTGRES_URL", "") + st, err := Open(filepath.Join(t.TempDir(), "accounts.db")) + if err != nil { + t.Fatalf("open default local account storage without PostgreSQL: %v", err) + } + defer st.Close() + u := mustCreate(t, st, "local@example.invalid", "Local account") + want := NotifyState{NotifyKey: NotifyKey{UserID: u.ID, Event: NotifyNodeBattery, Subject: "node-local"}, State: NotifyBad, ChangedAt: time.Unix(1234567890, 0).UTC()} + if err := st.WriteNotifyStates([]NotifyState{want}); err != nil { + t.Fatal(err) + } + got, err := st.AllNotifyStates() + if err != nil || len(got) != 1 || got[0] != want { + t.Fatalf("out-of-order notification parameters changed: %+v, %v", got, err) + } +} + +func TestModerncNumberedParametersPreservePositions(t *testing.T) { + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + var second, first, repeated int + var null sql.NullString + if err := db.QueryRow(`SELECT ?2,?1,?2,?3`, 11, 22, nil).Scan(&second, &first, &repeated, &null); err != nil { + t.Fatal(err) + } + if second != 22 || first != 11 || repeated != 22 || null.Valid { + t.Fatal("SQLite numbered parameters did not retain logical argument positions") + } +} + +func TestOpenStorageUsesOnlySelectedAccountTarget(t *testing.T) { + target := filepath.Join(t.TempDir(), "accounts.db") + setup, err := Open(target) + if err != nil { + t.Fatal(err) + } + setup.Close() + st, err := OpenStorage(dbconfig.Storage{Backend: dbconfig.SQLite, UsersDBPath: target, UsersDatabaseURL: "postgres://inactive.invalid/accounts", WriterDatabaseURL: "postgres://inactive.invalid/accounts"}) + if err != nil { + t.Fatal(err) + } + if st.Backend() != dbconfig.SQLite { + t.Fatal("inactive URL changed selected backend") + } + st.Close() + before, err := os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + for _, backend := range []dbconfig.Backend{dbconfig.Postgres, ""} { + if st, err := OpenStorage(dbconfig.Storage{Backend: backend, UsersDBPath: target}); err == nil { + st.Close() + t.Fatal("missing explicit account backend/URL fell back to SQLite") + } + } + after, err := os.ReadFile(target) + if err != nil || string(before) != string(after) { + t.Fatal("invalid selection mutated existing account file") + } + if st, err := OpenStorage(dbconfig.Storage{Backend: dbconfig.SQLite, UsersDBPath: target, DBPath: target}); err == nil { + st.Close() + t.Fatal("selected telemetry file accepted as account storage") + } +} + +func TestSQLiteAccountLiteralFilenameAndAliasIsolation(t *testing.T) { + names := []string{"accounts # % ü.db"} + if runtime.GOOS != "windows" { + names = append(names, "accounts ?mode=memory.db") + } + for _, name := range names { + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), name) + setup, err := Open(path) + if err != nil { + t.Fatal(err) + } + setup.Close() + st, err := OpenStorage(dbconfig.Storage{Backend: dbconfig.SQLite, UsersDBPath: path}) + if err != nil { + t.Fatal(err) + } + defer st.Close() + mustCreate(t, st, "literal@example.invalid", "Literal") + if _, err := os.Stat(path); err != nil { + t.Fatal("literal account file was not created:", err) + } + var actual string + var n int + if err := st.db.QueryRow(`PRAGMA database_list`).Scan(&n, &actual, &actual); err != nil { + t.Fatal(err) + } + if !samePath(path, actual) { + t.Fatal("URI opened another filesystem target") + } + alias := filepath.Join(filepath.Dir(path), "alias.db") + if err := os.Link(path, alias); err != nil { + t.Fatal(err) + } + if bad, err := OpenStorage(dbconfig.Storage{Backend: dbconfig.SQLite, UsersDBPath: alias, DBPath: path}); err == nil { + bad.Close() + t.Fatal("hardlink alias bypassed telemetry isolation") + } + }) + } +} diff --git a/internal/users/sqlite_bootstrap_test.go b/internal/users/sqlite_bootstrap_test.go new file mode 100644 index 000000000..be4bf15fd --- /dev/null +++ b/internal/users/sqlite_bootstrap_test.go @@ -0,0 +1,51 @@ +package users + +import ( + "context" + "database/sql" + "errors" + "net/url" + "path/filepath" + "testing" + "time" + + "github.com/meshcore-analyzer/dbconfig" +) + +func TestSQLiteWALBootstrapWaitIsBounded(t *testing.T) { + path := filepath.Join(t.TempDir(), "journal.db") + dsn, err := dbconfig.SQLiteURI(path, url.Values{"_pragma": {"busy_timeout(0)"}}) + if err != nil { + t.Fatal(err) + } + owner, err := sql.Open("sqlite", dsn) + if err != nil { + t.Fatal(err) + } + defer owner.Close() + owner.SetMaxOpenConns(1) + mustSQL(t, owner, `CREATE TABLE preserved(value TEXT);INSERT INTO preserved VALUES('keep');BEGIN IMMEDIATE`) + defer owner.Exec(`ROLLBACK`) + other, err := sql.Open("sqlite", dsn) + if err != nil { + t.Fatal(err) + } + defer other.Close() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + if err = enableSQLiteWAL(ctx, other); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("locked journal wait did not honor its bound: %v", err) + } + if _, err = owner.Exec(`COMMIT`); err != nil { + t.Fatal(err) + } + ctx2, cancel2 := context.WithTimeout(context.Background(), time.Second) + defer cancel2() + if err = enableSQLiteWAL(ctx2, other); err != nil { + t.Fatal("journal did not become usable after lock release", err) + } + var value string + if err = other.QueryRow(`SELECT value FROM preserved`).Scan(&value); err != nil || value != "keep" { + t.Fatal("bootstrap changed data", err) + } +} diff --git a/internal/users/sqlite_identity_test.go b/internal/users/sqlite_identity_test.go new file mode 100644 index 000000000..eab28a454 --- /dev/null +++ b/internal/users/sqlite_identity_test.go @@ -0,0 +1,384 @@ +package users + +import ( + "context" + "database/sql" + "fmt" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" +) + +func legacySQLiteFixture(t *testing.T, version int, auto bool) (string, *sql.DB) { + t.Helper() + path := filepath.Join(t.TempDir(), "legacy.db") + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + t.Cleanup(func() { db.Close() }) + mustSQL(t, db, `CREATE TABLE schema_version(version INTEGER NOT NULL)`, fmt.Sprintf(`INSERT INTO schema_version VALUES(%d)`, version)) + for _, migration := range LegacyMigrations()[:version] { + for _, statement := range migration { + if auto { + statement = strings.Replace(statement, "id INTEGER PRIMARY KEY,", "id INTEGER PRIMARY KEY AUTOINCREMENT,", 1) + } + mustSQL(t, db, statement) + } + } + return path, db +} +func mustSQL(t *testing.T, db *sql.DB, statements ...string) { + t.Helper() + for _, stmt := range statements { + if _, err := db.Exec(stmt); err != nil { + t.Fatalf("fixture SQL: %v", err) + } + } +} +func sqliteRows(t *testing.T, db *sql.DB, query string) [][]any { + t.Helper() + rows, err := db.Query(query) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + cols, err := rows.Columns() + if err != nil { + t.Fatal(err) + } + var result [][]any + for rows.Next() { + values := make([]any, len(cols)) + dest := make([]any, len(cols)) + for i := range values { + dest[i] = &values[i] + } + if err := rows.Scan(dest...); err != nil { + t.Fatal(err) + } + result = append(result, values) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return result +} +func seedSQLiteAccounts(t *testing.T, db *sql.DB) { + mustSQL(t, db, + `INSERT INTO users VALUES(0,'zero@example.invalid','Zero','opaque-hash','admin','active',1,NULL,123,NULL,0)`, + `INSERT INTO users VALUES(-7,'negative@example.invalid','Negative','opaque-hash-2','user','pending',-1,NULL,NULL,2,1)`, + `INSERT INTO sessions VALUES(0,'session-hash',0,'csrf',1,9999999999,2,'agent')`, + `INSERT INTO tokens VALUES('live-token',0,'reset',NULL,9999999999,NULL),('used-token',0,'email_change','next@example.invalid',9999999999,3)`, + `INSERT INTO audit_log VALUES(-1,1,77,'user.old',89,' { "raw": true } ')`, + `INSERT INTO mail_log VALUES(0,0,'zero@example.invalid','reset','provider-id',1,'sent',2,'')`, + `INSERT INTO mail_events(rowid,mail_id,event,at,reason) VALUES(0,0,'sent',2,''),(14,0,'delivered',3,'delivery')`, + `INSERT INTO user_settings VALUES(0,' { "saved": 1 } ',2,'generation',3)`, + `INSERT INTO proposals VALUES(0,'hashtag_channel','#test','approved',0,NULL,'note',1,3)`, + `INSERT INTO notification_prefs VALUES(0,1,'node.offline','unsubscribe',3)`, + `INSERT INTO notification_watches VALUES(0,'public-key',3)`, + `INSERT INTO notification_state VALUES(0,'node.offline','public-key','bad',3)`) +} + +func TestSQLiteIdentityUpgradePreservesAllAccountRows(t *testing.T) { + path, db := legacySQLiteFixture(t, 5, false) + seedSQLiteAccounts(t, db) + tables := Tables() + before := make(map[string][][]any) + for _, table := range tables { + projection := "*" + if table == "mail_events" { + projection = "rowid,*" + } + before[table] = sqliteRows(t, db, `SELECT `+projection+` FROM `+sqliteQuote(table)+` ORDER BY 1`) + } + mustSQL(t, db, `PRAGMA foreign_keys=ON`) + if err := ApplySQLite(db); err != nil { + t.Fatal(err) + } + for _, table := range tables { + if got := sqliteRows(t, db, `SELECT * FROM `+sqliteQuote(table)+` ORDER BY 1`); !reflect.DeepEqual(got, before[table]) { + t.Fatalf("%s rows changed: got=%v want=%v", table, got, before[table]) + } + } + var fk int + if err := db.QueryRow(`PRAGMA foreign_keys`).Scan(&fk); err != nil || fk != 1 { + t.Fatalf("foreign_keys=%d,%v", fk, err) + } + if err := AssertSQLiteReady(db); err != nil { + t.Fatal(err) + } + // Historical references to deleted users remain meaningful after the upgrade. + st, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer st.Close() + next := mustCreate(t, st, "next@example.invalid", "Next") + if next.ID != 124 { + t.Fatalf("next user id=%d; want123+1", next.ID) + } + // Existing rowid=0 becomes an explicit ID; next mail-event allocation uses14+1. + mustSQL(t, db, `INSERT INTO mail_events(mail_id,event,at) VALUES(0,'opened',4)`) + var eventID int64 + if err := db.QueryRow(`SELECT id FROM mail_events WHERE event='opened'`).Scan(&eventID); err != nil || eventID != 15 { + t.Fatalf("mail event id=%d,%v", eventID, err) + } + // Original FK actions and uniqueness survive the rebuild. + mustSQL(t, db, `DELETE FROM users WHERE id=0`) + for _, table := range []string{"sessions", "tokens", "user_settings", "notification_prefs", "notification_watches", "notification_state"} { + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM ` + sqliteQuote(table)).Scan(&n); err != nil || n != 0 { + t.Fatalf("%s cascade count=%d,%v", table, n, err) + } + } + var mailUser, proposer sql.NullInt64 + if err := db.QueryRow(`SELECT user_id FROM mail_log WHERE id=0`).Scan(&mailUser); err != nil || mailUser.Valid { + t.Fatal("mail SET NULL failed") + } + if err := db.QueryRow(`SELECT proposer_id FROM proposals WHERE id=0`).Scan(&proposer); err != nil || proposer.Valid { + t.Fatal("proposal SET NULL failed") + } + if _, err := db.Exec(`INSERT INTO mail_events(mail_id,event,at) VALUES(0,'opened',4)`); err == nil { + t.Fatal("mail event uniqueness lost") + } + if _, err := db.Exec(`INSERT INTO users(email,display_name,password_hash,role,created_at) VALUES('invalid','x','x','invalid',1)`); err == nil { + t.Fatal("role constraint lost") + } + mustSQL(t, db, `DELETE FROM mail_log WHERE id=0`) + var n int + db.QueryRow(`SELECT count(*) FROM mail_events`).Scan(&n) + if n != 0 { + t.Fatal("mail-event cascade lost") + } +} + +func TestSQLiteIdentityMigrationEveryLegacyVersion(t *testing.T) { + for version := 1; version <= 5; version++ { + t.Run(fmt.Sprint(version), func(t *testing.T) { + _, db := legacySQLiteFixture(t, version, false) + mustSQL(t, db, `INSERT INTO users(id,email,display_name,password_hash,created_at) VALUES(0,'legacy','Legacy','hash',1)`) + if err := ApplySQLite(db); err != nil { + t.Fatal(err) + } + if err := ApplySQLite(db); err != nil { + t.Fatal("idempotence:", err) + } + var id int64 + if err := db.QueryRow(`SELECT id FROM users WHERE email='legacy'`).Scan(&id); err != nil || id != 0 { + t.Fatal("legacy zero ID lost") + } + }) + } +} + +func TestSQLiteIdentityMigrationPreservesDeletedSequenceHighWater(t *testing.T) { + path, db := legacySQLiteFixture(t, 5, true) + seedSQLiteAccounts(t, db) + for _, table := range sqliteIdentities { + if table.name == "mail_events" { + continue + } + mustSQL(t, db, `UPDATE sqlite_sequence SET seq=900 WHERE name='`+table.name+`'`) + } + if err := ApplySQLite(db); err != nil { + t.Fatal(err) + } + for _, table := range sqliteIdentities { + if table.name == "mail_events" { + continue + } + var seq int64 + if err := db.QueryRow(`SELECT seq FROM sqlite_sequence WHERE name=?1`, table.name).Scan(&seq); err != nil || seq != 900 { + t.Fatalf("%s seq=%d,%v", table.name, seq, err) + } + } + st, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer st.Close() + u := mustCreate(t, st, "allocate@example.invalid", "Allocate") + if u.ID != 901 { + t.Fatalf("next id=%d; want901", u.ID) + } + if err := st.Delete(u.ID); err != nil { + t.Fatal(err) + } + snapshot := filepath.Join(t.TempDir(), "copy.db") + if err := st.Snapshot(snapshot); err != nil { + t.Fatal(err) + } + restored, err := Open(snapshot) + if err != nil { + t.Fatal(err) + } + defer restored.Close() + if got := mustCreate(t, restored, "later@example.invalid", "Later"); got.ID != 902 { + t.Fatalf("snapshot reused deleted id: %d", got.ID) + } +} + +func TestSQLiteIdentityMigrationRefusesExtensionsWithoutMutation(t *testing.T) { + for _, ddl := range []string{`CREATE TRIGGER extension AFTER INSERT ON users BEGIN UPDATE users SET display_name='changed' WHERE id=NEW.id; END`, `ALTER TABLE users ADD COLUMN generated TEXT GENERATED ALWAYS AS (email) VIRTUAL`, `CREATE UNIQUE INDEX custom_constraint ON users(display_name)`} { + t.Run(strings.Fields(ddl)[1], func(t *testing.T) { + _, db := legacySQLiteFixture(t, 5, false) + mustSQL(t, db, ddl) + before := sqliteRows(t, db, `SELECT type,name,sql FROM sqlite_schema ORDER BY type,name`) + if err := ApplySQLite(db); err == nil { + t.Fatal("unsupported extension accepted") + } + after := sqliteRows(t, db, `SELECT type,name,sql FROM sqlite_schema ORDER BY type,name`) + if !reflect.DeepEqual(before, after) { + t.Fatal("refused migration changed source schema") + } + var version int + db.QueryRow(`SELECT version FROM schema_version`).Scan(&version) + if version != 5 { + t.Fatal("refused migration changed version") + } + }) + } +} + +func TestSQLiteIdentityMigrationFailureRollsBackAndRestoresPragma(t *testing.T) { + _, db := legacySQLiteFixture(t, 5, false) + mustSQL(t, db, `INSERT INTO tokens VALUES('orphan',999,'reset',NULL,1,NULL)`, `PRAGMA foreign_keys=ON`) + before := sqliteRows(t, db, `SELECT type,name,sql FROM sqlite_schema ORDER BY type,name`) + if err := ApplySQLite(db); err == nil || !strings.Contains(err.Error(), "foreign keys") { + t.Fatalf("late migration error=%v", err) + } + if after := sqliteRows(t, db, `SELECT type,name,sql FROM sqlite_schema ORDER BY type,name`); !reflect.DeepEqual(before, after) { + t.Fatal("failed table rebuild was not rolled back") + } + var fk, version int + db.QueryRow(`PRAGMA foreign_keys`).Scan(&fk) + db.QueryRow(`SELECT version FROM schema_version`).Scan(&version) + if fk != 1 || version != 5 { + t.Fatalf("rollback state FK=%d version=%d", fk, version) + } + if rows := sqliteRows(t, db, `SELECT token_hash,user_id FROM tokens`); len(rows) != 1 || rows[0][0] != "orphan" { + t.Fatal("source row changed on rollback") + } +} + +func TestSQLiteReadyRejectsMissingAccountObjects(t *testing.T) { + for _, statement := range []string{`DROP TABLE tokens`, `DROP INDEX proposals_kind_subject`, `DROP TABLE notification_state`} { + t.Run(statement, func(t *testing.T) { + st, err := Open(filepath.Join(t.TempDir(), "accounts.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + mustSQL(t, st.db, statement) + if err := AssertSQLiteReady(st.db); err == nil { + t.Fatal("incomplete current account schema accepted") + } + }) + } +} + +func TestSQLiteImmediateTransactionsAcrossIndependentPools(t *testing.T) { + path := filepath.Join(t.TempDir(), "accounts.db") + first, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer first.Close() + second, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer second.Close() + // Begin itself must reserve the writer before either caller reads a quota. + tx, err := first.db.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + // A native NOWAIT connection deterministically detects the reserved writer; + // no goroutine sleeps or scheduling order are needed to prove IMMEDIATE. + conn, err := second.db.Conn(context.Background()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if _, err := conn.ExecContext(context.Background(), `PRAGMA busy_timeout=0`); err != nil { + t.Fatal(err) + } + if _, err := conn.BeginTx(context.Background(), nil); err == nil { + t.Fatal("second write transaction began before first committed") + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + next, err := conn.BeginTx(context.Background(), nil) + if err != nil { + t.Fatal(err) + } + next.Rollback() + conn.Close() + // Explicit read-only snapshot transactions are not promoted to writers. + read, err := first.db.BeginTx(context.Background(), &sql.TxOptions{ReadOnly: true}) + if err != nil { + t.Fatal(err) + } + defer read.Rollback() + if _, err := second.CreatePending("writer@example.invalid", "Writer", "hash"); err != nil { + t.Fatal(err) + } + if first.db.Stats().MaxOpenConnections != 1 || second.db.Stats().MaxOpenConnections != 1 { + t.Fatal("account pool grew") + } + +} + +func TestSQLiteEmptyVersionTableFromInterruptedLegacyInitialization(t *testing.T) { + _, db := legacySQLiteFixture(t, 0, false) + mustSQL(t, db, `DELETE FROM schema_version`) + if err := ApplySQLite(db); err != nil { + t.Fatal("legacy initialization could stop between CREATE and INSERT:", err) + } + if err := AssertSQLiteReady(db); err != nil { + t.Fatal(err) + } +} + +func TestSQLiteAccountBootstrapAcrossPools(t *testing.T) { + path := filepath.Join(t.TempDir(), "accounts.db") + start := make(chan struct{}) + var wg sync.WaitGroup + for i := 0; i < 4; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + st, err := Open(path) + if err != nil { + t.Errorf("bootstrap: %v", err) + return + } + defer st.Close() + if _, err := st.CreatePending(fmt.Sprintf("worker%d@example.invalid", i), "Concurrent", "hash"); err != nil { + t.Error(err) + } + }(i) + } + close(start) + wg.Wait() + st, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer st.Close() + var n int + if err := st.db.QueryRow(`SELECT count(*) FROM users`).Scan(&n); err != nil || n != 4 { + t.Fatalf("accounts=%d,%v", n, err) + } + if err := AssertSQLiteReady(st.db); err != nil { + t.Fatal(err) + } +} diff --git a/internal/users/sqlite_import_test.go b/internal/users/sqlite_import_test.go new file mode 100644 index 000000000..e0fc6e291 --- /dev/null +++ b/internal/users/sqlite_import_test.go @@ -0,0 +1,32 @@ +package users + +import ( + "database/sql" + "errors" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbconfig" +) + +func TestSQLitePendingImportRefusesBeforeAccountSchemaChanges(t *testing.T) { + for name, check := range map[string]func(*sql.DB) error{"apply": ApplySQLite, "ready": AssertSQLiteReady} { + t.Run(name, func(t *testing.T) { + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "partial.db")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`CREATE TABLE corescope_reverse_progress (table_name TEXT)`); err != nil { + t.Fatal(err) + } + if err := check(db); !errors.Is(err, dbconfig.ErrSQLiteImportIncomplete) { + t.Errorf("pending account import was not identified: %v", err) + } + var objects int + if err := db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE name <> 'corescope_reverse_progress'`).Scan(&objects); err != nil || objects != 0 { + t.Errorf("pending source changed before refusal: objects=%d, err=%v", objects, err) + } + }) + } +} diff --git a/internal/users/sqlite_open.go b/internal/users/sqlite_open.go new file mode 100644 index 000000000..359a91c9e --- /dev/null +++ b/internal/users/sqlite_open.go @@ -0,0 +1,134 @@ +package users + +import ( + "context" + "database/sql" + "errors" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "github.com/meshcore-analyzer/dbconfig" + "modernc.org/sqlite" +) + +func openSQLite(path string, forbidden ...string) (*Store, error) { + return openSQLiteMode(path, true, forbidden...) +} +func openSQLiteMode(path string, create bool, forbidden ...string) (*Store, error) { + if !create && strings.TrimSpace(path) == "" { + return nil, errors.New("users: accounts are not initialized; run explicit account setup") + } + if strings.TrimSpace(path) == "" || strings.Contains(path, "://") || strings.HasPrefix(strings.ToLower(path), "file:") { + return nil, errors.New("users: SQLite account storage requires a local filesystem path") + } + abs, err := filepath.Abs(path) + if err != nil { + return nil, errors.New("users: cannot resolve account database path") + } + for _, other := range forbidden { + if strings.TrimSpace(other) != "" && samePath(abs, other) { + return nil, errors.New("users: refusing to open the measurement database") + } + } + if create { + if err := os.MkdirAll(filepath.Dir(abs), 0755); err != nil { + return nil, err + } + } else { + info, err := os.Stat(abs) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + return nil, errors.New("users: initialized SQLite accounts are missing or empty; restore the original store") + } + } + // Immediate write transactions serialize read-before-write across processes. + // The pinned modernc driver leaves explicit read-only transactions deferred. + dsn, err := sqliteAccountDSN(abs, create) + if err != nil { + return nil, err + } + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, errors.New("users: cannot open SQLite account storage") + } + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + if !create { + if err := validateExistingSQLiteAccounts(db); err != nil { + db.Close() + return nil, err + } + } + + if err := ApplySQLite(db); err != nil { + db.Close() + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := enableSQLiteWAL(ctx, db); err != nil { + db.Close() + return nil, err + } + + return &Store{db: db, target: abs, backend: dbconfig.SQLite, now: time.Now}, nil +} + +func samePath(abs, other string) bool { + otherAbs, err := filepath.Abs(other) + if err != nil { + return false + } + a, ea := os.Stat(abs) + b, eb := os.Stat(otherAbs) + if ea == nil && eb == nil { + return os.SameFile(a, b) + } + return strings.EqualFold(filepath.Clean(abs), filepath.Clean(otherAbs)) +} + +func sqliteAccountDSN(path string, create bool) (string, error) { + q := url.Values{"_pragma": {"busy_timeout(5000)", "foreign_keys(1)"}, "_txlock": {"immediate"}} + if !create { + q.Set("mode", "rw") + } + return dbconfig.SQLiteURI(path, q) +} + +func validateExistingSQLiteAccounts(db *sql.DB) error { + if err := dbconfig.AssertSQLiteImportComplete(db); err != nil { + return err + } + var count, version int + if err := db.QueryRow(`SELECT count(*),COALESCE(MAX(version),0) FROM schema_version`).Scan(&count, &version); err != nil || count != 1 || version < 1 || version > SQLiteSchemaVersion { + return errors.New("users: initialized SQLite accounts have no supported schema; restore or explicitly upgrade the original store") + } + return validateSQLiteLayout(db, version) +} + +// journal_mode may return SQLITE_BUSY immediately while another first opener +// changes the journal or creates its schema, even with busy_timeout enabled. +// Retry only that transient native result, bounded by the startup deadline. +func enableSQLiteWAL(ctx context.Context, db *sql.DB) error { + for { + var mode string + err := db.QueryRowContext(ctx, `PRAGMA journal_mode=WAL`).Scan(&mode) + if err == nil { + if mode != "wal" { + return errors.New("users: SQLite accounts require WAL mode") + } + return nil + } + var busy *sqlite.Error + if !errors.As(err, &busy) || busy.Code()&0xff != 5 { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(10 * time.Millisecond): + } + } +} diff --git a/internal/users/sqlite_runtime_open_test.go b/internal/users/sqlite_runtime_open_test.go new file mode 100644 index 000000000..2cb6e6e46 --- /dev/null +++ b/internal/users/sqlite_runtime_open_test.go @@ -0,0 +1,216 @@ +package users + +import ( + "crypto/sha256" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/meshcore-analyzer/dbconfig" +) + +func TestSelectedSQLiteAccountsCannotInitializeReplacement(t *testing.T) { + for _, kind := range []string{"missing", "empty", "header only", "wrong schema"} { + t.Run(kind, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "accounts.db") + original, err := Open(path) // Explicit setup/fixture creation. + if err != nil { + t.Fatal(err) + } + user := mustCreate(t, original, "preserved@example.invalid", "Preserved") + token, session, err := original.CreateSession(user.ID, time.Hour, "test") + if err != nil { + t.Fatal(err) + } + mustSQL(t, original.db, `INSERT INTO audit_log(id,at,action,detail) VALUES(900,1,'seed','{}'); DELETE FROM audit_log WHERE id=900`) + original.Close() + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.SQLite, StateDir: dir, DBPath: filepath.Join(dir, "telemetry.db"), UsersDBPath: path}) + if err != nil { + t.Fatal(err) + } + preserved := path + ".preserved" + if err := os.Rename(path, preserved); err != nil { + t.Fatal(err) + } + before, err := os.ReadFile(preserved) + if err != nil { + t.Fatal(err) + } + if kind == "empty" { + if err := os.WriteFile(path, nil, 0600); err != nil { + t.Fatal(err) + } + } else if kind == "header only" || kind == "wrong schema" { + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + query := `PRAGMA user_version=1` + if kind == "wrong schema" { + query = `CREATE TABLE unrelated(value TEXT); INSERT INTO unrelated VALUES('keep')` + } + if _, err := db.Exec(query); err != nil { + t.Fatal(err) + } + db.Close() + } + badBefore, _ := os.ReadFile(path) + raw, err := selected.ApplyTo(dbconfig.StorageInputs{BaseDir: dir, UsersDBPath: filepath.Join(dir, "inactive-old-setting.db")}) + if err != nil { + t.Fatal(err) + } + storage, err := dbconfig.ResolveStorage(raw) + if err != nil { + t.Fatal(err) + } + if storage.UsersDBPath != path { + t.Fatal("recorded initialized account target disappeared from resolution") + } + if st, err := OpenStorage(storage); err == nil { + st.Close() + t.Fatal("selected account storage initialized a replacement") + } + badAfter, statErr := os.ReadFile(path) + if kind == "missing" { + if !errors.Is(statErr, os.ErrNotExist) { + t.Fatal("runtime created missing accounts", statErr) + } + } else { + if statErr != nil || sha256.Sum256(badBefore) != sha256.Sum256(badAfter) { + t.Fatal("runtime modified rejected account target", statErr) + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + } + after, err := os.ReadFile(preserved) + if err != nil || sha256.Sum256(before) != sha256.Sum256(after) { + t.Fatal("preserved accounts changed", err) + } + if err := os.Rename(preserved, path); err != nil { + t.Fatal(err) + } + restored, err := OpenStorage(storage) + if err != nil { + t.Fatal(err) + } + defer restored.Close() + if got, err := restored.GetByID(user.ID); err != nil || got.Email != user.Email { + t.Fatal("user lost after restoring original store", err) + } + if got, err := restored.LookupSession(token); err != nil || got.ID != session.ID { + t.Fatal("session lost after restoring original store", err) + } + var sequence int + if err := restored.db.QueryRow(`SELECT seq FROM sqlite_sequence WHERE name='audit_log'`).Scan(&sequence); err != nil || sequence != 900 { + t.Fatal("allocator high-water lost", sequence, err) + } + }) + } +} + +func TestSQLiteAccountTargetRemovedAfterDSNValidation(t *testing.T) { + for _, replaceEmpty := range []bool{false, true} { + path := filepath.Join(t.TempDir(), "accounts.db") + setup, err := Open(path) + if err != nil { + t.Fatal(err) + } + setup.Close() + dsn, err := sqliteAccountDSN(path, false) + if err != nil { + t.Fatal(err) + } + if err = os.Rename(path, path+".preserved"); err != nil { + t.Fatal(err) + } + if replaceEmpty { + if err = os.WriteFile(path, nil, 0600); err != nil { + t.Fatal(err) + } + } + db, err := sql.Open("sqlite", dsn) + if err != nil { + t.Fatal(err) + } + if err = validateExistingSQLiteAccounts(db); err == nil { + t.Fatal("raced missing/empty target accepted") + } + db.Close() + info, err := os.Stat(path) + if replaceEmpty { + if err != nil || info.Size() != 0 { + t.Fatal("raced empty accounts were initialized", err) + } + } else if !errors.Is(err, os.ErrNotExist) { + t.Fatal("raced missing accounts were recreated", err) + } + } +} + +func TestSelectedSQLiteAccountsUpgradeSupportedInitializedVersions(t *testing.T) { + for version := 1; version < SQLiteSchemaVersion; version++ { + t.Run(fmt.Sprint(version), func(t *testing.T) { + path, db := legacySQLiteFixture(t, version, false) + mustSQL(t, db, `INSERT INTO users(id,email,display_name,password_hash,role,status,created_at) VALUES(91,'legacy@example.invalid','Legacy','hash','admin','active',1)`) + db.Close() + st, err := OpenStorage(dbconfig.Storage{Backend: dbconfig.SQLite, UsersDBPath: path}) + if err != nil { + t.Fatal(err) + } + defer st.Close() + if user, err := st.GetByID(91); err != nil || user.Email != "legacy@example.invalid" { + t.Fatal("supported initialized account row lost", err) + } + if err = AssertSQLiteReady(st.db); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestSelectedSQLiteAccountsNeedExplicitInitialization(t *testing.T) { + dir := t.TempDir() + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.SQLite, StateDir: dir, DBPath: filepath.Join(dir, "telemetry.db")}) + if err != nil { + t.Fatal(err) + } + raw, err := selected.ApplyTo(dbconfig.StorageInputs{BaseDir: dir, UsersDBPath: filepath.Join(dir, "unused.db")}) + if err != nil { + t.Fatal(err) + } + uninitialized, err := dbconfig.ResolveStorage(raw) + if err != nil { + t.Fatal(err) + } + if uninitialized.UsersDBPath != "" { + t.Fatal("uninitialized selected accounts received an inferred file") + } + if st, err := OpenStorage(uninitialized); err == nil { + st.Close() + t.Fatal("uninitialized accounts were opened") + } + path := filepath.Join(t.TempDir(), "new", "accounts.db") + if st, err := OpenStorage(dbconfig.Storage{Backend: dbconfig.SQLite, UsersDBPath: path}); err == nil { + st.Close() + t.Fatal("selected runtime performed first initialization") + } + if _, err := os.Stat(filepath.Dir(path)); !errors.Is(err, os.ErrNotExist) { + t.Fatal("runtime created account directory", err) + } + setup, err := Open(path) + if err != nil { + t.Fatal(err) + } + setup.Close() + runtime, err := OpenStorage(dbconfig.Storage{Backend: dbconfig.SQLite, UsersDBPath: path}) + if err != nil { + t.Fatal("explicitly initialized accounts failed to open", err) + } + runtime.Close() +} diff --git a/internal/users/sqlite_schema.go b/internal/users/sqlite_schema.go new file mode 100644 index 000000000..e2730dee4 --- /dev/null +++ b/internal/users/sqlite_schema.go @@ -0,0 +1,410 @@ +package users + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "strings" + "unicode" + + "github.com/meshcore-analyzer/dbconfig" +) + +// Version 6 adds non-reusing physical identities. PostgreSQL retains its +// existing version 5 layout; the five historical SQLite migrations are intact. +const SQLiteSchemaVersion = 6 + +var sqliteIdentities = []struct{ name, columns, body string }{ + {"users", "id,email,display_name,password_hash,role,status,created_at,activated_at,activated_by,last_login_at,email_bouncing", `( + id INTEGER PRIMARY KEY AUTOINCREMENT, email TEXT NOT NULL UNIQUE, display_name TEXT NOT NULL, password_hash TEXT NOT NULL, + role TEXT NOT NULL DEFAULT 'user' CHECK (role IN ('user','admin')), + status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','active','disabled')), + created_at INTEGER NOT NULL, activated_at INTEGER, activated_by INTEGER, last_login_at INTEGER, + email_bouncing INTEGER NOT NULL DEFAULT 0)`}, + {"sessions", "id,token_hash,user_id,csrf_token,created_at,expires_at,last_seen_at,user_agent", `( + id INTEGER PRIMARY KEY AUTOINCREMENT, token_hash TEXT NOT NULL UNIQUE, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, csrf_token TEXT NOT NULL, + created_at INTEGER NOT NULL, expires_at INTEGER NOT NULL, last_seen_at INTEGER NOT NULL, + user_agent TEXT NOT NULL DEFAULT '')`}, + {"audit_log", "id,at,actor_user_id,action,target_user_id,detail", `( + id INTEGER PRIMARY KEY AUTOINCREMENT, at INTEGER NOT NULL, actor_user_id INTEGER, + action TEXT NOT NULL, target_user_id INTEGER, detail TEXT NOT NULL DEFAULT '{}')`}, + {"mail_log", "id,user_id,to_email,purpose,provider_message_id,sent_at,last_event,last_event_at,last_reason", `( + id INTEGER PRIMARY KEY AUTOINCREMENT, user_id INTEGER REFERENCES users(id) ON DELETE SET NULL, + to_email TEXT NOT NULL, purpose TEXT NOT NULL, provider_message_id TEXT, sent_at INTEGER NOT NULL, + last_event TEXT NOT NULL DEFAULT 'sent', last_event_at INTEGER NOT NULL, last_reason TEXT NOT NULL DEFAULT '')`}, + {"mail_events", "id,mail_id,event,at,reason", `( + id INTEGER PRIMARY KEY AUTOINCREMENT, mail_id INTEGER NOT NULL REFERENCES mail_log(id) ON DELETE CASCADE, + event TEXT NOT NULL, at INTEGER NOT NULL, reason TEXT NOT NULL DEFAULT '', UNIQUE (mail_id,event,at))`}, + {"proposals", "id,kind,subject,status,proposer_id,reviewer_id,note,created_at,decided_at", `( + id INTEGER PRIMARY KEY AUTOINCREMENT, kind TEXT NOT NULL, subject TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending','approved','rejected','revoked')), + proposer_id INTEGER REFERENCES users(id) ON DELETE SET NULL, reviewer_id INTEGER REFERENCES users(id) ON DELETE SET NULL, + note TEXT NOT NULL DEFAULT '', created_at INTEGER NOT NULL, decided_at INTEGER)`}, +} + +type sqliteSchemaQueryer interface { + QueryContext(context.Context, string, ...any) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...any) *sql.Row +} + +func sqliteQuote(name string) string { return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` } + +// Normalize only for schema comparison, never for execution or query rewriting. +// Keep quoted content intact so different defaults/check literals cannot match. +func normalizedSQLiteDDL(text string) string { + var out strings.Builder + var quote rune + for _, r := range text { + if quote != 0 { + out.WriteRune(r) + if r == quote { + quote = 0 + } + continue + } + if r == '\'' || r == '"' || r == '`' { + quote = r + out.WriteRune(r) + continue + } + if !unicode.IsSpace(r) { + out.WriteRune(unicode.ToUpper(r)) + } + } + return out.String() +} + +func sqliteObjects(q sqliteSchemaQueryer) (map[string]string, error) { + rows, err := q.QueryContext(context.Background(), `SELECT type,name,sql FROM sqlite_schema WHERE sql IS NOT NULL`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]string{} + for rows.Next() { + var kind, name, ddl string + if err := rows.Scan(&kind, &name, &ddl); err != nil { + return nil, err + } + if kind == "table" && (name == "schema_version" || name == "sqlite_sequence" || name == "sqlite_stat1" || name == "sqlite_stat2" || name == "sqlite_stat3" || name == "sqlite_stat4") { + continue + } + out[kind+":"+name] = normalizedSQLiteDDL(ddl) + } + return out, rows.Err() +} + +// Legacy account schemas are defined by the original append-only migrations. +// Refuse unsupported extensions rather than lose their columns or constraints +// during a table rebuild. Current version 6 never repeats this migration. +func validateSQLiteLayout(q sqliteSchemaQueryer, version int) error { + expected, err := sql.Open("sqlite", ":memory:") + if err != nil { + return err + } + defer expected.Close() + expected.SetMaxOpenConns(1) + legacyVersion := version + if legacyVersion > len(legacyMigrations) { + legacyVersion = len(legacyMigrations) + } + for _, migration := range legacyMigrations[:legacyVersion] { + for _, statement := range migration { + if _, err := expected.Exec(statement); err != nil { + return err + } + } + } + want, err := sqliteObjects(expected) + if err != nil { + return err + } + if version == SQLiteSchemaVersion { + for _, table := range sqliteIdentities { + want["table:"+table.name] = normalizedSQLiteDDL(`CREATE TABLE ` + table.name + table.body) + } + } + got, err := sqliteObjects(q) + if err != nil { + return err + } + if len(got) != len(want) { + return errors.New("users: unsupported SQLite account schema; preserve the database and review its extra or missing objects before upgrading") + } + for name, ddl := range want { + actual, ok := got[name] + if !ok { + return errors.New("users: incomplete SQLite account schema") + } + if version == SQLiteSchemaVersion && strings.HasPrefix(name, "table:") { + tableName := strings.TrimPrefix(name, "table:") + quoted := strings.Replace(ddl, "CREATETABLE"+strings.ToUpper(tableName)+"(", "CREATETABLE"+sqliteQuote(tableName)+"(", 1) + if actual == quoted { + continue + } + } + if actual == ddl { + continue + } + // Accept an already non-reusing allocator for a known INTEGER PK, and + // preserve its sequence high water. Hidden mail-event rowids are v1-v5. + safeIdentity := name == "table:users" || name == "table:sessions" || name == "table:audit_log" || name == "table:mail_log" || name == "table:proposals" + if safeIdentity && actual == strings.Replace(ddl, "IDINTEGERPRIMARYKEY", "IDINTEGERPRIMARYKEYAUTOINCREMENT", 1) { + continue + } + return errors.New("users: unsupported SQLite account columns or constraints; the original database was not changed") + } + return nil +} + +// ApplySQLite atomically upgrades known SQLite account layouts through version 6. +// A reserved connection keeps FK settings local; BEGIN IMMEDIATE serializes +// competing migration/writer processes before any read-modify-write work. +func ApplySQLite(db *sql.DB) (err error) { + if err := dbconfig.AssertSQLiteImportComplete(db); err != nil { + return err + } + var v int + if e := db.QueryRow(`SELECT version FROM schema_version`).Scan(&v); e == nil && v == SQLiteSchemaVersion { + return AssertSQLiteReady(db) + } + ctx := context.Background() + conn, err := db.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + var foreignKeys int + if err := conn.QueryRowContext(ctx, `PRAGMA foreign_keys`).Scan(&foreignKeys); err != nil { + return err + } + if _, err := conn.ExecContext(ctx, `PRAGMA foreign_keys=OFF`); err != nil { + return err + } + defer func() { + if _, restore := conn.ExecContext(ctx, fmt.Sprintf(`PRAGMA foreign_keys=%d`, foreignKeys)); restore != nil { + _ = conn.Raw(func(any) error { return driver.ErrBadConn }) + err = errors.Join(err, errors.New("users: cannot restore SQLite connection foreign-key enforcement")) + } + }() + if _, err := conn.ExecContext(ctx, `BEGIN IMMEDIATE`); err != nil { + return err + } + defer conn.ExecContext(ctx, `ROLLBACK`) + var hasVersion int + if err := conn.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_schema WHERE type='table' AND name='schema_version'`).Scan(&hasVersion); err != nil { + return err + } + if hasVersion == 0 { + objects, err := sqliteObjects(conn) + if err != nil { + return err + } + if len(objects) != 0 { + return errors.New("users: refusing to initialize accounts in an existing unrelated SQLite database") + } + if _, err := conn.ExecContext(ctx, `CREATE TABLE schema_version(version INTEGER NOT NULL); INSERT INTO schema_version VALUES(0)`); err != nil { + return err + } + } + var count int + if err := conn.QueryRowContext(ctx, `SELECT count(*),COALESCE(MAX(version),0) FROM schema_version`).Scan(&count, &v); err != nil { + return err + } + // The historical initializer committed CREATE before the version-0 row. + // Recover only its empty layout, never reinterpret occupied data as fresh. + if count == 0 { + if err := validateSQLiteLayout(conn, 0); err != nil { + return err + } + if _, err := conn.ExecContext(ctx, `INSERT INTO schema_version(version) VALUES(0)`); err != nil { + return err + } + count = 1 + } + if count != 1 || v < 0 || v > SQLiteSchemaVersion { + return errors.New("users: invalid or unsupported SQLite account schema version") + } + if v == SQLiteSchemaVersion { + if err := assertSQLiteReady(conn); err != nil { + return err + } + _, err := conn.ExecContext(ctx, `COMMIT`) + return err + } + if err := validateSQLiteLayout(conn, v); err != nil { + return err + } + for i := v; i < len(legacyMigrations); i++ { + for _, statement := range legacyMigrations[i] { + if _, err := conn.ExecContext(ctx, statement); err != nil { + return fmt.Errorf("users: SQLite migration %d: %w", i+1, err) + } + } + } + high, err := sqliteAccountHighWater(conn) + if err != nil { + return err + } + indexRows, err := conn.QueryContext(ctx, `SELECT sql FROM sqlite_schema WHERE type='index' AND sql IS NOT NULL + AND tbl_name IN ('users','sessions','audit_log','mail_log','mail_events','proposals') ORDER BY name`) + if err != nil { + return err + } + var indexes []string + for indexRows.Next() { + var statement string + if err := indexRows.Scan(&statement); err != nil { + indexRows.Close() + return err + } + indexes = append(indexes, statement) + } + indexErr := indexRows.Err() + indexRows.Close() + if indexErr != nil { + return indexErr + } + for _, table := range sqliteIdentities { + temporary := "corescope_identity_" + table.name + if _, err := conn.ExecContext(ctx, `CREATE TABLE `+sqliteQuote(temporary)+table.body); err != nil { + return err + } + columns := table.columns + sourceColumns := columns + if table.name == "mail_events" { + sourceColumns = "rowid," + strings.TrimPrefix(columns, "id,") + } + if _, err := conn.ExecContext(ctx, `INSERT INTO `+sqliteQuote(temporary)+` (`+columns+`) SELECT `+sourceColumns+` FROM `+sqliteQuote(table.name)); err != nil { + return err + } + if _, err := conn.ExecContext(ctx, `DROP TABLE `+sqliteQuote(table.name)); err != nil { + return err + } + if _, err := conn.ExecContext(ctx, `ALTER TABLE `+sqliteQuote(temporary)+` RENAME TO `+sqliteQuote(table.name)); err != nil { + return err + } + res, err := conn.ExecContext(ctx, `UPDATE sqlite_sequence SET seq=?2 WHERE name=?1`, table.name, high[table.name]) + if err != nil { + return err + } + n, err := res.RowsAffected() + if err != nil { + return err + } + if n == 0 { + if _, err := conn.ExecContext(ctx, `INSERT INTO sqlite_sequence(name,seq) VALUES(?1,?2)`, table.name, high[table.name]); err != nil { + return err + } + } + } + // Recreate the exact declared indexes lost when their owning tables changed. + for _, statement := range indexes { + if _, err := conn.ExecContext(ctx, statement); err != nil { + return err + } + } + rows, err := conn.QueryContext(ctx, `PRAGMA foreign_key_check`) + if err != nil { + return err + } + invalid := rows.Next() + rowErr := rows.Err() + rows.Close() + if rowErr != nil { + return rowErr + } + if invalid { + return errors.New("users: SQLite account identity migration found invalid foreign keys; all changes were rolled back") + } + if _, err := conn.ExecContext(ctx, `UPDATE schema_version SET version=?1`, SQLiteSchemaVersion); err != nil { + return err + } + if err := assertSQLiteReady(conn); err != nil { + return err + } + _, err = conn.ExecContext(ctx, `COMMIT`) + return err +} + +func sqliteAccountHighWater(q sqliteSchemaQueryer) (map[string]int64, error) { + ctx := context.Background() + high := map[string]int64{} + var hasSequence int + if err := q.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_schema WHERE name='sqlite_sequence' AND type='table'`).Scan(&hasSequence); err != nil { + return nil, err + } + if hasSequence != 0 { + rows, err := q.QueryContext(ctx, `SELECT name,MAX(seq),count(*) FROM sqlite_sequence GROUP BY name`) + if err != nil { + return nil, err + } + for rows.Next() { + var name string + var seq int64 + var count int + if err := rows.Scan(&name, &seq, &count); err != nil { + rows.Close() + return nil, err + } + if count != 1 || seq < 0 { + rows.Close() + return nil, errors.New("users: invalid SQLite account sequence state") + } + high[name] = seq + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + } + for _, table := range sqliteIdentities { + column := "id" + if table.name == "mail_events" { + column = "rowid" + } + var n int64 + if err := q.QueryRowContext(ctx, `SELECT COALESCE(MAX(`+sqliteQuote(column)+`),0) FROM `+sqliteQuote(table.name)).Scan(&n); err != nil { + return nil, err + } + if n > high[table.name] { + high[table.name] = n + } + } + for _, ref := range []struct{ target, table, column string }{ + {"users", "audit_log", "actor_user_id"}, {"users", "audit_log", "target_user_id"}, {"users", "users", "activated_by"}, + {"users", "sessions", "user_id"}, {"users", "tokens", "user_id"}, {"users", "mail_log", "user_id"}, + {"users", "proposals", "proposer_id"}, {"users", "proposals", "reviewer_id"}, + {"users", "user_settings", "user_id"}, {"users", "notification_prefs", "user_id"}, + {"users", "notification_watches", "user_id"}, {"users", "notification_state", "user_id"}, {"mail_log", "mail_events", "mail_id"}, + } { + var n int64 + if err := q.QueryRowContext(ctx, `SELECT COALESCE(MAX(`+sqliteQuote(ref.column)+`),0) FROM `+sqliteQuote(ref.table)).Scan(&n); err != nil { + return nil, err + } + if n > high[ref.target] { + high[ref.target] = n + } + } + return high, nil +} + +func AssertSQLiteReady(db *sql.DB) error { + if err := dbconfig.AssertSQLiteImportComplete(db); err != nil { + return err + } + return assertSQLiteReady(db) +} + +func assertSQLiteReady(q sqliteSchemaQueryer) error { + ctx := context.Background() + var count, version int + if err := q.QueryRowContext(ctx, `SELECT count(*),COALESCE(MAX(version),0) FROM schema_version`).Scan(&count, &version); err != nil || count != 1 || version != SQLiteSchemaVersion { + return errors.New("users: SQLite account schema needs a supported upgrade") + } + return validateSQLiteLayout(q, SQLiteSchemaVersion) +} diff --git a/internal/users/stats.go b/internal/users/stats.go index 2883b5076..6093f62e9 100644 --- a/internal/users/stats.go +++ b/internal/users/stats.go @@ -1,6 +1,9 @@ package users -import "time" +import ( + "github.com/meshcore-analyzer/dbconfig" + "time" +) // Admin overview thresholds (docs/specs/2026-10-07-admin-dashboard-design.md). // Constants in this version; they belong in config later (AGENTS.md rule 8). @@ -65,14 +68,14 @@ func (s *Store) Stats(now time.Time) (Stats, error) { COUNT(*) FILTER (WHERE status = 'pending'), COUNT(*) FILTER (WHERE status = 'disabled'), COUNT(*) FILTER (WHERE role = 'admin' AND status = 'active'), - COUNT(*) FILTER (WHERE status = 'pending' AND created_at < $1), + COUNT(*) FILTER (WHERE status = 'pending' AND created_at < `+s.p(1)+`), COUNT(*) FILTER (WHERE email_bouncing != 0) FROM users`, t-secs(StuckPendingAfter)). Scan(&st.Total, &st.Active, &st.Pending, &st.Disabled, &st.Admins, &st.StuckPending, &st.Bouncing); err != nil { return Stats{}, err } - if err := s.db.QueryRow(`SELECT COUNT(*) FILTER (WHERE at >= $1), COUNT(*) FROM audit_log - WHERE action = 'user.register' AND at >= $2 AND at <= $3`, t-secs(7*day), t-secs(30*day), t). + if err := s.db.QueryRow(`SELECT COUNT(*) FILTER (WHERE at >= `+s.p(1)+`), COUNT(*) FROM audit_log + WHERE action = 'user.register' AND at >= `+s.p(2)+` AND at <= `+s.p(3), t-secs(7*day), t-secs(30*day), t). Scan(&st.New7d, &st.New30d); err != nil { return Stats{}, err } @@ -81,16 +84,20 @@ func (s *Store) Stats(now time.Time) (Stats, error) { return Stats{}, err } st.NewPerDay = perDay - if err := s.db.QueryRow(`SELECT COUNT(*) FILTER (WHERE last >= $1), COUNT(*) FROM ( - SELECT GREATEST(COALESCE(u.last_login_at, 0), + greatest := "GREATEST" + if s.backend == dbconfig.SQLite { + greatest = "MAX" + } + if err := s.db.QueryRow(`SELECT COUNT(*) FILTER (WHERE last >= `+s.p(1)+`), COUNT(*) FROM ( + SELECT `+greatest+`(COALESCE(u.last_login_at, 0), COALESCE((SELECT MAX(last_seen_at) FROM sessions s WHERE s.user_id = u.id), 0)) AS last FROM users u - ) activity WHERE last >= $2`, t-secs(7*day), t-secs(30*day)). + ) activity WHERE last >= `+s.p(2), t-secs(7*day), t-secs(30*day)). Scan(&st.Active7d, &st.Active30d); err != nil { return Stats{}, err } if err := s.db.QueryRow(`SELECT COUNT(*) FILTER (WHERE action = 'user.login'), COUNT(*) FILTER (WHERE action = 'user.login.failed') - FROM audit_log WHERE action IN ('user.login', 'user.login.failed') AND at >= $1`, t-secs(day)). + FROM audit_log WHERE action IN ('user.login', 'user.login.failed') AND at >= `+s.p(1), t-secs(day)). Scan(&st.Logins24h, &st.FailedLogins24h); err != nil { return Stats{}, err } @@ -107,8 +114,8 @@ func (s *Store) Stats(now time.Time) (Stats, error) { // statsPerDayDays days, today included. func (s *Store) registrationsPerDay(now time.Time) ([]DayCount, error) { first := now.UTC().Truncate(24*time.Hour).AddDate(0, 0, -(statsPerDayDays - 1)) - rows, err := s.db.Query(`SELECT (at - $1) / 86400, COUNT(*) FROM audit_log - WHERE action = 'user.register' AND at >= $2 AND at <= $3 GROUP BY 1`, unix(first), unix(first), unix(now)) + rows, err := s.db.Query(`SELECT (at - `+s.p(1)+`) / 86400, COUNT(*) FROM audit_log + WHERE action = 'user.register' AND at >= `+s.p(2)+` AND at <= `+s.p(3)+` GROUP BY 1`, unix(first), unix(first), unix(now)) if err != nil { return nil, err } @@ -134,7 +141,7 @@ func (s *Store) registrationsPerDay(now time.Time) ([]DayCount, error) { func (s *Store) mailCounts(since int64) (MailCounts, error) { var mc MailCounts - rows, err := s.db.Query(`SELECT last_event, COUNT(*) FROM mail_log WHERE sent_at >= $1 GROUP BY last_event`, since) + rows, err := s.db.Query(`SELECT last_event, COUNT(*) FROM mail_log WHERE sent_at >= `+s.p(1)+` GROUP BY last_event`, since) if err != nil { return mc, err } @@ -166,8 +173,8 @@ func (s *Store) mailCounts(since int64) (MailCounts, error) { func (s *Store) guessedAccounts(since int64) ([]GuessedAccount, error) { rows, err := s.db.Query(`SELECT a.target_user_id, COALESCE(u.display_name, ''), COUNT(*) AS n FROM audit_log a LEFT JOIN users u ON u.id = a.target_user_id - WHERE a.action = 'user.login.failed' AND a.at >= $1 AND a.target_user_id IS NOT NULL - GROUP BY a.target_user_id,u.display_name HAVING COUNT(*) >= $2 ORDER BY n DESC, a.target_user_id LIMIT $3`, + WHERE a.action = 'user.login.failed' AND a.at >= `+s.p(1)+` AND a.target_user_id IS NOT NULL + GROUP BY a.target_user_id,u.display_name HAVING COUNT(*) >= `+s.p(2)+` ORDER BY n DESC, a.target_user_id LIMIT `+s.p(3), since, GuessingThreshold, statsGuessingLimit) if err != nil { return nil, err diff --git a/internal/users/store.go b/internal/users/store.go index 0ebd482dc..6014d1743 100644 --- a/internal/users/store.go +++ b/internal/users/store.go @@ -1,6 +1,5 @@ -// Package users is CoreScope's optional PostgreSQL account store. Account -// state lives in a separate database from telemetry; runtime connections -// never bootstrap schemas or write telemetry (#1283). +// Package users owns optional account state, separate from telemetry. One +// account implementation uses native SQLite or PostgreSQL storage (#1283). package users import ( @@ -11,7 +10,9 @@ import ( "time" "github.com/jackc/pgx/v5/pgconn" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/pgutil" + "modernc.org/sqlite" ) var ( @@ -25,20 +26,40 @@ var ( ErrAccountChanged = errors.New("users: account changed since it was read") ) -// A SQL NULL limit means every row for full account exports. -var noLimit any = nil - // Store owns a separate account database. Safe for concurrent use. type Store struct { - db *sql.DB - databaseURL string - now func() time.Time + db *sql.DB + target string + backend dbconfig.Backend + now func() time.Time } -// Open connects using the account runtime role, never applying migrations. +// OpenStorage consumes the resolved installation choice. Missing PostgreSQL +// settings never cause a SQLite fallback, and inactive settings are ignored. +func OpenStorage(storage dbconfig.Storage) (*Store, error) { + switch storage.Backend { + case dbconfig.SQLite: + return openSQLiteMode(storage.UsersDBPath, false, storage.DBPath) + case dbconfig.Postgres: + return openPostgres(storage.UsersDatabaseURL, storage.ReaderDatabaseURL, storage.WriterDatabaseURL) + default: + return nil, errors.New("users: account storage backend must be selected") + } +} + +// Open retains compatibility for an unambiguous SQLite path or PostgreSQL URL. +// Application startup should pass its resolved choice to OpenStorage instead. +func Open(target string, forbidden ...string) (*Store, error) { + if strings.HasPrefix(strings.ToLower(target), "postgres:") || strings.HasPrefix(strings.ToLower(target), "postgresql:") { + return openPostgres(target, forbidden...) + } + return openSQLite(target, forbidden...) +} + +// openPostgres connects using the account runtime role, never applying migrations. // Every forbidden URL is checked against the effective PostgreSQL database, // so alternate hostnames, credentials or search paths cannot bypass isolation. -func Open(databaseURL string, forbidden ...string) (*Store, error) { +func openPostgres(databaseURL string, forbidden ...string) (*Store, error) { if _, err := pgutil.ParseConfig(databaseURL); err != nil { return nil, err } @@ -70,7 +91,26 @@ func Open(databaseURL string, forbidden ...string) (*Store, error) { db.Close() return nil, err } - return &Store{db: db, databaseURL: databaseURL, now: time.Now}, nil + return &Store{db: db, target: databaseURL, backend: dbconfig.Postgres, now: time.Now}, nil +} + +// Backend identifies the native snapshot/query format without exposing targets. +func (s *Store) Backend() dbconfig.Backend { return s.backend } + +func (s *Store) p(position int) string { return s.backend.Parameter(position) } + +func (s *Store) noLimit() any { + if s.backend == dbconfig.SQLite { + return -1 + } + return nil +} + +func (s *Store) forUpdate() string { + if s.backend == dbconfig.Postgres { + return " FOR UPDATE" + } + return "" } func assertRuntimePrivileges(db *sql.DB) error { @@ -94,9 +134,9 @@ func assertRuntimePrivileges(db *sql.DB) error { // lockUser serializes account-level read/modify/write operations, including // the initial insert where no settings, tokens or watch rows exist yet. -func lockUser(tx *sql.Tx, id int64) error { +func (s *Store) lockUser(tx *sql.Tx, id int64) error { var found int64 - err := tx.QueryRow(`SELECT id FROM users WHERE id=$1 FOR UPDATE`, id).Scan(&found) + err := tx.QueryRow(`SELECT id FROM users WHERE id=`+s.p(1)+s.forUpdate(), id).Scan(&found) if errors.Is(err, sql.ErrNoRows) { return ErrNotFound } @@ -145,5 +185,9 @@ func expectOne(res sql.Result, err error) error { func isUniqueViolation(err error) bool { var pgerr *pgconn.PgError - return errors.As(err, &pgerr) && pgerr.Code == "23505" + if errors.As(err, &pgerr) { + return pgerr.Code == "23505" + } + var sqliteErr *sqlite.Error + return errors.As(err, &sqliteErr) && (sqliteErr.Code() == 2067 || sqliteErr.Code() == 1555) } diff --git a/internal/users/store_test.go b/internal/users/store_test.go index 4fa448296..fcb130ebb 100644 --- a/internal/users/store_test.go +++ b/internal/users/store_test.go @@ -1,17 +1,18 @@ package users import ( - "github.com/meshcore-analyzer/pgutil/pgtest" + "github.com/meshcore-analyzer/dbconfig" "net/url" + "path/filepath" "strings" "sync" "testing" ) func TestOpenCreatesSchemaAndIsIdempotent(t *testing.T) { - dsn := pgtest.NewSchema(t) + dsn := testTarget(t, false) db := testOwner(t, dsn) - if err := Apply(db); err != nil { + if err := applyTestSchema(t, db); err != nil { t.Fatal(err) } runtimeDSN := testRuntimeURL(t, dsn) @@ -20,19 +21,26 @@ func TestOpenCreatesSchemaAndIsIdempotent(t *testing.T) { if err != nil { t.Fatal(err) } - if v, err := st.SchemaVersion(); err != nil || v != CurrentSchemaVersion { + if v, err := st.SchemaVersion(); err != nil || v != schemaVersion(t) { t.Fatalf("schema=%d,%v", v, err) } st.Close() - if err := Apply(db); err != nil { + if err := applyTestSchema(t, db); err != nil { t.Fatal(err) } } } func TestOpenRefusesForbiddenPath(t *testing.T) { - dsn := pgtest.NewSchema(t) + dsn := testTarget(t, false) alias, _ := url.Parse(dsn) + if testBackend(t) == dbconfig.SQLite { + if st, err := Open(dsn, filepath.Join(filepath.Dir(dsn), ".", filepath.Base(dsn))); err == nil { + st.Close() + t.Fatal("same SQLite target accepted") + } + return + } q := alias.Query() q.Set("application_name", "different-client") q.Set("search_path", "public") @@ -46,28 +54,28 @@ func TestOpenRefusesForbiddenPath(t *testing.T) { } func TestOpenRejectsNewerSchema(t *testing.T) { - dsn := pgtest.NewSchema(t) + dsn := testTarget(t, false) db := testOwner(t, dsn) - if err := Apply(db); err != nil { + if err := applyTestSchema(t, db); err != nil { t.Fatal(err) } runtimeDSN := testRuntimeURL(t, dsn) if _, err := db.Exec(`UPDATE schema_version SET version=999`); err != nil { t.Fatal(err) } - if st, err := Open(runtimeDSN); err == nil || !strings.Contains(err.Error(), "newer") { + if st, err := Open(runtimeDSN); err == nil { if st != nil { st.Close() } t.Fatalf("Open with newer schema error=%v", err) } - if err := Apply(db); err == nil { + if err := applyTestSchema(t, db); err == nil { t.Fatal("bootstrap accepted a newer schema") } } func TestOpenRejectsDSNCharacters(t *testing.T) { - for _, dsn := range []string{"users.db", "users?.db", "file:users.db", "postgresql://u:p@host/a?host=elsewhere", "postgresql://u:p@host1,host2/a"} { + for _, dsn := range []string{"", "file:users.db", "postgresql://u:p@host/a?host=elsewhere", "postgresql://u:p@host1,host2/a"} { if st, err := Open(dsn); err == nil { st.Close() t.Fatal("unsupported database address accepted") @@ -76,14 +84,15 @@ func TestOpenRejectsDSNCharacters(t *testing.T) { } func TestApplyConcurrentAndIncomplete(t *testing.T) { - dsn := pgtest.NewSchema(t) + requirePostgres(t) + dsn := testTarget(t, false) db := testOwner(t, dsn) var wg sync.WaitGroup for i := 0; i < 4; i++ { wg.Add(1) go func() { defer wg.Done() - if err := Apply(db); err != nil { + if err := applyTestSchema(t, db); err != nil { t.Errorf("concurrent bootstrap: %v", err) } }() @@ -95,7 +104,7 @@ func TestApplyConcurrentAndIncomplete(t *testing.T) { if err := AssertReady(db); err == nil { t.Fatal("incomplete import accepted") } - if err := Apply(db); err == nil { + if err := applyTestSchema(t, db); err == nil { t.Fatal("bootstrap completed an unverified import") } } diff --git a/internal/users/users.go b/internal/users/users.go index 5a90d9aba..7a7f38979 100644 --- a/internal/users/users.go +++ b/internal/users/users.go @@ -72,7 +72,7 @@ func scanUser(row rowScanner) (*User, error) { func (s *Store) CreatePending(email, displayName, passwordHash string) (*User, error) { var id int64 err := s.db.QueryRow(`INSERT INTO users (email, display_name, password_hash, role, status, created_at) - VALUES ($1, $2, $3, 'user', 'pending', $4) RETURNING id`, email, displayName, passwordHash, unix(s.now())).Scan(&id) + VALUES (`+s.p(1)+`, `+s.p(2)+`, `+s.p(3)+`, 'user', 'pending', `+s.p(4)+`) RETURNING id`, email, displayName, passwordHash, unix(s.now())).Scan(&id) if err != nil { if isUniqueViolation(err) { return nil, ErrEmailTaken @@ -83,40 +83,40 @@ func (s *Store) CreatePending(email, displayName, passwordHash string) (*User, e } func (s *Store) GetByID(id int64) (*User, error) { - return scanUser(s.db.QueryRow(`SELECT `+userCols+` FROM users WHERE id = $1`, id)) + return scanUser(s.db.QueryRow(`SELECT `+userCols+` FROM users WHERE id = `+s.p(1), id)) } // GetByEmail looks up a normalized address. func (s *Store) GetByEmail(email string) (*User, error) { - return scanUser(s.db.QueryRow(`SELECT `+userCols+` FROM users WHERE email = $1`, email)) + return scanUser(s.db.QueryRow(`SELECT `+userCols+` FROM users WHERE email = `+s.p(1), email)) } // Activate moves a pending user to active with role. by is the admin who // activated manually, nil for link activation. ErrNotFound if not pending. func (s *Store) Activate(id int64, role Role, by *int64) error { - return expectOne(s.db.Exec(`UPDATE users SET status = 'active', role = $1, activated_at = $2, activated_by = $3 - WHERE id = $4 AND status = 'pending'`, string(role), unix(s.now()), nullInt(by), id)) + return expectOne(s.db.Exec(`UPDATE users SET status = 'active', role = `+s.p(1)+`, activated_at = `+s.p(2)+`, activated_by = `+s.p(3)+` + WHERE id = `+s.p(4)+` AND status = 'pending'`, string(role), unix(s.now()), nullInt(by), id)) } func (s *Store) SetStatus(id int64, st Status) error { - return expectOne(s.db.Exec(`UPDATE users SET status = $1 WHERE id = $2`, string(st), id)) + return expectOne(s.db.Exec(`UPDATE users SET status = `+s.p(1)+` WHERE id = `+s.p(2), string(st), id)) } func (s *Store) SetRole(id int64, r Role) error { - return expectOne(s.db.Exec(`UPDATE users SET role = $1 WHERE id = $2`, string(r), id)) + return expectOne(s.db.Exec(`UPDATE users SET role = `+s.p(1)+` WHERE id = `+s.p(2), string(r), id)) } func (s *Store) SetPassword(id int64, hash string) error { - return expectOne(s.db.Exec(`UPDATE users SET password_hash = $1 WHERE id = $2`, hash, id)) + return expectOne(s.db.Exec(`UPDATE users SET password_hash = `+s.p(1)+` WHERE id = `+s.p(2), hash, id)) } func (s *Store) SetDisplayName(id int64, name string) error { - return expectOne(s.db.Exec(`UPDATE users SET display_name = $1 WHERE id = $2`, name, id)) + return expectOne(s.db.Exec(`UPDATE users SET display_name = `+s.p(1)+` WHERE id = `+s.p(2), name, id)) } // SetEmail changes the address (normalized) and clears the bounce flag. func (s *Store) SetEmail(id int64, email string) error { - err := expectOne(s.db.Exec(`UPDATE users SET email = $1, email_bouncing = 0 WHERE id = $2`, email, id)) + err := expectOne(s.db.Exec(`UPDATE users SET email = `+s.p(1)+`, email_bouncing = 0 WHERE id = `+s.p(2), email, id)) if isUniqueViolation(err) { return ErrEmailTaken } @@ -128,11 +128,11 @@ func (s *Store) SetEmailBouncing(id int64, v bool) error { if v { b = 1 } - return expectOne(s.db.Exec(`UPDATE users SET email_bouncing = $1 WHERE id = $2`, b, id)) + return expectOne(s.db.Exec(`UPDATE users SET email_bouncing = `+s.p(1)+` WHERE id = `+s.p(2), b, id)) } func (s *Store) TouchLogin(id int64) error { - return expectOne(s.db.Exec(`UPDATE users SET last_login_at = $1 WHERE id = $2`, unix(s.now()), id)) + return expectOne(s.db.Exec(`UPDATE users SET last_login_at = `+s.p(1)+` WHERE id = `+s.p(2), unix(s.now()), id)) } // Delete removes a user; sessions and tokens cascade. Mail-log rows survive @@ -144,16 +144,16 @@ func (s *Store) Delete(id int64) error { } defer tx.Rollback() var exists int - if err := tx.QueryRow(`SELECT 1 FROM users WHERE id = $1 FOR UPDATE`, id).Scan(&exists); err != nil { + if err := tx.QueryRow(`SELECT 1 FROM users WHERE id = `+s.p(1)+s.forUpdate(), id).Scan(&exists); err != nil { if errors.Is(err, sql.ErrNoRows) { return ErrNotFound } return err } - if err := hashMailLogTx(tx, id); err != nil { + if err := s.hashMailLogTx(tx, id); err != nil { return err } - if _, err := tx.Exec(`DELETE FROM users WHERE id = $1`, id); err != nil { + if _, err := tx.Exec(`DELETE FROM users WHERE id = `+s.p(1), id); err != nil { return err } return tx.Commit() @@ -172,11 +172,11 @@ func (s *Store) List(f ListFilter) ([]User, error) { q := `SELECT ` + userCols + ` FROM users WHERE 1=1` var args []any if f.Status != "" { - q += fmt.Sprintf(` AND status = $%d`, len(args)+1) + q += fmt.Sprintf(` AND status = %s`, s.p(len(args)+1)) args = append(args, string(f.Status)) } if f.Role != "" { - q += fmt.Sprintf(` AND role = $%d`, len(args)+1) + q += fmt.Sprintf(` AND role = %s`, s.p(len(args)+1)) args = append(args, string(f.Role)) } if f.Bouncing { @@ -184,7 +184,7 @@ func (s *Store) List(f ListFilter) ([]User, error) { } if t := strings.TrimSpace(f.Query); t != "" { like := "%" + escapeLike(strings.ToLower(t)) + "%" - q += fmt.Sprintf(` AND (email LIKE $%d ESCAPE '\' OR lower(display_name) LIKE $%d ESCAPE '\')`, len(args)+1, len(args)+2) + q += fmt.Sprintf(` AND (email LIKE %s ESCAPE '\' OR lower(display_name) LIKE %s ESCAPE '\')`, s.p(len(args)+1), s.p(len(args)+2)) args = append(args, like, like) } q += ` ORDER BY created_at DESC, id DESC LIMIT 1000` @@ -226,7 +226,7 @@ func (s *Store) UsersByID(ids []int64) (map[int64]User, error) { for i, id := range ids { args[i] = id } - ph := placeholders(len(ids)) + ph := s.placeholders(len(ids)) rows, err := s.db.Query(`SELECT `+userCols+` FROM users WHERE id IN (`+ph+`)`, args...) if err != nil { return nil, err @@ -251,9 +251,9 @@ func (s *Store) PruneStalePending(maxAge time.Duration) (int64, error) { return 0, err } defer tx.Rollback() - rows, err := tx.Query(`SELECT id FROM users WHERE status = 'pending' AND created_at < $1 + rows, err := tx.Query(`SELECT id FROM users WHERE status = 'pending' AND created_at < `+s.p(1)+` AND NOT EXISTS (SELECT 1 FROM tokens t WHERE t.user_id = users.id AND t.purpose = 'activate' - AND t.used_at IS NULL AND t.expires_at > $2) ORDER BY id FOR UPDATE`, now-int64(maxAge/time.Second), now) + AND t.used_at IS NULL AND t.expires_at > `+s.p(2)+`) ORDER BY id`+s.forUpdate(), now-int64(maxAge/time.Second), now) if err != nil { return 0, err } @@ -276,17 +276,17 @@ func (s *Store) PruneStalePending(maxAge time.Duration) (int64, error) { // Recheck with a fresh READ COMMITTED snapshot after owning the user // lock; its original NOT EXISTS snapshot cannot see that new token. var eligible bool - if err := tx.QueryRow(`SELECT EXISTS(SELECT 1 FROM users WHERE id=$1 AND status='pending' AND created_at<$2 - AND NOT EXISTS(SELECT 1 FROM tokens WHERE user_id=$1 AND purpose='activate' AND used_at IS NULL AND expires_at>$3))`, id, now-int64(maxAge/time.Second), now).Scan(&eligible); err != nil { + if err := tx.QueryRow(`SELECT EXISTS(SELECT 1 FROM users WHERE id=`+s.p(1)+` AND status='pending' AND created_at<`+s.p(2)+` + AND NOT EXISTS(SELECT 1 FROM tokens WHERE user_id=`+s.p(1)+` AND purpose='activate' AND used_at IS NULL AND expires_at>`+s.p(3)+`))`, id, now-int64(maxAge/time.Second), now).Scan(&eligible); err != nil { return 0, err } if !eligible { continue } - if err := hashMailLogTx(tx, id); err != nil { + if err := s.hashMailLogTx(tx, id); err != nil { return 0, err } - if _, err := tx.Exec(`DELETE FROM users WHERE id = $1`, id); err != nil { + if _, err := tx.Exec(`DELETE FROM users WHERE id = `+s.p(1), id); err != nil { return 0, err } removed++ @@ -298,8 +298,8 @@ func (s *Store) PruneStalePending(maxAge time.Duration) (int64, error) { // user with HashedEmail of that row's own address (rows may predate an email // change). Rows already hashed are skipped. The cursor is closed before any // UPDATE because the store uses a single connection. -func hashMailLogTx(tx *sql.Tx, userID int64) error { - rows, err := tx.Query(`SELECT id, to_email FROM mail_log WHERE user_id = $1 AND to_email NOT LIKE 'sha256:%'`, userID) +func (s *Store) hashMailLogTx(tx *sql.Tx, userID int64) error { + rows, err := tx.Query(`SELECT id, to_email FROM mail_log WHERE user_id = `+s.p(1)+` AND to_email NOT LIKE 'sha256:%'`, userID) if err != nil { return err } @@ -321,7 +321,7 @@ func hashMailLogTx(tx *sql.Tx, userID int64) error { return err } for _, r := range list { - if _, err := tx.Exec(`UPDATE mail_log SET to_email = $1 WHERE id = $2`, HashedEmail(r.email), r.id); err != nil { + if _, err := tx.Exec(`UPDATE mail_log SET to_email = `+s.p(1)+` WHERE id = `+s.p(2), HashedEmail(r.email), r.id); err != nil { return err } } diff --git a/manage.sh b/manage.sh index 01c36a4dd..5bd2cd0fd 100755 --- a/manage.sh +++ b/manage.sh @@ -7,6 +7,7 @@ # # Idempotent: safe to cancel and re-run at any point. # Each step checks what's already done and skips it. +set +x set -e IMAGE_NAME="corescope" @@ -64,7 +65,7 @@ NC='\033[0m' log() { printf '%b\n' "${GREEN}✓${NC} $1"; } warn() { printf '%b\n' "${YELLOW}⚠${NC} $1"; } -err() { printf '%b\n' "${RED}✗${NC} $1"; } +err() { printf '%b\n' "${RED}✗${NC} $1" >&2; } info() { printf '%b\n' "${CYAN}→${NC} $1"; } step() { printf '%b\n' "\n${BOLD}[$1/$TOTAL_STEPS] $2${NC}"; } @@ -75,36 +76,151 @@ is_true() { esac } +# The base files require no PostgreSQL credentials and are also the read-only +# discovery boundary before the recorded backend is known. +dc_prod_base() { + local file=docker-compose.yml + if is_true "${DISABLE_MOSQUITTO:-false}"; then file=docker-compose.no-mosquitto.yml; fi + $DC -f "$file" "$@" +} + +dc_staging_base() { + local file="$STAGING_COMPOSE_FILE" + if is_true "${DISABLE_MOSQUITTO:-false}"; then file=docker-compose.staging.no-mosquitto.yml; fi + $DC -f "$file" -p corescope-staging "$@" +} + +dc_postgres() { + local environment="$1" file override + shift + require_database_credentials || return 1 + case "$environment" in + prod) + file=docker-compose.yml; override=docker-compose.postgres.yml + if is_true "${DISABLE_MOSQUITTO:-false}"; then file=docker-compose.no-mosquitto.yml; fi + $DC -f "$file" -f "$override" "$@" + ;; + staging) + file="$STAGING_COMPOSE_FILE"; override=docker-compose.staging.postgres.yml + if is_true "${DISABLE_MOSQUITTO:-false}"; then file=docker-compose.staging.no-mosquitto.yml; fi + $DC -f "$file" -f "$override" -p corescope-staging "$@" + ;; + *) err "Unknown storage environment"; return 1 ;; + esac +} + +storage_at() { + local environment="$1" data_directory="$2" service + local volumes=() + shift 2 + if [ -n "$data_directory" ]; then volumes=(-v "$data_directory:/app/data"); fi + case "$environment" in prod) service=prod ;; staging) service=staging-go ;; *) err "Unknown storage environment"; return 1 ;; esac + "dc_${environment}_base" run --rm --no-deps "${volumes[@]}" --entrypoint /app/storage.sh "$service" "$@" +} + +storage_action() { + local environment="$1" + shift + storage_at "$environment" "" "$@" +} + +storage_field() { + storage_action "$1" field "$2" +} + +selected_backend() { + local environment="$1" state backend job + state=$(storage_field "$environment" state) || { err "Cannot read storage selection. Build the current image with setup; keep the existing data and selection intact."; return 1; } + case "$state" in + ready) backend=$(storage_field "$environment" backend) || return 1 ;; + unrecorded) + # The packaged entrypoint performs explicit guarded setup. This hint + # selects its Compose dependencies; it never authorizes fresh data itself. + backend=${CORESCOPE_DB_BACKEND:-sqlite} + ;; + pending) + job=$(storage_field "$environment" job_id) || return 1 + err "Storage switch $job is pending; keep services stopped and use storage resume or abort."; return 1 + ;; + *) err "Storage selection is invalid; preserve it and inspect storage status."; return 1 ;; + esac + case "$backend" in sqlite|postgres) printf '%s\n' "$backend" ;; *) err "Choose sqlite or postgres during setup."; return 1 ;; esac +} + dc_prod() { - if is_true "${DISABLE_MOSQUITTO:-false}"; then - $DC -f docker-compose.no-mosquitto.yml "$@" - else - $DC "$@" - fi + local backend + backend=$(selected_backend prod) || return 1 + case "$backend" in sqlite) dc_prod_base "$@" ;; postgres) dc_postgres prod "$@" ;; esac } dc_staging() { - if is_true "${DISABLE_MOSQUITTO:-false}"; then - $DC -f docker-compose.staging.no-mosquitto.yml -p corescope-staging "$@" - else - $DC -f "$STAGING_COMPOSE_FILE" -p corescope-staging "$@" - fi + local backend + backend=$(selected_backend staging) || return 1 + case "$backend" in sqlite) dc_staging_base "$@" ;; postgres) dc_postgres staging "$@" ;; esac +} + +require_managed_postgres() { + storage_action "$1" managed-postgres +} + +sqlite_source_exists() { + local environment="$1" kind="$2" source service + if [ "$kind" = accounts ] && [ "$(storage_field "$environment" has_accounts)" = false ]; then return 3; fi + source=$(storage_field "$environment" "$kind.sqlite_path") || return 1 + service=prod; [ "$environment" != staging ] || service=staging-go + "dc_${environment}_base" run --rm --no-deps --entrypoint /bin/sh "$service" -eu -c ' + if [ -f "$1" ]; then exit 0; fi + # A nonnil recorded target is initialized. Missing it is data loss, not + # an intentionally absent optional store, even when no sidecar survives. + exit 1 + ' sh "$source" +} + +sqlite_dump_file() { + local environment="$1" kind="$2" destination="$3" temporary="$3.partial" source directory service + case "$kind" in telemetry|accounts) ;; *) err "Unknown SQLite store"; return 1 ;; esac + [ "$(basename "$destination")" = "$kind.db" ] || { err "Use the native store filename $kind.db"; return 1; } + source=$(storage_field "$environment" "$kind.sqlite_path") || return 1 + [ ! -e "$destination" ] || { err "Backup target already exists: $destination"; return 1; } + directory=$(cd "$(dirname "$destination")" && pwd -P) || return 1 + service=prod; [ "$environment" != staging ] || service=staging-go + ( + umask 077 + set -o noclobber + : > "$temporary" || exit 1 + trap 'rm -f -- "$temporary"' EXIT HUP INT TERM + # SQLite's native backup includes committed WAL frames and preserves rowids. + # The source is read-only; a fixed output name avoids dot-command quoting. + "dc_${environment}_base" run --rm --no-deps -v "$directory:/backup" --entrypoint sqlite3 "$service" \ + -readonly -cmd '.timeout 5000' "$source" ".backup '/backup/$kind.db.partial'" || exit 1 + sync -f "$temporary" || exit 1 + ln -- "$temporary" "$destination" || exit 1 + rm -- "$temporary" || exit 1 + trap - EXIT HUP INT TERM + ) } # Read the owner credential inside the database container, never from argv. -pg_exec() { +pg_container_exec() { local environment="$1" database="$2" shift 2 - case "$environment" in - prod) dc_prod exec -T postgres sh -c 'export PGHOST=127.0.0.1 PGPORT=5432 PGUSER=corescope_owner PGPASSWORD="$CORESCOPE_OWNER_PASSWORD" PGDATABASE="$1"; shift; exec "$@"' sh "$database" "$@" ;; - staging) dc_staging exec -T postgres sh -c 'export PGHOST=127.0.0.1 PGPORT=5432 PGUSER=corescope_owner PGPASSWORD="$CORESCOPE_OWNER_PASSWORD" PGDATABASE="$1"; shift; exec "$@"' sh "$database" "$@" ;; - *) err "Unknown PostgreSQL environment"; return 1 ;; - esac + case "$database" in corescope_telemetry|corescope_accounts) ;; *) err "Unsupported managed database target"; return 1 ;; esac + dc_postgres "$environment" exec -T postgres sh -c 'export PGHOST=127.0.0.1 PGPORT=5432 PGUSER=corescope_owner PGPASSWORD="$CORESCOPE_OWNER_PASSWORD" PGDATABASE="$1"; shift; exec "$@"' sh "$database" "$@" +} + +pg_exec() { + require_managed_postgres "$1" || return 1 + if [ "$2" = corescope_accounts ] && [ "$(storage_field "$1" has_accounts)" != true ]; then + err "No initialized account database is selected; managed account operations were refused." + return 1 + fi + pg_container_exec "$@" } pg_empty() { - local count - count=$(pg_exec "$1" "$2" psql -X -A -t -v ON_ERROR_STOP=1 -c "SELECT COUNT(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname NOT IN ('pg_catalog','information_schema') AND n.nspname NOT LIKE 'pg_toast%' AND c.relkind IN ('r','p','v','m','S','f')") || return 1 + local count execute=pg_exec + [ "${3:-}" != fresh-staging ] || execute=pg_container_exec + count=$("$execute" "$1" "$2" psql -X -A -t -v ON_ERROR_STOP=1 -c "SELECT COUNT(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname NOT IN ('pg_catalog','information_schema') AND n.nspname NOT LIKE 'pg_toast%' AND c.relkind IN ('r','p','v','m','S','f')") || return 1 [ "$count" = 0 ] } @@ -126,12 +242,13 @@ pg_dump_file() { } pg_restore_file() { - local environment="$1" database="$2" archive="$3" + local environment="$1" database="$2" archive="$3" scope="${4:-}" execute=pg_exec + [ "$scope" != fresh-staging ] || execute=pg_container_exec if [ "$(head -c 5 "$archive")" != PGDMP ]; then err "SQLite files require docs/postgresql-upgrade.md; expected a PostgreSQL archive."; return 1; fi - pg_empty "$environment" "$database" || { err "Refusing to overwrite a nonempty PostgreSQL destination."; return 1; } - pg_exec "$environment" "$database" pg_restore --no-password --exit-on-error --single-transaction --no-owner --no-privileges --dbname="$database" < "$archive" || return 1 - pg_exec "$environment" "$database" psql -X -v ON_ERROR_STOP=1 < docker/postgres-grants.sql >/dev/null || return 1 - pg_exec "$environment" "$database" psql -X -v ON_ERROR_STOP=1 -c ANALYZE >/dev/null || return 1 + pg_empty "$environment" "$database" "$scope" || { err "Refusing to overwrite a nonempty PostgreSQL destination."; return 1; } + "$execute" "$environment" "$database" pg_restore --no-password --exit-on-error --single-transaction --no-owner --no-privileges --dbname="$database" < "$archive" || return 1 + "$execute" "$environment" "$database" psql -X -v ON_ERROR_STOP=1 < docker/postgres-grants.sql >/dev/null || return 1 + "$execute" "$environment" "$database" psql -X -v ON_ERROR_STOP=1 -c ANALYZE >/dev/null || return 1 } require_database_credentials() { @@ -143,6 +260,57 @@ require_database_credentials() { done } +# Update only one private bootstrap setting, preserving unrelated .env content. +write_private_env() { + local key="$1" value="$2" temporary + umask 077 + temporary=$(mktemp .env.tmp.XXXXXX) || return 1 + if [ -f .env ]; then + awk -F= -v key="$key" '$1 != key { print }' .env > "$temporary" || { rm -f -- "$temporary"; return 1; } + fi + printf '%s=%s\n' "$key" "$value" >> "$temporary" + chmod 600 "$temporary" + mv -- "$temporary" .env + export "$key=$value" +} + +prepare_database_credentials() { + local directory="${PROD_POSTGRES_DATA_DIR:-$HOME/corescope-postgres}" key value occupied=false + if [ -d "$directory" ] && [ -n "$(find "$directory" -mindepth 1 -maxdepth 1 -print -quit)" ]; then occupied=true; fi + for key in POSTGRES_ADMIN_PASSWORD CORESCOPE_OWNER_PASSWORD CORESCOPE_READER_PASSWORD CORESCOPE_WRITER_PASSWORD CORESCOPE_ACCOUNTS_PASSWORD CORESCOPE_CHANNELS_PASSWORD; do + value=$(printenv "$key" || true) + if [ -z "$value" ]; then + if $occupied; then err "Existing PostgreSQL storage needs its matching private .env; changing passwords does not rotate existing roles. Keep the volume intact."; return 1; fi + value=$(openssl rand -hex 32) || return 1 + write_private_env "$key" "$value" || return 1 + fi + done + require_database_credentials +} + +choose_storage() { + local state backend reply + state=$(storage_field prod state) || return 1 + if [ "$state" = ready ]; then + backend=$(storage_field prod backend) || return 1 + info "Keeping recorded $backend storage. Use storage switch for an explicit offline backend change." + return 0 + fi + if [ "$state" != unrecorded ]; then selected_backend prod >/dev/null; return 1; fi + echo "Storage for this installation:" + echo " 1) SQLite (default; no separate database service)" + echo " 2) PostgreSQL (separate local service and private role credentials)" + read -r -p "Choose [1]: " reply + case "${reply:-1}" in + 1|sqlite) backend=sqlite ;; + 2|postgres) backend=postgres ;; + *) err "Choose 1 (SQLite) or 2 (PostgreSQL)."; return 1 ;; + esac + write_private_env CORESCOPE_DB_BACKEND "$backend" || return 1 + if [ "$backend" = postgres ]; then prepare_database_credentials || return 1; fi + info "Setup will validate existing stores before adopting them, or initialize only a provably fresh $backend installation." +} + confirm() { read -p " $1 [y/N] " -n 1 -r echo @@ -813,7 +981,6 @@ cmd_setup() { write_env_managed_values "$selected_http" "$selected_https" "$selected_mqtt" "$selected_data_dir" "$selected_disable_mosquitto" log "Saved negotiated ports to .env" - require_database_credentials show_env_port_summary "$selected_http" "$selected_https" "$selected_mqtt" "$selected_data_dir" "$selected_disable_mosquitto" echo " Resolved port mapping:" @@ -846,16 +1013,18 @@ cmd_setup() { if [ -n "$IMAGE_EXISTS" ] && is_done "build"; then log "Image already built." if confirm "Rebuild? (only needed if you updated the code)"; then - dc_prod build prod + dc_prod_base build prod log "Image rebuilt." fi else info "This takes 1-2 minutes the first time..." - dc_prod build prod + dc_prod_base build prod log "Image built." fi mark_done "build" + choose_storage || return 1 + # ── Step 5: Start container ── step 5 "Starting container" @@ -867,18 +1036,18 @@ cmd_setup() { fi fi - # Detect existing data directories - if [ -d "$PROD_DATA" ] && [ -f "$PROD_DATA/meshcore.db" ]; then - info "Found existing data at $PROD_DATA/ — will use bind mount." - fi - - if docker ps --format '{{.Names}}' | grep -q "^corescope-prod$"; then - log "Container already running." + local backend + backend=$(selected_backend prod) || return 1 + dc_prod_base stop prod || return 1 + if [ "$backend" = postgres ]; then + require_database_credentials || return 1 + dc_postgres prod up -d --wait postgres || return 1 + dc_postgres prod run --rm --no-deps --entrypoint /app/storage.sh bootstrap setup -backend=postgres || return 1 else - mkdir -p "$PROD_DATA" - dc_prod up -d prod - log "Container started." + storage_action prod setup -backend=sqlite || return 1 fi + dc_prod up -d prod || return 1 + log "Validated storage and application started." mark_done "container" # ── Step 6: Verify ── @@ -933,23 +1102,52 @@ cmd_setup() { # Copy telemetry into an empty staging database; accounts stay independent. prepare_staging_db() { + local backend state path destination copy_dir attempt contents mkdir -p "$STAGING_DATA" - dc_staging up -d postgres - local attempt - for attempt in $(seq 1 30); do - if pg_exec staging corescope_telemetry psql -X -A -t -c 'SELECT 1' >/dev/null 2>&1; then break; fi - [ "$attempt" -lt 30 ] || { err "Staging PostgreSQL did not become ready"; return 1; } - sleep 1 - done - if ! pg_empty staging corescope_telemetry; then info "Keeping existing staging PostgreSQL data; automatic replacement is refused."; return 0; fi - local copy_dir + if container_running "$STAGING_CONTAINER"; then info "Keeping running staging data."; return 0; fi + state=$(storage_field staging state) || return 1 + if [ "$state" = ready ]; then info "Keeping recorded staging storage; automatic replacement is refused."; return 0; fi + [ "$state" = unrecorded ] || { err "Staging recovery must finish before cloning data."; return 1; } + contents=$(find "$STAGING_DATA" -mindepth 1 -maxdepth 1 ! -name config.json ! -name theme.json ! -name Caddyfile ! -name .env -print -quit) + if [ -n "$contents" ]; then info "Keeping existing staging files; packaged setup will validate them before startup."; return 0; fi + backend=$(selected_backend prod) || return 1 + prepare_staging_config copy_dir=$(mktemp -d) chmod 700 "$copy_dir" - pg_dump_file prod corescope_telemetry "$copy_dir/telemetry.dump" || return 1 - pg_restore_file staging corescope_telemetry "$copy_dir/telemetry.dump" || return 1 - rm -f -- "$copy_dir/telemetry.dump" + if [ "$backend" = sqlite ]; then + path=$(storage_field prod telemetry.sqlite_path) || return 1 + case "$path" in /app/data/*) ;; *) err "Managed staging requires telemetry inside the persistent data mount."; return 1 ;; esac + destination=$(realpath -m -- "$STAGING_DATA/${path#/app/data/}") || return 1 + case "$destination" in "$(realpath -m -- "$STAGING_DATA")"/*) ;; *) err "Staging target escaped its data directory."; return 1 ;; esac + [ ! -e "$destination" ] || { err "Staging telemetry already exists."; return 1; } + sqlite_dump_file prod telemetry "$copy_dir/telemetry.db" || return 1 + mkdir -p "$(dirname "$destination")" + cp -- "$copy_dir/telemetry.db" "$destination" || return 1 + chmod 600 "$destination" + # Only telemetry is cloned. A new account target is independently initialized. + storage_action staging setup -backend=sqlite -sqlite-path "$path" -users-sqlite-path /app/data/users.db || return 1 + else + # Explicit fresh staging only: start the local database, never bootstrap its + # schema before checking emptiness and restoring the native telemetry dump. + dc_postgres staging up -d postgres + for attempt in $(seq 1 30); do + if pg_container_exec staging corescope_telemetry psql -X -A -t -c 'SELECT 1' >/dev/null 2>&1; then break; fi + [ "$attempt" -lt 30 ] || { err "Staging PostgreSQL did not become ready"; return 1; } + sleep 1 + done + if ! pg_empty staging corescope_telemetry fresh-staging; then + info "Keeping existing staging PostgreSQL data; validating its adoption before startup." + rmdir "$copy_dir" + dc_postgres staging run --rm bootstrap + return $? + fi + pg_dump_file prod corescope_telemetry "$copy_dir/telemetry.dump" || return 1 + pg_restore_file staging corescope_telemetry "$copy_dir/telemetry.dump" fresh-staging || return 1 + dc_postgres staging run --rm bootstrap || return 1 + fi + rm -f -- "$copy_dir/telemetry.db" "$copy_dir/telemetry.dump" rmdir -- "$copy_dir" - log "Telemetry restored into the empty staging database. Staging accounts remain independent." + log "Telemetry cloned into fresh staging storage. Staging accounts remain independent." } # Copy config.prod.json → config.staging.json with siteName change @@ -1091,7 +1289,6 @@ ensure_config() { } cmd_start() { - require_database_credentials local WITH_STAGING=false if [ "$1" = "--with-staging" ]; then WITH_STAGING=true @@ -1138,19 +1335,19 @@ cmd_stop() { case "$TARGET" in prod) info "Stopping production container (corescope-prod)..." - dc_prod stop prod + dc_prod_base stop prod log "Production stopped." ;; staging) info "Stopping staging container (${STAGING_CONTAINER})..." - dc_staging rm -sf staging-go 2>/dev/null || true + dc_staging_base rm -sf staging-go 2>/dev/null || true docker rm -f "$STAGING_CONTAINER" meshcore-staging-go corescope-staging meshcore-staging 2>/dev/null || true log "Staging stopped and cleaned up." ;; all) info "Stopping all containers..." - dc_prod stop prod - dc_staging rm -sf staging-go 2>/dev/null || true + dc_prod_base stop prod + dc_staging_base rm -sf staging-go 2>/dev/null || true docker rm -f "$STAGING_CONTAINER" meshcore-staging-go corescope-staging meshcore-staging 2>/dev/null || true log "All containers stopped." ;; @@ -1172,7 +1369,7 @@ cmd_restart() { staging) info "Restarting staging container (${STAGING_CONTAINER})..." # Stop and remove old container - dc_staging rm -sf staging-go 2>/dev/null || true + dc_staging_base rm -sf staging-go 2>/dev/null || true docker rm -f "$STAGING_CONTAINER" 2>/dev/null || true # Wait for container to be fully gone and memory to be reclaimed # This prevents OOM when old + new containers overlap on small VMs @@ -1195,7 +1392,7 @@ cmd_restart() { all) info "Restarting all containers..." dc_prod up -d --force-recreate prod - dc_staging rm -sf staging-go 2>/dev/null || true + dc_staging_base rm -sf staging-go 2>/dev/null || true docker rm -f "$STAGING_CONTAINER" 2>/dev/null || true dc_staging up -d staging-go log "All containers restarted." @@ -1262,11 +1459,20 @@ cmd_status() { fi echo "" - # Disk usage - local database size - for database in corescope_telemetry corescope_accounts; do - if size=$(pg_exec prod "$database" psql -X -A -t -c 'SELECT pg_size_pretty(pg_database_size(current_database()))' 2>/dev/null); then info "Production $database: $size"; else warn "PostgreSQL size unavailable for $database"; fi - done + local backend database size target + storage_action prod status || return 1 + backend=$(selected_backend prod) || return 1 + if [ "$backend" = postgres ]; then + for database in corescope_telemetry corescope_accounts; do + if [ "$database" = corescope_accounts ] && [ "$(storage_field prod has_accounts)" = false ]; then continue; fi + if size=$(pg_exec prod "$database" psql -X -A -t -c 'SELECT pg_size_pretty(pg_database_size(current_database()))' 2>/dev/null); then info "Production $database: $size"; else warn "PostgreSQL size unavailable for $database"; fi + done + else + for database in telemetry accounts; do + target=$(storage_field prod "$database.sqlite_path") || continue + if size=$(dc_prod_base run --rm --no-deps --entrypoint /bin/sh prod -c 'stat -c %s "$1"' sh "$target" 2>/dev/null); then info "Production SQLite $database: $size bytes (main file; WAL may be additional)"; fi + done + fi } @@ -1397,7 +1603,7 @@ cmd_update() { migrate_config auto info "Rebuilding image..." - dc_prod build prod + dc_prod_base build prod info "Restarting with new image..." dc_prod up -d --force-recreate prod @@ -1411,38 +1617,208 @@ cmd_update() { # ─── Backup ─────────────────────────────────────────────────────────────── +backup_state() { + local destination="$1" backend="$2" data_root target relative database_root + local excludes=() + data_root=$(cd "$PROD_DATA" && pwd -P) || return 1 + destination=$(realpath -m -- "$destination") || return 1 + case "$destination" in "$data_root"|"$data_root"/*) err "Choose a backup directory outside the live data directory."; return 1 ;; esac + if [ "$backend" = sqlite ]; then + for target in telemetry accounts; do + if [ "$target" = accounts ] && [ "$(storage_field prod has_accounts)" = false ]; then continue; fi + relative=$(storage_field prod "$target.sqlite_path") || return 1 + case "$relative" in + /app/data/*) + relative=${relative#/app/data/} + excludes+=("--exclude=./$relative" "--exclude=./$relative-wal" "--exclude=./$relative-shm" "--exclude=./$relative-journal") + ;; + esac + done + else + database_root=$(realpath -m -- "${PROD_POSTGRES_DATA_DIR:-$HOME/corescope-postgres}") || return 1 + case "$database_root" in + "$data_root") err "PostgreSQL storage and application state must use separate directories."; return 1 ;; + "$data_root"/*) excludes+=("--exclude=./${database_root#"$data_root"/}") ;; + esac + fi + # Preserve queues, selection/history, settings and custom files as recovery + # material. The selected live databases are supplied only by native snapshots. + ( + umask 077 + set -o noclobber + exec 3>"$destination/state.tar.partial" || exit 1 + trap 'rm -f -- "$destination/state.tar.partial"' EXIT HUP INT TERM + tar -C "$data_root" "${excludes[@]}" -cf - . >&3 || exit 1 + exec 3>&- + sync -f "$destination/state.tar.partial" || exit 1 + ln -- "$destination/state.tar.partial" "$destination/state.tar" || exit 1 + rm -- "$destination/state.tar.partial" || exit 1 + trap - EXIT HUP INT TERM + ) +} + cmd_backup() { - local timestamp backup_dir source name + local timestamp backup_dir source name backend status has_accounts timestamp=$(date +%Y%m%d-%H%M%S) - backup_dir="$1" + backup_dir="${1:-./backups/corescope-$timestamp}" [ -n "$backup_dir" ] || backup_dir="./backups/corescope-$timestamp" + backend=$(selected_backend prod) || return 1 + [ "$(storage_field prod state)" = ready ] || { err "Complete validated setup before taking a native backup."; return 1; } umask 077 + if [ -d "$backup_dir" ] && [ -n "$(find "$backup_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]; then err "Choose a new empty backup directory; prior recovery files were preserved."; return 1; fi mkdir -p "$backup_dir" - info "Writing native PostgreSQL archives to $backup_dir/" - pg_dump_file prod corescope_telemetry "$backup_dir/telemetry.dump" || return 1 - pg_dump_file prod corescope_accounts "$backup_dir/accounts.dump" || return 1 + chmod 700 "$backup_dir" + case "$backend" in + sqlite) + info "Writing native SQLite snapshots to $backup_dir/" + sqlite_dump_file prod telemetry "$backup_dir/telemetry.db" || return 1 + if sqlite_source_exists prod accounts; then + sqlite_dump_file prod accounts "$backup_dir/accounts.db" || return 1 + else + status=$? + [ "$status" = 3 ] || { err "Account storage is inaccessible or incomplete; backup refused."; return 1; } + : > "$backup_dir/accounts.absent" + fi + ;; + postgres) + info "Writing native PostgreSQL archives to $backup_dir/" + has_accounts=$(storage_field prod has_accounts) || return 1 + case "$has_accounts" in true|false) ;; *) err "Cannot read the recorded account target."; return 1 ;; esac + pg_dump_file prod corescope_telemetry "$backup_dir/telemetry.dump" || return 1 + if [ "$has_accounts" = true ]; then + pg_dump_file prod corescope_accounts "$backup_dir/accounts.dump" || return 1 + else + : > "$backup_dir/accounts.absent" + fi + ;; + esac + backup_state "$backup_dir" "$backend" || return 1 + printf '%s\n' "$backend" > "$backup_dir/backend.txt" for name in config.json theme.json; do source="$PROD_DATA/$name" [ -f "$source" ] || source="$name" if [ -f "$source" ]; then cp -- "$source" "$backup_dir/$name"; chmod 600 "$backup_dir/$name"; fi done + if [ -f .env ]; then cp -- .env "$backup_dir/compose.env"; chmod 600 "$backup_dir/compose.env"; fi if [ -f caddy-config/Caddyfile ]; then cp -- caddy-config/Caddyfile "$backup_dir/Caddyfile"; chmod 600 "$backup_dir/Caddyfile"; fi - log "Both database archives and available configuration saved. Keep the bundle private and encrypted off-host." + log "Native $backend backups and available configuration saved. Keep the bundle private and encrypted off-host." } # ─── Restore ────────────────────────────────────────────────────────────── -cmd_restore() { - local bundle="$1" file source destination stamp - if [ ! -d "$bundle" ]; then err "Usage: ./manage.sh restore . SQLite files require docs/postgresql-upgrade.md."; return 1; fi - for file in telemetry.dump accounts.dump; do - if [ ! -f "$bundle/$file" ] || [ "$(head -c 5 "$bundle/$file")" != PGDMP ]; then err "Backup must contain native telemetry.dump and accounts.dump; SQLite files are not accepted."; return 1; fi +stage_backup_state() { + local bundle="$1" destination="$2" stage + bundle=$(cd "$bundle" && pwd -P) || return 1 + [ -f "$bundle/state.tar" ] || { err "This restore requires the matching private state archive; use the native recovery guide for older snapshots."; return 1; } + mkdir -p "$(dirname "$destination")" + stage=$(mktemp -d "$destination.restore-XXXXXXXX") || return 1 + chmod 700 "$stage" + # Managed archives contain relative regular files/directories only. Keep an + # unsupported archive and its staged recovery material for manual inspection. + tar -tf "$bundle/state.tar" > "$stage/.restore-names" || return 1 + LC_ALL=C tar -tvf "$bundle/state.tar" > "$stage/.restore-types" || return 1 + if ! awk '/^\// || /(^|\/)\.\.(\/|$)/ { bad=1 } END { exit bad }' "$stage/.restore-names" || + ! awk 'substr($0,1,1)!="-" && substr($0,1,1)!="d" { bad=1 } END { exit bad }' "$stage/.restore-types"; then + err "State archive contains unsupported paths or links. Preserve it for manual recovery." + return 1 + fi + rm -- "$stage/.restore-names" "$stage/.restore-types" + tar --no-same-owner --no-same-permissions -xf "$bundle/state.tar" -C "$stage" || return 1 + printf '%s\n' "$stage" +} + +restore_sqlite_bundle() { + local bundle="$1" destination stage state backend kind target relative result device + bundle=$(cd "$bundle" && pwd -P) || return 1 + [ -f "$bundle/state.tar" ] || { err "This restore requires the matching private state archive; use the native recovery guide for older snapshots."; return 1; } + if container_running corescope-prod; then err "Stop CoreScope before restoring."; return 1; fi + destination=$(realpath -m -- "$PROD_DATA") || return 1 + if [ -L "$PROD_DATA" ] || { [ -d "$destination" ] && [ -n "$(find "$destination" -mindepth 1 -maxdepth 1 -print -quit)" ]; }; then + err "SQLite restore requires a new empty state directory. Preserve the current directory and select a fresh PROD_DATA_DIR." + return 1 + fi + if ! confirm "Restore both native SQLite snapshots and state into this empty directory?"; then return 0; fi + stage=$(stage_backup_state "$bundle" "$destination") || return 1 + state=$(storage_at prod "$stage" field state) || return 1 + backend=$(storage_at prod "$stage" field backend) || return 1 + [ "$state" = ready ] && [ "$backend" = sqlite ] || { err "State archive is not a complete SQLite installation snapshot."; return 1; } + if [ "$(storage_at prod "$stage" field has_accounts)" = true ] && [ -e "$bundle/accounts.absent" ]; then + err "The selection records initialized accounts; an absence marker cannot replace their native snapshot." + return 1 + fi + for kind in telemetry accounts; do + if [ "$kind" = accounts ] && [ "$(storage_at prod "$stage" field has_accounts)" = false ]; then + [ -f "$bundle/accounts.absent" ] && [ ! -e "$bundle/accounts.db" ] || { err "Account backup and selection disagree."; return 1; } + continue + fi + target=$(storage_at prod "$stage" field "$kind.sqlite_path") || return 1 + case "$target" in /app/data/*) ;; *) err "Managed restore requires SQLite targets inside the persistent data mount."; return 1 ;; esac + relative=${target#/app/data/} + target=$(realpath -m -- "$stage/$relative") || return 1 + case "$target" in "$stage"/*) ;; *) err "SQLite target escaped the staged state directory."; return 1 ;; esac + [ ! -e "$target" ] && [ ! -e "$target-wal" ] && [ ! -e "$target-shm" ] || { err "State archive already contains a selected database; native snapshot restore was refused."; return 1; } + if [ "$kind" = accounts ] && [ -f "$bundle/accounts.absent" ]; then + [ ! -e "$bundle/accounts.db" ] || { err "Account backup metadata is inconsistent."; return 1; } + continue + fi + [ -f "$bundle/$kind.db" ] && [ "$(head -c 15 "$bundle/$kind.db")" = 'SQLite format 3' ] || { err "A native SQLite snapshot is missing or invalid."; return 1; } + result=$(dc_prod_base run --rm --no-deps -v "$bundle:/backup:ro" --entrypoint sqlite3 prod -readonly "/backup/$kind.db" 'PRAGMA quick_check') || return 1 + [ "$result" = ok ] || { err "SQLite snapshot integrity check failed."; return 1; } + mkdir -p "$(dirname "$target")" + cp -- "$bundle/$kind.db" "$target" || return 1 + chmod 600 "$target" done + # Validate both schemas on the staged mount before publishing any restored + # selection or starting a process. This also handles supported old schemas. + storage_at prod "$stage" setup -backend=sqlite || return 1 + device=$(stat -c %d "$stage") || return 1 + [ ! -e "$destination" ] || { [ -d "$destination" ] && [ "$(stat -c %d "$destination")" = "$device" ] && [ -z "$(find "$destination" -mindepth 1 -maxdepth 1 -print -quit)" ]; } || { + err "Restore destination changed or is a separate mount; staged recovery remains at $stage." + return 1 + } + mv -T -- "$stage" "$destination" || { err "Restore publication failed; preserve staged recovery at $stage."; return 1; } + log "SQLite state restored. Services remain stopped: review restored accounts, sessions and tokens before start." +} + +cmd_restore() { + local bundle="$1" file source destination stamp has_accounts stage database + local databases=(corescope_telemetry) + if [ ! -d "$bundle" ]; then err "Usage: ./manage.sh restore ; see docs/storage.md for older standalone snapshots."; return 1; fi + local backend + backend=$(cat "$bundle/backend.txt" 2>/dev/null || printf postgres) + case "$backend" in + sqlite) restore_sqlite_bundle "$bundle"; return $? ;; + postgres) ;; + *) err "Unknown backup backend."; return 1 ;; + esac if container_running corescope-prod; then err "Stop CoreScope before restoring; leave target PostgreSQL running."; return 1; fi - if ! pg_empty prod corescope_telemetry || ! pg_empty prod corescope_accounts; then err "Restore requires empty PostgreSQL databases. Preserve the current cluster and configure a fresh destination."; return 1; fi - if ! confirm "Restore both native archives into these empty databases?"; then return 0; fi - pg_restore_file prod corescope_telemetry "$bundle/telemetry.dump" || return 1 - pg_restore_file prod corescope_accounts "$bundle/accounts.dump" || return 1 + require_managed_postgres prod || return 1 + has_accounts=$(storage_field prod has_accounts) || return 1 + case "$has_accounts" in + true) + [ ! -e "$bundle/accounts.absent" ] || { err "The selection records initialized accounts; their native archive is required."; return 1; } + databases+=(corescope_accounts) + ;; + false) + [ -f "$bundle/accounts.absent" ] && [ ! -e "$bundle/accounts.dump" ] || { err "Account backup and selection disagree."; return 1; } + destination=$(realpath -m -- "$PROD_DATA") || return 1 + stage=$(stage_backup_state "$bundle" "$destination") || return 1 + storage_at prod "$stage" managed-postgres || return 1 + [ "$(storage_at prod "$stage" field has_accounts)" = false ] || { err "Account absence marker disagrees with the retained backup selection."; return 1; } + ;; + *) err "Cannot read the recorded account target."; return 1 ;; + esac + for database in "${databases[@]}"; do + file="${database#corescope_}.dump" + if [ ! -f "$bundle/$file" ] || [ "$(head -c 5 "$bundle/$file")" != PGDMP ]; then err "Backup must contain a native $file; SQLite files are not accepted."; return 1; fi + done + for database in "${databases[@]}"; do + if ! pg_empty prod "$database"; then err "Restore requires empty PostgreSQL databases. Preserve the current cluster and configure a fresh destination."; return 1; fi + done + if ! confirm "Restore the selected native archives into these empty databases?"; then return 0; fi + for database in "${databases[@]}"; do + pg_restore_file prod "$database" "$bundle/${database#corescope_}.dump" || return 1 + done mkdir -p "$PROD_DATA" caddy-config stamp="$(date +%Y%m%d-%H%M%S)-$$" for file in config.json theme.json Caddyfile; do @@ -1500,6 +1876,68 @@ cmd_reset() { # ─── Help ───────────────────────────────────────────────────────────────── +cmd_storage() { + local action="${1:-status}" argument="${2:-}" state source destination target requires_postgres=false + local STORAGE_OPERATION="$action" extra=() + case "$action" in + status) storage_action prod status; return $? ;; + switch) + case "$argument" in sqlite|postgres) ;; *) err "Usage: ./manage.sh storage switch sqlite|postgres"; return 1 ;; esac + state=$(storage_field prod state) || return 1 + [ "$state" = ready ] || { err "Finish setup or explicitly resume/abort the pending job before another switch."; return 1; } + source=$(storage_field prod backend) || return 1 + [ "$source" != "$argument" ] || { info "Already using recorded $source storage."; return 0; } + [ "$source" != postgres ] || require_managed_postgres prod || return 1 + prepare_database_credentials || return 1 + if ! confirm "Stop production writers and run the verified offline switch to $argument?"; then return 0; fi + dc_prod_base stop prod || return 1 + # Starting only PostgreSQL leaves the conversion destination uninitialized. + dc_postgres prod up -d --wait postgres || return 1 + if [ "$argument" = sqlite ]; then + destination="$(storage_field prod state_dir)/sqlite-$(date -u +%Y%m%d-%H%M%S)-$$" || return 1 + extra=(-sqlite-path "$destination/meshcore.db" -users-sqlite-path "$destination/users.db") + fi + if ! dc_postgres prod run --rm --no-deps --entrypoint /app/storage.sh bootstrap switch "-backend=$argument" "${extra[@]}"; then + err "Switch failed. Services remain stopped; preserve source/recovery files and inspect storage status for its resume/abort job ID." + return 1 + fi + write_private_env CORESCOPE_DB_BACKEND "$argument" || return 1 + [ "$argument" != sqlite ] || dc_postgres prod stop postgres || return 1 + log "Verified backend switch completed. Services remain stopped; inspect storage status and run start when ready." + ;; + resume|abort) + [[ "$argument" =~ ^[0-9a-f]{32}$ ]] || { err "Use storage $action from storage status."; return 1; } + state=$(storage_field prod state) || return 1 + [ "$state" = pending ] || { err "No pending storage job matches this recovery action."; return 1; } + [ "$(storage_field prod job_id)" = "$argument" ] || { err "Job ID does not match the pending switch."; return 1; } + if [ "$action" = abort ]; then + if ! confirm "Keep writers stopped and abort storage job $argument?"; then return 0; fi + dc_prod_base stop prod || return 1 + storage_action prod abort "-job-id=$argument" || return 1 + log "Storage job aborted before publication. Source, staged targets and recovery files remain; inspect status before setup/start." + return 0 + fi + source=$(storage_field prod source_backend) || { err "Recovery source identity is unavailable; inspect storage status."; return 1; } + target=$(storage_field prod target_backend) || { err "Recovery target identity is unavailable; inspect storage status."; return 1; } + case "$source:$target" in :sqlite|:postgres|sqlite:sqlite|sqlite:postgres|postgres:sqlite|postgres:postgres) ;; *) err "The job has no bound target yet. Preserve recovery files, abort this job, then start a new setup/switch."; return 1 ;; esac + if [ "$source" = postgres ] || [ "$target" = postgres ]; then requires_postgres=true; require_database_credentials || return 1; fi + if ! confirm "Keep writers stopped and $action storage job $argument?"; then return 0; fi + dc_prod_base stop prod || return 1 + if $requires_postgres; then + dc_postgres prod up -d --wait postgres || return 1 + dc_postgres prod run --rm --no-deps --entrypoint /app/storage.sh bootstrap "$action" "-job-id=$argument" || return 1 + else + storage_action prod "$action" "-job-id=$argument" || return 1 + fi + source=$(storage_field prod backend) || return 1 + write_private_env CORESCOPE_DB_BACKEND "$source" || return 1 + if $requires_postgres && [ "$source" = sqlite ]; then dc_postgres prod stop postgres || return 1; fi + log "Storage recovery completed. Original and staged recovery files remain; validate before start." + ;; + *) err "Usage: ./manage.sh storage status|switch sqlite|postgres|resume JOB_ID|abort JOB_ID"; return 1 ;; + esac +} + cmd_help() { echo "" echo "CoreScope — Management Script" @@ -1521,8 +1959,11 @@ cmd_help() { printf '%b\n' " ${BOLD}Maintain${NC}" echo " update [version] Update to version (no arg=latest tag, 'latest'=master tip, or e.g. v3.1.0)" echo " promote Promote staging → production (backup + restart)" - echo " backup [dir] Native PostgreSQL archives + config + theme" - echo " restore Restore native archives into empty PostgreSQL databases" + echo " backup [dir] Native selected-backend snapshots + private state/config" + echo " restore Restore native backups into fresh/empty targets" + echo " storage status Show recorded backend and pending recovery job" + echo " storage switch sqlite|postgres Verified offline conversion; keeps recovery copies" + echo " storage resume|abort JOB_ID Explicit interrupted-switch recovery" echo " mqtt-test Check if MQTT data is flowing" echo "" echo "Prod uses docker-compose.yml; staging uses ${STAGING_COMPOSE_FILE}." @@ -1533,6 +1974,7 @@ cmd_help() { case "${1:-help}" in setup) cmd_setup ;; + storage) shift; cmd_storage "$@" ;; start) cmd_start "$2" ;; stop) cmd_stop "$2" ;; restart) cmd_restart "$2" ;; diff --git a/public/perf.js b/public/perf.js index e077e146f..cef098ae7 100644 --- a/public/perf.js +++ b/public/perf.js @@ -117,11 +117,11 @@ function renderVersionCard(health) { try { // #1258: /api/health was awaited AFTER Promise.all, adding a full RTT // (~50-200ms) on every 5s refresh. Issue it in parallel with the rest. - const [server, client, ioStats, postgresStats, writeSources, health] = await Promise.all([ + const [server, client, ioStats, databaseStats, writeSources, health] = await Promise.all([ fetch('/api/perf').then(r => r.json()), Promise.resolve(window.apiPerf ? window.apiPerf() : null), fetch('/api/perf/io').then(r => r.json()).catch(() => null), - fetch('/api/perf/postgres').then(r => r.json()).catch(() => null), + fetch('/api/perf/database').then(r => r.json()).catch(() => null), fetch('/api/perf/write-sources').then(r => r.json()).catch(() => null), fetch('/api/health').then(r => r.json()).catch(() => null) ]); @@ -247,18 +247,28 @@ function renderVersionCard(health) { } // Database-wide PostgreSQL counters and this server's connection pool. - if (postgresStats && postgresStats.engine === 'postgresql') { - const hitRate = postgresStats.cacheHitRate == null ? null : Number(postgresStats.cacheHitRate) * 100; + if (databaseStats && databaseStats.engine === 'postgresql') { + const hitRate = databaseStats.cacheHitRate == null ? null : Number(databaseStats.cacheHitRate) * 100; const hitFlag = hitRate > 0 && hitRate < 90 ? ' ' : ''; const hitText = hitRate != null && Number.isFinite(hitRate) ? hitRate.toFixed(1) + '%' : '—'; html += `

PostgreSQL connections

-
${Number(postgresStats.inUseConnections || 0)} / ${Number(postgresStats.openConnections || 0)}
Connections in use
-
${Number(postgresStats.connectionWaitCount || 0).toLocaleString()}
Connection waits
-
${Number(postgresStats.connectionWaitMs || 0).toFixed(1)}ms
Total connection wait
+
${Number(databaseStats.inUseConnections || 0)} / ${Number(databaseStats.openConnections || 0)}
Connections in use
+
${Number(databaseStats.connectionWaitCount || 0).toLocaleString()}
Connection waits
+
${Number(databaseStats.connectionWaitMs || 0).toFixed(1)}ms
Total connection wait
${hitText}${hitFlag}
Database Block Cache Hit Rate
`; } + if (databaseStats && databaseStats.engine === 'sqlite') { + const walMB = databaseStats.walSizeMB; + html += `

SQLite storage

+
${walMB == null ? '—' : Number(walMB).toFixed(1) + 'MB'}
WAL Size
+
${databaseStats.pageCount == null ? '—' : Number(databaseStats.pageCount).toLocaleString()}
Page Count
+
${databaseStats.pageSize == null ? '—' : Number(databaseStats.pageSize)}
Page Size
+
${databaseStats.plannerStats == null ? 'Unknown' : databaseStats.plannerStats ? 'Available' : 'Pending'}
Planner Statistics
+
`; + } + // Cache stats if (server.cache) { const c = server.cache; @@ -298,24 +308,29 @@ function renderVersionCard(health) {
`; } - // PostgreSQL size includes this database's tables and indexes, not cluster WAL. - if (server.postgres && server.postgres.rows && server.postgres.dbSizeMB != null) { - const sq = server.postgres; - html += `

PostgreSQL data

+ // Native engine sizes retain their distinct meanings; PostgreSQL excludes cluster WAL. + const selectedDatabase = server.database || server.sqlite || server.postgres; + const databaseLabel = selectedDatabase && selectedDatabase.engine === 'sqlite' ? 'SQLite' : 'PostgreSQL'; + if (selectedDatabase && selectedDatabase.rows && selectedDatabase.dbSizeMB != null) { + const sq = selectedDatabase; + html += `

${databaseLabel} data

${sq.dbSizeMB}MB
DB Size
${(sq.rows.transmissions || 0).toLocaleString()}
Transmissions
${(sq.rows.observations || 0).toLocaleString()}
Observations
${sq.rows.nodes || 0}
Nodes
${sq.rows.observers || 0}
Observers
`; + if (sq.engine === 'sqlite' && sq.freelistMB != null) { + html += `
${Number(sq.freelistMB).toFixed(1)}MB
Reusable Free Space
`; + } html += `
`; if (sq.sampledAt) { const sampled = new Date(sq.sampledAt); const sampleTime = Number.isFinite(sampled.getTime()) ? sampled.toLocaleString() : 'unknown time'; const interval = Number(sq.sampleIntervalSeconds); - html += `
${sq.stale ? 'Last verified PostgreSQL sample' : 'PostgreSQL sample'}: ${sampleTime}${Number.isFinite(interval) && interval > 0 ? ' · refreshes every ' + interval + 's' : ''}${sq.stale ? ' · refresh unavailable' : ''}
`; + html += `
${sq.stale ? 'Last verified ' + databaseLabel + ' sample' : databaseLabel + ' sample'}: ${sampleTime}${Number.isFinite(interval) && interval > 0 ? ' · refreshes every ' + interval + 's' : ''}${sq.stale ? ' · refresh unavailable' : ''}
`; } - } else if (server.postgres) { - html += '

PostgreSQL data

PostgreSQL storage unavailable
'; + } else if (selectedDatabase) { + html += `

${databaseLabel} data

${databaseLabel} storage unavailable
`; } // Server endpoints table — sort by total time (count * avg) DESC. diff --git a/scripts/check-dockerfile-internal-pkgs.sh b/scripts/check-dockerfile-internal-pkgs.sh index c23918932..0e38aed8f 100644 --- a/scripts/check-dockerfile-internal-pkgs.sh +++ b/scripts/check-dockerfile-internal-pkgs.sh @@ -2,7 +2,7 @@ # scripts/check-dockerfile-internal-pkgs.sh # # Asserts every internal/ referenced via a "replace" directive in -# cmd/server/go.mod or cmd/ingestor/go.mod has a matching +# each shipped command's go.mod has a matching # "COPY internal//" line in Dockerfile for each builder section that # needs it. # @@ -18,7 +18,7 @@ set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" DOCKERFILE="$ROOT/Dockerfile" -MODS=("$ROOT/cmd/server/go.mod" "$ROOT/cmd/ingestor/go.mod") +MODS=("$ROOT/cmd/server/go.mod" "$ROOT/cmd/ingestor/go.mod" "$ROOT/cmd/decrypt/go.mod" "$ROOT/cmd/migrate/go.mod") ERRORS=0 # PKG_COUNT[pkg] = number of go.mod files that reference internal/. diff --git a/scripts/postgres-benchmark/README.md b/scripts/postgres-benchmark/README.md index 1220af267..c155592e0 100644 --- a/scripts/postgres-benchmark/README.md +++ b/scripts/postgres-benchmark/README.md @@ -97,7 +97,12 @@ python3 scripts/postgres-benchmark/run.py --repo . \ --output /tmp/corescope-bench-supplemental-50eps ``` -The baseline is fixed to `9dbc287579a237ffa744dd0c91fa7227d09763ac`. Floating +The baseline is fixed to `3e21b179ecccae005aa5f6f45bc095353c94b343`, including +upstream's store-lock and reach-query fixes (#2147, #2155). Earlier reports used `9dbc287`; their results +are historical and must not be pooled with this comparison. The candidate's +verified PostgreSQL import is explicitly adopted into its persistent selection +before either timed startup; selection setup is outside the startup measurement. +Floating revisions and existing output directories are refused. Primary defaults to 60 seconds of warmup and 180 measured seconds per backend, with 50 offered ingest events/second and 20 HTTP requests/second: 12,000 scheduled ingest events diff --git a/scripts/postgres-benchmark/run.py b/scripts/postgres-benchmark/run.py index 0d59077df..5a045deef 100644 --- a/scripts/postgres-benchmark/run.py +++ b/scripts/postgres-benchmark/run.py @@ -29,7 +29,7 @@ import urllib.parse import urllib.request -BASELINE = "9dbc287579a237ffa744dd0c91fa7227d09763ac" +BASELINE = "3e21b179ecccae005aa5f6f45bc095353c94b343" FIRMWARE = "a366955cb2f67b8e6842d4f00d2b6a554dddd88a" HERE = Path(__file__).resolve().parent PROCESSES = [] @@ -700,6 +700,14 @@ def server_settings(backend, sqlite_path, state_dir, run_env): return config, env +def record_postgres_selection(binaries, config_dir, state_dir, run_env, owner_url, log, budget): + """Adopt the verified imported target before measuring runtime startup.""" + command([binaries / "migrate", "-storage-action=adopt", "-backend=postgres", "-offline", + "-selection-file", state_dir / "storage-selection.json", "-config-dir", config_dir], + cwd=config_dir, env=dict(run_env, CORESCOPE_DATABASE_URL=owner_url), + log=log, timeout=180, budget=budget) + + def retained_counts(source, since_epoch): db = sqlite3.connect(f"file:{source}?mode=ro", uri=True) try: @@ -1021,6 +1029,8 @@ def main(argv=None): retained_expected = retained_counts(sqlite_path, int(time.time())-168*3600) if args.corpus in ("S", "B") else None app_config, server_env = server_settings(backend, sqlite_path, state_dir, run_env) write_json(config_dir / "config.json", app_config) + if backend == "postgres": + record_postgres_selection(binaries, config_dir, state_dir, run_env, urls["owner"], run_dir / "selection.log", budget) resources.phase="startup_full" launched_ns=time.monotonic_ns() server = spawn([binaries / (backend + "-server"), "-config-dir", config_dir, "-public", roots[backend] / "public", "-port", app_config["port"], "-poll-ms", 1000], cwd=roots[backend], env=server_env, log=run_dir / "server.log", budget=budget) diff --git a/scripts/postgres-benchmark/test_harness.py b/scripts/postgres-benchmark/test_harness.py index dd05292d1..43179eba2 100644 --- a/scripts/postgres-benchmark/test_harness.py +++ b/scripts/postgres-benchmark/test_harness.py @@ -36,6 +36,26 @@ def read(self, size=-1): class HarnessTests(unittest.TestCase): + def test_selected_postgres_setup_is_offline_and_keeps_owner_out_of_argv(self): + owner = "postgres://bench_owner:private@127.0.0.1:25432/bench_p00" + runtime = {"CORESCOPE_WRITER_DATABASE_URL": "writer", "CORESCOPE_READER_DATABASE_URL": "reader"} + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + with mock.patch.object(bench, "command") as command: + bench.record_postgres_selection(root / "bin", root, root / "state", runtime, owner, root / "setup.log", None) + args = [str(value) for value in command.call_args.args[0]] + self.assertIn("-storage-action=adopt", args) + self.assertIn("-backend=postgres", args) + self.assertIn("-offline", args) + self.assertIn(str(root / "state" / "storage-selection.json"), args) + self.assertNotIn(owner, " ".join(args)) + self.assertEqual(command.call_args.kwargs["env"]["CORESCOPE_DATABASE_URL"], owner) + self.assertEqual(command.call_args.kwargs["env"]["CORESCOPE_WRITER_DATABASE_URL"], "writer") + self.assertNotIn("CORESCOPE_DATABASE_URL", runtime) + with mock.patch.object(bench, "command", side_effect=RuntimeError("selection refused")): + with self.assertRaisesRegex(RuntimeError, "selection refused"): + bench.record_postgres_selection(root / "bin", root, root / "state", runtime, owner, root / "setup.log", None) + def test_default_common_rate_and_emitted_worker_configs(self): parse = bench.argparse.ArgumentParser.parse_args for extra, expected_rate in (([], 50), (["--ingest-rate", "100"], 100)): diff --git a/scripts/test-postgres-compose.sh b/scripts/test-postgres-compose.sh index b5124f8e7..36b72a7dc 100755 --- a/scripts/test-postgres-compose.sh +++ b/scripts/test-postgres-compose.sh @@ -8,6 +8,8 @@ repo=$(cd "$(dirname "$0")/.." && pwd -P) temp_root=$(realpath -e "${RUNNER_TEMP:?RUNNER_TEMP is required}") [[ -x "$repo/docker/postgres-init.sh" ]] || { echo 'PostgreSQL initialization must be executable, not sourced by the vendor entrypoint.' >&2; exit 1; } image=$1 +backend=${2:-postgres} +case "$backend" in sqlite|postgres) ;; *) echo 'Choose sqlite or postgres for the packaged smoke.' >&2; exit 1 ;; esac image_id=$(timeout 15s docker image inspect --format '{{.Id}}' "$image") umask 077 work=$(mktemp -d "$temp_root/corescope-pg-smoke-XXXXXXXX") @@ -15,7 +17,9 @@ project=$(basename "$work" | tr '[:upper:]' '[:lower:]') owned_project=false phase=configuration dc=(docker compose --ansi never --project-name "$project" --env-file "$work/.env" - -f "$repo/docker-compose.example.yml" -f "$work/override.yml") + -f "$repo/docker-compose.example.yml") +if [ "$backend" = postgres ]; then dc+=(-f "$repo/docker-compose.example.postgres.yml"); fi +dc+=(-f "$work/override.yml") project_resources() { timeout 10s docker container ls -aq --filter "label=com.docker.compose.project=$project" || return 1 @@ -57,7 +61,7 @@ PY if [[ $(realpath -e "$work") == "$temp_root"/corescope-pg-smoke-* && ! -L "$work" ]]; then if $cleanup_ok; then sudo rm -rf -- "$work/state" "$work/postgres" || result=1 - rm -f -- "$work/.env" "$work/compose.json" "$work/override.yml" "$work/packet.json" "$work/stats.json" "$work/states.json" "$work/health.json" "$work/proxy-health.json" "$work/actions.raw.log" "$work/container.raw.log" || result=1 + rm -f -- "$work/.env" "$work/compose.json" "$work/override.yml" "$work/packet.json" "$work/stats.json" "$work/states.json" "$work/health.json" "$work/proxy-health.json" "$work/actions.raw.log" "$work/container.raw.log" "$work/recreate.yml" "$work/counts.txt" || result=1 fi if [ "$result" -eq 0 ]; then rm -f -- "$work/failure.log" @@ -76,12 +80,18 @@ trap finish EXIT trap 'exit 130' INT trap 'exit 143' TERM +touch "$work/.env" +if [ "$backend" = postgres ]; then for key in POSTGRES_ADMIN_PASSWORD CORESCOPE_OWNER_PASSWORD CORESCOPE_READER_PASSWORD CORESCOPE_WRITER_PASSWORD CORESCOPE_ACCOUNTS_PASSWORD CORESCOPE_CHANNELS_PASSWORD; do password=$(openssl rand -hex 32) export "$key=$password" printf '%s=%s\n' "$key" "${!key}" >> "$work/.env" done unset password +else + unset POSTGRES_ADMIN_PASSWORD CORESCOPE_OWNER_PASSWORD CORESCOPE_READER_PASSWORD CORESCOPE_WRITER_PASSWORD CORESCOPE_ACCOUNTS_PASSWORD CORESCOPE_CHANNELS_PASSWORD +fi +export CORESCOPE_STATE_DIR=/app/data/state export CORESCOPE_IMAGE="$image" DATA_DIR="$work/state" POSTGRES_DATA_DIR="$work/postgres" export DISABLE_MOSQUITTO=false # Exercise the example's default Caddy/port-80 path, including when the runner @@ -92,7 +102,7 @@ python3 - "$repo" "$work" <<'PY' import json,re,sys from pathlib import Path repo,work=map(Path,sys.argv[1:]) -config={"siteName":"CI packaged PostgreSQL smoke","port":3000,"stateDir":"/app/data", +config={"siteName":"CI packaged native storage smoke","port":3000,"stateDir":"/app/data/state", "mqttSources":[{"name":"ci-smoke","broker":"mqtt://localhost:1883","topics":["meshcore/SJC/ci-smoke/packets"]}], "userManagement":{"enabled":False},"autoRegionKeys":{"enabled":False}} (work/"state/config.json").write_text(json.dumps(config),encoding="utf-8") @@ -104,12 +114,6 @@ bytes.fromhex(raw) PY cat > "$work/override.yml" <<'YAML' services: - postgres: - restart: "no" - volumes: - - postgres-smoke:/var/lib/postgresql - bootstrap: - pull_policy: never corescope: pull_policy: never restart: "no" @@ -117,18 +121,30 @@ services: networks: default: internal: true +YAML +if [ "$backend" = postgres ]; then + cat >> "$work/override.yml" <<'YAML' volumes: postgres-smoke: YAML + python3 - "$work/override.yml" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); text=p.read_text() +text=text.replace('services:\n','services:\n postgres:\n restart: "no"\n volumes:\n - postgres-smoke:/var/lib/postgresql\n bootstrap:\n pull_policy: never\n',1) +p.write_text(text) +PY +fi timeout 30s "${dc[@]}" config --format json > "$work/compose.json" 2>> "$work/actions.raw.log" -python3 - "$repo" "$work" <<'PY' +python3 - "$repo" "$work" "$backend" <<'PY' import json,sys from pathlib import Path -repo,work=map(Path,sys.argv[1:]) +repo,work=map(Path,sys.argv[1:3]); backend=sys.argv[3] config=json.loads((work/"compose.json").read_text()) assert config["name"]==work.name.lower() -assert set(config["services"])=={"postgres","bootstrap","corescope"} +assert set(config["services"])==({"postgres","bootstrap","corescope"} if backend=="postgres" else {"corescope"}) +assert config["services"]["corescope"]["environment"]["CORESCOPE_STATE_DIR"]=="/app/data/state" assert set(config["networks"])=={"default"} assert config["networks"]["default"]["internal"] is True assert config["services"]["corescope"]["environment"]["DISABLE_CADDY"] == "false", "smoke must use the default Caddy listener" @@ -139,18 +155,35 @@ for service in config["services"].values(): for mount in service.get("volumes",[]): if mount["type"]=="bind": assert Path(mount["source"]).resolve() in allowed, "unexpected host mount" -for volume in config["volumes"].values(): +for volume in config.get("volumes",{}).values(): assert not volume.get("external") and volume["name"].startswith(config["name"]+"_") PY existing=$(project_resources) [[ -z "$existing" ]] || { echo 'Smoke project already exists; refusing to touch it.' >&2; exit 1; } owned_project=true +if [ "$backend" = postgres ]; then + phase=nil-account-adoption + # Generate a tiny native telemetry fixture with seed/account state outside the + # active selection, then adopt it as a legacy telemetry-only installation. + # No selection JSON is authored or edited by this test. + timeout 60s "${dc[@]}" run --rm --no-deps --entrypoint /bin/sh corescope -eu -c ' + /app/corescope-migrate -storage-action=init -backend=sqlite -offline -config-dir /app \ + -selection-file /app/data/generator/storage-selection.json \ + -sqlite-path /app/data/meshcore.db -users-sqlite-path /app/data/generator/users.db + /app/corescope-migrate -storage-action=adopt -backend=sqlite -offline -config-dir /app \ + -selection-file /app/data/state/storage-selection.json \ + -sqlite-path /app/data/meshcore.db -users-sqlite-path /app/data/uninitialized-users.db + ' >> "$work/actions.raw.log" 2>&1 + timeout 90s "${dc[@]}" up -d --wait --wait-timeout 60 postgres >> "$work/actions.raw.log" 2>&1 + timeout 120s "${dc[@]}" run --rm --no-deps --entrypoint /app/storage.sh bootstrap switch -backend=postgres >> "$work/actions.raw.log" 2>&1 +fi phase=startup timeout 210s "${dc[@]}" up -d --wait --wait-timeout 180 >> "$work/actions.raw.log" 2>&1 -bootstrap=$(timeout 10s "${dc[@]}" ps -a -q bootstrap) application=$(timeout 10s "${dc[@]}" ps -q corescope) phase=bootstrap-order [[ $(timeout 10s docker inspect --format '{{.Image}}' "$application") == "$image_id" ]] +if [ "$backend" = postgres ]; then +bootstrap=$(timeout 10s "${dc[@]}" ps -a -q bootstrap) timeout 10s docker inspect --format '{{json .State}}' "$bootstrap" "$application" > "$work/states.json" python3 - "$work/states.json" <<'PY' import datetime,json,sys @@ -160,9 +193,13 @@ assert app["Running"] stamp=lambda value: datetime.datetime.fromisoformat(value.replace("Z","+00:00")) assert stamp(app["StartedAt"]) >= stamp(bootstrap["FinishedAt"]), "application preceded bootstrap completion" PY +fi timeout 10s "${dc[@]}" exec -T corescope pg_dump --version | grep -Eq '^pg_dump .* 18[.]' phase=schema-readiness +[[ $(timeout 10s "${dc[@]}" exec -T corescope /app/storage.sh field backend) == "$backend" ]] +[[ $(timeout 10s "${dc[@]}" exec -T corescope /app/storage.sh field state_dir) == /app/data/state ]] +if [ "$backend" = postgres ]; then owner_sql() { timeout 10s "${dc[@]}" exec -T postgres sh -eu -c ' export PGHOST=127.0.0.1 PGUSER=corescope_owner PGPASSWORD="$CORESCOPE_OWNER_PASSWORD" PGDATABASE="$1" @@ -173,11 +210,18 @@ owner_sql() { SELECT version=1 AND ready AND current_setting('server_version_num')='180006' FROM corescope_schema WHERE kind='telemetry'; SQL ) == t ]] +[[ $(timeout 10s "${dc[@]}" exec -T corescope /app/storage.sh field has_accounts) == false ]] +# An explicit test inspection proves bootstrap did not initialize the absent +# account target. Runtime bootstrap/readiness/grants must select telemetry only. [[ $(owner_sql corescope_accounts <<'SQL' -SELECT version=1 AND ready AND (SELECT version=5 FROM schema_version) FROM corescope_schema WHERE kind='accounts'; +SELECT count(*)=0 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relkind IN ('r','p','v','m','S','f'); SQL ) == t ]] +else + timeout 10s "${dc[@]}" exec -T corescope sqlite3 -readonly /app/data/meshcore.db "SELECT count(*) FROM transmissions;" | grep -qx 0 +fi + phase=mqtt-ingestion timeout 10s "${dc[@]}" exec -T corescope mosquitto_pub -h 127.0.0.1 -q 1 -r -t meshcore/SJC/ci-smoke/packets -s < "$work/packet.json" >> "$work/actions.raw.log" 2>&1 for attempt in $(seq 1 30); do @@ -198,6 +242,7 @@ PY sleep 1 done phase=runtime-roles +if [ "$backend" = postgres ]; then [[ $(owner_sql corescope_telemetry <<'SQL' SELECT (SELECT count(*)=1 FROM transmissions) AND (SELECT count(*)=1 FROM observations) @@ -208,4 +253,33 @@ SELECT (SELECT count(*)=1 FROM transmissions) AND NOT has_schema_privilege('corescope_writer','public','CREATE'); SQL ) == t ]] -echo 'Packaged Compose bootstrap, restricted PostgreSQL runtime, MQTT ingestion, direct and Caddy HTTP readiness, API visibility and pg_dump availability passed.' +else + timeout 10s "${dc[@]}" exec -T corescope sqlite3 -readonly /app/data/meshcore.db 'SELECT count(*) FROM transmissions; SELECT count(*) FROM observations;' > "$work/counts.txt" + [ "$(cat "$work/counts.txt")" = $'1\n1' ] +fi +if [ "$backend" = postgres ]; then + phase=first-account-setup + timeout 60s "${dc[@]}" stop corescope >> "$work/actions.raw.log" 2>&1 + timeout 90s "${dc[@]}" run --rm --no-deps --entrypoint /app/storage.sh bootstrap setup -backend=postgres >> "$work/actions.raw.log" 2>&1 + timeout 210s "${dc[@]}" up -d --force-recreate --wait --wait-timeout 180 >> "$work/actions.raw.log" 2>&1 + [[ $(timeout 10s "${dc[@]}" exec -T corescope /app/storage.sh field has_accounts) == true ]] + [[ $(owner_sql corescope_accounts <<'SQL' +SELECT version=1 AND ready AND (SELECT version=5 FROM schema_version) FROM corescope_schema WHERE kind='accounts'; +SQL +) == t ]] +fi +phase=recreation +before=$(timeout 10s "${dc[@]}" exec -T corescope /app/storage.sh field generation) +# Exercise stale bootstrap hints and persistent custom state on actual recreate. +other=sqlite; [ "$backend" != sqlite ] || other=postgres +printf 'services:\n corescope:\n environment:\n CORESCOPE_DB_BACKEND: %s\n' "$other" > "$work/recreate.yml" +dc+=(-f "$work/recreate.yml") +timeout 180s "${dc[@]}" up -d --no-deps --force-recreate --wait --wait-timeout 150 corescope >> "$work/actions.raw.log" 2>&1 +[[ $(timeout 10s "${dc[@]}" exec -T corescope /app/storage.sh field generation) == "$before" ]] +[[ $(timeout 10s "${dc[@]}" exec -T corescope /app/storage.sh field backend) == "$backend" ]] +timeout 10s "${dc[@]}" exec -T corescope wget -qO- http://127.0.0.1:80/api/stats > "$work/stats.json" +python3 - "$work/stats.json" <<'PY' +import json,sys +stats=json.load(open(sys.argv[1])); assert stats['totalTransmissions']==1 and stats['totalObservations']>=1 +PY +echo "Packaged $backend setup, persistent selection on recreate, native runtime, MQTT ingestion, direct/Caddy readiness, API visibility and native clients passed." diff --git a/scripts/test-storage-compose.js b/scripts/test-storage-compose.js new file mode 100644 index 000000000..370048a53 --- /dev/null +++ b/scripts/test-storage-compose.js @@ -0,0 +1,93 @@ +// Validate actual Compose models without starting a Docker daemon or service. +// Run: node scripts/test-storage-compose.js +// Optional portable executable: CORESCOPE_COMPOSE_BIN=/path/to/docker-compose +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const { spawnSync } = require('node:child_process'); +const root = path.resolve(__dirname, '..'); +const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'corescope-storage-config-')); +const candidates = process.env.CORESCOPE_COMPOSE_BIN + ? [[process.env.CORESCOPE_COMPOSE_BIN, []]] + : [['docker', ['compose']], ['docker-compose', []]]; +const executable = candidates.find(([bin, args]) => spawnSync(bin, [...args, 'version'], { encoding: 'utf8', timeout: 10000 }).status === 0); +assert(executable, 'Docker Compose CLI required for model validation (no daemon needed); set CORESCOPE_COMPOSE_BIN for a portable executable'); +const [bin, prefix] = executable; +const passwordKeys = ['POSTGRES_ADMIN_PASSWORD', 'CORESCOPE_OWNER_PASSWORD', 'CORESCOPE_READER_PASSWORD', 'CORESCOPE_WRITER_PASSWORD', 'CORESCOPE_ACCOUNTS_PASSWORD', 'CORESCOPE_CHANNELS_PASSWORD']; +const clean = { ...process.env }; +for (const key of Object.keys(clean)) if (/^(PG|POSTGRES_|CORESCOPE_)/.test(key)) delete clean[key]; +Object.assign(clean, { + DATA_DIR: path.join(temp, 'example-state'), + PROD_DATA_DIR: path.join(temp, 'prod-state'), + STAGING_DATA_DIR: path.join(temp, 'staging-state'), + POSTGRES_DATA_DIR: path.join(temp, 'example-postgres'), + PROD_POSTGRES_DATA_DIR: path.join(temp, 'prod-postgres'), + STAGING_POSTGRES_DATA_DIR: path.join(temp, 'staging-postgres'), +}); +const credentials = Object.fromEntries(passwordKeys.map(key => [key, crypto.randomBytes(24).toString('hex')])); +const emptyEnv = path.join(temp, 'empty.env'); +fs.writeFileSync(emptyEnv, ''); +const cases = [ + ['docker-compose.yml', 'prod', 'docker-compose.postgres.yml', 'corescope', clean.PROD_POSTGRES_DATA_DIR], + ['docker-compose.no-mosquitto.yml', 'prod', 'docker-compose.postgres.yml', 'corescope', clean.PROD_POSTGRES_DATA_DIR], + ['docker-compose.example.yml', 'corescope', 'docker-compose.example.postgres.yml', 'corescope', clean.POSTGRES_DATA_DIR], + ['docker-compose.staging.yml', 'staging-go', 'docker-compose.staging.postgres.yml', 'corescope-staging', clean.STAGING_POSTGRES_DATA_DIR], + ['docker-compose.staging.no-mosquitto.yml', 'staging-go', 'docker-compose.staging.postgres.yml', 'corescope-staging', clean.STAGING_POSTGRES_DATA_DIR], +]; +function model(files, env) { + return spawnSync(bin, [...prefix, '--env-file', emptyEnv, ...files.flatMap(file => ['-f', path.join(root, file)]), 'config', '--format', 'json'], { cwd: root, env, encoding: 'utf8', timeout: 20000 }); +} +function parsed(result, label) { + // Deliberately never print a rendered model: it contains generated passwords. + assert.equal(result.status, 0, `${label} failed Compose validation: ${result.stderr || result.error?.message}`); + return JSON.parse(result.stdout); +} +try { + for (const [base, appName, override, project, postgresPath] of cases) { + const sqlite = parsed(model([base], clean), `${base} with all PostgreSQL variables unset`); + assert.equal(sqlite.name, project); + assert.deepEqual(Object.keys(sqlite.services), [appName], `${base} started an optional service by default`); + const app = sqlite.services[appName]; + assert.equal(app.environment.CORESCOPE_DB_BACKEND, 'sqlite'); + assert.equal(app.environment.CORESCOPE_STATE_DIR, '/app/data'); + assert(!app.depends_on?.postgres && !app.depends_on?.bootstrap, 'SQLite app depends on PostgreSQL'); + assert(!Object.keys(app.environment).some(key => /DATABASE_URL|PASSWORD/.test(key)), 'default app needs PostgreSQL credentials'); + const missing = model([base, override], clean); + assert.notEqual(missing.status, 0, `${override} accepted missing required credentials`); + const pg = parsed(model([base, override], { ...clean, ...credentials }), `${base} + ${override}`); + assert.equal(pg.name, project); + assert.deepEqual(Object.keys(pg.services).sort(), [appName, 'bootstrap', 'postgres'].sort()); + assert(!pg.services.postgres.ports?.length, 'database port exposed on host'); + assert.equal(pg.services.postgres.image, 'postgres:18.6-alpine3.24'); + const data = pg.services.postgres.volumes.find(volume => volume.target === '/var/lib/postgresql'); + assert.equal(path.resolve(data.source), path.resolve(postgresPath), 'wrong environment persistent database directory'); + const init = pg.services.postgres.volumes.find(volume => volume.target === '/docker-entrypoint-initdb.d/10-corescope.sh'); + assert.equal(path.resolve(init.source), path.resolve(root, 'docker/postgres-init.sh'), 'extends resolved the init script outside this checkout'); + assert(init.read_only); + const runtime = pg.services[appName]; + assert.equal(runtime.environment.CORESCOPE_DB_BACKEND, 'postgres'); + for (const key of passwordKeys) assert(!(key in runtime.environment), 'owner/admin credentials reached runtime'); + const roles = { CORESCOPE_READER_DATABASE_URL: ['corescope_reader','corescope_telemetry'], CORESCOPE_WRITER_DATABASE_URL: ['corescope_writer','corescope_telemetry'], CORESCOPE_USERS_DATABASE_URL: ['corescope_accounts','corescope_accounts'], CORESCOPE_APPROVED_CHANNELS_DATABASE_URL: ['corescope_channels','corescope_accounts'] }; + for (const [key, [user, db]] of Object.entries(roles)) { + const target = new URL(runtime.environment[key]); + assert.equal(target.username, user); assert.equal(target.hostname, 'postgres'); assert.equal(target.port, '5432', 'managed role endpoint must be explicitly pinned'); assert.equal(target.searchParams.get('search_path'), 'public', 'managed schema must be pinned'); assert.equal(target.pathname, '/' + db); + assert(![credentials.POSTGRES_ADMIN_PASSWORD, credentials.CORESCOPE_OWNER_PASSWORD].includes(target.password), 'owner credential in runtime URL'); + } + assert.equal(runtime.depends_on.bootstrap.condition, 'service_completed_successfully'); + assert.equal(pg.services.bootstrap.depends_on.postgres.condition, 'service_healthy'); + assert.equal(pg.services.bootstrap.environment.CORESCOPE_STATE_DIR, runtime.environment.CORESCOPE_STATE_DIR); + const runtimeState = runtime.volumes.find(volume => volume.target === '/app/data'); + const setupState = pg.services.bootstrap.volumes.find(volume => volume.target === '/app/data'); + assert.equal(setupState.source, runtimeState.source); assert(!setupState.read_only, 'bootstrap cannot atomically commit shared selection'); + if (base.includes('no-mosquitto')) assert.equal(runtime.environment.DISABLE_MOSQUITTO, 'true'); + if (base === 'docker-compose.staging.yml') assert(pg.networks['meshcore-net'].external, 'staging lost its existing broker network'); + console.log(`PASS ${base}: SQLite default and explicit PostgreSQL override`); + } + const custom = parsed(model(['docker-compose.example.yml','docker-compose.example.postgres.yml'], { ...clean, ...credentials, CORESCOPE_STATE_DIR: '/app/data/custom-state' }), 'custom shared state directory'); + for (const service of ['corescope','bootstrap']) assert.equal(custom.services[service].environment.CORESCOPE_STATE_DIR, '/app/data/custom-state'); +} finally { + assert(path.resolve(temp).startsWith(path.resolve(os.tmpdir()) + path.sep) && path.basename(temp).startsWith('corescope-storage-config-')); + fs.rmSync(temp, { recursive: true }); +} diff --git a/test-all.sh b/test-all.sh index c650b924a..709431299 100755 --- a/test-all.sh +++ b/test-all.sh @@ -201,5 +201,7 @@ node tests/unit/test-ws-stale-watchdog-1074.js node tests/unit/test-xss-escape-sinks.js node tests/unit/test-postgres-ops.js +node tests/unit/test-storage-entrypoint.js +node tests/unit/test-storage-manage.js echo "All standalone frontend suites passed" diff --git a/tests/e2e/test-postgres-backup-e2e.js b/tests/e2e/test-postgres-backup-e2e.js index fbb3a6972..694437f18 100644 --- a/tests/e2e/test-postgres-backup-e2e.js +++ b/tests/e2e/test-postgres-backup-e2e.js @@ -1,5 +1,5 @@ /** - * Native PostgreSQL backup and restored-browser-session proof. + * Native SQLite/PostgreSQL backup and restored-browser-session proof. * Run test-user-management-e2e.js first to create its synthetic accounts. * * BASE_URL=http://localhost:13582 E2E_BACKUP_DIR= @@ -22,6 +22,9 @@ const { chromium } = require('playwright'); const BASE = process.env.BASE_URL || 'http://localhost:13582'; const DIR = process.env.E2E_BACKUP_DIR || ''; const MODE = process.argv[2]; +const BACKEND = process.env.CORESCOPE_TEST_BACKEND || 'postgres'; +assert(['sqlite','postgres'].includes(BACKEND), 'explicit supported backup backend required'); +const EXTENSION = BACKEND === 'sqlite' ? 'db' : 'dump'; const HASH = 'fae0c9e6d357a814'; const ACCOUNTS = ['admin@e2e.test', 'sync@e2e.test']; let passed = 0; @@ -103,16 +106,17 @@ async function telemetry(page) { const anonymous = await (await browser.newContext()).newPage(); // Telemetry falls back to the API-key gate; account backups use withAdmin. - for (const [route, filename, anonymousStatus] of [['/api/backup', 'telemetry.dump', 403], ['/api/admin/users-backup', 'accounts.dump', 401]]) { + for (const [route, filename, anonymousStatus] of [['/api/backup', 'telemetry.' + EXTENSION, 403], ['/api/admin/users-backup', 'accounts.' + EXTENSION, 401]]) { assert.equal((await anonymous.request.get(BASE + route)).status(), anonymousStatus, route + ' anonymous denial'); assert.equal((await pages[1].request.get(BASE + route)).status(), 403, route + ' non-admin denial'); const response = await pages[0].request.get(BASE + route); assert.equal(response.status(), 200, route + ' admin download'); - assert.match(response.headers()['content-disposition'], /^attachment; filename="corescope-[^"]+\.dump"$/); + assert.match(response.headers()['content-disposition'], new RegExp('^attachment; filename="corescope-[^"]+\\.' + EXTENSION + '"$')); assert.equal(response.headers()['content-type'], 'application/octet-stream'); assert.equal(response.headers()['cache-control'], 'no-store'); const body = await response.body(); - assert.equal(body.subarray(0, 5).toString(), 'PGDMP', route + ' native archive magic'); + const magic = BACKEND === 'sqlite' ? 'SQLite format 3\0' : 'PGDMP'; + assert.equal(body.subarray(0, Buffer.byteLength(magic)).toString(), magic, route + ' native archive magic'); if (MODE === 'capture') fs.writeFileSync(path.join(DIR, filename), body, { mode: 0o600, flag: 'wx' }); pass(MODE + ': protected native ' + filename + ' download'); } diff --git a/tests/unit/test-issue-1956-release-routing.js b/tests/unit/test-issue-1956-release-routing.js index 312de970a..f4e50ec71 100644 --- a/tests/unit/test-issue-1956-release-routing.js +++ b/tests/unit/test-issue-1956-release-routing.js @@ -170,7 +170,7 @@ function context(ref = 'refs/tags/v9.8.7', event = 'workflow_dispatch', inputs = // That is what keeps a gate red rather than skipped, which branch protection // would count as passing. function route(ctx, failedJob, ingestor = 'true') { - for (const name of ['changes', 'go-test', 'race-test', 'e2e-shard', 'e2e-test', 'image-check', 'build-and-publish', 'release-artifacts', 'deploy', 'publish']) { + for (const name of ['changes', 'go-backend', 'go-test', 'race-backend', 'race-test', 'e2e-shard', 'e2e-test', 'image-check', 'build-and-publish', 'release-artifacts', 'deploy', 'publish']) { const job = block(deploy, name, 2); assert.ok(job, `missing ${name} job`); const needs = value(job, 'needs', 4).replace(/[\[\]\s]/g, '').split(',').filter(Boolean); @@ -193,6 +193,17 @@ function route(ctx, failedJob, ingestor = 'true') { return ctx.needs; } +// SQLite must have no PostgreSQL service; selected PG must get the pinned one. +const serviceImage = value(block(deploy, 'postgres', 6), 'image', 8); +assert.equal(evaluate(serviceImage.startsWith('${{') ? serviceImage : JSON.stringify(serviceImage), { matrix: { backend: 'sqlite' } }), '', 'default SQLite CI starts a PostgreSQL service'); +assert.equal(evaluate(serviceImage.startsWith('${{') ? serviceImage : JSON.stringify(serviceImage), { matrix: { backend: 'postgres' } }), 'postgres:18.6-alpine3.24'); +for (const [gate, worker] of [['go-test','go-backend'], ['race-test','race-backend']]) { + const definition = block(deploy, worker, 2); + assert.ok(definition.includes('backend: [sqlite, postgres]'), `${worker} must exercise both native backends`); + assert.equal(value(definition, 'CORESCOPE_TEST_BACKEND', 6), '${{ matrix.backend }}'); + assert.ok(value(block(deploy, gate, 2), 'needs', 4).includes(worker), `${gate} does not wait for both backends`); +} + for (const [name, edge] of [['matching', 'a'.repeat(40)], ['missing', null], ['mismatched', 'b'.repeat(40)]]) { const ctx = context(); const { commands } = runSteps(block(fast, 'retag-or-fallback', 2), ctx, edge); @@ -282,25 +293,28 @@ for (const [ref, event] of [['refs/heads/master', 'push'], ['refs/heads/master', assert.equal(noRace['build-and-publish'].result, 'success', `${event}: a skipped race-test must not block`); } assert.equal(route(context(), 'go-test')['release-artifacts'].result, 'skipped', 'failed Go validation must block release'); +assert.equal(route(context(), 'go-backend')['go-test'].result, 'failure', 'failed backend matrix must fail the stable Go gate'); +assert.equal(route(context(), 'race-backend')['race-test'].result, 'failure', 'failed race matrix must fail the stable race gate'); const dispatchInput = block(deploy, 'images_published', 6); assert.equal(value(dispatchInput, 'type', 8), 'boolean', 'dispatch flag must retain boolean semantics'); assert.equal(value(dispatchInput, 'default', 8), 'false', 'manual and fallback dispatches must build images by default'); const release = block(deploy, 'release-artifacts', 2); const builds = runSteps(release, context(), null).commands.filter(command => command[0] === 'go'); -// Exercise the actual shell: native PostgreSQL tools use pure Go; the offline -// importer alone retains SQLite/cgo and a static musl cross-compiler. +// Exercise the actual release shell: both tools support native SQLite and use +// the same static musl cross-compiler for both architectures. assert.deepEqual(builds.map(command => command.slice(1, 5)), [ - ['linux', 'amd64', '0', ''], ['linux', 'amd64', '1', 'zig cc -target x86_64-linux-musl'], - ['linux', 'arm64', '0', ''], + ['linux', 'amd64', '1', 'zig cc -target x86_64-linux-musl'], + ['linux', 'arm64', '1', 'zig cc -target aarch64-linux-musl'], ['linux', 'arm64', '1', 'zig cc -target aarch64-linux-musl'], ]); for (const command of builds) { - const importer = command.includes('netgo,osusergo,sqlite_omit_load_extension'); + const importer = command.some(arg => arg.includes('corescope-migrate-linux-')); + assert.ok(command.includes('netgo,osusergo,sqlite_omit_load_extension'), 'SQLite extension loading must be omitted for static linking'); assert.ok(command.includes(importer ? "-ldflags=-s -w -extldflags '-static -Wl,-s'" - : '-ldflags=-s -w -X main.version=v9.8.7'), 'static linking/version flags must match the tool'); + : "-ldflags=-s -w -extldflags '-static -Wl,-s' -X main.version=v9.8.7"), 'static linking/version flags must match the tool'); assert.ok(command.includes('-tags'), 'runtime resolver tags must survive'); } const upload = steps(release).filter(step => step.includes('uses: softprops/action-gh-release@v2')); @@ -433,15 +447,19 @@ console.log('PASS opted-in PR benchmark uses exact head and matching workflow/ha // the same name; replacing the first matching substring silently changes login. const restoreStep = steps(block(deploy, 'e2e-shard', 2)).find(step => value(step, '- name', 6) === 'Restore native HTTP backups and verify browser sessions'); const restoreScript = value(restoreStep, 'run', 8); -const switchURLs = restoreScript.split('# Switch restored database URLs.\n')[1]?.split('# Restart restored servers.')[0]; +const switchURLs = restoreScript.split('# Switch restored database URLs.\n')[1]?.split('# End restored database URLs.')[0]; assert.ok(switchURLs, 'missing restore URL transformation'); const sourceURLs = { CORESCOPE_USERS_DATABASE_URL: 'postgresql://corescope_accounts:p%40ss%2F%3A%3F%23@db.example.invalid:5433/corescope_accounts?sslmode=verify-full&application_name=corescope_accounts', CORESCOPE_READER_DATABASE_URL: 'postgresql://corescope_reader:p%2540%3A%2F@[::1]:5432/corescope_telemetry?sslmode=require&application_name=reader%3Bsmoke' }; +sourceURLs.CORESCOPE_WRITER_DATABASE_URL = sourceURLs.CORESCOPE_READER_DATABASE_URL.replace('corescope_reader:', 'corescope_writer:'); +sourceURLs.CORESCOPE_APPROVED_CHANNELS_DATABASE_URL = sourceURLs.CORESCOPE_USERS_DATABASE_URL.replace('corescope_accounts:', 'corescope_channels:'); +sourceURLs.CORESCOPE_DATABASE_URL = sourceURLs.CORESCOPE_READER_DATABASE_URL.replace('corescope_reader:', 'corescope_owner:'); +sourceURLs.CORESCOPE_USERS_OWNER_DATABASE_URL = sourceURLs.CORESCOPE_USERS_DATABASE_URL.replace('corescope_accounts:', 'corescope_owner:'); const rewritten = spawnSync(bash, ['--noprofile', '--norc', '-e', '-o', 'pipefail'], { input: (process.platform === 'win32' ? 'python3(){ python "$@"; }\n' : '') + switchURLs + - '\nprintf "%s\\n" "$CORESCOPE_USERS_DATABASE_URL" "$CORESCOPE_READER_DATABASE_URL"\n', + '\nprintf "%s\\n" ' + Object.keys(sourceURLs).map(key => '"$' + key + '"').join(' ') + '\n', encoding: 'utf8', env: { ...process.env, ...sourceURLs } }); assert.equal(rewritten.status, 0, rewritten.stderr); @@ -452,6 +470,8 @@ for (const [index, key] of Object.keys(sourceURLs).entries()) { for (const field of ['protocol', 'username', 'password', 'host', 'search', 'hash']) { assert.equal(restored[field], original[field], 'restore URL changed ' + field); } - assert.equal(restored.pathname, key === 'CORESCOPE_USERS_DATABASE_URL' ? '/corescope_restore_accounts' : '/corescope_restore_telemetry'); + assert.equal(restored.pathname, /USERS|APPROVED_CHANNELS/.test(key) ? '/corescope_restore_accounts' : '/corescope_restore_telemetry'); } console.log('PASS native backup restore changes only database URL paths'); +const goMatrix = block(deploy, 'go-backend', 2); +assert(goMatrix.includes('cd internal/dbconfig') && goMatrix.includes('cd ../sqliteutil'), 'new shared storage modules need their own explicit tests'); diff --git a/tests/unit/test-perf-disk-io-1120.js b/tests/unit/test-perf-disk-io-1120.js index 61fc1b7cc..14d025e20 100644 --- a/tests/unit/test-perf-disk-io-1120.js +++ b/tests/unit/test-perf-disk-io-1120.js @@ -50,7 +50,7 @@ function stubFetch(sb, perfData, healthData, ioData, postgresData, sourcesData) if (url === '/api/perf') return Promise.resolve({ json: () => Promise.resolve(perfData) }); if (url === '/api/health') return Promise.resolve({ json: () => Promise.resolve(healthData) }); if (url === '/api/perf/io') return Promise.resolve({ json: () => Promise.resolve(ioData) }); - if (url === '/api/perf/postgres') return Promise.resolve({ json: () => Promise.resolve(postgresData) }); + if (url === '/api/perf/database') return Promise.resolve({ json: () => Promise.resolve(postgresData) }); if (url === '/api/perf/write-sources') return Promise.resolve({ json: () => Promise.resolve(sourcesData) }); return Promise.resolve({ json: () => Promise.resolve({}) }); }; @@ -82,6 +82,26 @@ const sourcesData = { console.log('\n🧪 perf.js — Disk I/O + Write Sources (#1120)\n'); (async () => { +await test('Selected SQLite renders native diagnostics without PostgreSQL labels', async () => { + const sb = loadPerf(); + const fetched = []; + const sqlite = {engine:'sqlite', dbSizeMB:12, walSizeMB:2, freelistMB:1, + rows:{transmissions:3, observations:4, nodes:2, observers:1}}; + sb.ctx.fetch = url => { + fetched.push(url); + const data = url === '/api/perf' ? {...basePerf, database:sqlite, sqlite} : + url === '/api/perf/database' ? {engine:'sqlite', pageCount:300, pageSize:4096, cacheSize:-2000, walSizeMB:2, journalMode:'wal', plannerStats:true} : null; + return Promise.resolve({json:()=>Promise.resolve(data)}); + }; + sb.pages.perf.init({set innerHTML(v) {}}); + await new Promise(r=>setTimeout(r,30)); + const html=sb.getHtml(); + assert.ok(fetched.includes('/api/perf/database'), 'must fetch selected-backend diagnostics'); + assert.ok(!fetched.includes('/api/perf/postgres'), 'must not assume a PostgreSQL installation'); + assert.ok(html.includes('SQLite data') && html.includes('WAL Size') && html.includes('Planner Statistics'),html); + assert.ok(!html.includes('PostgreSQL'), 'SQLite must not render PostgreSQL labels'); +}); + await test('Renders Disk I/O section', async () => { const sb = loadPerf(); stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, postgresData, sourcesData); diff --git a/tests/unit/test-postgres-ops.js b/tests/unit/test-postgres-ops.js index b1579dc95..92212f078 100644 --- a/tests/unit/test-postgres-ops.js +++ b/tests/unit/test-postgres-ops.js @@ -9,7 +9,7 @@ const { spawnSync } = require('child_process'); const root = path.resolve(__dirname, '../..'); const source = fs.readFileSync(path.join(root, 'manage.sh'), 'utf8').replace(/\r/g, ''); const bash = require('../../scripts/bash-path')(); -const functions = ['pg_exec', 'pg_empty', 'pg_dump_file', 'pg_restore_file', 'cmd_backup', 'cmd_restore', 'prepare_staging_db', 'prepare_staging_config', 'write_env_managed_values', 'is_true', 'cmd_start'] +const functions = ['pg_container_exec', 'pg_exec', 'pg_empty', 'pg_dump_file', 'pg_restore_file', 'sqlite_source_exists', 'sqlite_dump_file', 'backup_state', 'stage_backup_state', 'restore_sqlite_bundle', 'cmd_backup', 'cmd_restore', 'prepare_staging_db', 'prepare_staging_config', 'write_env_managed_values', 'is_true', 'cmd_start'] .map(name => source.match(new RegExp(`^${name}\\(\\)\\s*\\{[\\s\\S]*?^}`, 'm'))?.[0] || '').join('\n'); const quote = value => "'" + value.replace(/'/g, "'\\''") + "'"; const posix = value => value.replace(/\\/g, '/'); @@ -33,8 +33,13 @@ for (const endpoint of ['http://127.0.0.1:3000/api/healthz', 'http://127.0.0.1:8 assert(smoke.includes(endpoint), `packaged smoke does not exercise ${endpoint}`); } assert(smoke.includes('proxy_health.get("ready") is True'), 'packaged smoke does not validate readiness through Caddy'); +const nilPreparation=smoke.slice(smoke.indexOf('phase=nil-account-adoption'),smoke.indexOf('phase=startup')); +assert(nilPreparation.includes('-storage-action=adopt')&&nilPreparation.includes('-users-sqlite-path /app/data/uninitialized-users.db'), 'packaged smoke lacks native adoption of telemetry without accounts'); +assert(nilPreparation.includes('bootstrap switch -backend=postgres'),'packaged nil-account fixture bypasses the verified switch'); +const firstAccount=smoke.slice(smoke.indexOf('phase=first-account-setup'),smoke.indexOf('phase=recreation')); +assert(firstAccount.indexOf('stop corescope')>=0&&firstAccount.indexOf('stop corescope')&2; } confirm(){ return 0; }; container_running(){ return 1; } docker(){ return 0; }; require_database_credentials(){ :; } +selected_backend(){ printf '%s\\n' "$BACKEND"; } +require_managed_postgres(){ [ "$BACKEND" = postgres ]; } +storage_field(){ case "$2" in has_accounts) echo "$HAS_ACCOUNTS";;state) if [ "$1" = staging ]; then echo unrecorded; else echo ready; fi;;backend) echo "$BACKEND";;telemetry.sqlite_path) echo '/app/data/custom telemetry.db';;accounts.sqlite_path) echo '/app/data/users.db';;state_dir) echo '/app/data';;esac; } +storage_action(){ :; } +storage_at(){ shift 2; if [ "$1" = field ]; then if [ "$2" = has_accounts ]; then echo "$BACKUP_HAS_ACCOUNTS"; else storage_field prod "$2"; fi; elif [ "$1" = managed-postgres ]; then [ "$BACKEND" = postgres ]; fi; } +dc_staging_base(){ dc_prod_base "$@"; } +dc_prod_base(){ + printf 'base %s\\n' "$*" >> "$CALLS" + case "$*" in + *'--entrypoint sqlite3'*) + case "$*" in *'PRAGMA quick_check'*) printf 'ok\\n'; return;;esac + local destination='' previous='' part + for part in "$@"; do [ "$previous" != -v ] || destination="\${part%:/backup}"; previous="$part"; done + case "$*" in *telemetry.db.partial*) printf 'SQLite format 3\\0native WAL snapshot' >| "$destination/telemetry.db.partial";;*accounts.db.partial*) printf 'SQLite format 3\\0native accounts snapshot' >| "$destination/accounts.db.partial";;esac + ;; + *) :;; + esac +} preflight_validate_prod_ports(){ return 0; }; migrate_config(){ :; }; ensure_config(){ :; } database_reply(){ case "$*" in @@ -64,6 +90,7 @@ dc_prod(){ fi database_reply "$@" } +dc_postgres(){ local environment="$1"; shift; "dc_$environment" "$@"; } dc_staging(){ printf 'staging %s\\n' "$*" >> "$CALLS"; database_reply "$@"; } ${functions} ${command} @@ -77,6 +104,38 @@ try { fs.writeFileSync(path.join(temp, 'docker/postgres-grants.sql'), 'SELECT 1;'); fs.writeFileSync(path.join(temp, 'state/meshcore.db'), 'SQLite format 3\0legacy recovery'); fs.writeFileSync(path.join(temp, 'state/config.json'), '{"siteName":"test"}'); + const sqliteBackup = path.join(temp, 'sqlite-backup'); + const sqlite = run(`cmd_backup ${quote(posix(sqliteBackup))}`, '0', '0', '1', 'sqlite'); + assert.strictEqual(sqlite.status, 0, sqlite.stderr || sqlite.error?.message); + for (const name of ['telemetry.db','accounts.db']) assert(fs.existsSync(path.join(sqliteBackup,name)), 'SQLite native snapshot missing: '+name); + assert(fs.readFileSync(path.join(sqliteBackup,'telemetry.db')).includes(Buffer.from('native WAL snapshot')), 'backup copied the live main file instead of taking a native snapshot'); + const sqliteCalls = fs.readFileSync(path.join(temp,'calls'),'utf8'); + assert(sqliteCalls.includes('--entrypoint sqlite3') && sqliteCalls.includes('-readonly') && sqliteCalls.includes('/app/data/custom telemetry.db'), 'backup ignored the selected path or read-only boundary'); + assert(!sqliteCalls.includes('pg_dump'), 'SQLite backup contacted PostgreSQL'); + assert(fs.existsSync(path.join(sqliteBackup,'config.json')), 'SQLite backup omitted configuration'); + assert(fs.existsSync(path.join(sqliteBackup,'state.tar')), 'SQLite backup omitted persistent selection/queue/state recovery archive'); + const missingRecorded = run(` + storage_field(){ case "$2" in has_accounts) echo true;;state) echo ready;;backend) echo sqlite;;*) printf '%s/missing-recorded.db\\n' "$PROD_DATA";;esac; } + sqlite_dump_file(){ :; } + dc_prod_base(){ while [ "$1" != -c ]; do shift; done; shift; bash -eu -c "$@"; } + cmd_backup ${quote(posix(path.join(temp,'missing-recorded-backup')))} + `, '0','0','1','sqlite'); + assert.notStrictEqual(missingRecorded.status,0,'backup labelled a missing recorded account database as intentionally absent'); + const badAccounts = path.join(temp,'bad-account-backup'); + fs.cpSync(sqliteBackup,badAccounts,{recursive:true}); + fs.unlinkSync(path.join(badAccounts,'accounts.db')); + fs.writeFileSync(path.join(badAccounts,'accounts.absent'),''); + const badRestore = run(`PROD_DATA=${quote(posix(path.join(temp,'bad-restored-state')))}; cmd_restore ${quote(posix(badAccounts))}`,'0','0','1','sqlite'); + assert.notStrictEqual(badRestore.status,0,'restore skipped initialized accounts based on an absence marker'); + const restoredSQLite = path.join(temp,'restored-sqlite'); + const sqliteRestore = run(`PROD_DATA=${quote(posix(restoredSQLite))}; cmd_restore ${quote(posix(sqliteBackup))}`, '0', '0', '1', 'sqlite'); + assert.strictEqual(sqliteRestore.status,0,sqliteRestore.stderr || sqliteRestore.error?.message); + assert(fs.readFileSync(path.join(restoredSQLite,'custom telemetry.db')).includes(Buffer.from('native WAL snapshot')), 'SQLite restore ignored the recorded target'); + assert(fs.existsSync(path.join(restoredSQLite,'users.db')), 'SQLite restore lost accounts'); + assert(fs.existsSync(path.join(restoredSQLite,'config.json')), 'SQLite restore lost persistent state'); + const repeatedSQLite = run(`PROD_DATA=${quote(posix(restoredSQLite))}; cmd_restore ${quote(posix(sqliteBackup))}`, '0', '0', '1', 'sqlite'); + assert.notStrictEqual(repeatedSQLite.status,0,'SQLite restore overwrote existing state'); + fs.writeFileSync(path.join(temp,'calls'),''); const backup = path.join(temp, 'backup'); const made = run(`cmd_backup ${quote(posix(backup))}`); assert.strictEqual(made.status, 0, made.stderr || made.error?.message); @@ -96,9 +155,50 @@ try { const calls = fs.readFileSync(path.join(temp, 'calls'), 'utf8'); assert(calls.includes('pg_restore'), 'native restore was not invoked'); assert(!calls.includes('must-not-appear-in-argv'), 'database password appeared in command arguments'); + const nilBackup=path.join(temp,'nil-account-backup'); + fs.writeFileSync(path.join(temp,'calls'),''); + const nilMade=run(`cmd_backup ${quote(posix(nilBackup))}`,'0','0','1','postgres','false'); + assert.strictEqual(nilMade.status,0,nilMade.stderr||nilMade.error?.message); + assert(fs.existsSync(path.join(nilBackup,'accounts.absent')),'nullable account target was not recorded as absent'); + assert(!fs.existsSync(path.join(nilBackup,'accounts.dump')),'backup dumped an unselected account database'); + assert(!fs.readFileSync(path.join(temp,'calls'),'utf8').includes('corescope_accounts'),'nil-account backup contacted the account database'); + fs.writeFileSync(path.join(temp,'calls'),''); + const nilRestore=run(`cmd_restore ${quote(posix(nilBackup))}`,'0','0','1','postgres','false'); + assert.strictEqual(nilRestore.status,0,nilRestore.stderr||nilRestore.error?.message); + const nilCalls=fs.readFileSync(path.join(temp,'calls'),'utf8'); + assert(nilCalls.includes('pg_restore'),'telemetry-only restore did not restore telemetry'); + assert(!nilCalls.includes('corescope_accounts'),'telemetry-only restore read or wrote the unselected account database'); + for(const [bundle,selected,recorded] of [[nilBackup,'true','false'],[nilBackup,'false','true'],[backup,'false','true']]){ + fs.writeFileSync(path.join(temp,'calls'),''); + const mismatch=run(`cmd_restore ${quote(posix(bundle))}`,'0','0','1','postgres',selected,recorded); + assert.notStrictEqual(mismatch.status,0,'restore accepted account snapshot/selection disagreement'); + assert(!fs.readFileSync(path.join(temp,'calls'),'utf8').includes('pg_restore'),'mismatched selection reached restore writes'); + } + const legacyBundle=path.join(temp,'legacy-two-dump-backup'); + fs.cpSync(backup,legacyBundle,{recursive:true});fs.unlinkSync(path.join(legacyBundle,'state.tar')); + const legacyBundleRestore=run(`cmd_restore ${quote(posix(legacyBundle))}`); + assert.strictEqual(legacyBundleRestore.status,0,'existing two-archive PG backup support regressed: '+legacyBundleRestore.stderr); + const absentWithoutSelection=path.join(temp,'absence-without-selection'); + fs.cpSync(nilBackup,absentWithoutSelection,{recursive:true});fs.unlinkSync(path.join(absentWithoutSelection,'state.tar')); + fs.writeFileSync(path.join(temp,'calls'),''); + const unprovedAbsence=run(`cmd_restore ${quote(posix(absentWithoutSelection))}`,'0','0','1','postgres','false'); + assert.notStrictEqual(unprovedAbsence.status,0,'restore trusted a bare absence marker without its retained selection'); + assert(!fs.readFileSync(path.join(temp,'calls'),'utf8').includes('pg_restore'),'unproved account absence reached restore writes'); + fs.writeFileSync(path.join(temp,'calls'),''); + const unselectedSQL=run('pg_exec prod corescope_accounts psql -c SELECT','0','0','1','postgres','false'); + assert.notStrictEqual(unselectedSQL.status,0,'generic managed executor contacted an unselected account target'); + assert(!fs.readFileSync(path.join(temp,'calls'),'utf8').includes('corescope_accounts'),'unselected database reached the container boundary'); const legacy = run(`cmd_restore ${quote(posix(path.join(temp, 'state/meshcore.db')))}`); assert.notStrictEqual(legacy.status, 0, 'SQLite restore accepted without offline import'); fs.writeFileSync(path.join(temp, 'calls'), ''); + const sqliteStage = run('require_managed_postgres(){ :; }; prepare_staging_db', '0', '0', '1', 'sqlite'); + assert.strictEqual(sqliteStage.status,0,sqliteStage.stderr || sqliteStage.error?.message); + assert(fs.existsSync(path.join(temp,'staging/custom telemetry.db')), 'SQLite staging did not clone native telemetry'); + assert(!fs.existsSync(path.join(temp,'staging/users.db')), 'staging copied production accounts'); + assert(!fs.readFileSync(path.join(temp,'calls'),'utf8').includes('pg_dump'), 'SQLite staging required PostgreSQL'); + // The independent PG cases below use another fresh staging directory. + fs.renameSync(path.join(temp,'staging'),path.join(temp,'staging-sqlite-retained')); + fs.writeFileSync(path.join(temp, 'calls'), ''); const staged = run('prepare_staging_db'); assert.strictEqual(staged.status, 0, staged.stderr || staged.error?.message); assert(!fs.readFileSync(path.join(temp, 'calls'), 'utf8').includes('corescope_accounts'), 'staging copied production accounts'); diff --git a/tests/unit/test-storage-entrypoint.js b/tests/unit/test-storage-entrypoint.js new file mode 100644 index 000000000..2b9657068 --- /dev/null +++ b/tests/unit/test-storage-entrypoint.js @@ -0,0 +1,134 @@ +// Run the packaged startup guard against a fake migration/process boundary. +// No Docker daemon, host state, database, or credentials are used. +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const root = path.resolve(__dirname, '../..'); +const bash = require('../../scripts/bash-path')(); +const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'corescope-storage-entry-')); +const app = path.join(temp, 'app'); +const posix = value => value.replace(/\\/g, '/').replace(/^([A-Za-z]):/, (_, drive) => '/' + drive.toLowerCase()); +const shellQuote = value => "'" + value.replace(/'/g, "'\\''") + "'"; +const calls = path.join(temp, 'calls'); +fs.mkdirSync(path.join(app, 'data'), { recursive: true }); +fs.mkdirSync(path.join(temp, 'bin')); +fs.writeFileSync(path.join(app, 'data/config.json'), '{}'); +fs.writeFileSync(path.join(temp, 'mountinfo'), `1 0 8:1 / ${posix(app)}/data rw - ext4 /dev/owned rw\n`); +const tool = path.join(temp, 'tools.js'); +fs.writeFileSync(tool, ` +const fs=require('fs'),path=require('path'); +const [name,...args]=process.argv.slice(2); +const calls=process.env.CALLS; +if(name!=='jq')fs.appendFileSync(calls,JSON.stringify([name,...args])+'\\n'); +const field=flag=>args.find(v=>v.startsWith(flag+'='))?.slice(flag.length+1)||args[args.indexOf(flag)+1]; +if(name==='corescope-migrate'){ + if(args.includes('-check-ready')){ + fs.appendFileSync(calls,JSON.stringify(['readiness-accounts',Boolean(process.env.CORESCOPE_USERS_DATABASE_URL)])+'\\n');process.exit(0); + } + const selection=field('-selection-file');const action=field('-storage-action'); + if(action==='status'){ + let report={version:1,state:'unrecorded',backend:'',generation:'',job_id:'',source_backend:'',target_backend:''}; + if(fs.existsSync(selection))report=JSON.parse(fs.readFileSync(selection,'utf8')); + if(process.env.SCENARIO==='pending')report={...report,state:'pending',job_id:'0123456789abcdef0123456789abcdef'}; + if(process.env.SCENARIO==='corrupt'){console.log(JSON.stringify({...report,state:'corrupt'}));process.exit(1)}; + console.log(JSON.stringify(report)); + }else if(action==='setup'){ + if(process.env.SCENARIO==='fail-setup')process.exit(1); + const backend=field('-backend')||'sqlite';fs.mkdirSync(path.dirname(selection),{recursive:true}); + fs.writeFileSync(selection,JSON.stringify({version:1,state:'ready',backend,generation:'one',job_id:''}));console.log(JSON.stringify({verified:true})); + }else{process.stderr.write('unexpected migration action');process.exit(2)} +}else if(name==='jq'){ + const value=JSON.parse(fs.readFileSync(0,'utf8'));const query=args.at(-1); + if(query==='.accounts != null'){console.log(value.accounts!=null);process.exit(0)} + if(query.includes('.telemetry.postgres ==')){ + const canonical=(target,database)=>target&&Object.keys(target).length===4&&target.host==='postgres'&&target.port===5432&&target.database===database&&target.schema==='public'; + const valid=value.state==='ready'&&value.backend==='postgres'&&canonical(value.telemetry?.postgres,'corescope_telemetry')&& + ((query.includes('.accounts == null or')&&value.accounts===null)||canonical(value.accounts?.postgres,'corescope_accounts')); + if(!valid)process.exit(1);console.log(true);process.exit(0); + } + const key=query.replace(/^\\./,'').replace(/ \\/\\/.*$/,''); + if(!/^[a-z_.]+$/.test(key))throw Error('unhandled jq boundary query: '+query); + let selected=key.split('.').reduce((obj,key)=>obj?.[key],value); + if(selected==null||selected===''){process.exit(1)}else{console.log(selected)} +}else if(name==='psql'){ + fs.appendFileSync(calls,JSON.stringify(['grant-database',process.env.PGDATABASE])+'\\n'); +} +`); +for (const name of ['corescope-migrate','jq','supervisord','psql']) { + const file = path.join(name === 'corescope-migrate' ? app : path.join(temp, 'bin'), name); + fs.writeFileSync(file, `#!/bin/sh\nexec node ${shellQuote(posix(tool))} ${name} "$@"\n`, { mode: 0o755 }); +} +function materialize() { + let source = fs.readFileSync(path.join(root, 'docker/entrypoint-go.sh'), 'utf8'); + source = source.replaceAll('/app', posix(app)).replace('/usr/bin/supervisord', posix(path.join(temp, 'bin/supervisord'))); + fs.writeFileSync(path.join(temp, 'entrypoint.sh'), source, { mode: 0o755 }); + const helper = path.join(root, 'docker/storage.sh'); + if (fs.existsSync(helper)) fs.writeFileSync(path.join(app, 'storage.sh'), fs.readFileSync(helper, 'utf8').replaceAll('/app', posix(app)).replace('/proc/self/mountinfo', posix(path.join(temp, 'mountinfo'))), { mode: 0o755 }); + fs.writeFileSync(path.join(app,'postgres-bootstrap.sh'),fs.readFileSync(path.join(root,'docker/postgres-bootstrap.sh'),'utf8').replaceAll('/app',posix(app)),{mode:0o755}); +} +function run(extra = {}, action = [], bootstrap = false) { + fs.writeFileSync(calls, ''); + const env = { ...process.env, CALLS: calls, CORESCOPE_STATE_DIR: posix(path.join(app, 'data/custom')), CORESCOPE_DB_BACKEND: 'sqlite', SCENARIO: '', ...extra }; + const inherited = env.PATH || env.Path || ''; + delete env.Path; + env.PATH = path.join(temp, 'bin') + path.delimiter + inherited; + const result = spawnSync(bash, ['--noprofile','--norc',posix(bootstrap ? path.join(app,'postgres-bootstrap.sh') : action.length ? path.join(app,'storage.sh') : path.join(temp,'entrypoint.sh')), ...action], { cwd: app, env, encoding: 'utf8', timeout: 15000 }); + result.calls = fs.readFileSync(calls,'utf8').trim().split('\n').filter(Boolean).map(JSON.parse); + return result; +} +try { + materialize(); + const fresh = run(); + assert.equal(fresh.status,0,fresh.stderr || fresh.error?.message); + assert(fresh.calls.some(call => call.includes('setup') || call.includes('-storage-action=setup')), 'default Docker startup did not invoke guarded fresh/legacy setup'); + const last = fresh.calls.at(-1); assert.equal(last?.[0], 'supervisord', 'supervisor started before setup completed'); + const setup = fresh.calls.find(call => call.includes('setup') || call.includes('-storage-action=setup')); + assert(setup.includes('-offline'), 'guarded setup must be offline, before child processes'); + assert(setup.some(arg => arg.includes('/data/custom/storage-selection.json')), 'custom persistent selection path was ignored'); + const recreated = run({ CORESCOPE_DB_BACKEND: 'postgres' }); + assert.equal(recreated.status,0,recreated.stderr); + assert(!recreated.calls.some(call => call.includes('setup') || call.includes('-storage-action=setup')), 'recreate interpreted a stale bootstrap backend as new setup'); + assert.equal(recreated.calls.at(-1)?.[0],'supervisord'); + for (const scenario of ['pending','corrupt']) { + const blocked = run({ SCENARIO: scenario }); + assert.notEqual(blocked.status,0,scenario+' selection started'); + assert(!blocked.calls.some(call => call[0]==='supervisord'),scenario+' selection launched application'); + } + const corruptField = run({ SCENARIO: 'corrupt' }, ['field','backend']); + assert.notEqual(corruptField.status,0,'field reader hid the CLI corrupt-status failure'); + const failed = run({ SCENARIO: 'fail-setup', CORESCOPE_STATE_DIR: posix(path.join(app,'data/new')) }); + assert.notEqual(failed.status,0,'failed setup started the server'); + assert(!failed.calls.some(call=>call[0]==='supervisord')); + const ephemeral = run({ CORESCOPE_STATE_DIR: posix(path.join(temp,'ephemeral')) }); + assert.notEqual(ephemeral.status,0,'ephemeral selection directory accepted'); + assert.equal(ephemeral.calls.length,0,'unsupported path reached migration/runtime before rejection'); + for (const file of fs.readdirSync(path.join(root,'docker')).filter(name => /^supervisord-go.*[.]conf$/.test(name))) { + const config = fs.readFileSync(path.join(root,'docker',file),'utf8'); + assert(!/corescope-server[^\n]*-state-dir/.test(config), file+' overrides shared selection directory'); + } + const selectionFile=path.join(app,'data/custom/storage-selection.json'); + const target=database=>({postgres:{host:'postgres',port:5432,database,schema:'public'}}); + const pgReport={version:1,state:'ready',backend:'postgres',generation:'one',telemetry:target('corescope_telemetry'),accounts:null}; + for(const accounts of [null,target('corescope_accounts')]){ + fs.writeFileSync(selectionFile,JSON.stringify({...pgReport,accounts})); + const guard=run({},['managed-postgres']); + assert.equal(guard.status,0,'canonical PostgreSQL with optional accounts was rejected: '+guard.stderr); + const started=run({CORESCOPE_OWNER_PASSWORD:'synthetic-owner',CORESCOPE_USERS_OWNER_DATABASE_URL:accounts?'synthetic-owner-url':'',CORESCOPE_USERS_DATABASE_URL:'runtime-account-url'},[],true); + assert.equal(started.status,0,started.stderr||started.error?.message); + assert.deepEqual(started.calls.filter(call=>call[0]==='grant-database').map(call=>call[1]),accounts?['corescope_telemetry','corescope_accounts']:['corescope_telemetry']); + assert.deepEqual(started.calls.find(call=>call[0]==='readiness-accounts'),['readiness-accounts',accounts!==null]); + assert(!started.calls.some(call=>call.includes('-storage-action=setup')),'ready PostgreSQL was implicitly initialized'); + } + for(const report of [{...pgReport,telemetry:target('unselected_telemetry')},{...pgReport,accounts:target('unselected_accounts')},{...pgReport,accounts:{sqlite_path:'/app/data/users.db'}}]){ + fs.writeFileSync(selectionFile,JSON.stringify(report)); + const rejected=run({CORESCOPE_OWNER_PASSWORD:'synthetic-owner'},[],true); + assert.notEqual(rejected.status,0,'custom/non-PostgreSQL target passed the managed guard'); + assert(!rejected.calls.some(call=>call[0]==='psql'||call.includes('-check-ready')),'rejected target reached database operations'); + } + console.log('PASS packaged SQLite setup, recorded choice on recreate, blocked journals/errors and persistent custom state'); +} finally { + assert(path.resolve(temp).startsWith(path.resolve(os.tmpdir())+path.sep)&&path.basename(temp).startsWith('corescope-storage-entry-')); + fs.rmSync(temp,{recursive:true}); +} diff --git a/tests/unit/test-storage-manage.js b/tests/unit/test-storage-manage.js new file mode 100644 index 000000000..8396a2f04 --- /dev/null +++ b/tests/unit/test-storage-manage.js @@ -0,0 +1,103 @@ +// Exercise the actual management shell against a fake Compose/process boundary. +const assert=require('node:assert/strict'); +const fs=require('node:fs'),os=require('node:os'),path=require('node:path'); +const {spawnSync}=require('node:child_process'); +const root=path.resolve(__dirname,'../..'); +const source=fs.readFileSync(path.join(root,'manage.sh'),'utf8').replace(/\r/g,''); +const bash=require('../../scripts/bash-path')(); +const temp=fs.mkdtempSync(path.join(os.tmpdir(),'corescope-storage-manage-')); +const quote=value=>"'"+value.replace(/'/g,"'\\''")+"'"; +const posix=value=>value.replace(/\\/g,'/'); +const names=['is_true','dc_prod_base','dc_staging_base','dc_postgres','storage_at','storage_action','storage_field','selected_backend','require_managed_postgres','choose_storage','dc_prod','dc_staging','cmd_start','cmd_update','cmd_stop','cmd_storage']; +const functions=names.map(name=>source.match(new RegExp(`^${name}\\(\\)\\s*\\{[\\s\\S]*?^}`,'m'))?.[0]||'').join('\n'); +function run(command,backend='sqlite',state='ready',input=''){ + const log=path.join(temp,'calls');fs.writeFileSync(log,''); + const script=` +set -e +DC=fake_compose +PROD_DATA=${quote(posix(path.join(temp,'prod')))} +STAGING_DATA=${quote(posix(path.join(temp,'staging')))} +STAGING_COMPOSE_FILE=docker-compose.staging.yml +STAGING_CONTAINER=corescope-staging-go +RECORD_BACKEND=${quote(backend)} +RECORD_STATE=${quote(state)} +CORESCOPE_DB_BACKEND=sqlite +CALLS=${quote(posix(log))} +log(){ :; }; info(){ :; }; warn(){ :; }; err(){ printf '%s\\n' "$*" >&2; } +confirm(){ return 0; } +migrate_config(){ :; }; ensure_config(){ :; }; preflight_validate_prod_ports(){ :; } +write_private_env(){ export "$1=$2"; }; +prepare_database_credentials(){ printf 'credentials prepared\\n' >> "$CALLS"; }; +require_database_credentials(){ [ "$RECORD_BACKEND" = postgres ] || [ "\${STORAGE_OPERATION:-}" = switch ] || { err 'PostgreSQL credentials are not configured'; return 1; }; } +container_running(){ return 1; } +prepare_staging_db(){ :; };prepare_staging_config(){ :; } +docker(){ if [ "$1" = ps ];then return 0;fi;printf 'docker %s\\n' "$*" >> "$CALLS"; } +git(){ case "$1" in describe) echo v1.0.0;;rev-parse) echo fixture;;*) printf 'git %s\\n' "$*" >> "$CALLS";;esac; } +fake_compose(){ + case "$*" in + *'/app/storage.sh prod field state'|*'/app/storage.sh staging-go field state') printf '%s\\n' "$RECORD_STATE";; + *'/app/storage.sh prod field backend'|*'/app/storage.sh staging-go field backend') printf '%s\\n' "$RECORD_BACKEND";; + *'/app/storage.sh prod field has_accounts') printf 'false\\n';; + *'/app/storage.sh prod field state_dir') printf '/app/data\\n';; + *'/app/storage.sh prod field source_backend'|*'/app/storage.sh prod field target_backend') printf '%s\\n' "$RECORD_BACKEND";; + *'/app/storage.sh prod field job_id'|*'/app/storage.sh staging-go field job_id') printf '0123456789abcdef0123456789abcdef\\n';; + *) printf 'compose %s\\n' "$*" >> "$CALLS";; + esac +} +${functions} +${input ? `printf %s ${quote(input)} | ${command}` : command} +`; + const result=spawnSync(bash,['--noprofile','--norc'],{input:script,cwd:temp,encoding:'utf8',timeout:20000}); + result.calls=fs.readFileSync(log,'utf8');return result; +} +try{ + const sqlite=run("cmd_start ''"); + assert.equal(sqlite.status,0,sqlite.stderr||sqlite.error?.message); + assert(sqlite.calls.includes('up -d prod'),'SQLite app never started'); + assert(!sqlite.calls.includes('.postgres.yml'),'SQLite default selected a PostgreSQL overlay'); + const pg=run("cmd_start ''",'postgres'); + assert.equal(pg.status,0,pg.stderr); + assert(pg.calls.includes('-f docker-compose.postgres.yml'),'recorded PostgreSQL was replaced by stale SQLite bootstrap hint'); + const pending=run("cmd_start ''",'postgres','pending'); + assert.notEqual(pending.status,0,'pending conversion started'); + assert(!pending.calls.includes('up -d'),'pending conversion reached application startup'); + const update=run('cmd_update latest','postgres'); + assert.equal(update.status,0,update.stderr); + assert(update.calls.includes('build prod'),'update did not build its source'); + assert(update.calls.includes('docker-compose.postgres.yml'),'update discarded recorded backend'); + assert(!/storage.sh (switch|init)/.test(update.calls),'ordinary update changed backend'); + const defaultChoice=run('choose_storage','sqlite','unrecorded','\n'); + assert.equal(defaultChoice.status,0,defaultChoice.stderr); + assert(!defaultChoice.calls.includes('credentials prepared'),'default setup prepared PostgreSQL credentials'); + const postgresChoice=run('choose_storage','postgres','unrecorded','2\n'); + assert.equal(postgresChoice.status,0,postgresChoice.stderr); + assert(postgresChoice.calls.includes('credentials prepared'),'explicit PostgreSQL setup did not prepare its credentials'); + const keep=run('choose_storage','postgres','ready','1\n'); + assert.equal(keep.status,0,keep.stderr); + assert(!keep.calls.includes('credentials prepared'),'setup reconfigured credentials for an existing recorded backend'); + const switched=run('cmd_storage switch postgres','sqlite'); + assert.equal(switched.status,0,switched.stderr); + assert(switched.calls.includes('stop prod'),'conversion did not stop owned app processes'); + assert(switched.calls.includes('up -d --wait postgres'),'conversion did not prepare only the database service'); + assert(switched.calls.includes('bootstrap switch -backend=postgres'),'conversion bypassed verified CLI action'); + assert(!switched.calls.includes('up -d bootstrap'),'conversion pre-initialized its target schema'); + assert(!switched.calls.includes('up -d prod'),'conversion automatically restarted writers before operator validation'); + const reversed=run('cmd_storage switch sqlite','postgres'); + assert.equal(reversed.status,0,reversed.stderr); + assert(reversed.calls.includes('prod managed-postgres'),'reverse conversion skipped the recorded target guard'); + assert(reversed.calls.includes('bootstrap switch -backend=sqlite'),'reverse conversion bypassed the verified CLI'); + assert(!reversed.calls.includes('corescope_accounts'),'reverse routing touched the unselected account database'); + assert(reversed.calls.includes('stop postgres'),'completed reverse conversion left the managed PG service running'); + const sqliteResume=run('cmd_storage resume 0123456789abcdef0123456789abcdef','sqlite','pending'); + assert.equal(sqliteResume.status,0,sqliteResume.stderr); + assert(sqliteResume.calls.includes('prod resume -job-id='),'same-backend SQLite recovery bypassed the CLI'); + assert(!sqliteResume.calls.includes('.postgres.yml')&&!sqliteResume.calls.includes('up -d --wait postgres'),'SQLite-only recovery required PostgreSQL'); + const pgAbort=run('require_database_credentials(){ return 1; }; cmd_storage abort 0123456789abcdef0123456789abcdef','postgres','pending'); + assert.equal(pgAbort.status,0,pgAbort.stderr); + assert(pgAbort.calls.includes('prod abort -job-id='),'abort bypassed metadata-only CLI'); + assert(!pgAbort.calls.includes('.postgres.yml')&&!pgAbort.calls.includes('up -d --wait postgres'),'metadata abort started PostgreSQL'); + const stop=run('cmd_stop prod','postgres','pending'); + assert.equal(stop.status,0,stop.stderr); + assert(stop.calls.includes('stop prod'),'operator cannot stop app while recovery journal exists'); + console.log('PASS SQLite default start, authoritative PG routing/update, pending start refusal and stop during recovery'); +}finally{assert(path.resolve(temp).startsWith(path.resolve(os.tmpdir())+path.sep)&&path.basename(temp).startsWith('corescope-storage-manage-'));fs.rmSync(temp,{recursive:true});} From c03a2585eef3819cf06640911b2f7308d99339db Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 16:14:20 -0400 Subject: [PATCH 18/67] fix(build): complete native SQLite server dependency checksums --- cmd/server/go.mod | 1 + cmd/server/go.sum | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/cmd/server/go.mod b/cmd/server/go.mod index 072136857..98310c11e 100644 --- a/cmd/server/go.mod +++ b/cmd/server/go.mod @@ -58,6 +58,7 @@ require ( require ( github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect diff --git a/cmd/server/go.sum b/cmd/server/go.sum index 489fcca40..c370cf1a4 100644 --- a/cmd/server/go.sum +++ b/cmd/server/go.sum @@ -3,6 +3,10 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= @@ -32,6 +36,8 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -39,15 +45,33 @@ golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= +modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= +modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= +modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= +modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= +modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= From c77084aecbd5fd3911036a3b86ceeb18a8a4ddd1 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 16:44:57 -0400 Subject: [PATCH 19/67] test(ci): initialize first account fixtures explicitly --- .github/workflows/deploy.yml | 5 +++++ tests/unit/test-postgres-ops.js | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 46e5f454f..8f8df412b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -457,6 +457,7 @@ jobs: needs: [changes] runs-on: ubuntu-24.04 strategy: + fail-fast: false matrix: shard: [1, 2, 3] backend: [sqlite, postgres] @@ -705,6 +706,10 @@ jobs: } JSON if [ "$CORESCOPE_TEST_BACKEND" = sqlite ]; then + # Existing telemetry is adopted with no account target; initialize + # first accounts explicitly before starting the strict runtime. + ./corescope-migrate -storage-action=adopt -backend=sqlite -offline -config-dir /tmp/cs-um \ + -selection-file /tmp/cs-um/storage-selection.json -sqlite-path "$E2E_SQLITE" -users-sqlite-path /tmp/cs-um/users.db ./corescope-migrate -storage-action=setup -backend=sqlite -offline -config-dir /tmp/cs-um \ -selection-file /tmp/cs-um/storage-selection.json -sqlite-path "$E2E_SQLITE" -users-sqlite-path /tmp/cs-um/users.db else diff --git a/tests/unit/test-postgres-ops.js b/tests/unit/test-postgres-ops.js index 92212f078..e60ba5dd2 100644 --- a/tests/unit/test-postgres-ops.js +++ b/tests/unit/test-postgres-ops.js @@ -7,6 +7,17 @@ const path = require('path'); const { spawnSync } = require('child_process'); const root = path.resolve(__dirname, '../..'); +const workflow=fs.readFileSync(path.join(root,'.github/workflows/deploy.yml'),'utf8').replace(/\r/g,''); +const e2eJob=workflow.split('\n e2e-shard:\n')[1]?.split(/^ [a-z][a-z0-9-]*:\s*$/m)[0]; +const userStep=workflow.split('- name: Start user-management E2E server (fake mailer)')[1]?.split('\n - name:')[0]; +const sqliteSetup=userStep?.split('if [ "$CORESCOPE_TEST_BACKEND" = sqlite ]; then')[1]?.split(/^\s*else\s*$/m)[0]; +const fixtureCommands=sqliteSetup?.replace(/\\\n\s*/g,' ').match(/\.\/corescope-migrate[^\n]+/g)||[]; +assert.deepStrictEqual({actions:fixtureCommands.map(command=>command.match(/-storage-action=(\w+)/)?.[1]),failFastDisabled:/strategy:\n\s+fail-fast: false\n/.test(e2eJob||'')}, + {actions:['adopt','setup'],failFastDisabled:true},'UM fixture must adopt legacy telemetry then initialize accounts; all independent E2E shards must run'); +const fixtureArgs='-backend=sqlite -offline -config-dir /tmp/cs-um -selection-file /tmp/cs-um/storage-selection.json -sqlite-path "$E2E_SQLITE" -users-sqlite-path /tmp/cs-um/users.db'; +for(const command of fixtureCommands){ + assert.strictEqual(command.replace(/^\.\/corescope-migrate -storage-action=\w+\s+/,'').replace(/\s+/g,' '),fixtureArgs,'account setup changed the adopted selection, SQLite paths or config directory'); +} const source = fs.readFileSync(path.join(root, 'manage.sh'), 'utf8').replace(/\r/g, ''); const bash = require('../../scripts/bash-path')(); const functions = ['pg_container_exec', 'pg_exec', 'pg_empty', 'pg_dump_file', 'pg_restore_file', 'sqlite_source_exists', 'sqlite_dump_file', 'backup_state', 'stage_backup_state', 'restore_sqlite_bundle', 'cmd_backup', 'cmd_restore', 'prepare_staging_db', 'prepare_staging_config', 'write_env_managed_values', 'is_true', 'cmd_start'] From c6fee4e8872b6c0caf7af0aadc3013d48b704882 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 16:44:57 -0400 Subject: [PATCH 20/67] test(auth): use the native account SQLite driver for fixture access --- cmd/server/auth_fixture_test.go | 48 ++++++++++++++++++++++++++--- cmd/server/auth_login_audit_test.go | 2 +- cmd/server/go.mod | 2 +- 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/cmd/server/auth_fixture_test.go b/cmd/server/auth_fixture_test.go index c2c6da393..fe87e3641 100644 --- a/cmd/server/auth_fixture_test.go +++ b/cmd/server/auth_fixture_test.go @@ -2,12 +2,14 @@ package main import ( "bytes" + "database/sql" "encoding/json" "io" "net/http" "net/http/httptest" "net/url" "regexp" + "strings" "testing" "time" @@ -16,6 +18,7 @@ import ( "github.com/meshcore-analyzer/mailer" "github.com/meshcore-analyzer/pgutil/pgtest" "github.com/meshcore-analyzer/users" + "modernc.org/sqlite" ) const ( @@ -39,6 +42,43 @@ type client struct { me meResponse } +// Account owners must use the same SQLite library as users.Store, not the +// telemetry driver: https://sqlite.org/howtocorrupt.html#multiple_copies_of_sqlite_linked_into_the_same_application +func openAccountFixtureSQL(target string) (*sql.DB, error) { + if strings.HasPrefix(target, "postgres://") || strings.HasPrefix(target, "postgresql://") { + return openFixtureSQL(target) + } + uri, err := dbconfig.SQLiteURI(target, url.Values{"mode": {"rw"}, "_pragma": {"busy_timeout(5000)"}}) + if err != nil { + return nil, err + } + db, err := sql.Open("sqlite", uri) + if err == nil { + db.SetMaxOpenConns(1) + } + return db, err +} + +// Account fixture owners share the live users.Store file. Mixing SQLite +// implementations in one process bypasses their per-library lock bookkeeping. +func TestAccountFixtureUsesAccountSQLiteDriver(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite account driver boundary") + } + f := newAuthFixture(t) + db, err := openAccountFixtureSQL(f.ownerURL) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, ok := db.Driver().(*sqlite.Driver); !ok { + t.Fatalf("account fixture uses %T; users.Store uses modernc.org/sqlite", db.Driver()) + } + if err := db.Ping(); err != nil { + t.Fatal(err) + } +} + func newTestAuthService(t *testing.T, adminEmails ...string) (*authService, *mailer.Fake) { a, fake, _ := newTestAuthServiceWithURL(t, postgresTestDSN(t), adminEmails...) return a, fake @@ -53,7 +93,7 @@ func newTestAuthServiceWithURL(t *testing.T, ownerURL string, adminEmails ...str t.Helper() runtimeURL := ownerURL if testBackend(t) == dbconfig.Postgres { - owner, err := openFixtureSQL(ownerURL) + owner, err := openAccountFixtureSQL(ownerURL) if err != nil { t.Fatal(err) } @@ -222,7 +262,7 @@ func (f *authFixture) login(t *testing.T, email, password string) *client { // store call that touches it fails with a DB error (not ErrNotFound). func (f *authFixture) breakTable(t *testing.T, table string) { t.Helper() - db, err := openFixtureSQL(f.ownerURL) + db, err := openAccountFixtureSQL(f.ownerURL) if err != nil { t.Fatal(err) } @@ -236,7 +276,7 @@ func (f *authFixture) breakTable(t *testing.T, table string) { // from users.db (the raw tokens are not observable when no mail left). func (f *authFixture) unusedTokens(t *testing.T, uid int64, p users.Purpose) int { t.Helper() - db, err := openFixtureSQL(f.ownerURL) + db, err := openAccountFixtureSQL(f.ownerURL) if err != nil { t.Fatal(err) } @@ -252,7 +292,7 @@ func (f *authFixture) unusedTokens(t *testing.T, uid int64, p users.Purpose) int // simulate a concurrent writer or a failing statement). func (f *authFixture) execDB(t *testing.T, stmt string) { t.Helper() - db, err := openFixtureSQL(f.ownerURL) + db, err := openAccountFixtureSQL(f.ownerURL) if err != nil { t.Fatal(err) } diff --git a/cmd/server/auth_login_audit_test.go b/cmd/server/auth_login_audit_test.go index ff2dbf24a..29f6076ca 100644 --- a/cmd/server/auth_login_audit_test.go +++ b/cmd/server/auth_login_audit_test.go @@ -133,7 +133,7 @@ func TestLoginAnswersBeforeTheAuditWrite(t *testing.T) { f := newAuthFixture(t) dave := f.registerAndActivate(t, "dave@example.org", "Dave", pw) - db, err := openFixtureSQL(f.ownerURL) + db, err := openAccountFixtureSQL(f.ownerURL) if err != nil { t.Fatal(err) } diff --git a/cmd/server/go.mod b/cmd/server/go.mod index 98310c11e..8cec7a049 100644 --- a/cmd/server/go.mod +++ b/cmd/server/go.mod @@ -54,6 +54,7 @@ replace github.com/meshcore-analyzer/users => ../../internal/users require ( github.com/jackc/pgx/v5 v5.7.6 github.com/meshcore-analyzer/channel v0.0.0 + modernc.org/sqlite v1.34.5 ) require ( @@ -69,7 +70,6 @@ require ( modernc.org/libc v1.55.3 // indirect modernc.org/mathutil v1.6.0 // indirect modernc.org/memory v1.8.0 // indirect - modernc.org/sqlite v1.34.5 // indirect ) replace github.com/meshcore-analyzer/channel => ../../internal/channel From f7ffdfb69d32d319a460e0ff5132b4d1f0f17f00 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 18:29:29 -0400 Subject: [PATCH 21/67] fix(sqlite): restore observer planner statistics after identity migration --- cmd/server/db.go | 28 ++-- cmd/server/observer_planner_test.go | 211 ++++++++++++++++++++++++++++ internal/dbschema/sqlite.go | 22 ++- internal/dbschema/sqlite_test.go | 15 ++ 4 files changed, 260 insertions(+), 16 deletions(-) create mode 100644 cmd/server/observer_planner_test.go diff --git a/cmd/server/db.go b/cmd/server/db.go index 6c7fadc56..297490533 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -2401,23 +2401,25 @@ func (db *DB) GetMaxObservationID() int { return maxID } -// GetObserverPacketCounts returns packetsLastHour for all observers (batch query). -func (db *DB) GetObserverPacketCounts(sinceEpoch int64) map[string]int { - counts := make(map[string]int) - var rows *sql.Rows - var err error +// observerPacketCountsSQL is shared with the native planner regression. +func (db *DB) observerPacketCountsSQL() string { if db.isV3 { - rows, err = db.conn.Query(`SELECT obs.id, COUNT(*) as cnt + return `SELECT obs.id, COUNT(*) as cnt FROM observations o JOIN observers obs ON obs.rowid = o.observer_idx - WHERE o.timestamp > `+db.parameter(1)+` - GROUP BY obs.id`, sinceEpoch) - } else { - rows, err = db.conn.Query(`SELECT o.observer_id, COUNT(*) as cnt - FROM observations o - WHERE o.observer_id IS NOT NULL AND o.timestamp > `+db.parameter(1)+` - GROUP BY o.observer_id`, sinceEpoch) + WHERE o.timestamp > ` + db.parameter(1) + ` + GROUP BY obs.id` } + return `SELECT o.observer_id, COUNT(*) as cnt + FROM observations o + WHERE o.observer_id IS NOT NULL AND o.timestamp > ` + db.parameter(1) + ` + GROUP BY o.observer_id` +} + +// GetObserverPacketCounts returns packetsLastHour for all observers (batch query). +func (db *DB) GetObserverPacketCounts(sinceEpoch int64) map[string]int { + counts := make(map[string]int) + rows, err := db.conn.Query(db.observerPacketCountsSQL(), sinceEpoch) if err != nil { return counts } diff --git a/cmd/server/observer_planner_test.go b/cmd/server/observer_planner_test.go new file mode 100644 index 000000000..d49208966 --- /dev/null +++ b/cmd/server/observer_planner_test.go @@ -0,0 +1,211 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + "time" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" +) + +func TestGetObserverPacketCountsWindowAndIdentity(t *testing.T) { + db := setupTestDB(t) + for _, query := range []string{ + `INSERT INTO observers(rowid,id,inactive) VALUES(1,'obs-a',0),(2,'obs-b',1)`, + `INSERT INTO transmissions(id,raw_hex,hash,first_seen) VALUES(1,'00','counts','2026-01-01')`, + `INSERT INTO observations(transmission_id,observer_idx,timestamp) VALUES + (1,1,100),(1,1,101),(1,1,102),(1,2,101),(1,NULL,101),(1,99,101)`, + } { + if _, err := db.conn.Exec(query); err != nil { + t.Fatal(err) + } + } + // Strict cutoff and inner-join identity semantics are shared by both engines. + want := map[string]int{"obs-a": 2, "obs-b": 1} + if got := db.GetObserverPacketCounts(100); !reflect.DeepEqual(got, want) { + t.Fatalf("observer counts=%v, want=%v", got, want) + } +} + +// Match the B corpus's cardinalities: 128 observers, 16 observations per +// transmission on average, eight days, and one eighth in the recent window. +// SQL-only data exercises the real endpoint pipeline without a PacketStore. +func observerPlannerFixture(t testing.TB, observations int) (*sql.DB, string, int64, map[string]int) { + t.Helper() + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite identity migration and planner fixture") + } + path := filepath.Join(t.TempDir(), "observers.db") + dsn, err := legacy.WriterDSN(path) + if err != nil { + t.Fatal(err) + } + w, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal(err) + } + w.SetMaxOpenConns(1) + t.Cleanup(func() { w.Close() }) + if err = legacy.ApplyBase(w); err != nil { + t.Fatal(err) + } + if err = legacy.Apply(w, nil); err != nil { + t.Fatal(err) + } + if err = dbschema.EnsureSQLiteObserverTimeIndex(context.Background(), w); err != nil { + t.Fatal(err) + } + epoch := time.Now().Unix() + tx, err := w.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + for _, query := range []string{ + `WITH RECURSIVE n(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM n WHERE i<128) + INSERT INTO observers(rowid,id,name,iata,last_seen,first_seen,packet_count) + SELECT i,printf('%064x',i),printf('Synthetic observer %d',i),'AAA','2026-01-01','2026-01-01',0 FROM n`, + `INSERT INTO nodes(public_key,name,role,lat,lon) SELECT id,name,'repeater',20,30 FROM observers`, + fmt.Sprintf(`WITH RECURSIVE n(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM n WHERE i<%d) + INSERT INTO transmissions(id,raw_hex,hash,first_seen) SELECT i,'00',printf('fixture-%%d',i),'2026-01-01' FROM n`, observations/16), + fmt.Sprintf(`WITH RECURSIVE n(i) AS (VALUES(0) UNION ALL SELECT i+1 FROM n WHERE i+1<%d) + INSERT INTO observations(transmission_id,observer_idx,timestamp,path_json) + SELECT i/16+1,i%%128+1,CASE WHEN (i/160)%%8=0 THEN %d-2700+(i/16)%%1800 + ELSE %d-((i/160)%%8)*86400-43200+(i/16)%%3600 END,'[]' FROM n`, observations, epoch, epoch), + } { + if _, err = tx.Exec(query); err != nil { + t.Fatal(err) + } + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + if _, err = w.Exec(`ANALYZE`); err != nil { + t.Fatal(err) + } + want := make(map[string]int) + for i := 0; i < observations; i++ { + if (i/160)%8 == 0 { + want[fmt.Sprintf("%064x", i%128+1)]++ + } + } + return w, path, epoch - 3600, want +} + +func observerPlannerReader(t testing.TB, path string, since int64, want map[string]int) *DB { + t.Helper() + db, err := OpenDB(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + if got := db.GetObserverPacketCounts(since); !reflect.DeepEqual(got, want) { + t.Fatalf("recent observation counts changed: got %d observer groups, want %d", len(got), len(want)) + } + return db +} + +func observerAggregatePlan(t testing.TB, db *DB, since int64) []string { + t.Helper() + rows, err := db.conn.Query("EXPLAIN QUERY PLAN "+db.observerPacketCountsSQL(), since) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var plan []string + for rows.Next() { + var id, parent, unused int + var detail string + if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { + t.Fatal(err) + } + plan = append(plan, detail) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return plan +} + +func TestObserverAggregatePlannerAfterSQLiteUpgrade(t *testing.T) { + for _, repair := range []bool{false, true} { + name := "legacy adoption" + if repair { + name = "already rebuilt without statistics" + } + t.Run(name, func(t *testing.T) { + w, path, since, want := observerPlannerFixture(t, 128000) + if repair { + if err := dbschema.ApplySQLite(w, nil); err != nil { + t.Fatal(err) + } + if _, err := w.Exec(`DELETE FROM sqlite_stat1 WHERE tbl='observers'; ANALYZE sqlite_schema`); err != nil { + t.Fatal(err) + } + } + if err := dbschema.ApplySQLite(w, nil); err != nil { + t.Fatal(err) + } + // A new reader is important: the old writer can retain pre-rebuild + // planner estimates and conceal the statistics lost by DROP TABLE. + db := observerPlannerReader(t, path, since, want) + plan := observerAggregatePlan(t, db, since) + if strings.Contains(strings.Join(plan, "\n"), "TEMP B-TREE FOR GROUP BY") { + t.Fatalf("observer statistics lost after upgrade; aggregate sorts the observation window: %q", plan) + } + if !strings.Contains(strings.Join(plan, "\n"), "observer_idx=? AND timestamp>?") { + t.Fatalf("aggregate no longer uses observer/time range lookups: %q", plan) + } + }) + } +} + +// Run explicitly with -benchtime=50x. Reports uncached response-build latency, +// not full concurrent HTTP latency or a replacement for the paired workload. +func BenchmarkObserverListSQLiteUpgrade(b *testing.B) { + w, path, since, want := observerPlannerFixture(b, 2048000) + var reference ObserverListResponse + for _, stage := range []string{"legacy", "upgraded"} { + if stage == "upgraded" { + if err := dbschema.ApplySQLite(w, nil); err != nil { + b.Fatal(err) + } + } + b.Run(stage, func(b *testing.B) { + db := observerPlannerReader(b, path, since, want) + srv := &Server{db: db, cfg: &Config{}} + b.Logf("observations=2048000 transmissions=128000 observers=128 recent=%d; plan=%q", 256000, observerAggregatePlan(b, db, since)) + response, err := srv.buildObserversDefaultResponse() + if err != nil || len(response.Observers) != 128 { + b.Fatalf("response rows=%d err=%v", len(response.Observers), err) + } + response.ServerTime = "" + if stage == "legacy" { + reference = response + } else if !reflect.DeepEqual(reference, response) { + b.Fatal("observer response semantics changed after identity migration") + } + durations := make([]time.Duration, 0, b.N) + b.ResetTimer() + for i := 0; i < b.N; i++ { + start := time.Now() + if _, err := srv.buildObserversDefaultResponse(); err != nil { + b.Fatal(err) + } + durations = append(durations, time.Since(start)) + } + b.StopTimer() + sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] }) + b.ReportMetric(float64(durations[(len(durations)-1)/2])/float64(time.Millisecond), "p50-ms") + b.ReportMetric(float64(durations[(len(durations)*95+99)/100-1])/float64(time.Millisecond), "p95-ms") + }) + } +} diff --git a/internal/dbschema/sqlite.go b/internal/dbschema/sqlite.go index 8b2f2044d..13f3c8d39 100644 --- a/internal/dbschema/sqlite.go +++ b/internal/dbschema/sqlite.go @@ -74,10 +74,26 @@ func ApplySQLite(db *sql.DB, logf Logger) error { } // The identity migration's corpus/FK scans run once. Ordinary restarts // retain existing schema maintenance without rescanning all observations. - if !needsIdentity { - return AssertSQLiteReady(db) + if needsIdentity { + if err := migrateSQLiteObserverIdentity(db); err != nil { + return err + } + } + // Rebuilding observers drops its sqlite_stat1 entries and can turn the + // recent-packet aggregate into a scan/sort of the observation window. + // Refresh only observers, also repairing previously rebuilt databases. + // Do not create sqlite_stat1 on a fresh database: its absence tells the + // ingestor that the initial whole-database analysis still needs to run. + var analyzed int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='table' AND name='sqlite_stat1'`).Scan(&analyzed); err != nil { + return err + } + if analyzed != 0 { + if _, err := db.Exec(`ANALYZE observers`); err != nil { + return fmt.Errorf("refresh SQLite observer planner statistics: %w", err) + } } - return migrateSQLiteObserverIdentity(db) + return AssertSQLiteReady(db) } // EnsureSQLiteAsyncMigrations retains the upstream ingestor's exact ledger diff --git a/internal/dbschema/sqlite_test.go b/internal/dbschema/sqlite_test.go index bd55113e8..74eac0e82 100644 --- a/internal/dbschema/sqlite_test.go +++ b/internal/dbschema/sqlite_test.go @@ -109,6 +109,21 @@ func TestSQLiteFreshAndReadOnlyReadiness(t *testing.T) { } } +func TestSQLiteFreshSchemaDoesNotSuppressInitialPlannerAnalysis(t *testing.T) { + db, _ := sqliteTestDB(t) + for range 2 { + if err := ApplySQLite(db, nil); err != nil { + t.Fatal(err) + } + } + // The ingestor uses sqlite_stat1's presence to decide whether its initial + // whole-database ANALYZE is required. A table-only refresh must not mask it. + var tables int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name='sqlite_stat1'`).Scan(&tables); err != nil || tables != 0 { + t.Fatalf("fresh database appears fully analyzed: tables=%d err=%v", tables, err) + } +} + func TestSQLiteObserverIdentityPreservesOldRowsAndReferences(t *testing.T) { db, _ := sqliteTestDB(t) if err := legacy.ApplyBase(db); err != nil { From 8c79c4e5f6d9d5a7083f38db1a6fcb2737c8a1fd Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 18:29:45 -0400 Subject: [PATCH 22/67] test: await background readiness before startup API assertions --- cmd/server/analytics_recompute_after_load_test.go | 13 +++++++++++-- cmd/server/paths_observations_race_test.go | 10 ++++++++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/cmd/server/analytics_recompute_after_load_test.go b/cmd/server/analytics_recompute_after_load_test.go index 7c29d7456..34ee530bf 100644 --- a/cmd/server/analytics_recompute_after_load_test.go +++ b/cmd/server/analytics_recompute_after_load_test.go @@ -433,9 +433,18 @@ func TestAnalyticsRecomputers_FullDataRightAfterStartupLoad(t *testing.T) { t.Fatalf("fixture precondition: %d packets in memory after load, want %d", inMemory, totalRows) } - waitForTest(t, "rf gate opens after full load", func() bool { return get("/api/analytics/rf").Code == http.StatusOK }) + // Post-load passes run sequentially: RF completing does not imply that + // topology or channels has published its full snapshot yet. + for _, name := range []string{"rf", "topology", "channels"} { + rc := all[name] + waitForTest(t, name+" first full pass finishes", func() bool { return !rc.FirstPassDoneAt_1659().IsZero() }) + } + rfResponse := get("/api/analytics/rf") + if rfResponse.Code != http.StatusOK { + t.Fatalf("rf after full load: got %d, want 200", rfResponse.Code) + } var rf map[string]interface{} - if err := json.Unmarshal(get("/api/analytics/rf").Body.Bytes(), &rf); err != nil { + if err := json.Unmarshal(rfResponse.Body.Bytes(), &rf); err != nil { t.Fatalf("rf body: %v", err) } if got, _ := rf["totalTransmissions"].(float64); int(got) != totalRows { diff --git a/cmd/server/paths_observations_race_test.go b/cmd/server/paths_observations_race_test.go index c4fbe840c..d491785c5 100644 --- a/cmd/server/paths_observations_race_test.go +++ b/cmd/server/paths_observations_race_test.go @@ -4,6 +4,7 @@ import ( "net/http/httptest" "sync" "testing" + "time" "github.com/gorilla/mux" ) @@ -13,6 +14,9 @@ import ( // ingest appends to under the write lock. Run with -race. func TestNodePathsDoesNotReadObservationsUnlocked(t *testing.T) { store := newResolvedPathStore(t) + if !store.WaitIndexesReady(5 * time.Second) { + t.Fatal("background indexes not ready within 5s") + } srv := NewServer(store.db, &Config{Port: 3000}, NewHub()) srv.store = store router := mux.NewRouter() @@ -26,6 +30,10 @@ func TestNodePathsDoesNotReadObservationsUnlocked(t *testing.T) { stop := make(chan struct{}) var wg sync.WaitGroup wg.Add(1) + defer func() { + close(stop) + wg.Wait() + }() go func() { // stands in for ingest adding an observation to the tx defer wg.Done() for i := 0; ; i++ { @@ -50,6 +58,4 @@ func TestNodePathsDoesNotReadObservationsUnlocked(t *testing.T) { t.Fatalf("paths: status %d: %s", w.Code, w.Body.String()) } } - close(stop) - wg.Wait() } From f15211faea412f98aa847127ed9f7d45742f98c4 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 18:29:59 -0400 Subject: [PATCH 23/67] docs: publish optional backend measurements and sizing tradeoffs --- .../optional-backend-b5-2026-10-09.json | 37670 ++++++++++++++++ .../optional-backend-b5-2026-10-09.md | 91 + docs/postgresql-performance.md | 4 +- docs/storage.md | 4 +- 4 files changed, 37766 insertions(+), 3 deletions(-) create mode 100644 docs/performance/optional-backend-b5-2026-10-09.json create mode 100644 docs/performance/optional-backend-b5-2026-10-09.md diff --git a/docs/performance/optional-backend-b5-2026-10-09.json b/docs/performance/optional-backend-b5-2026-10-09.json new file mode 100644 index 000000000..56c7e9b3b --- /dev/null +++ b/docs/performance/optional-backend-b5-2026-10-09.json @@ -0,0 +1,37670 @@ +{ + "title": "Optional backend B5 evidence, exact c03 candidate", + "measured_candidate": "c03a2585eef3819cf06640911b2f7308d99339db", + "baseline": "3e21b179ecccae005aa5f6f45bc095353c94b343", + "methodology": { + "independent_unit": "paired run, five pairs per study", + "ratio": "median of within-pair candidate/baseline ratios", + "absolutes": "median of per-run nearest-rank quantiles", + "p99_minimum_samples_per_leg": 1000, + "logical_digest_exclusions": { + "transmissions": [ + "created_at" + ], + "observers": [ + "last_seen", + "last_packet_at", + "first_seen" + ], + "_async_migrations": [ + "started_at", + "ended_at" + ], + "scope_match_totals": [ + "updated_unix", + "since_unix" + ] + }, + "cross_study_backend_ranking": false, + "service": "completed minus started/dispatched", + "scheduled_to_complete": "completed minus scheduled; includes queue wait", + "resource_window": "first/last samples inside exact measured schedule window", + "resource_counter_scope": "raw memory.events counters are cumulative; only measured deltas are per-window", + "body_parity_limit": "static/isolated controls and data digests matched; dynamic HTTP body equality not universally established" + }, + "source_hashes": { + "refs": { + "baseline": "3e21b179ecccae005aa5f6f45bc095353c94b343", + "candidate": "c03a2585eef3819cf06640911b2f7308d99339db", + "sqlite_harness": "4de419413c0a6fd93ecc2361cd704cd2a59e673e" + }, + "production": { + "baseline": "8b8fd97e7e8b834dc7829e7615f2c192faef9def8e20987874fa06606e182f7f", + "candidate": "74bcf9bd678e4b320e3da198c02a4f2696453360c81dfa1fa5483e3e9c100ada" + }, + "sqlite_harness": { + "ref": "4de419413c0a6fd93ecc2361cd704cd2a59e673e", + "sha256": "f06f836de930a16d428b7c644231d8460e1a7ab736eed855e5d3eae7f2e23e96", + "files": { + "scripts/sqlite-benchmark/.gitignore": "32dae3052f331ee34d628ef535709b301259a45df7c7522c4d35dcf49873f00b", + "scripts/sqlite-benchmark/README.md": "edbd48971c60fb7aad1e111e429a0081801210ac72ecda8566789402cb2c2053", + "scripts/sqlite-benchmark/overlays/clock_linux_test.go": "f9f4bec0b594e8d5821e898e74c36d436e7ecbbb87acdfe9f200c23b37e04a30", + "scripts/sqlite-benchmark/overlays/clock_other_test.go": "88a4366455cd183dc86fb3bcb71b9825a8220974b2cf7fff137a858625635356", + "scripts/sqlite-benchmark/overlays/common_test.go": "eed3138244b560c78cf9231502c7998175405e36690dd91ae2f8a4e2afd5fab1", + "scripts/sqlite-benchmark/overlays/ingestor_test.go": "3e1011ae8b5c7e9184565dd794ee5b66ed73b8b7ceed6038eaf0c6434f96dfda", + "scripts/sqlite-benchmark/overlays/profiles_test.go": "33df8868db54d836e872e6a9791f33410dfe7fc823808f1e1af4ef90db60e0f4", + "scripts/sqlite-benchmark/overlays/server_test.go": "971ec39ea9de6ab27dabb415bf898c1e848e13fb139a0ac0c9a591a5e89c5418", + "scripts/sqlite-benchmark/overlays/sqlite_adapter_test.go": "5e5077109d0357a5603054a39d94623107c00805f428ccc8bbc5438d09346f6e", + "scripts/sqlite-benchmark/run.py": "a882ed9fcac4f348d03b00eab220da57588ac86bbe2adb97546ca13aa4ad4248", + "scripts/sqlite-benchmark/test_harness.py": "c41f0ed43624d588a952973b1b2204f9b536b2844c9f8d7fe06da42e60f242c5", + "scripts/sqlite-benchmark/test_sqlite_controls.py": "b06775795ca7dec29d34cd2b34b4dcdbb0d66a5d0a6341679fbdd4724a2dec89" + } + }, + "postgres_harness": { + "ref": "c03a2585eef3819cf06640911b2f7308d99339db", + "sha256": "7c234ba61c93351688d7068c9d8405f7d2906558d797854738eb0ff059f35d84", + "files": { + "scripts/postgres-benchmark/.gitignore": "32dae3052f331ee34d628ef535709b301259a45df7c7522c4d35dcf49873f00b", + "scripts/postgres-benchmark/README.md": "c82f10b3492e4eddbc22c5696f676143a59556fb6059281836fc7c9dd9863563", + "scripts/postgres-benchmark/overlays/clock_linux_test.go": "f9f4bec0b594e8d5821e898e74c36d436e7ecbbb87acdfe9f200c23b37e04a30", + "scripts/postgres-benchmark/overlays/clock_other_test.go": "88a4366455cd183dc86fb3bcb71b9825a8220974b2cf7fff137a858625635356", + "scripts/postgres-benchmark/overlays/common_test.go": "e82310c29c6a594824ca6ae4dae139a4232a962576f0f2ffcea6b90053a22a20", + "scripts/postgres-benchmark/overlays/ingestor_test.go": "0b8a02040d83bd858d99c72f163ddead269e8414b7f1981e36e4571d37e7a4e1", + "scripts/postgres-benchmark/overlays/postgres_adapter_test.go": "2c3d7f2fc0377519482b6ed7f78267345ed0a8f16eabab429d5fbba0ff242331", + "scripts/postgres-benchmark/overlays/server_test.go": "a2a5c1b74b26ea3a5f118c21e4bdbbb5ab221a478ab94c371e7f3a950f0874a0", + "scripts/postgres-benchmark/overlays/sqlite_adapter_test.go": "ca35f4f96ce121334e9193e4020c947c95811097ed1fb2ac7ef45fdc3a295a94", + "scripts/postgres-benchmark/run.py": "a400fea3c42bd22237b6d0cc27bb94982250f4b398c8693e0d288292ba66ec7f", + "scripts/postgres-benchmark/test_harness.py": "73a9efadee400ed290051bee4e84baf4f958c237e0d19620a13aaf98997df103" + } + } + }, + "studies": { + "sqlite_before_after": { + "run_id": 37985636346, + "artifact_id": 11645874366, + "baseline": "3e21b179ecccae005aa5f6f45bc095353c94b343", + "candidate": "c03a2585eef3819cf06640911b2f7308d99339db", + "harness": "4de419413c0a6fd93ecc2361cd704cd2a59e673e", + "eligibility": "passed independent raw-log/manifest checks", + "manifest": { + "baseline_production_hash_after": "8b8fd97e7e8b834dc7829e7615f2c192faef9def8e20987874fa06606e182f7f", + "baseline_production_hash_before": "8b8fd97e7e8b834dc7829e7615f2c192faef9def8e20987874fa06606e182f7f", + "baseline_sha": "3e21b179ecccae005aa5f6f45bc095353c94b343", + "baseline_storage_selection_required": false, + "binary_sha256": { + "baseline-ingestor.test": "2a83cf0e0ead533ca88248d1923e908fa83fe92ab0d3b4dfd0b60fbb553e1108", + "baseline-server": "80edb7b741613974dd88dbec21f756fabf63cf83d5b9bf365b2eb306c0d84f30", + "baseline-server.test": "a0bb965ff4435a166bc524e0913b4c1909296bdb299e94b425cc3b1c7edf64f1", + "candidate-ingestor.test": "df9a2a13650012567254b3109958e0cc02b1d56c2e7923ead2a3a5a767bd541a", + "candidate-migrate": "646d8cee19bbddfa3b64d7ef141be14a44fad0f0b9454c4476f82741467dde67", + "candidate-server": "6e81b75e618b9e7014d7827a0ec6e37d5caa692820d57b7bb2463b99eb09dfb5", + "candidate-server.test": "4714697219799f4b0dd91ffe9fa4d5359c0c37ef070bc579534c9b7be1042aed" + }, + "cache_policy": { + "mixed": "application warmup before measured requests", + "sql": "channel application cache explicitly cleared; OS cache warm", + "startup": "new process, OS cache warm" + }, + "candidate_production_hash_after": "74bcf9bd678e4b320e3da198c02a4f2696453360c81dfa1fa5483e3e9c100ada", + "candidate_production_hash_before": "74bcf9bd678e4b320e3da198c02a4f2696453360c81dfa1fa5483e3e9c100ada", + "candidate_sha": "c03a2585eef3819cf06640911b2f7308d99339db", + "candidate_storage_selection_required": true, + "canonical_file_sha256": "01f94c2f6e078cccdb1f656189a56a0ff1978fa42976936d04861f4bb63baaf9", + "capacity_guards": { + "available_disk_bytes": 89770668032, + "available_ram_bytes": 15723311104, + "minimum_disk_bytes": 12884901888, + "minimum_ram_bytes": 8589934592 + }, + "comparison_eligible": true, + "comparison_kind": "source-pinned SQLite before/after", + "compiler": "cc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0", + "completed_pairs": 5, + "corpus": "B", + "declared_loaded_counts": null, + "engine": "native CGO SQLite", + "epoch": 1791577164, + "firmware_reference": "a366955cb2f67b8e6842d4f00d2b6a554dddd88a", + "go_version": "go version go1.27.2 linux/amd64", + "harness_sha256": "f06f836de930a16d428b7c644231d8460e1a7ab736eed855e5d3eae7f2e23e96", + "harness_source_sha": "4de419413c0a6fd93ecc2361cd704cd2a59e673e", + "harness_text_normalization": "UTF-8 with LF line endings", + "host": { + "architecture": "x86_64", + "cpus": 4, + "kernel": "6.17.0-1022-azure", + "ram_bytes": 16766414848, + "system": "Linux" + }, + "measured_rows": 930, + "order": [ + [ + 0, + "baseline" + ], + [ + 0, + "candidate" + ], + [ + 1, + "candidate" + ], + [ + 1, + "baseline" + ], + [ + 2, + "baseline" + ], + [ + 2, + "candidate" + ], + [ + 3, + "candidate" + ], + [ + 3, + "baseline" + ], + [ + 4, + "baseline" + ], + [ + 4, + "candidate" + ] + ], + "profile": "primary", + "profiling": false, + "resource_budget": { + "allowed_cpus": [ + 0, + 1, + 2, + 3 + ], + "cpu_limit": 3.0, + "enforced": true, + "inactive_charge_reset_bytes": 3301376, + "memory_limit_bytes": 6442450944, + "memory_peak_reset": true, + "resource_counters": [ + "cgroup.procs", + "memory.current", + "memory.events", + "cpu.stat", + "io.stat" + ] + }, + "seed": 20261008, + "shape": { + "days": 8, + "nodes": 2000, + "observations": 2048000, + "observers": 128, + "transmissions": 128000 + }, + "source_databases_unchanged": true, + "status": "completed", + "table_manifest": [ + { + "columns": [ + "public_key", + "name", + "role", + "lat", + "lon", + "last_seen", + "first_seen", + "advert_count", + "battery_mv", + "temperature_c", + "foreign_advert", + "default_scope", + "configured_scope", + "configured_scope_at", + "multibyte_sup", + "multibyte_evidence" + ], + "name": "nodes" + }, + { + "columns": [ + "public_key", + "name", + "role", + "lat", + "lon", + "last_seen", + "first_seen", + "advert_count", + "battery_mv", + "temperature_c", + "foreign_advert", + "default_scope", + "configured_scope", + "configured_scope_at", + "multibyte_sup", + "multibyte_evidence" + ], + "name": "inactive_nodes" + }, + { + "columns": [ + "rowid", + "id", + "name", + "iata", + "last_seen", + "first_seen", + "packet_count", + "model", + "firmware", + "client_version", + "radio", + "battery_mv", + "uptime_secs", + "noise_floor", + "inactive", + "last_packet_at", + "clock_skew_seconds", + "clock_skew_count_24h", + "clock_last_naive_at", + "can_relay", + "can_relay_seen" + ], + "name": "observers" + }, + { + "columns": [ + "id", + "raw_hex", + "hash", + "first_seen", + "route_type", + "payload_type", + "payload_version", + "decoded_json", + "from_pubkey", + "last_seen", + "created_at", + "code1", + "code2", + "channel_hash", + "scope_name" + ], + "name": "transmissions" + }, + { + "columns": [ + "id", + "transmission_id", + "observer_idx", + "direction", + "snr", + "rssi", + "score", + "path_json", + "timestamp", + "resolved_path", + "raw_hex" + ], + "name": "observations" + }, + { + "columns": [ + "observer_id", + "timestamp", + "noise_floor", + "tx_air_secs", + "rx_air_secs", + "recv_errors", + "battery_mv", + "packets_sent", + "packets_recv" + ], + "name": "observer_metrics" + }, + { + "columns": [ + "node_a", + "node_b", + "count", + "last_seen" + ], + "name": "neighbor_edges" + }, + { + "columns": [ + "id", + "hash", + "raw_hex", + "reason", + "observer_id", + "observer_name", + "node_pubkey", + "node_name", + "dropped_at" + ], + "name": "dropped_packets" + }, + { + "columns": [ + "id", + "rx_pubkey", + "heard_key", + "heard_keylen", + "rssi", + "snr", + "lat", + "lon", + "pos_acc_m", + "rx_at", + "ingested_at", + "src" + ], + "name": "client_receptions" + }, + { + "columns": [ + "pubkey", + "name", + "last_seen" + ], + "name": "client_observers" + }, + { + "columns": [ + "id", + "rx_pubkey", + "rx_at", + "ingested_at", + "pkt_hash", + "route_type", + "payload_type", + "code1", + "code2", + "scope_name", + "hash_size", + "hop_count", + "path_json", + "forwarder", + "snr", + "rssi", + "lat", + "lon", + "pos_acc_m" + ], + "name": "client_rx_observations" + }, + { + "columns": [ + "id", + "rx_pubkey", + "sampled_at", + "ingested_at", + "lat", + "lon", + "pos_acc_m", + "stationary", + "uptime_secs", + "battery_mv", + "queue_len", + "errors", + "noise_floor", + "last_rssi", + "last_snr", + "tx_air_secs", + "rx_air_secs", + "recv", + "sent", + "flood_rx", + "direct_rx", + "flood_tx", + "direct_tx", + "recv_errors" + ], + "name": "client_rf_samples" + }, + { + "columns": [ + "id", + "target", + "rx_pubkey", + "observed_at", + "ingested_at", + "regions_csv", + "truncated", + "lat", + "lon", + "pos_acc_m", + "repeater_clock" + ], + "name": "node_declared_regions" + }, + { + "columns": [ + "id", + "since_unix", + "unique_matches", + "explicit_over_derived", + "ambiguous", + "none_matches", + "updated_unix" + ], + "name": "scope_match_totals" + }, + { + "columns": [ + "name" + ], + "name": "_migrations" + }, + { + "columns": [ + "name", + "status", + "started_at", + "ended_at", + "error" + ], + "name": "_async_migrations" + }, + { + "columns": [ + "id", + "tx_id", + "bit" + ], + "name": "advert_route_evidence" + }, + { + "columns": [ + "id", + "tx_cursor", + "obs_cursor" + ], + "name": "advert_evidence_backfill" + } + ], + "workload": { + "http_concurrency": 64, + "http_rate": 20, + "ingest_rate": 50, + "ingestor_memory_mib": 512, + "measured_seconds": 180, + "packet_store_mib": 2048, + "queue_limit": 1024, + "retention_hours": 168, + "server_memory_mib": 3072, + "warmup_seconds": 60 + } + }, + "checks": { + "pairs": 5, + "legs": 10, + "source_hashes_recomputed": true, + "harness_hash_recomputed": true, + "summary_rows_recomputed": 930, + "all_18_table_comparable_digests_pair_equal": true, + "only_approved_identity_marker_normalized": true, + "complete_ingest_events": 120000, + "complete_http_requests": 48000, + "errors": 0, + "drops": 0, + "full_retained_loads": true, + "sql_rows": 2000, + "sql_output_parity": true + }, + "sqlite_settings": { + "busy_timeout": "5000", + "cache_size": "-2000", + "cache_spill": "483", + "foreign_keys": "1", + "journal_mode": "wal", + "journal_size_limit": "-1", + "mmap_size": "0", + "page_size": "4096", + "sqlite_version": "3.53.4", + "synchronous": "2", + "wal_autocheckpoint": "1000" + }, + "runs": [ + { + "pair": 0, + "variant": "baseline", + "storage_setup_ns": 0, + "window_ns": [ + 760182069475, + 940182069475 + ], + "observed_window_seconds": 178.661750551, + "resource_samples": 179, + "cpu_seconds": 53.042935, + "sampled_memory_peak_bytes": 4239302656, + "sampled_pss_peak_bytes": 3073052672, + "sampled_rss_peak_bytes": 3076358144, + "storage_peak_bytes": 1145506704, + "wal_peak_bytes": 40582032, + "min_free_disk_bytes": 85978206208, + "io_delta_per_device": { + "8:0": { + "rbytes": 0, + "wbytes": 2480648192, + "rios": 0, + "wios": 220705, + "dbytes": 0, + "dios": 0 + } + }, + "memory_event_deltas": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "full_leg_kernel_peak_bytes": 4382474240, + "full_leg_memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "measured_ingest_counts": { + "new": 4500, + "additional": 2700, + "duplicate": 1350, + "late": 450 + }, + "measured_http_counts": { + "nodes_50": 252, + "nodes_region": 216, + "nodes_2000": 252, + "channels": 180, + "messages_0": 180, + "messages_1000": 180, + "observers": 180, + "metrics_dense_24h": 180, + "metrics_sparse_7d": 180, + "paths": 180, + "reach": 180, + "neighbors": 180, + "rx_coverage": 180, + "packets_memory": 180, + "packet_detail_memory": 180, + "packet_detail_sql": 135, + "analytics_rf": 108, + "analytics_topology": 108, + "analytics_channels": 72, + "analytics_rf_filtered": 72, + "stats": 108, + "healthz": 72, + "packet_detail_pruned": 45 + }, + "negative_visibility_samples": 0, + "duplicate_ws_receipts": 0, + "ingest_drain_seconds": 0, + "http_drain_seconds": 0, + "full_stream_writer_wait_hold": { + "mqtt_handler": { + "count": 12000, + "contention_total": 34, + "wait_ms_p50": 0.000171, + "wait_ms_p95": 0.00026, + "wait_ms_p99": 0.000391, + "wait_ms_max": 456.511963, + "hold_ms_p50": 0.854577, + "hold_ms_p95": 1.349072, + "hold_ms_p99": 3.401298, + "hold_ms_max": 24.250548 + }, + "prune_packets": { + "count": 65, + "contention_total": 0, + "wait_ms_p50": 1.194743, + "wait_ms_p95": 2.254522, + "wait_ms_p99": 4.006908, + "wait_ms_max": 5.766021, + "hold_ms_p50": 197.35194, + "hold_ms_p95": 251.352796, + "hold_ms_p99": 253.083773, + "hold_ms_max": 253.125389 + } + }, + "ingest_scheduled_during_retention": { + "service": { + "n": 562, + "p50": 537014, + "p95": 202624634, + "p99": null, + "maximum": 457723410 + }, + "scheduled_to_complete": { + "n": 562, + "p50": 5002811135, + "p95": 9143465540, + "p99": null, + "maximum": 9650673206 + } + }, + "average_cores": 0.29689026798636786 + }, + { + "pair": 0, + "variant": "candidate", + "storage_setup_ns": 42421987180, + "window_ns": [ + 1193063960671, + 1373063960671 + ], + "observed_window_seconds": 178.719542127, + "resource_samples": 179, + "cpu_seconds": 52.752416, + "sampled_memory_peak_bytes": 4207431680, + "sampled_pss_peak_bytes": 3048941568, + "sampled_rss_peak_bytes": 3052183552, + "storage_peak_bytes": 1145494344, + "wal_peak_bytes": 40569672, + "min_free_disk_bytes": 85974056960, + "io_delta_per_device": { + "8:0": { + "rbytes": 0, + "wbytes": 2405064704, + "rios": 0, + "wios": 206760, + "dbytes": 0, + "dios": 0 + } + }, + "memory_event_deltas": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "full_leg_kernel_peak_bytes": 4364914688, + "full_leg_memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "measured_ingest_counts": { + "new": 4500, + "additional": 2700, + "duplicate": 1350, + "late": 450 + }, + "measured_http_counts": { + "nodes_50": 252, + "nodes_region": 216, + "nodes_2000": 252, + "channels": 180, + "messages_0": 180, + "messages_1000": 180, + "observers": 180, + "metrics_dense_24h": 180, + "metrics_sparse_7d": 180, + "paths": 180, + "reach": 180, + "neighbors": 180, + "rx_coverage": 180, + "packets_memory": 180, + "packet_detail_memory": 180, + "packet_detail_sql": 135, + "analytics_rf": 108, + "analytics_channels": 72, + "analytics_rf_filtered": 72, + "analytics_topology": 108, + "stats": 108, + "healthz": 72, + "packet_detail_pruned": 45 + }, + "negative_visibility_samples": 0, + "duplicate_ws_receipts": 0, + "ingest_drain_seconds": 0, + "http_drain_seconds": 0, + "full_stream_writer_wait_hold": { + "mqtt_handler": { + "count": 12000, + "contention_total": 53, + "wait_ms_p50": 0.000151, + "wait_ms_p95": 0.000241, + "wait_ms_p99": 0.000331, + "wait_ms_max": 305.357275, + "hold_ms_p50": 0.60985, + "hold_ms_p95": 0.89997, + "hold_ms_p99": 3.01691, + "hold_ms_max": 56.090983 + }, + "prune_packets": { + "count": 65, + "contention_total": 0, + "wait_ms_p50": 1.440905, + "wait_ms_p95": 1.99388, + "wait_ms_p99": 2.73874, + "wait_ms_max": 56.432992, + "hold_ms_p50": 190.149555, + "hold_ms_p95": 211.956382, + "hold_ms_p99": 230.297711, + "hold_ms_max": 251.682349 + } + }, + "ingest_scheduled_during_retention": { + "service": { + "n": 555, + "p50": 376504, + "p95": 200919993, + "p99": null, + "maximum": 306260021 + }, + "scheduled_to_complete": { + "n": 555, + "p50": 3904089140, + "p95": 7437894874, + "p99": null, + "maximum": 7908859033 + } + }, + "average_cores": 0.29516870607531864 + }, + { + "pair": 1, + "variant": "baseline", + "storage_setup_ns": 0, + "window_ns": [ + 2100411216777, + 2280411216777 + ], + "observed_window_seconds": 178.713126174, + "resource_samples": 179, + "cpu_seconds": 54.890594, + "sampled_memory_peak_bytes": 4281593856, + "sampled_pss_peak_bytes": 3111116800, + "sampled_rss_peak_bytes": 3114422272, + "storage_peak_bytes": 1150121704, + "wal_peak_bytes": 41080552, + "min_free_disk_bytes": 84839493632, + "io_delta_per_device": { + "8:0": { + "rbytes": 0, + "wbytes": 2480660480, + "rios": 0, + "wios": 219922, + "dbytes": 0, + "dios": 0 + } + }, + "memory_event_deltas": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "full_leg_kernel_peak_bytes": 4482162688, + "full_leg_memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "measured_ingest_counts": { + "new": 4500, + "additional": 2700, + "duplicate": 1350, + "late": 450 + }, + "measured_http_counts": { + "nodes_50": 252, + "nodes_region": 216, + "nodes_2000": 252, + "channels": 180, + "messages_0": 180, + "messages_1000": 180, + "observers": 180, + "metrics_dense_24h": 180, + "metrics_sparse_7d": 180, + "paths": 180, + "reach": 180, + "neighbors": 180, + "rx_coverage": 180, + "packets_memory": 180, + "packet_detail_memory": 180, + "packet_detail_sql": 135, + "analytics_rf": 108, + "analytics_topology": 108, + "analytics_channels": 72, + "analytics_rf_filtered": 72, + "stats": 108, + "healthz": 72, + "packet_detail_pruned": 45 + }, + "negative_visibility_samples": 0, + "duplicate_ws_receipts": 0, + "ingest_drain_seconds": 0, + "http_drain_seconds": 0, + "full_stream_writer_wait_hold": { + "mqtt_handler": { + "count": 12000, + "contention_total": 37, + "wait_ms_p50": 0.000181, + "wait_ms_p95": 0.000351, + "wait_ms_p99": 0.000511, + "wait_ms_max": 453.582406, + "hold_ms_p50": 0.893651, + "hold_ms_p95": 1.365559, + "hold_ms_p99": 3.555453, + "hold_ms_max": 53.610984 + }, + "prune_packets": { + "count": 65, + "contention_total": 0, + "wait_ms_p50": 1.415699, + "wait_ms_p95": 2.394928, + "wait_ms_p99": 3.243317, + "wait_ms_max": 52.084262, + "hold_ms_p50": 194.172412, + "hold_ms_p95": 248.72853, + "hold_ms_p99": 251.707798, + "hold_ms_max": 252.307538 + } + }, + "ingest_scheduled_during_retention": { + "service": { + "n": 573, + "p50": 588100, + "p95": 206092617, + "p99": null, + "maximum": 454546950 + }, + "scheduled_to_complete": { + "n": 573, + "p50": 4989899156, + "p95": 9232737648, + "p99": null, + "maximum": 9711899985 + } + }, + "average_cores": 0.3071436059294101 + }, + { + "pair": 1, + "variant": "candidate", + "storage_setup_ns": 42588190282, + "window_ns": [ + 1709636463206, + 1889636463206 + ], + "observed_window_seconds": 179.696897687, + "resource_samples": 180, + "cpu_seconds": 52.677365, + "sampled_memory_peak_bytes": 4234752000, + "sampled_pss_peak_bytes": 3081181184, + "sampled_rss_peak_bytes": 3084419072, + "storage_peak_bytes": 1140081264, + "wal_peak_bytes": 31040112, + "min_free_disk_bytes": 84851814400, + "io_delta_per_device": { + "8:0": { + "rbytes": 0, + "wbytes": 2400497664, + "rios": 0, + "wios": 204870, + "dbytes": 0, + "dios": 0 + } + }, + "memory_event_deltas": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "full_leg_kernel_peak_bytes": 4343148544, + "full_leg_memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "measured_ingest_counts": { + "new": 4500, + "additional": 2700, + "duplicate": 1350, + "late": 450 + }, + "measured_http_counts": { + "nodes_50": 252, + "nodes_region": 216, + "nodes_2000": 252, + "channels": 180, + "messages_0": 180, + "messages_1000": 180, + "observers": 180, + "metrics_dense_24h": 180, + "metrics_sparse_7d": 180, + "paths": 180, + "reach": 180, + "neighbors": 180, + "rx_coverage": 180, + "packets_memory": 180, + "packet_detail_memory": 180, + "packet_detail_sql": 135, + "analytics_rf": 108, + "analytics_topology": 108, + "analytics_channels": 72, + "analytics_rf_filtered": 72, + "stats": 108, + "healthz": 72, + "packet_detail_pruned": 45 + }, + "negative_visibility_samples": 0, + "duplicate_ws_receipts": 0, + "ingest_drain_seconds": 0, + "http_drain_seconds": 0, + "full_stream_writer_wait_hold": { + "mqtt_handler": { + "count": 12000, + "contention_total": 52, + "wait_ms_p50": 0.00016, + "wait_ms_p95": 0.00026, + "wait_ms_p99": 0.000401, + "wait_ms_max": 347.715653, + "hold_ms_p50": 0.610621, + "hold_ms_p95": 0.880449, + "hold_ms_p99": 3.122106, + "hold_ms_max": 42.704898 + }, + "prune_packets": { + "count": 65, + "contention_total": 0, + "wait_ms_p50": 1.468866, + "wait_ms_p95": 2.161183, + "wait_ms_p99": 18.199173, + "wait_ms_max": 42.979234, + "hold_ms_p50": 187.307329, + "hold_ms_p95": 247.173419, + "hold_ms_p99": 249.359406, + "hold_ms_max": 251.735406 + } + }, + "ingest_scheduled_during_retention": { + "service": { + "n": 547, + "p50": 380832, + "p95": 192285989, + "p99": null, + "maximum": 348461987 + }, + "scheduled_to_complete": { + "n": 547, + "p50": 3934013655, + "p95": 7304109242, + "p99": null, + "maximum": 7670495810 + } + }, + "average_cores": 0.2931456562580985 + }, + { + "pair": 2, + "variant": "baseline", + "storage_setup_ns": 0, + "window_ns": [ + 2577183328019, + 2757183328019 + ], + "observed_window_seconds": 178.69029541, + "resource_samples": 179, + "cpu_seconds": 54.771503, + "sampled_memory_peak_bytes": 4303065088, + "sampled_pss_peak_bytes": 3128674304, + "sampled_rss_peak_bytes": 3131949056, + "storage_peak_bytes": 1154509744, + "wal_peak_bytes": 41290672, + "min_free_disk_bytes": 83706896384, + "io_delta_per_device": { + "8:0": { + "rbytes": 0, + "wbytes": 2483662848, + "rios": 0, + "wios": 221292, + "dbytes": 0, + "dios": 0 + } + }, + "memory_event_deltas": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "full_leg_kernel_peak_bytes": 4501434368, + "full_leg_memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "measured_ingest_counts": { + "new": 4500, + "additional": 2700, + "duplicate": 1350, + "late": 450 + }, + "measured_http_counts": { + "nodes_50": 252, + "nodes_region": 216, + "nodes_2000": 252, + "channels": 180, + "messages_0": 180, + "messages_1000": 180, + "observers": 180, + "metrics_dense_24h": 180, + "metrics_sparse_7d": 180, + "paths": 180, + "reach": 180, + "neighbors": 180, + "rx_coverage": 180, + "packet_detail_memory": 180, + "packets_memory": 180, + "packet_detail_sql": 135, + "analytics_rf": 108, + "analytics_topology": 108, + "analytics_channels": 72, + "analytics_rf_filtered": 72, + "stats": 108, + "healthz": 72, + "packet_detail_pruned": 45 + }, + "negative_visibility_samples": 0, + "duplicate_ws_receipts": 0, + "ingest_drain_seconds": 0, + "http_drain_seconds": 0, + "full_stream_writer_wait_hold": { + "mqtt_handler": { + "count": 12000, + "contention_total": 36, + "wait_ms_p50": 0.000181, + "wait_ms_p95": 0.00036, + "wait_ms_p99": 0.000521, + "wait_ms_max": 454.294341, + "hold_ms_p50": 0.849348, + "hold_ms_p95": 1.345921, + "hold_ms_p99": 3.573482, + "hold_ms_max": 36.290652 + }, + "prune_packets": { + "count": 65, + "contention_total": 0, + "wait_ms_p50": 1.171998, + "wait_ms_p95": 2.170339, + "wait_ms_p99": 2.757074, + "wait_ms_max": 3.311238, + "hold_ms_p50": 191.549178, + "hold_ms_p95": 249.769581, + "hold_ms_p99": 252.709787, + "hold_ms_max": 253.415236 + } + }, + "ingest_scheduled_during_retention": { + "service": { + "n": 557, + "p50": 545089, + "p95": 201381939, + "p99": null, + "maximum": 455084418 + }, + "scheduled_to_complete": { + "n": 557, + "p50": 5010201766, + "p95": 9189747985, + "p99": null, + "maximum": 9675575670 + } + }, + "average_cores": 0.30651638285295957 + }, + { + "pair": 2, + "variant": "candidate", + "storage_setup_ns": 42031647873, + "window_ns": [ + 3009484766433, + 3189484766433 + ], + "observed_window_seconds": 178.752809694, + "resource_samples": 179, + "cpu_seconds": 53.588029, + "sampled_memory_peak_bytes": 4312424448, + "sampled_pss_peak_bytes": 3144603648, + "sampled_rss_peak_bytes": 3147845632, + "storage_peak_bytes": 1154633344, + "wal_peak_bytes": 41414272, + "min_free_disk_bytes": 83696173056, + "io_delta_per_device": { + "8:0": { + "rbytes": 0, + "wbytes": 2397114368, + "rios": 0, + "wios": 205463, + "dbytes": 0, + "dios": 0 + } + }, + "memory_event_deltas": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "full_leg_kernel_peak_bytes": 4423811072, + "full_leg_memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "measured_ingest_counts": { + "new": 4500, + "additional": 2700, + "duplicate": 1350, + "late": 450 + }, + "measured_http_counts": { + "nodes_50": 252, + "nodes_region": 216, + "nodes_2000": 252, + "channels": 180, + "messages_0": 180, + "messages_1000": 180, + "observers": 180, + "metrics_dense_24h": 180, + "metrics_sparse_7d": 180, + "paths": 180, + "reach": 180, + "neighbors": 180, + "rx_coverage": 180, + "packets_memory": 180, + "packet_detail_memory": 180, + "packet_detail_sql": 135, + "analytics_rf": 108, + "analytics_topology": 108, + "analytics_channels": 72, + "analytics_rf_filtered": 72, + "stats": 108, + "healthz": 72, + "packet_detail_pruned": 45 + }, + "negative_visibility_samples": 0, + "duplicate_ws_receipts": 0, + "ingest_drain_seconds": 0, + "http_drain_seconds": 0, + "full_stream_writer_wait_hold": { + "mqtt_handler": { + "count": 12000, + "contention_total": 52, + "wait_ms_p50": 0.000161, + "wait_ms_p95": 0.000331, + "wait_ms_p99": 0.000571, + "wait_ms_max": 393.58385, + "hold_ms_p50": 0.632493, + "hold_ms_p95": 0.914592, + "hold_ms_p99": 2.943933, + "hold_ms_max": 41.423197 + }, + "prune_packets": { + "count": 65, + "contention_total": 0, + "wait_ms_p50": 1.445022, + "wait_ms_p95": 2.176058, + "wait_ms_p99": 11.643554, + "wait_ms_max": 42.089495, + "hold_ms_p50": 189.569043, + "hold_ms_p95": 248.840622, + "hold_ms_p99": 252.348851, + "hold_ms_max": 283.523711 + } + }, + "ingest_scheduled_during_retention": { + "service": { + "n": 554, + "p50": 394046, + "p95": 199526908, + "p99": null, + "maximum": 394502837 + }, + "scheduled_to_complete": { + "n": 554, + "p50": 4220651812, + "p95": 7765791879, + "p99": null, + "maximum": 8176987177 + } + }, + "average_cores": 0.29978845698557277 + }, + { + "pair": 3, + "variant": "baseline", + "storage_setup_ns": 0, + "window_ns": [ + 3919149953989, + 4099149953989 + ], + "observed_window_seconds": 179.698954814, + "resource_samples": 180, + "cpu_seconds": 54.683538, + "sampled_memory_peak_bytes": 4244541440, + "sampled_pss_peak_bytes": 3077771264, + "sampled_rss_peak_bytes": 3081072640, + "storage_peak_bytes": 1149024544, + "wal_peak_bytes": 29927712, + "min_free_disk_bytes": 82562383872, + "io_delta_per_device": { + "8:0": { + "rbytes": 0, + "wbytes": 2491469824, + "rios": 0, + "wios": 222514, + "dbytes": 0, + "dios": 0 + } + }, + "memory_event_deltas": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "full_leg_kernel_peak_bytes": 4631904256, + "full_leg_memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "measured_ingest_counts": { + "new": 4500, + "additional": 2700, + "duplicate": 1350, + "late": 450 + }, + "measured_http_counts": { + "nodes_50": 252, + "nodes_region": 216, + "nodes_2000": 252, + "channels": 180, + "messages_0": 180, + "messages_1000": 180, + "observers": 180, + "metrics_dense_24h": 180, + "metrics_sparse_7d": 180, + "paths": 180, + "reach": 180, + "neighbors": 180, + "rx_coverage": 180, + "packets_memory": 180, + "packet_detail_memory": 180, + "packet_detail_sql": 135, + "analytics_rf": 108, + "analytics_topology": 108, + "analytics_channels": 72, + "analytics_rf_filtered": 72, + "stats": 108, + "healthz": 72, + "packet_detail_pruned": 45 + }, + "negative_visibility_samples": 0, + "duplicate_ws_receipts": 0, + "ingest_drain_seconds": 0, + "http_drain_seconds": 0, + "full_stream_writer_wait_hold": { + "mqtt_handler": { + "count": 12000, + "contention_total": 36, + "wait_ms_p50": 0.000191, + "wait_ms_p95": 0.000391, + "wait_ms_p99": 0.000571, + "wait_ms_max": 459.143269, + "hold_ms_p50": 0.908047, + "hold_ms_p95": 1.362956, + "hold_ms_p99": 3.145478, + "hold_ms_max": 136.86449 + }, + "prune_packets": { + "count": 65, + "contention_total": 1, + "wait_ms_p50": 1.547109, + "wait_ms_p95": 2.698906, + "wait_ms_p99": 86.582682, + "wait_ms_max": 137.840795, + "hold_ms_p50": 189.885605, + "hold_ms_p95": 248.758701, + "hold_ms_p99": 252.759674, + "hold_ms_max": 255.723493 + } + }, + "ingest_scheduled_during_retention": { + "service": { + "n": 564, + "p50": 535240, + "p95": 200362193, + "p99": null, + "maximum": 460374822 + }, + "scheduled_to_complete": { + "n": 564, + "p50": 4897799347, + "p95": 9018405919, + "p99": null, + "maximum": 9484113140 + } + }, + "average_cores": 0.30430637761138335 + }, + { + "pair": 3, + "variant": "candidate", + "storage_setup_ns": 42498975928, + "window_ns": [ + 3528105890856, + 3708105890856 + ], + "observed_window_seconds": 178.773447195, + "resource_samples": 179, + "cpu_seconds": 53.516078, + "sampled_memory_peak_bytes": 4220968960, + "sampled_pss_peak_bytes": 3061602304, + "sampled_rss_peak_bytes": 3064844288, + "storage_peak_bytes": 1147516624, + "wal_peak_bytes": 28419792, + "min_free_disk_bytes": 82567188480, + "io_delta_per_device": { + "8:0": { + "rbytes": 0, + "wbytes": 2415157248, + "rios": 0, + "wios": 209442, + "dbytes": 0, + "dios": 0 + } + }, + "memory_event_deltas": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "full_leg_kernel_peak_bytes": 4415995904, + "full_leg_memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "measured_ingest_counts": { + "new": 4500, + "additional": 2700, + "duplicate": 1350, + "late": 450 + }, + "measured_http_counts": { + "nodes_50": 252, + "nodes_region": 216, + "nodes_2000": 252, + "channels": 180, + "messages_0": 180, + "messages_1000": 180, + "observers": 180, + "metrics_dense_24h": 180, + "metrics_sparse_7d": 180, + "paths": 180, + "reach": 180, + "neighbors": 180, + "rx_coverage": 180, + "packets_memory": 180, + "packet_detail_memory": 180, + "packet_detail_sql": 135, + "analytics_rf": 108, + "analytics_topology": 108, + "analytics_channels": 72, + "analytics_rf_filtered": 72, + "stats": 108, + "healthz": 72, + "packet_detail_pruned": 45 + }, + "negative_visibility_samples": 0, + "duplicate_ws_receipts": 0, + "ingest_drain_seconds": 0, + "http_drain_seconds": 0, + "full_stream_writer_wait_hold": { + "mqtt_handler": { + "count": 12000, + "contention_total": 52, + "wait_ms_p50": 0.000161, + "wait_ms_p95": 0.000331, + "wait_ms_p99": 0.000551, + "wait_ms_max": 390.762443, + "hold_ms_p50": 0.619989, + "hold_ms_p95": 0.851054, + "hold_ms_p99": 2.982739, + "hold_ms_max": 8.613833 + }, + "prune_packets": { + "count": 65, + "contention_total": 0, + "wait_ms_p50": 1.33699, + "wait_ms_p95": 1.863291, + "wait_ms_p99": 1.902419, + "wait_ms_max": 2.315791, + "hold_ms_p50": 194.085683, + "hold_ms_p95": 235.153204, + "hold_ms_p99": 244.279856, + "hold_ms_max": 251.152517 + } + }, + "ingest_scheduled_during_retention": { + "service": { + "n": 561, + "p50": 373859, + "p95": 201350742, + "p99": null, + "maximum": 391714554 + }, + "scheduled_to_complete": { + "n": 561, + "p50": 4232500719, + "p95": 7759900816, + "p99": null, + "maximum": 8171590163 + } + }, + "average_cores": 0.29935137929978206 + }, + { + "pair": 4, + "variant": "baseline", + "storage_setup_ns": 0, + "window_ns": [ + 4397102493933, + 4577102493933 + ], + "observed_window_seconds": 178.13994789, + "resource_samples": 178, + "cpu_seconds": 54.596186, + "sampled_memory_peak_bytes": 4297109504, + "sampled_pss_peak_bytes": 3114579968, + "sampled_rss_peak_bytes": 3117817856, + "storage_peak_bytes": 1161523040, + "wal_peak_bytes": 40046432, + "min_free_disk_bytes": 81409437696, + "io_delta_per_device": { + "8:0": { + "rbytes": 0, + "wbytes": 2454142976, + "rios": 0, + "wios": 215544, + "dbytes": 0, + "dios": 0 + } + }, + "memory_event_deltas": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "full_leg_kernel_peak_bytes": 4321443840, + "full_leg_memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "measured_ingest_counts": { + "new": 4500, + "additional": 2700, + "duplicate": 1350, + "late": 450 + }, + "measured_http_counts": { + "nodes_50": 252, + "nodes_region": 216, + "nodes_2000": 252, + "channels": 180, + "messages_0": 180, + "messages_1000": 180, + "observers": 180, + "metrics_dense_24h": 180, + "metrics_sparse_7d": 180, + "paths": 180, + "reach": 180, + "neighbors": 180, + "rx_coverage": 180, + "packets_memory": 180, + "packet_detail_memory": 180, + "packet_detail_sql": 135, + "analytics_rf": 108, + "analytics_topology": 108, + "analytics_channels": 72, + "analytics_rf_filtered": 72, + "stats": 108, + "healthz": 72, + "packet_detail_pruned": 45 + }, + "negative_visibility_samples": 0, + "duplicate_ws_receipts": 0, + "ingest_drain_seconds": 0, + "http_drain_seconds": 0, + "full_stream_writer_wait_hold": { + "mqtt_handler": { + "count": 12000, + "contention_total": 34, + "wait_ms_p50": 0.00018, + "wait_ms_p95": 0.000331, + "wait_ms_p99": 0.000481, + "wait_ms_max": 502.832104, + "hold_ms_p50": 0.867437, + "hold_ms_p95": 1.303679, + "hold_ms_p99": 3.446865, + "hold_ms_max": 121.308279 + }, + "prune_packets": { + "count": 65, + "contention_total": 0, + "wait_ms_p50": 1.40924, + "wait_ms_p95": 2.016999, + "wait_ms_p99": 2.232534, + "wait_ms_max": 2.856998, + "hold_ms_p50": 189.495793, + "hold_ms_p95": 252.204571, + "hold_ms_p99": 272.815667, + "hold_ms_max": 278.769418 + } + }, + "ingest_scheduled_during_retention": { + "service": { + "n": 548, + "p50": 524471, + "p95": 204819541, + "p99": null, + "maximum": 503924346 + }, + "scheduled_to_complete": { + "n": 548, + "p50": 4911074486, + "p95": 8951609783, + "p99": null, + "maximum": 9321026618 + } + }, + "average_cores": 0.3064791847458758 + }, + { + "pair": 4, + "variant": "candidate", + "storage_setup_ns": 42543284490, + "window_ns": [ + 4830414045950, + 5010414045950 + ], + "observed_window_seconds": 178.715161965, + "resource_samples": 179, + "cpu_seconds": 55.870671, + "sampled_memory_peak_bytes": 4276379648, + "sampled_pss_peak_bytes": 3099301888, + "sampled_rss_peak_bytes": 3102543872, + "storage_peak_bytes": 1149439080, + "wal_peak_bytes": 27962472, + "min_free_disk_bytes": 81404821504, + "io_delta_per_device": { + "8:0": { + "rbytes": 0, + "wbytes": 2407985152, + "rios": 0, + "wios": 208345, + "dbytes": 0, + "dios": 0 + } + }, + "memory_event_deltas": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "full_leg_kernel_peak_bytes": 4420767744, + "full_leg_memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "measured_ingest_counts": { + "new": 4500, + "additional": 2700, + "duplicate": 1350, + "late": 450 + }, + "measured_http_counts": { + "nodes_50": 252, + "nodes_region": 216, + "nodes_2000": 252, + "channels": 180, + "messages_0": 180, + "messages_1000": 180, + "observers": 180, + "metrics_dense_24h": 180, + "metrics_sparse_7d": 180, + "paths": 180, + "reach": 180, + "neighbors": 180, + "rx_coverage": 180, + "packets_memory": 180, + "packet_detail_memory": 180, + "packet_detail_sql": 135, + "analytics_rf": 108, + "analytics_topology": 108, + "analytics_channels": 72, + "analytics_rf_filtered": 72, + "stats": 108, + "healthz": 72, + "packet_detail_pruned": 45 + }, + "negative_visibility_samples": 0, + "duplicate_ws_receipts": 0, + "ingest_drain_seconds": 0, + "http_drain_seconds": 0, + "full_stream_writer_wait_hold": { + "mqtt_handler": { + "count": 12000, + "contention_total": 55, + "wait_ms_p50": 0.000231, + "wait_ms_p95": 0.00043, + "wait_ms_p99": 0.000581, + "wait_ms_max": 393.339476, + "hold_ms_p50": 0.662569, + "hold_ms_p95": 0.958532, + "hold_ms_p99": 3.285973, + "hold_ms_max": 41.142223 + }, + "prune_packets": { + "count": 65, + "contention_total": 0, + "wait_ms_p50": 1.402519, + "wait_ms_p95": 1.83063, + "wait_ms_p99": 1.951396, + "wait_ms_max": 2.334247, + "hold_ms_p50": 191.436879, + "hold_ms_p95": 250.847717, + "hold_ms_p99": 251.819876, + "hold_ms_max": 251.8354 + } + }, + "ingest_scheduled_during_retention": { + "service": { + "n": 571, + "p50": 385160, + "p95": 201969397, + "p99": null, + "maximum": 394260097 + }, + "scheduled_to_complete": { + "n": 571, + "p50": 4261354505, + "p95": 8011962017, + "p99": null, + "maximum": 8414649715 + } + }, + "average_cores": 0.3126241242527696 + } + ], + "paired": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 644512, + "candidate_median": 639856, + "ratio_median": 0.9925407062969733, + "ratio_min": 0.987214241301688, + "ratio_max": 1.0246481058537311, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 644512, + "candidate": 660398, + "ratio": 1.0246481058537311, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 1, + "baseline": 637862, + "candidate": 633104, + "ratio": 0.9925407062969733, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 2, + "baseline": 644445, + "candidate": 639515, + "ratio": 0.9923500065948219, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 3, + "baseline": 648143, + "candidate": 639856, + "ratio": 0.987214241301688, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 4, + "baseline": 655620, + "candidate": 663994, + "ratio": 1.0127726426893628, + "n_baseline": 2700, + "n_candidate": 2700 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 2202443499, + "candidate_median": 1582919317, + "ratio_median": 0.7466051180823607, + "ratio_min": 0.6440398995516599, + "ratio_max": 0.9585397244206861, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 2202443499, + "candidate": 1574359063, + "ratio": 0.714823814420131, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 1, + "baseline": 2212632281, + "candidate": 1425023472, + "ratio": 0.6440398995516599, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 2, + "baseline": 2120155995, + "candidate": 1582919317, + "ratio": 0.7466051180823607, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 3, + "baseline": 2233491441, + "candidate": 1705687674, + "ratio": 0.7636866847523326, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 4, + "baseline": 1994253387, + "candidate": 1911571092, + "ratio": 0.9585397244206861, + "n_baseline": 2700, + "n_candidate": 2700 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "metric": "p99", + "pairs": 5, + "baseline_median": 8038159893, + "candidate_median": 6763119610, + "ratio_median": 0.841376596139825, + "ratio_min": 0.7889092583413672, + "ratio_max": 0.8975616749524219, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 8035375154, + "candidate": 6627818487, + "ratio": 0.8248299998414735, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 1, + "baseline": 8256866783, + "candidate": 6513918650, + "ratio": 0.7889092583413672, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 2, + "baseline": 8038159893, + "candidate": 6763119610, + "ratio": 0.841376596139825, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 3, + "baseline": 8065629546, + "candidate": 6798159041, + "ratio": 0.84285535335198, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 4, + "baseline": 7866327633, + "candidate": 7060514206, + "ratio": 0.8975616749524219, + "n_baseline": 2700, + "n_candidate": 2700 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 9480543244, + "candidate_median": 8013062675, + "ratio_median": 0.8382518132200261, + "ratio_min": 0.806970764343259, + "ratio_max": 0.8924302002624368, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 9630692402, + "candidate": 7888882206, + "ratio": 0.8191396710335926, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 1, + "baseline": 9480543244, + "candidate": 7650521228, + "ratio": 0.806970764343259, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 2, + "baseline": 9559254807, + "candidate": 8013062675, + "ratio": 0.8382518132200261, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 3, + "baseline": 9290805521, + "candidate": 8034874710, + "ratio": 0.8648200300650767, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 4, + "baseline": 9231621314, + "candidate": 8238577658, + "ratio": 0.8924302002624368, + "n_baseline": 2700, + "n_candidate": 2700 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 1427650, + "candidate_median": 1208265, + "ratio_median": 0.8372568807145082, + "ratio_min": 0.8011379233065418, + "ratio_max": 0.9148560012857743, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1449640, + "candidate": 1212374, + "ratio": 0.8363276399657846, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 1, + "baseline": 1416023, + "candidate": 1185575, + "ratio": 0.8372568807145082, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 2, + "baseline": 1427650, + "candidate": 1203827, + "ratio": 0.8432227786922565, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 3, + "baseline": 1508186, + "candidate": 1208265, + "ratio": 0.8011379233065418, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 4, + "baseline": 1375047, + "candidate": 1257970, + "ratio": 0.9148560012857743, + "n_baseline": 2700, + "n_candidate": 2700 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 2202796298, + "candidate_median": 1583136443, + "ratio_median": 0.7465792456626409, + "ratio_min": 0.6328304370142596, + "ratio_max": 0.949144513912817, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 2202796298, + "candidate": 1574797442, + "ratio": 0.7149083387464454, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 1, + "baseline": 2252208637, + "candidate": 1425266176, + "ratio": 0.6328304370142596, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 2, + "baseline": 2120520296, + "candidate": 1583136443, + "ratio": 0.7465792456626409, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 3, + "baseline": 2233809315, + "candidate": 1705918796, + "ratio": 0.7636814765453693, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 4, + "baseline": 2014245323, + "candidate": 1911809898, + "ratio": 0.949144513912817, + "n_baseline": 2700, + "n_candidate": 2700 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "scheduled-to-complete", + "metric": "p99", + "pairs": 5, + "baseline_median": 8065927713, + "candidate_median": 6802849495, + "ratio_median": 0.8421907124004059, + "ratio_min": 0.7889017685897322, + "ratio_max": 0.9024898624620016, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 8035661008, + "candidate": 6647805252, + "ratio": 0.8272879163744833, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 1, + "baseline": 8257312216, + "candidate": 6514208211, + "ratio": 0.7889017685897322, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 2, + "baseline": 8077564137, + "candidate": 6802849495, + "ratio": 0.8421907124004059, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 3, + "baseline": 8065927713, + "candidate": 6807444725, + "ratio": 0.8439754194707596, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 4, + "baseline": 7866608738, + "candidate": 7099534638, + "ratio": 0.9024898624620016, + "n_baseline": 2700, + "n_candidate": 2700 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 9481113970, + "candidate_median": 8033041345, + "ratio_median": 0.8403011548271302, + "ratio_min": 0.8090289637136384, + "ratio_max": 0.8898579668737061, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 9650673206, + "candidate": 7908859033, + "ratio": 0.8195137131037571, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 1, + "baseline": 9481113970, + "candidate": 7670495810, + "ratio": 0.8090289637136384, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 2, + "baseline": 9559717131, + "candidate": 8033041345, + "ratio": 0.8403011548271302, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 3, + "baseline": 9291368223, + "candidate": 8035156877, + "ratio": 0.8647980237302021, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 4, + "baseline": 9258607582, + "candidate": 8238845719, + "ratio": 0.8898579668737061, + "n_baseline": 2700, + "n_candidate": 2700 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 722460, + "candidate_median": 548395, + "ratio_median": 0.7372374702699548, + "ratio_min": 0.7273750989804187, + "ratio_max": 0.7983060469826462, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 707578, + "candidate": 532685, + "ratio": 0.7528286634123729, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 1, + "baseline": 722460, + "candidate": 530962, + "ratio": 0.734936190238906, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 2, + "baseline": 753867, + "candidate": 555779, + "ratio": 0.7372374702699548, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 3, + "baseline": 753937, + "candidate": 548395, + "ratio": 0.7273750989804187, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 4, + "baseline": 716903, + "candidate": 572308, + "ratio": 0.7983060469826462, + "n_baseline": 2700, + "n_candidate": 2700 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 1131857, + "candidate_median": 767374, + "ratio_median": 0.6956126083065264, + "ratio_min": 0.6356543427497413, + "ratio_max": 0.7325491814799725, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1081251, + "candidate": 766019, + "ratio": 0.7084562233930882, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 1, + "baseline": 1139792, + "candidate": 745351, + "ratio": 0.6539359813018516, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 2, + "baseline": 1207219, + "candidate": 767374, + "ratio": 0.6356543427497413, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 3, + "baseline": 1131857, + "candidate": 787334, + "ratio": 0.6956126083065264, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 4, + "baseline": 1124102, + "candidate": 823460, + "ratio": 0.7325491814799725, + "n_baseline": 2700, + "n_candidate": 2700 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "metric": "p99", + "pairs": 5, + "baseline_median": 3522194, + "candidate_median": 3444216, + "ratio_median": 0.9891890904324364, + "ratio_min": 0.9387123752110644, + "ratio_max": 1.0524166842365796, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 3448276, + "candidate": 3410997, + "ratio": 0.9891890904324364, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 1, + "baseline": 3588359, + "candidate": 3368437, + "ratio": 0.9387123752110644, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 2, + "baseline": 3539988, + "candidate": 3540810, + "ratio": 1.0002322041769633, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 3, + "baseline": 3522194, + "candidate": 3444216, + "ratio": 0.9778609582549967, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 4, + "baseline": 3490816, + "candidate": 3673793, + "ratio": 1.0524166842365796, + "n_baseline": 2700, + "n_candidate": 2700 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 397978418, + "candidate_median": 251913582, + "ratio_median": 0.6761656602401428, + "ratio_min": 0.5806576375706886, + "ratio_max": 0.8668783667297525, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 397978418, + "candidate": 231089208, + "ratio": 0.5806576375706886, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 1, + "baseline": 360470925, + "candidate": 243738061, + "ratio": 0.6761656602401428, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 2, + "baseline": 455084418, + "candidate": 394502837, + "ratio": 0.8668783667297525, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 3, + "baseline": 400959940, + "candidate": 251913582, + "ratio": 0.6282761863940821, + "n_baseline": 2700, + "n_candidate": 2700 + }, + { + "pair": 4, + "baseline": 396628156, + "candidate": 295014433, + "ratio": 0.7438060776502211, + "n_baseline": 2700, + "n_candidate": 2700 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 649442, + "candidate_median": 648628, + "ratio_median": 0.9987466163260152, + "ratio_min": 0.9640966891261789, + "ratio_max": 1.0127643550965204, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 646703, + "candidate": 654116, + "ratio": 1.0114627580202968, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 1, + "baseline": 647684, + "candidate": 624430, + "ratio": 0.9640966891261789, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 2, + "baseline": 649442, + "candidate": 648628, + "ratio": 0.9987466163260152, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 3, + "baseline": 658437, + "candidate": 640251, + "ratio": 0.9723800454713207, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 4, + "baseline": 650640, + "candidate": 658945, + "ratio": 1.0127643550965204, + "n_baseline": 1350, + "n_candidate": 1350 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 2124011490, + "candidate_median": 1503958099, + "ratio_median": 0.7366136619743419, + "ratio_min": 0.5728419385046213, + "ratio_max": 0.9374306634465158, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 2124011490, + "candidate": 1495775761, + "ratio": 0.70422206661415, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 1, + "baseline": 2349913944, + "candidate": 1346129259, + "ratio": 0.5728419385046213, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 2, + "baseline": 2041718986, + "candidate": 1503958099, + "ratio": 0.7366136619743419, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 3, + "baseline": 2154924440, + "candidate": 1646629605, + "ratio": 0.7641240567117054, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 4, + "baseline": 1955074462, + "candidate": 1832746750, + "ratio": 0.9374306634465158, + "n_baseline": 1350, + "n_candidate": 1350 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "metric": "p99", + "pairs": 5, + "baseline_median": 7986939705, + "candidate_median": 6767733324, + "ratio_median": 0.8457302777531798, + "ratio_min": 0.7868342025324561, + "ratio_max": 0.9286432610759828, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 7956802642, + "candidate": 6593438239, + "ratio": 0.8286542391030942, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 1, + "baseline": 8178460854, + "candidate": 6435092724, + "ratio": 0.7868342025324561, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 2, + "baseline": 8255237194, + "candidate": 6981704045, + "ratio": 0.8457302777531798, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 3, + "baseline": 7986939705, + "candidate": 6767733324, + "ratio": 0.8473499956138707, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 4, + "baseline": 7787632442, + "candidate": 7231932387, + "ratio": 0.9286432610759828, + "n_baseline": 1350, + "n_candidate": 1350 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 9362885572, + "candidate_median": 7916778057, + "ratio_median": 0.863032878368613, + "ratio_min": 0.8150252004382822, + "ratio_max": 0.8950381208398084, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 9532924186, + "candidate": 7790697449, + "ratio": 0.8172411001066573, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 1, + "baseline": 9362885572, + "candidate": 7630987690, + "ratio": 0.8150252004382822, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 2, + "baseline": 9441690663, + "candidate": 8157014458, + "ratio": 0.8639357874713715, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 3, + "baseline": 9173205628, + "candidate": 7916778057, + "ratio": 0.863032878368613, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 4, + "baseline": 9238636577, + "candidate": 8268931921, + "ratio": 0.8950381208398084, + "n_baseline": 1350, + "n_candidate": 1350 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 1341792, + "candidate_median": 1127770, + "ratio_median": 0.8251466433900542, + "ratio_min": 0.7853166946308725, + "ratio_max": 0.9222135896225383, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1366751, + "candidate": 1127770, + "ratio": 0.8251466433900542, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 1, + "baseline": 1341792, + "candidate": 1102537, + "ratio": 0.8216899489637738, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 2, + "baseline": 1332747, + "candidate": 1136601, + "ratio": 0.8528257801368152, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 3, + "baseline": 1430400, + "candidate": 1123317, + "ratio": 0.7853166946308725, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 4, + "baseline": 1279388, + "candidate": 1179869, + "ratio": 0.9222135896225383, + "n_baseline": 1350, + "n_candidate": 1350 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 2124433819, + "candidate_median": 1504126393, + "ratio_median": 0.7365390091313467, + "ratio_min": 0.572805479742956, + "ratio_max": 0.9374022513881998, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 2124433819, + "candidate": 1495995051, + "ratio": 0.7041852928627286, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 1, + "baseline": 2350371560, + "candidate": 1346305709, + "ratio": 0.572805479742956, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 2, + "baseline": 2042154420, + "candidate": 1504126393, + "ratio": 0.7365390091313467, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 3, + "baseline": 2155316603, + "candidate": 1722054024, + "ratio": 0.7989796123702018, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 4, + "baseline": 1955350659, + "candidate": 1832950110, + "ratio": 0.9374022513881998, + "n_baseline": 1350, + "n_candidate": 1350 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "scheduled-to-complete", + "metric": "p99", + "pairs": 5, + "baseline_median": 7987331878, + "candidate_median": 6930493016, + "ratio_median": 0.8457322678831231, + "ratio_min": 0.7868305671925141, + "ratio_max": 0.9311622096025949, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 7957348352, + "candidate": 6593640738, + "ratio": 0.8286228585610123, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 1, + "baseline": 8178752690, + "candidate": 6435292618, + "ratio": 0.7868305671925141, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 2, + "baseline": 8255489165, + "candidate": 6981933574, + "ratio": 0.8457322678831231, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 3, + "baseline": 7987331878, + "candidate": 6930493016, + "ratio": 0.8676856204121283, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 4, + "baseline": 7788033432, + "candidate": 7251922419, + "ratio": 0.9311622096025949, + "n_baseline": 1350, + "n_candidate": 1350 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 9442091492, + "candidate_median": 7916974795, + "ratio_median": 0.8480391708742682, + "ratio_min": 0.785763031516639, + "ratio_max": 0.8892696192920582, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 9533358999, + "candidate": 7790904857, + "ratio": 0.8172255820657992, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 1, + "baseline": 9711899985, + "candidate": 7631251974, + "ratio": 0.785763031516639, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 2, + "baseline": 9442091492, + "candidate": 8176987177, + "ratio": 0.8660143977558484, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 3, + "baseline": 9335623951, + "candidate": 7916974795, + "ratio": 0.8480391708742682, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 4, + "baseline": 9321026618, + "candidate": 8288905792, + "ratio": 0.8892696192920582, + "n_baseline": 1350, + "n_candidate": 1350 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 632172, + "candidate_median": 473485, + "ratio_median": 0.7319898055353599, + "ratio_min": 0.715247104177115, + "ratio_max": 0.7968116272153781, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 622063, + "candidate": 462785, + "ratio": 0.7439519791403765, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 1, + "baseline": 631323, + "candidate": 462122, + "ratio": 0.7319898055353599, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 2, + "baseline": 664652, + "candidate": 476591, + "ratio": 0.7170534354820267, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 3, + "baseline": 661988, + "candidate": 473485, + "ratio": 0.715247104177115, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 4, + "baseline": 632172, + "candidate": 503722, + "ratio": 0.7968116272153781, + "n_baseline": 1350, + "n_candidate": 1350 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 884543, + "candidate_median": 635227, + "ratio_median": 0.7110397128285357, + "ratio_min": 0.7024710403361809, + "ratio_max": 0.7771549828555536, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 833509, + "candidate": 592658, + "ratio": 0.7110397128285357, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 1, + "baseline": 875068, + "candidate": 636507, + "ratio": 0.7273800436080397, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 2, + "baseline": 904275, + "candidate": 635227, + "ratio": 0.7024710403361809, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 3, + "baseline": 891837, + "candidate": 633194, + "ratio": 0.709988484442785, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 4, + "baseline": 884543, + "candidate": 687427, + "ratio": 0.7771549828555536, + "n_baseline": 1350, + "n_candidate": 1350 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "metric": "p99", + "pairs": 5, + "baseline_median": 1392424, + "candidate_median": 2066614, + "ratio_median": 1.7036118040224948, + "ratio_min": 0.5793183184386647, + "ratio_max": 2.745380095705373, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 1213078, + "candidate": 2066614, + "ratio": 1.7036118040224948, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 1, + "baseline": 1392424, + "candidate": 2811575, + "ratio": 2.0191945844081975, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 2, + "baseline": 1238175, + "candidate": 3399261, + "ratio": 2.745380095705373, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 3, + "baseline": 2495300, + "candidate": 1445573, + "ratio": 0.5793183184386647, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 4, + "baseline": 1926854, + "candidate": 2020788, + "ratio": 1.0487499312350599, + "n_baseline": 1350, + "n_candidate": 1350 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 395600421, + "candidate_median": 252460577, + "ratio_median": 0.6967041484815552, + "ratio_min": 0.5322614739103317, + "ratio_max": 1.0803976783870675, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 351204256, + "candidate": 208586299, + "ratio": 0.5939173442135052, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 1, + "baseline": 395600421, + "candidate": 303746627, + "ratio": 0.7678116879456001, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 2, + "baseline": 408040985, + "candidate": 284283847, + "ratio": 0.6967041484815552, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 3, + "baseline": 438021246, + "candidate": 233141834, + "ratio": 0.5322614739103317, + "n_baseline": 1350, + "n_candidate": 1350 + }, + { + "pair": 4, + "baseline": 233673750, + "candidate": 252460577, + "ratio": 1.0803976783870675, + "n_baseline": 1350, + "n_candidate": 1350 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 645136, + "candidate_median": 654628, + "ratio_median": 1.0108899954221398, + "ratio_min": 0.9924500612484991, + "ratio_max": 1.0358560055554178, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 652039, + "candidate": 671002, + "ratio": 1.0290826162238762, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 1, + "baseline": 645136, + "candidate": 668268, + "ratio": 1.0358560055554178, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 2, + "baseline": 659608, + "candidate": 654628, + "ratio": 0.9924500612484991, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 3, + "baseline": 633506, + "candidate": 634949, + "ratio": 1.0022778000524069, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 4, + "baseline": 640037, + "candidate": 647007, + "ratio": 1.0108899954221398, + "n_baseline": 450, + "n_candidate": 450 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 2084747996, + "candidate_median": 1464316679, + "ratio_median": 0.7312541342566878, + "ratio_min": 0.5654084768168238, + "ratio_max": 0.9361628838729088, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 2084747996, + "candidate": 1456368609, + "ratio": 0.6985825681541991, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 1, + "baseline": 2310725312, + "candidate": 1306503679, + "ratio": 0.5654084768168238, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 2, + "baseline": 2002473026, + "candidate": 1464316679, + "ratio": 0.7312541342566878, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 3, + "baseline": 2115603569, + "candidate": 1702075544, + "ratio": 0.8045342562947813, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 4, + "baseline": 1915641533, + "candidate": 1793352502, + "ratio": 0.9361628838729088, + "n_baseline": 450, + "n_candidate": 450 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 9382643845, + "candidate_median": 7857525815, + "ratio_median": 0.843475186526083, + "ratio_min": 0.7813052016950169, + "ratio_max": 0.8869211928399658, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 9473926600, + "candidate": 7731460786, + "ratio": 0.8160777587193888, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 1, + "baseline": 9691928388, + "candidate": 7572354064, + "ratio": 0.7813052016950169, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 2, + "baseline": 9382643845, + "candidate": 8117933939, + "ratio": 0.8652075122009494, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 3, + "baseline": 9315657343, + "candidate": 7857525815, + "ratio": 0.843475186526083, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 4, + "baseline": 9301066092, + "candidate": 8249312633, + "ratio": 0.8869211928399658, + "n_baseline": 450, + "n_candidate": 450 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 1602736, + "candidate_median": 1287547, + "ratio_median": 0.8033431581994789, + "ratio_min": 0.7593592544121617, + "ratio_max": 0.8844123049428274, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1635804, + "candidate": 1283585, + "ratio": 0.7846814166000328, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 1, + "baseline": 1587590, + "candidate": 1288865, + "ratio": 0.8118374391373088, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 2, + "baseline": 1602736, + "candidate": 1287547, + "ratio": 0.8033431581994789, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 3, + "baseline": 1655821, + "candidate": 1257363, + "ratio": 0.7593592544121617, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 4, + "baseline": 1510775, + "candidate": 1336148, + "ratio": 0.8844123049428274, + "n_baseline": 450, + "n_candidate": 450 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 2085364740, + "candidate_median": 1617994840, + "ratio_median": 0.8077391171375407, + "ratio_min": 0.652702214290913, + "ratio_max": 0.9360454722466189, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 2085364740, + "candidate": 1456729414, + "ratio": 0.6985489809327072, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 1, + "baseline": 2311435670, + "candidate": 1508679180, + "ratio": 0.652702214290913, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 2, + "baseline": 2003115617, + "candidate": 1617994840, + "ratio": 0.8077391171375407, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 3, + "baseline": 2116068228, + "candidate": 1800727152, + "ratio": 0.8509778315144194, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 4, + "baseline": 1916278664, + "candidate": 1793723967, + "ratio": 0.9360454722466189, + "n_baseline": 450, + "n_candidate": 450 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 9383283380, + "candidate_median": 7857823281, + "ratio_median": 0.8434182494421244, + "ratio_min": 0.7812991279215437, + "ratio_max": 0.8868577998367054, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 9474596392, + "candidate": 7731790042, + "ratio": 0.8160548188130144, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 1, + "baseline": 9692653842, + "candidate": 7572861994, + "ratio": 0.7812991279215437, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 2, + "baseline": 9383283380, + "candidate": 8118327515, + "ratio": 0.865190486765412, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 3, + "baseline": 9316638911, + "candidate": 7857823281, + "ratio": 0.8434182494421244, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 4, + "baseline": 9302149768, + "candidate": 8249684077, + "ratio": 0.8868577998367054, + "n_baseline": 450, + "n_candidate": 450 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 922585, + "candidate_median": 622381, + "ratio_median": 0.6650212478720288, + "ratio_min": 0.6553413954859667, + "ratio_max": 0.7305507310102232, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 898289, + "candidate": 609010, + "ratio": 0.6779666677427866, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 1, + "baseline": 922585, + "candidate": 612856, + "ratio": 0.6642813399307381, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 2, + "baseline": 942212, + "candidate": 626591, + "ratio": 0.6650212478720288, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 3, + "baseline": 949705, + "candidate": 622381, + "ratio": 0.6553413954859667, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 4, + "baseline": 909700, + "candidate": 664582, + "ratio": 0.7305507310102232, + "n_baseline": 450, + "n_candidate": 450 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 1257985, + "candidate_median": 810586, + "ratio_median": 0.6584728462586382, + "ratio_min": 0.6239239736562837, + "ratio_max": 0.6773042250195127, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1231009, + "candidate": 810586, + "ratio": 0.6584728462586382, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 1, + "baseline": 1257985, + "candidate": 784887, + "ratio": 0.6239239736562837, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 2, + "baseline": 1241431, + "candidate": 820734, + "ratio": 0.6611193050600477, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 3, + "baseline": 1267429, + "candidate": 806009, + "ratio": 0.6359401591726243, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 4, + "baseline": 1304278, + "candidate": 883393, + "ratio": 0.6773042250195127, + "n_baseline": 450, + "n_candidate": 450 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 394739948, + "candidate_median": 204935601, + "ratio_median": 0.5295223760042563, + "ratio_min": 0.46479650094187813, + "ratio_max": 0.7914687824287755, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 305256394, + "candidate": 204935601, + "ratio": 0.6713556375169655, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 1, + "baseline": 440272560, + "candidate": 348461987, + "ratio": 0.7914687824287755, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 2, + "baseline": 444018689, + "candidate": 206378333, + "ratio": 0.46479650094187813, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 3, + "baseline": 302084341, + "candidate": 159960418, + "ratio": 0.5295223760042563, + "n_baseline": 450, + "n_candidate": 450 + }, + { + "pair": 4, + "baseline": 394739948, + "candidate": 202962440, + "ratio": 0.5141674690599087, + "n_baseline": 450, + "n_candidate": 450 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 646832, + "candidate_median": 644511, + "ratio_median": 0.9935686347986978, + "ratio_min": 0.9897965922092282, + "ratio_max": 1.008005169812254, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 644244, + "candidate": 639137, + "ratio": 0.9920728792196746, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 1, + "baseline": 640455, + "candidate": 636336, + "ratio": 0.9935686347986978, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 2, + "baseline": 646895, + "candidate": 650648, + "ratio": 1.0058015597585388, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 3, + "baseline": 651155, + "candidate": 644511, + "ratio": 0.9897965922092282, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 4, + "baseline": 646832, + "candidate": 652010, + "ratio": 1.008005169812254, + "n_baseline": 4500, + "n_candidate": 4500 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 2007351053, + "candidate_median": 1638348211, + "ratio_median": 0.8425730859742175, + "ratio_min": 0.6937850211428023, + "ratio_max": 0.9619962659120237, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 2007351053, + "candidate": 1528817910, + "ratio": 0.7616096385907044, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 1, + "baseline": 2195626166, + "candidate": 1523292546, + "ratio": 0.6937850211428023, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 2, + "baseline": 1944458277, + "candidate": 1638348211, + "ratio": 0.8425730859742175, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 3, + "baseline": 2037960445, + "candidate": 1803872381, + "ratio": 0.8851361101858873, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 4, + "baseline": 1799422521, + "candidate": 1731037746, + "ratio": 0.9619962659120237, + "n_baseline": 4500, + "n_candidate": 4500 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "metric": "p99", + "pairs": 5, + "baseline_median": 8099369629, + "candidate_median": 6825031055, + "ratio_median": 0.8426620055174173, + "ratio_min": 0.7708948940322259, + "ratio_max": 0.9284983540844857, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 8152529524, + "candidate": 6455940286, + "ratio": 0.7918941313851782, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 1, + "baseline": 8334967372, + "candidate": 6425383789, + "ratio": 0.7708948940322259, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 2, + "baseline": 8099369629, + "candidate": 6825031055, + "ratio": 0.8426620055174173, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 3, + "baseline": 7908959932, + "candidate": 6935643349, + "ratio": 0.8769349457617154, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 4, + "baseline": 7709511144, + "candidate": 7158268408, + "ratio": 0.9284983540844857, + "n_baseline": 4500, + "n_candidate": 4500 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 9465586669, + "candidate_median": 8098349666, + "ratio_median": 0.8387204976018873, + "ratio_min": 0.7808494070263626, + "ratio_max": 0.9043872513966926, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 9465586669, + "candidate": 7740446722, + "ratio": 0.8177461147073039, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 1, + "baseline": 9672663570, + "candidate": 7552893613, + "ratio": 0.7808494070263626, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 2, + "baseline": 9655600035, + "candidate": 8098349666, + "ratio": 0.8387204976018873, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 3, + "baseline": 9464140040, + "candidate": 8151614118, + "ratio": 0.8613158811627221, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 4, + "baseline": 9282162912, + "candidate": 8394669803, + "ratio": 0.9043872513966926, + "n_baseline": 4500, + "n_candidate": 4500 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 1675108, + "candidate_median": 1379029, + "ratio_median": 0.8291692445323924, + "ratio_min": 0.7846970785869832, + "ratio_max": 0.9005734321562775, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1684931, + "candidate": 1356135, + "ratio": 0.8048608518687116, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 1, + "baseline": 1645418, + "candidate": 1364330, + "ratio": 0.8291692445323924, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 2, + "baseline": 1675108, + "candidate": 1396012, + "ratio": 0.8333862652437932, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 3, + "baseline": 1757403, + "candidate": 1379029, + "ratio": 0.7846970785869832, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 4, + "baseline": 1610827, + "candidate": 1450668, + "ratio": 0.9005734321562775, + "n_baseline": 4500, + "n_candidate": 4500 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 2027339200, + "candidate_median": 1662041806, + "ratio_median": 0.8460597112411483, + "ratio_min": 0.7026615437680845, + "ratio_max": 0.9686959958667419, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 2027339200, + "candidate": 1605073682, + "ratio": 0.7917144215432721, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 1, + "baseline": 2196340962, + "candidate": 1543284331, + "ratio": 0.7026615437680845, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 2, + "baseline": 1964449771, + "candidate": 1662041806, + "ratio": 0.8460597112411483, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 3, + "baseline": 2057944014, + "candidate": 1823863856, + "ratio": 0.8862553323085688, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 4, + "baseline": 1807594861, + "candidate": 1751009904, + "ratio": 0.9686959958667419, + "n_baseline": 4500, + "n_candidate": 4500 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "scheduled-to-complete", + "metric": "p99", + "pairs": 5, + "baseline_median": 8099952138, + "candidate_median": 6825552550, + "ratio_median": 0.8426657878604863, + "ratio_min": 0.773792869111885, + "ratio_max": 0.9261456957098311, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 8172518073, + "candidate": 6475931059, + "ratio": 0.792403394052427, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 1, + "baseline": 8354956471, + "candidate": 6465005739, + "ratio": 0.773792869111885, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 2, + "baseline": 8099952138, + "candidate": 6825552550, + "ratio": 0.8426657878604863, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 3, + "baseline": 7928947508, + "candidate": 6955634903, + "ratio": 0.8772456742817423, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 4, + "baseline": 7729500555, + "candidate": 7158643669, + "ratio": 0.9261456957098311, + "n_baseline": 4500, + "n_candidate": 4500 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 9485562153, + "candidate_median": 8098984242, + "ratio_median": 0.8370545090264381, + "ratio_min": 0.7808023907987849, + "ratio_max": 0.9064407023836621, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 9485562153, + "candidate": 7760420634, + "ratio": 0.8181297543388729, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 1, + "baseline": 9673912228, + "candidate": 7553413796, + "ratio": 0.7808023907987849, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 2, + "baseline": 9675575670, + "candidate": 8098984242, + "ratio": 0.8370545090264381, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 3, + "baseline": 9484113140, + "candidate": 8171590163, + "ratio": 0.8616082539690159, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 4, + "baseline": 9283177259, + "candidate": 8414649715, + "ratio": 0.9064407023836621, + "n_baseline": 4500, + "n_candidate": 4500 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 1006522, + "candidate_median": 712673, + "ratio_median": 0.6970132043381351, + "ratio_min": 0.6907436226428161, + "ratio_max": 0.7405332421944081, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 976114, + "candidate": 695891, + "ratio": 0.7129198024001295, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 1, + "baseline": 991735, + "candidate": 689399, + "ratio": 0.6951443682031995, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 2, + "baseline": 1039694, + "candidate": 718162, + "ratio": 0.6907436226428161, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 3, + "baseline": 1022467, + "candidate": 712673, + "ratio": 0.6970132043381351, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 4, + "baseline": 1006522, + "candidate": 745363, + "ratio": 0.7405332421944081, + "n_baseline": 4500, + "n_candidate": 4500 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 1571698, + "candidate_median": 1174145, + "ratio_median": 0.7446296934907343, + "ratio_min": 0.7076265976886249, + "ratio_max": 0.7630495364621013, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1571698, + "candidate": 1170333, + "ratio": 0.7446296934907343, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 1, + "baseline": 1550898, + "candidate": 1183412, + "ratio": 0.7630495364621013, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 2, + "baseline": 1659272, + "candidate": 1174145, + "ratio": 0.7076265976886249, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 3, + "baseline": 1578823, + "candidate": 1168742, + "ratio": 0.7402615746033596, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 4, + "baseline": 1561757, + "candidate": 1187744, + "ratio": 0.7605178014249336, + "n_baseline": 4500, + "n_candidate": 4500 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "metric": "p99", + "pairs": 5, + "baseline_median": 4254871, + "candidate_median": 4270934, + "ratio_median": 1.0174084103885412, + "ratio_min": 0.946914271659567, + "ratio_max": 1.0411156773832588, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 4254871, + "candidate": 4407299, + "ratio": 1.0358243528417195, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 1, + "baseline": 4197856, + "candidate": 4270934, + "ratio": 1.0174084103885412, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 2, + "baseline": 4330015, + "candidate": 4100153, + "ratio": 0.946914271659567, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 3, + "baseline": 4264602, + "candidate": 4439944, + "ratio": 1.0411156773832588, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 4, + "baseline": 4210359, + "candidate": 4211022, + "ratio": 1.0001574687574148, + "n_baseline": 4500, + "n_candidate": 4500 + } + ] + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 457723410, + "candidate_median": 306260021, + "ratio_median": 0.6690940736459164, + "ratio_min": 0.555940967154218, + "ratio_max": 0.8508600715788058, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 457723410, + "candidate": 306260021, + "ratio": 0.6690940736459164, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 1, + "baseline": 454546950, + "candidate": 252701271, + "ratio": 0.555940967154218, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 2, + "baseline": 444588052, + "candidate": 253928832, + "ratio": 0.5711553220058194, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 3, + "baseline": 460374822, + "candidate": 391714554, + "ratio": 0.8508600715788058, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 4, + "baseline": 503924346, + "candidate": 394260097, + "ratio": 0.7823795379792982, + "n_baseline": 4500, + "n_candidate": 4500 + } + ] + }, + { + "scenario": "handler-control", + "event_class": "200-valid-envelopes", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 1124282, + "candidate_median": 953422, + "ratio_median": 0.8517400306404364, + "ratio_min": 0.8283415913626487, + "ratio_max": 0.9149212580027104, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1151001, + "candidate": 953422, + "ratio": 0.8283415913626487, + "n_baseline": 200, + "n_candidate": 200 + }, + { + "pair": 1, + "baseline": 1069950, + "candidate": 978920, + "ratio": 0.9149212580027104, + "n_baseline": 200, + "n_candidate": 200 + }, + { + "pair": 2, + "baseline": 1092021, + "candidate": 930118, + "ratio": 0.8517400306404364, + "n_baseline": 200, + "n_candidate": 200 + }, + { + "pair": 3, + "baseline": 1124916, + "candidate": 965908, + "ratio": 0.8586490013476562, + "n_baseline": 200, + "n_candidate": 200 + }, + { + "pair": 4, + "baseline": 1124282, + "candidate": 943043, + "ratio": 0.8387957825527759, + "n_baseline": 200, + "n_candidate": 200 + } + ] + }, + { + "scenario": "handler-control", + "event_class": "200-valid-envelopes", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 2414298, + "candidate_median": 1819853, + "ratio_median": 0.7301687695553739, + "ratio_min": 0.6692835615056261, + "ratio_max": 0.8010595161501841, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 2636458, + "candidate": 1764538, + "ratio": 0.6692835615056261, + "n_baseline": 200, + "n_candidate": 200 + }, + { + "pair": 1, + "baseline": 2382405, + "candidate": 1819853, + "ratio": 0.7638722215576277, + "n_baseline": 200, + "n_candidate": 200 + }, + { + "pair": 2, + "baseline": 2272358, + "candidate": 1820294, + "ratio": 0.8010595161501841, + "n_baseline": 200, + "n_candidate": 200 + }, + { + "pair": 3, + "baseline": 2414298, + "candidate": 1762845, + "ratio": 0.7301687695553739, + "n_baseline": 200, + "n_candidate": 200 + }, + { + "pair": 4, + "baseline": 2700740, + "candidate": 1822918, + "ratio": 0.6749698230855247, + "n_baseline": 200, + "n_candidate": 200 + } + ] + }, + { + "scenario": "handler-control", + "event_class": "200-valid-envelopes", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 4462713, + "candidate_median": 4322900, + "ratio_median": 0.9686708511167982, + "ratio_min": 0.8058574303196183, + "ratio_max": 1.0198861744092245, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 5205812, + "candidate": 4519839, + "ratio": 0.8682293943769003, + "n_baseline": 200, + "n_candidate": 200 + }, + { + "pair": 1, + "baseline": 4053570, + "candidate": 4134180, + "ratio": 1.0198861744092245, + "n_baseline": 200, + "n_candidate": 200 + }, + { + "pair": 2, + "baseline": 4539226, + "candidate": 3657969, + "ratio": 0.8058574303196183, + "n_baseline": 200, + "n_candidate": 200 + }, + { + "pair": 3, + "baseline": 4373685, + "candidate": 4427515, + "ratio": 1.012307699342774, + "n_baseline": 200, + "n_candidate": 200 + }, + { + "pair": 4, + "baseline": 4462713, + "candidate": 4322900, + "ratio": 0.9686708511167982, + "n_baseline": 200, + "n_candidate": 200 + } + ] + }, + { + "scenario": "measured-resource", + "event_class": "8:0:dbytes", + "timing": "window", + "metric": "p50", + "pairs": 5, + "baseline_median": 0, + "candidate_median": 0, + "ratio_median": null, + "ratio_min": null, + "ratio_max": null, + "candidate_lower_pairs": 0, + "per_pair": [ + { + "pair": 0, + "baseline": 0, + "candidate": 0, + "ratio": null, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 0, + "candidate": 0, + "ratio": null, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 0, + "candidate": 0, + "ratio": null, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 0, + "candidate": 0, + "ratio": null, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 0, + "candidate": 0, + "ratio": null, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "measured-resource", + "event_class": "8:0:dios", + "timing": "window", + "metric": "p50", + "pairs": 5, + "baseline_median": 0, + "candidate_median": 0, + "ratio_median": null, + "ratio_min": null, + "ratio_max": null, + "candidate_lower_pairs": 0, + "per_pair": [ + { + "pair": 0, + "baseline": 0, + "candidate": 0, + "ratio": null, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 0, + "candidate": 0, + "ratio": null, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 0, + "candidate": 0, + "ratio": null, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 0, + "candidate": 0, + "ratio": null, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 0, + "candidate": 0, + "ratio": null, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "measured-resource", + "event_class": "8:0:rbytes", + "timing": "window", + "metric": "p50", + "pairs": 5, + "baseline_median": 0, + "candidate_median": 0, + "ratio_median": null, + "ratio_min": null, + "ratio_max": null, + "candidate_lower_pairs": 0, + "per_pair": [ + { + "pair": 0, + "baseline": 0, + "candidate": 0, + "ratio": null, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 0, + "candidate": 0, + "ratio": null, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 0, + "candidate": 0, + "ratio": null, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 0, + "candidate": 0, + "ratio": null, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 0, + "candidate": 0, + "ratio": null, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "measured-resource", + "event_class": "8:0:rios", + "timing": "window", + "metric": "p50", + "pairs": 5, + "baseline_median": 0, + "candidate_median": 0, + "ratio_median": null, + "ratio_min": null, + "ratio_max": null, + "candidate_lower_pairs": 0, + "per_pair": [ + { + "pair": 0, + "baseline": 0, + "candidate": 0, + "ratio": null, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 0, + "candidate": 0, + "ratio": null, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 0, + "candidate": 0, + "ratio": null, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 0, + "candidate": 0, + "ratio": null, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 0, + "candidate": 0, + "ratio": null, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "measured-resource", + "event_class": "8:0:wbytes", + "timing": "window", + "metric": "p50", + "pairs": 5, + "baseline_median": 2480660480, + "candidate_median": 2405064704, + "ratio_median": 0.9693704594513283, + "ratio_min": 0.9651528869670478, + "ratio_max": 0.9811918765730461, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 2480648192, + "candidate": 2405064704, + "ratio": 0.9695307507756424, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 2480660480, + "candidate": 2400497664, + "ratio": 0.9676848901144263, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 2483662848, + "candidate": 2397114368, + "ratio": 0.9651528869670478, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 2491469824, + "candidate": 2415157248, + "ratio": 0.9693704594513283, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 2454142976, + "candidate": 2407985152, + "ratio": 0.9811918765730461, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "measured-resource", + "event_class": "8:0:wios", + "timing": "window", + "metric": "p50", + "pairs": 5, + "baseline_median": 220705, + "candidate_median": 206760, + "ratio_median": 0.9368161120047122, + "ratio_min": 0.9284700757370352, + "ratio_max": 0.9666007868463051, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 220705, + "candidate": 206760, + "ratio": 0.9368161120047122, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 219922, + "candidate": 204870, + "ratio": 0.9315575522230609, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 221292, + "candidate": 205463, + "ratio": 0.9284700757370352, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 222514, + "candidate": 209442, + "ratio": 0.9412531346342252, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 215544, + "candidate": 208345, + "ratio": 0.9666007868463051, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "measured-resource", + "event_class": "cpu_seconds", + "timing": "window", + "metric": "p50", + "pairs": 5, + "baseline_median": 54.683538, + "candidate_median": 53.516078, + "ratio_median": 0.9786506132796309, + "ratio_min": 0.9596792667246414, + "ratio_max": 1.023343846766146, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 53.042935, + "candidate": 52.752416, + "ratio": 0.9945229463641104, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 54.890594, + "candidate": 52.677365, + "ratio": 0.9596792667246414, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 54.771503, + "candidate": 53.588029, + "ratio": 0.9783925228416681, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 54.683538, + "candidate": 53.516078, + "ratio": 0.9786506132796309, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 54.596186, + "candidate": 55.870671, + "ratio": 1.023343846766146, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "measured-resource", + "event_class": "sampled_memory_peak_bytes", + "timing": "window", + "metric": "p50", + "pairs": 5, + "baseline_median": 4281593856, + "candidate_median": 4234752000, + "ratio_median": 0.9944464012583654, + "ratio_min": 0.9890597152426407, + "ratio_max": 1.0021750449525155, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 4239302656, + "candidate": 4207431680, + "ratio": 0.9924820239114346, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 4281593856, + "candidate": 4234752000, + "ratio": 0.9890597152426407, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 4303065088, + "candidate": 4312424448, + "ratio": 1.0021750449525155, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 4244541440, + "candidate": 4220968960, + "ratio": 0.9944464012583654, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 4297109504, + "candidate": 4276379648, + "ratio": 0.995175860428806, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "measured-resource", + "event_class": "sampled_pss_peak_bytes", + "timing": "window", + "metric": "p50", + "pairs": 5, + "baseline_median": 3111116800, + "candidate_median": 3081181184, + "ratio_median": 0.9947465361740411, + "ratio_min": 0.9903778553090645, + "ratio_max": 1.005091403723179, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 3073052672, + "candidate": 3048941568, + "ratio": 0.9921540218884995, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 3111116800, + "candidate": 3081181184, + "ratio": 0.9903778553090645, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 3128674304, + "candidate": 3144603648, + "ratio": 1.005091403723179, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 3077771264, + "candidate": 3061602304, + "ratio": 0.9947465361740411, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 3114579968, + "candidate": 3099301888, + "ratio": 0.9950946579773289, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "measured-resource", + "event_class": "sampled_rss_peak_bytes", + "timing": "window", + "metric": "p50", + "pairs": 5, + "baseline_median": 3114422272, + "candidate_median": 3084419072, + "ratio_median": 0.9947328888682092, + "ratio_min": 0.9903663673774293, + "ratio_max": 1.0050756176795232, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 3076358144, + "candidate": 3052183552, + "ratio": 0.9921418148120533, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 3114422272, + "candidate": 3084419072, + "ratio": 0.9903663673774293, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 3131949056, + "candidate": 3147845632, + "ratio": 1.0050756176795232, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 3081072640, + "candidate": 3064844288, + "ratio": 0.9947328888682092, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 3117817856, + "candidate": 3102543872, + "ratio": 0.9951010659681077, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "measured-resource", + "event_class": "storage_peak_bytes", + "timing": "window", + "metric": "p50", + "pairs": 5, + "baseline_median": 1150121704, + "candidate_median": 1147516624, + "ratio_median": 0.9986876520541932, + "ratio_min": 0.9895964526024382, + "ratio_max": 1.0001070584294696, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 1145506704, + "candidate": 1145494344, + "ratio": 0.9999892100151341, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 1150121704, + "candidate": 1140081264, + "ratio": 0.9912701064895303, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 1154509744, + "candidate": 1154633344, + "ratio": 1.0001070584294696, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 1149024544, + "candidate": 1147516624, + "ratio": 0.9986876520541932, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 1161523040, + "candidate": 1149439080, + "ratio": 0.9895964526024382, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "measured-resource", + "event_class": "wal_peak_bytes", + "timing": "window", + "metric": "p50", + "pairs": 5, + "baseline_median": 40582032, + "candidate_median": 31040112, + "ratio_median": 0.9496145913192429, + "ratio_min": 0.69825126992587, + "ratio_max": 1.0029934121682496, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 40582032, + "candidate": 40569672, + "ratio": 0.9996954317122415, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 41080552, + "candidate": 31040112, + "ratio": 0.7555914049061463, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 41290672, + "candidate": 41414272, + "ratio": 1.0029934121682496, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 29927712, + "candidate": 28419792, + "ratio": 0.9496145913192429, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 40046432, + "candidate": 27962472, + "ratio": 0.69825126992587, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 451129, + "candidate_median": 355330, + "ratio_median": 0.8386338479257779, + "ratio_min": 0.5570471522118413, + "ratio_max": 1.164141404182655, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 662005, + "candidate": 368768, + "ratio": 0.5570471522118413, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 423701, + "candidate": 355330, + "ratio": 0.8386338479257779, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 451129, + "candidate": 338898, + "ratio": 0.7512219343026052, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 507142, + "candidate": 590385, + "ratio": 1.164141404182655, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 344645, + "candidate": 345747, + "ratio": 1.003197493072582, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 1040867, + "candidate_median": 1044359, + "ratio_median": 0.9928423365493642, + "ratio_min": 0.9629914292604146, + "ratio_max": 1.0807001663267932, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1077726, + "candidate": 1070012, + "ratio": 0.9928423365493642, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 1057304, + "candidate": 1044359, + "ratio": 0.9877565960215794, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 976288, + "candidate": 1036295, + "ratio": 1.0614644449178932, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 978796, + "candidate": 1057785, + "ratio": 1.0807001663267932, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 1040867, + "candidate": 1002346, + "ratio": 0.9629914292604146, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1088089, + "candidate_median": 1121808, + "ratio_median": 1.0563989464336667, + "ratio_min": 0.9253980000942529, + "ratio_max": 6.10642590724407, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 1808519, + "candidate": 3655181, + "ratio": 2.021090737780471, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 1080470, + "candidate": 6597810, + "ratio": 6.10642590724407, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 1061917, + "candidate": 1121808, + "ratio": 1.0563989464336667, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 1088089, + "candidate": 1104812, + "ratio": 1.0153691471929227, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 1145854, + "candidate": 1060371, + "ratio": 0.9253980000942529, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 34471349, + "candidate_median": 32441466, + "ratio_median": 0.951075398876905, + "ratio_min": 0.9263813666637368, + "ratio_max": 0.9756921985532201, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 33233528, + "candidate": 32425694, + "ratio": 0.9756921985532201, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 35019558, + "candidate": 32441466, + "ratio": 0.9263813666637368, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 33300250, + "candidate": 31917330, + "ratio": 0.958471182648779, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 34954148, + "candidate": 32487975, + "ratio": 0.9294454838378552, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 34471349, + "candidate": 32784852, + "ratio": 0.951075398876905, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 43435046, + "candidate_median": 42218430, + "ratio_median": 1.0102524614998654, + "ratio_min": 0.9118740421150556, + "ratio_max": 1.4716586372130374, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 47274491, + "candidate": 69571913, + "ratio": 1.4716586372130374, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 43037894, + "candidate": 62066738, + "ratio": 1.4421416159443117, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 41359336, + "candidate": 41783371, + "ratio": 1.0102524614998654, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 43435046, + "candidate": 42218430, + "ratio": 0.9719899916763067, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 45119589, + "candidate": 41143382, + "ratio": 0.9118740421150556, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 71476336, + "candidate_median": 59522083, + "ratio_median": 0.9638741044163676, + "ratio_min": 0.7847248773536558, + "ratio_max": 1.3855648667844065, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 74572601, + "candidate": 84736118, + "ratio": 1.1362902307779235, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 51542427, + "candidate": 71415376, + "ratio": 1.3855648667844065, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 71476336, + "candidate": 59522083, + "ratio": 0.8327522972078479, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 75380763, + "candidate": 59153160, + "ratio": 0.7847248773536558, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 48001938, + "candidate": 46267825, + "ratio": 0.9638741044163676, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 33691626, + "candidate_median": 31841481, + "ratio_median": 0.960998693566981, + "ratio_min": 0.9254371273628539, + "ratio_max": 0.9805329758784745, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 32647106, + "candidate": 32011564, + "ratio": 0.9805329758784745, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 34406963, + "candidate": 31841481, + "ratio": 0.9254371273628539, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 32809183, + "candidate": 31529582, + "ratio": 0.960998693566981, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 34098190, + "candidate": 31802888, + "ratio": 0.9326855179116545, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 33691626, + "candidate": 32544036, + "ratio": 0.965938420425301, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 43048959, + "candidate_median": 42105494, + "ratio_median": 1.0037892831061506, + "ratio_min": 0.9098897770824488, + "ratio_max": 1.4706360656650856, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 47107652, + "candidate": 69278212, + "ratio": 1.4706360656650856, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 42422789, + "candidate": 61917291, + "ratio": 1.4595290045640328, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 41241838, + "candidate": 41398115, + "ratio": 1.0037892831061506, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 43048959, + "candidate": 42105494, + "ratio": 0.9780839067444116, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 44806925, + "candidate": 40769363, + "ratio": 0.9098897770824488, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 70500048, + "candidate_median": 59001699, + "ratio_median": 0.9573179937401322, + "ratio_min": 0.7752858805916383, + "ratio_max": 1.392116621592873, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 72764082, + "candidate": 83640791, + "ratio": 1.1494790932702208, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 51255208, + "candidate": 71353227, + "ratio": 1.392116621592873, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 70500048, + "candidate": 59001699, + "ratio": 0.8369029621086216, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 75022669, + "candidate": 58164016, + "ratio": 0.7752858805916383, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 47399084, + "candidate": 45375996, + "ratio": 0.9573179937401322, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 605041, + "candidate_median": 597205, + "ratio_median": 0.9681360436730734, + "ratio_min": 0.8362447666384805, + "ratio_max": 1.1939045533351582, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 678818, + "candidate": 567658, + "ratio": 0.8362447666384805, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 724798, + "candidate": 672838, + "ratio": 0.9283110604609837, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 605041, + "candidate": 585762, + "ratio": 0.9681360436730734, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 567953, + "candidate": 597205, + "ratio": 1.0515042617963106, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 526032, + "candidate": 628032, + "ratio": 1.1939045533351582, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 1080672, + "candidate_median": 1053330, + "ratio_median": 0.9754026685242342, + "ratio_min": 0.962476038013831, + "ratio_max": 0.991719041485298, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1079548, + "candidate": 1052994, + "ratio": 0.9754026685242342, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 1081922, + "candidate": 1041324, + "ratio": 0.962476038013831, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 1090224, + "candidate": 1056188, + "ratio": 0.9687807276302851, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 1080672, + "candidate": 1071723, + "ratio": 0.991719041485298, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 1075586, + "candidate": 1053330, + "ratio": 0.9793080237191634, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1117408, + "candidate_median": 1102458, + "ratio_median": 0.982827680206747, + "ratio_min": 0.9528697030300743, + "ratio_max": 1.006286871044417, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 1101249, + "candidate": 1082338, + "ratio": 0.982827680206747, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 1114541, + "candidate": 1102458, + "ratio": 0.989158765805834, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 1139543, + "candidate": 1085836, + "ratio": 0.9528697030300743, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 1117408, + "candidate": 1124433, + "ratio": 1.006286871044417, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 1170431, + "candidate": 1141972, + "ratio": 0.9756850254308028, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 2393451, + "candidate_median": 2321885, + "ratio_median": 0.9685906375190015, + "ratio_min": 0.9272404462053138, + "ratio_max": 1.028206243228269, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 2388619, + "candidate": 2313594, + "ratio": 0.9685906375190015, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 2480697, + "candidate": 2321885, + "ratio": 0.9359808956918156, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 2452324, + "candidate": 2273894, + "ratio": 0.9272404462053138, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 2393451, + "candidate": 2384476, + "ratio": 0.9962501843572314, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 2315693, + "candidate": 2381010, + "ratio": 1.028206243228269, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 3207608, + "candidate_median": 3018167, + "ratio_median": 0.9409401024065285, + "ratio_min": 0.9057680298212982, + "ratio_max": 1.1313491636580286, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 3225375, + "candidate": 3317540, + "ratio": 1.028574971902492, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 3265049, + "candidate": 2957377, + "ratio": 0.9057680298212982, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 3182325, + "candidate": 2902401, + "ratio": 0.9120378968207207, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 3207608, + "candidate": 3018167, + "ratio": 0.9409401024065285, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 3058139, + "candidate": 3459823, + "ratio": 1.1313491636580286, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 4818077, + "candidate_median": 5605946, + "ratio_median": 0.8791209928498694, + "ratio_min": 0.8583449372021245, + "ratio_max": 3.682724831800506, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 7999593, + "candidate": 6982388, + "ratio": 0.8728429058828368, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 3888240, + "candidate": 14319318, + "ratio": 3.682724831800506, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 3961606, + "candidate": 3482731, + "ratio": 0.8791209928498694, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 4818077, + "candidate": 4135572, + "ratio": 0.8583449372021245, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 5683796, + "candidate": 5605946, + "ratio": 0.9863031678124972, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 1715958, + "candidate_median": 1654917, + "ratio_median": 0.9644274510215285, + "ratio_min": 0.9417489035087719, + "ratio_max": 0.9784060201250927, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1643421, + "candidate": 1607933, + "ratio": 0.9784060201250927, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 1670784, + "candidate": 1573459, + "ratio": 0.9417489035087719, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 1764001, + "candidate": 1667968, + "ratio": 0.9455595546714542, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 1715958, + "candidate": 1654917, + "ratio": 0.9644274510215285, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 1750049, + "candidate": 1698184, + "ratio": 0.9703636869596223, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 2407549, + "candidate_median": 2391528, + "ratio_median": 1.0164803267380154, + "ratio_min": 0.9706049945697083, + "ratio_max": 1.153121565020127, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 2280763, + "candidate": 2629997, + "ratio": 1.153121565020127, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 2489089, + "candidate": 2530110, + "ratio": 1.0164803267380154, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 2407549, + "candidate": 2347658, + "ratio": 0.9751236631113219, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 2463956, + "candidate": 2391528, + "ratio": 0.9706049945697083, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 2326269, + "candidate": 2388325, + "ratio": 1.0266761926501191, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 4472043, + "candidate_median": 5131323, + "ratio_median": 0.9839159045696391, + "ratio_min": 0.6733251446821956, + "ratio_max": 4.768807943414064, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 7531117, + "candidate": 5905009, + "ratio": 0.7840814317451182, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 2828406, + "candidate": 13488125, + "ratio": 4.768807943414064, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 2854186, + "candidate": 2808279, + "ratio": 0.9839159045696391, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 4472043, + "candidate": 3011139, + "ratio": 0.6733251446821956, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 4745351, + "candidate": 5131323, + "ratio": 1.0813368705497233, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 148782, + "candidate_median": 144164, + "ratio_median": 1.0183158980299638, + "ratio_min": 0.5971973094170404, + "ratio_max": 1.2857066042935301, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 181071, + "candidate": 148078, + "ratio": 0.817789706800095, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 124016, + "candidate": 136888, + "ratio": 1.1037930589601341, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 148782, + "candidate": 191290, + "ratio": 1.2857066042935301, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 141571, + "candidate": 144164, + "ratio": 1.0183158980299638, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 214080, + "candidate": 127848, + "ratio": 0.5971973094170404, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 929239, + "candidate_median": 1007081, + "ratio_median": 1.1186390153663375, + "ratio_min": 0.9291053259252464, + "ratio_max": 1.1693638271609288, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 1002868, + "candidate": 931770, + "ratio": 0.9291053259252464, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 929239, + "candidate": 1039483, + "ratio": 1.1186390153663375, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 851451, + "candidate": 995656, + "ratio": 1.1693638271609288, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 1043130, + "candidate": 1007081, + "ratio": 0.9654415077698849, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 917476, + "candidate": 1048164, + "ratio": 1.142442963085683, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1078157, + "candidate_median": 1180330, + "ratio_median": 1.1178022575354454, + "ratio_min": 0.9764269764269764, + "ratio_max": 2.4485330058609276, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 1078157, + "candidate": 2639903, + "ratio": 2.4485330058609276, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 1090908, + "candidate": 1065192, + "ratio": 0.9764269764269764, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 957641, + "candidate": 1180330, + "ratio": 1.232539124786846, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 1170037, + "candidate": 1307870, + "ratio": 1.1178022575354454, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 1063672, + "candidate": 1122413, + "ratio": 1.055224730932092, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 2225573, + "candidate_median": 2023558, + "ratio_median": 0.9125238131710356, + "ratio_min": 0.9085090783022602, + "ratio_max": 0.9253764074467141, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 2175273, + "candidate": 1994569, + "ratio": 0.9169281281016222, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 2134485, + "candidate": 1939199, + "ratio": 0.9085090783022602, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 2243513, + "candidate": 2076094, + "ratio": 0.9253764074467141, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 2225573, + "candidate": 2023558, + "ratio": 0.9092301173675273, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 2258729, + "candidate": 2061144, + "ratio": 0.9125238131710356, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 474084631, + "candidate_median": 451796208, + "ratio_median": 0.9208713559836956, + "ratio_min": 0.8681568646002024, + "ratio_max": 1.038954277111096, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 474084631, + "candidate": 436570957, + "ratio": 0.9208713559836956, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 462418199, + "candidate": 442691736, + "ratio": 0.9573406430744738, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 496181005, + "candidate": 451796208, + "ratio": 0.9105471661495789, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 525693164, + "candidate": 456384129, + "ratio": 0.8681568646002024, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 473815621, + "candidate": 492272766, + "ratio": 1.038954277111096, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 551283838, + "candidate_median": 544017611, + "ratio_median": 0.9794807119608172, + "ratio_min": 0.5882448303368111, + "ratio_max": 1.0489873603005935, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 851302646, + "candidate": 583649576, + "ratio": 0.6855958673949664, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 542182508, + "candidate": 544017611, + "ratio": 1.0033846591745819, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 551283838, + "candidate": 578289778, + "ratio": 1.0489873603005935, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 890228257, + "candidate": 523672170, + "ratio": 0.5882448303368111, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 543822231, + "candidate": 532663386, + "ratio": 0.9794807119608172, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 1906704, + "candidate_median": 1754023, + "ratio_median": 0.918200560531459, + "ratio_min": 0.9016302213587417, + "ratio_max": 0.9659653138189604, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1815824, + "candidate": 1754023, + "ratio": 0.9659653138189604, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 1902837, + "candidate": 1747186, + "ratio": 0.918200560531459, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 1952712, + "candidate": 1788112, + "ratio": 0.9157069757342609, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 1935320, + "candidate": 1744943, + "ratio": 0.9016302213587417, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 1906704, + "candidate": 1839589, + "ratio": 0.9648005143955223, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 473540996, + "candidate_median": 451742559, + "ratio_median": 0.9226032393051685, + "ratio_min": 0.8679204013670282, + "ratio_max": 1.0384108982192537, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 473081763, + "candidate": 436466767, + "ratio": 0.9226032393051685, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 462299261, + "candidate": 442218137, + "ratio": 0.9565625003237892, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 495823394, + "candidate": 451742559, + "ratio": 0.9110956934799248, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 525658525, + "candidate": 456229758, + "ratio": 0.8679204013670282, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 473540996, + "candidate": 491730131, + "ratio": 1.0384108982192537, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 550584916, + "candidate_median": 543718118, + "ratio_median": 0.9795330447534655, + "ratio_min": 0.5878091162667508, + "ratio_max": 1.0494832462863912, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 851160299, + "candidate": 583581541, + "ratio": 0.6856305935387619, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 542067749, + "candidate": 543718118, + "ratio": 1.0030445806876438, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 550584916, + "candidate": 577829645, + "ratio": 1.0494832462863912, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 890074964, + "candidate": 523194178, + "ratio": 0.5878091162667508, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 543713604, + "candidate": 532585442, + "ratio": 0.9795330447534655, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 651221, + "candidate_median": 665933, + "ratio_median": 1.071802704294391, + "ratio_min": 0.9080394423585911, + "ratio_max": 1.301906527000661, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 651221, + "candidate": 640247, + "ratio": 0.9831485778253466, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 684239, + "candidate": 621316, + "ratio": 0.9080394423585911, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 618465, + "candidate": 716515, + "ratio": 1.1585376698762258, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 672412, + "candidate": 720693, + "ratio": 1.071802704294391, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 511506, + "candidate": 665933, + "ratio": 1.301906527000661, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 1039898, + "candidate_median": 1064613, + "ratio_median": 1.0336387048804871, + "ratio_min": 1.0111629276447012, + "ratio_max": 1.045026631804611, + "candidate_lower_pairs": 0, + "per_pair": [ + { + "pair": 0, + "baseline": 1040673, + "candidate": 1087531, + "ratio": 1.045026631804611, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 1020178, + "candidate": 1056579, + "ratio": 1.0356810282127236, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 1052860, + "candidate": 1064613, + "ratio": 1.0111629276447012, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 1038298, + "candidate": 1073225, + "ratio": 1.0336387048804871, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 1039898, + "candidate": 1064360, + "ratio": 1.0235234609548245, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1096366, + "candidate_median": 1124468, + "ratio_median": 1.0295131885776287, + "ratio_min": 0.9621454895400279, + "ratio_max": 1.0461203733218678, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 1142845, + "candidate": 1176574, + "ratio": 1.0295131885776287, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 1096366, + "candidate": 1107066, + "ratio": 1.0097595146146452, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 1139917, + "candidate": 1096766, + "ratio": 0.9621454895400279, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 1079176, + "candidate": 1128948, + "ratio": 1.0461203733218678, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 1077103, + "candidate": 1124468, + "ratio": 1.0439744388419678, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 30174378, + "candidate_median": 30460396, + "ratio_median": 1.0080235623919658, + "ratio_min": 0.9991740809840447, + "ratio_max": 1.0425833411936647, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 29771568, + "candidate": 30460396, + "ratio": 1.0231371085325436, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 30240253, + "candidate": 30215277, + "ratio": 0.9991740809840447, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 30285550, + "candidate": 30528548, + "ratio": 1.0080235623919658, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 30174378, + "candidate": 30343804, + "ratio": 1.0056148961877525, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 29865153, + "candidate": 31136911, + "ratio": 1.0425833411936647, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 38174187, + "candidate_median": 38852038, + "ratio_median": 1.1197277524027283, + "ratio_min": 0.7685723633010295, + "ratio_max": 1.381215191301913, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 38174187, + "candidate": 52726767, + "ratio": 1.381215191301913, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 36293386, + "candidate": 49345405, + "ratio": 1.3596252771786022, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 48518233, + "candidate": 37289773, + "ratio": 0.7685723633010295, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 39291178, + "candidate": 38852038, + "ratio": 0.988823445303676, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 33622923, + "candidate": 37648520, + "ratio": 1.1197277524027283, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 61581397, + "candidate_median": 63549284, + "ratio_median": 1.311391027635953, + "ratio_min": 0.21041236387387466, + "ratio_max": 3.242864922674421, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 75288217, + "candidate": 244149518, + "ratio": 3.242864922674421, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 61581397, + "candidate": 85869128, + "ratio": 1.3944004550595044, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 242135372, + "candidate": 50948276, + "ratio": 0.21041236387387466, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 52002736, + "candidate": 50662693, + "ratio": 0.9742312981378518, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 48459447, + "candidate": 63549284, + "ratio": 1.311391027635953, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 29522529, + "candidate_median": 29720395, + "ratio_median": 1.013007930316539, + "ratio_min": 0.9977246042303424, + "ratio_max": 1.0353279100887283, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 29239469, + "candidate": 29720395, + "ratio": 1.0164478363133065, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 29623418, + "candidate": 29556013, + "ratio": 0.9977246042303424, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 29522529, + "candidate": 29906556, + "ratio": 1.013007930316539, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 29571181, + "candidate": 29683931, + "ratio": 1.0038128338533385, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 29410684, + "candidate": 30449702, + "ratio": 1.0353279100887283, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 37841922, + "candidate_median": 37723090, + "ratio_median": 1.101548753232249, + "ratio_min": 0.7626727231796214, + "ratio_max": 1.3868066235580228, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 37841922, + "candidate": 52391210, + "ratio": 1.3844753974177104, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 35225418, + "candidate": 48850843, + "ratio": 1.3868066235580228, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 47587779, + "candidate": 36293901, + "ratio": 0.7626727231796214, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 38917254, + "candidate": 37723090, + "ratio": 0.969315306778839, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 33342497, + "candidate": 36728386, + "ratio": 1.101548753232249, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 60539886, + "candidate_median": 63500687, + "ratio_median": 1.338990101153598, + "ratio_min": 0.2073906262181524, + "ratio_max": 3.244967581390484, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 75008615, + "candidate": 243400524, + "ratio": 3.244967581390484, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 60539886, + "candidate": 85248285, + "ratio": 1.4081342174975355, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 241810876, + "candidate": 50149309, + "ratio": 0.2073906262181524, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 51617984, + "candidate": 50261718, + "ratio": 0.9737249327676184, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 47424314, + "candidate": 63500687, + "ratio": 1.338990101153598, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 617123, + "candidate_median": 571243, + "ratio_median": 0.9113373946020091, + "ratio_min": 0.8595401565004059, + "ratio_max": 1.2500723167233239, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 459783, + "candidate": 574762, + "ratio": 1.2500723167233239, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 617123, + "candidate": 530442, + "ratio": 0.8595401565004059, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 636049, + "candidate": 574391, + "ratio": 0.9030609276958221, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 476038, + "candidate": 571243, + "ratio": 1.199994538251148, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 619675, + "candidate": 564733, + "ratio": 0.9113373946020091, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 1037938, + "candidate_median": 1050963, + "ratio_median": 1.0125489191069217, + "ratio_min": 0.9867884969963718, + "ratio_max": 1.0615695293767682, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 1001778, + "candidate": 1063457, + "ratio": 1.0615695293767682, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1047441, + "candidate": 1033738, + "ratio": 0.9869176402298554, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1076032, + "candidate": 1061816, + "ratio": 0.9867884969963718, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1016636, + "candidate": 1030668, + "ratio": 1.013802383547307, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1037938, + "candidate": 1050963, + "ratio": 1.0125489191069217, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1122627, + "candidate_median": 1125173, + "ratio_median": 1.0059076761006092, + "ratio_min": 0.9101378020169812, + "ratio_max": 1.082268643102295, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 1122627, + "candidate": 1214984, + "ratio": 1.082268643102295, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1091022, + "candidate": 1139647, + "ratio": 1.0445683038472184, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1127202, + "candidate": 1125173, + "ratio": 0.9981999677076513, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1108727, + "candidate": 1115277, + "ratio": 1.0059076761006092, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1211811, + "candidate": 1102915, + "ratio": 0.9101378020169812, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 1246918, + "candidate_median": 1173998, + "ratio_median": 0.9794659839235629, + "ratio_min": 0.8612458878611103, + "ratio_max": 1.1079524083295167, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1026230, + "candidate": 1137014, + "ratio": 1.1079524083295167, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1246918, + "candidate": 1073903, + "ratio": 0.8612458878611103, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1324396, + "candidate": 1173998, + "ratio": 0.8864403093938671, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1183862, + "candidate": 1195329, + "ratio": 1.0096860951698763, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1276029, + "candidate": 1249827, + "ratio": 0.9794659839235629, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 1755720, + "candidate_median": 1668232, + "ratio_median": 0.9006390540632903, + "ratio_min": 0.8838301184157058, + "ratio_max": 0.9899874807715876, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1691224, + "candidate": 1631389, + "ratio": 0.9646202986712582, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1755720, + "candidate": 1581270, + "ratio": 0.9006390540632903, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1956870, + "candidate": 1751539, + "ratio": 0.8950717216779858, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1746114, + "candidate": 1728631, + "ratio": 0.9899874807715876, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1887503, + "candidate": 1668232, + "ratio": 0.8838301184157058, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 22399221, + "candidate_median": 25463012, + "ratio_median": 1.028231052934673, + "ratio_min": 0.8691035937387748, + "ratio_max": 1.1964666340983707, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 21413463, + "candidate": 25620494, + "ratio": 1.1964666340983707, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 24763901, + "candidate": 25463012, + "ratio": 1.028231052934673, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 22399221, + "candidate": 21449505, + "ratio": 0.9576004897670325, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 22253661, + "candidate": 26331387, + "ratio": 1.183238434341208, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 26850783, + "candidate": 23336112, + "ratio": 0.8691035937387748, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 616533, + "candidate_median": 572490, + "ratio_median": 0.9589376121415756, + "ratio_min": 0.7825047483265292, + "ratio_max": 1.0231235373121124, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 497372, + "candidate": 508873, + "ratio": 1.0231235373121124, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 616533, + "candidate": 482440, + "ratio": 0.7825047483265292, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 648205, + "candidate": 572490, + "ratio": 0.8831928170871869, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 612507, + "candidate": 587356, + "ratio": 0.9589376121415756, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 637884, + "candidate": 630160, + "ratio": 0.9878912153306871, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 1023952, + "candidate_median": 891530, + "ratio_median": 0.9048957627488848, + "ratio_min": 0.7163266274935913, + "ratio_max": 0.9249464659207861, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1137516, + "candidate": 814833, + "ratio": 0.7163266274935913, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 984341, + "candidate": 890726, + "ratio": 0.9048957627488848, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1180056, + "candidate": 1017154, + "ratio": 0.8619540089622866, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 963872, + "candidate": 891530, + "ratio": 0.9249464659207861, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1023952, + "candidate": 940553, + "ratio": 0.9185518461802897, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 21507546, + "candidate_median": 25233559, + "ratio_median": 1.0629504817923898, + "ratio_min": 0.8828359185162294, + "ratio_max": 1.2361758692312026, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 20533461, + "candidate": 25382969, + "ratio": 1.2361758692312026, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 23739167, + "candidate": 25233559, + "ratio": 1.0629504817923898, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 21507546, + "candidate": 20835280, + "ratio": 0.9687427845092137, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 21214011, + "candidate": 25724069, + "ratio": 1.2125980796370852, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 25945682, + "candidate": 22905780, + "ratio": 0.8828359185162294, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "healthz", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 584044, + "candidate_median": 595458, + "ratio_median": 1.0104793466489703, + "ratio_min": 0.8574613035195113, + "ratio_max": 1.2647020978227919, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 584044, + "candidate": 637457, + "ratio": 1.0914537260891302, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 458523, + "candidate": 579895, + "ratio": 1.2647020978227919, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 614065, + "candidate": 620500, + "ratio": 1.0104793466489703, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 573840, + "candidate": 565755, + "ratio": 0.9859107068172313, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 694443, + "candidate": 595458, + "ratio": 0.8574613035195113, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "healthz", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 1079627, + "candidate_median": 1047873, + "ratio_median": 0.9684176977599043, + "ratio_min": 0.9513370265662194, + "ratio_max": 1.0368884037498394, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 1082958, + "candidate": 1062255, + "ratio": 0.9808829151269024, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 1027884, + "candidate": 1065801, + "ratio": 1.0368884037498394, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 1076204, + "candidate": 1042215, + "ratio": 0.9684176977599043, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 1101474, + "candidate": 1047873, + "ratio": 0.9513370265662194, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 1079627, + "candidate": 1039840, + "ratio": 0.9631474574089014, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "healthz", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1111964, + "candidate_median": 1139860, + "ratio_median": 1.0102545836137935, + "ratio_min": 0.9388208938013738, + "ratio_max": 1.0563316855118752, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 1111964, + "candidate": 1138339, + "ratio": 1.023719293070639, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 1083653, + "candidate": 1144697, + "ratio": 1.0563316855118752, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 1108677, + "candidate": 1079352, + "ratio": 0.9735495550101607, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 1214140, + "candidate": 1139860, + "ratio": 0.9388208938013738, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 1145439, + "candidate": 1157185, + "ratio": 1.0102545836137935, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "healthz", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 1801679, + "candidate_median": 1703211, + "ratio_median": 0.9453465350930993, + "ratio_min": 0.8845619908056769, + "ratio_max": 0.9823704347044818, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1669525, + "candidate": 1640092, + "ratio": 0.9823704347044818, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 1645746, + "candidate": 1575279, + "ratio": 0.957182335548742, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 1947071, + "candidate": 1722305, + "ratio": 0.8845619908056769, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 1801679, + "candidate": 1703211, + "ratio": 0.9453465350930993, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 1904300, + "candidate": 1763880, + "ratio": 0.9262616184424723, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "healthz", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 73901815, + "candidate_median": 2141034, + "ratio_median": 0.029118689439494822, + "ratio_min": 0.006422756826571689, + "ratio_max": 1.0075799337166382, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 343094727, + "candidate": 2203614, + "ratio": 0.006422756826571689, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 2100678, + "candidate": 2116601, + "ratio": 1.0075799337166382, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 251857210, + "candidate": 2119685, + "ratio": 0.008416217268507024, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 73901815, + "candidate": 2151924, + "ratio": 0.029118689439494822, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 2240395, + "candidate": 2141034, + "ratio": 0.9556502313208162, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "healthz", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 349813036, + "candidate_median": 2306296, + "ratio_median": 0.024886804013718993, + "ratio_min": 0.005063724091087339, + "ratio_max": 0.9458379798476869, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 651996616, + "candidate": 16226112, + "ratio": 0.024886804013718993, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 2519810, + "candidate": 2383332, + "ratio": 0.9458379798476869, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 349813036, + "candidate": 2263837, + "ratio": 0.006471562712145468, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 455454515, + "candidate": 2306296, + "ratio": 0.005063724091087339, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 2404307, + "candidate": 2272330, + "ratio": 0.9451080914375743, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "healthz", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 1160808, + "candidate_median": 1070398, + "ratio_median": 0.9221145960399997, + "ratio_min": 0.8685982996590671, + "ratio_max": 0.9671117623465505, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1041850, + "candidate": 1000207, + "ratio": 0.9600297547631617, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 1129891, + "candidate": 984167, + "ratio": 0.8710282673284414, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 1248926, + "candidate": 1084815, + "ratio": 0.8685982996590671, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 1160808, + "candidate": 1070398, + "ratio": 0.9221145960399997, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 1179297, + "candidate": 1140512, + "ratio": 0.9671117623465505, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "healthz", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 73544675, + "candidate_median": 1287918, + "ratio_median": 0.017367402874511308, + "ratio_min": 0.003841757371652418, + "ratio_max": 0.9390346867084592, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 342583061, + "candidate": 1316121, + "ratio": 0.003841757371652418, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 1390156, + "candidate": 1239725, + "ratio": 0.8917884036036243, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 251257891, + "candidate": 1362780, + "ratio": 0.0054238296539709475, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 73544675, + "candidate": 1277280, + "ratio": 0.017367402874511308, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 1371534, + "candidate": 1287918, + "ratio": 0.9390346867084592, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "healthz", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 349431524, + "candidate_median": 1417733, + "ratio_median": 0.023756377239964113, + "ratio_min": 0.0029539668286862435, + "ratio_max": 0.8574407226826637, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 651479131, + "candidate": 15476784, + "ratio": 0.023756377239964113, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 1, + "baseline": 1549117, + "candidate": 1328276, + "ratio": 0.8574407226826637, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 2, + "baseline": 349431524, + "candidate": 1559112, + "ratio": 0.004461852731981904, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 3, + "baseline": 454240375, + "candidate": 1341811, + "ratio": 0.0029539668286862435, + "n_baseline": 72, + "n_candidate": 72 + }, + { + "pair": 4, + "baseline": 1830961, + "candidate": 1417733, + "ratio": 0.7743108673532643, + "n_baseline": 72, + "n_candidate": 72 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 571881, + "candidate_median": 568931, + "ratio_median": 0.9948415841757289, + "ratio_min": 0.8903249807000014, + "ratio_max": 1.0971457146955184, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 607513, + "candidate": 540884, + "ratio": 0.8903249807000014, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 552613, + "candidate": 568942, + "ratio": 1.029548707685125, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 571881, + "candidate": 568931, + "ratio": 0.9948415841757289, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 650346, + "candidate": 600833, + "ratio": 0.9238666801979254, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 515926, + "candidate": 566046, + "ratio": 1.0971457146955184, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 1042840, + "candidate_median": 1055974, + "ratio_median": 1.009623550755723, + "ratio_min": 0.987631409732025, + "ratio_max": 1.020867055348855, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 1051292, + "candidate": 1038289, + "ratio": 0.987631409732025, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1037351, + "candidate": 1047334, + "ratio": 1.009623550755723, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1042840, + "candidate": 1064601, + "ratio": 1.020867055348855, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1036373, + "candidate": 1055974, + "ratio": 1.0189130747327457, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1057421, + "candidate": 1067488, + "ratio": 1.0095203329610438, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1132565, + "candidate_median": 1162617, + "ratio_median": 1.0050021350025773, + "ratio_min": 0.3914307348427179, + "ratio_max": 1.0506133396879407, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 2970173, + "candidate": 1162617, + "ratio": 0.3914307348427179, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1082679, + "candidate": 1137477, + "ratio": 1.0506133396879407, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1132565, + "candidate": 1163047, + "ratio": 1.0269141285489134, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1107727, + "candidate": 1113268, + "ratio": 1.0050021350025773, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1758475, + "candidate": 1192357, + "ratio": 0.6780630944426278, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 1897526, + "candidate_median": 1865561, + "ratio_median": 0.977772510984303, + "ratio_min": 0.9166871215302725, + "ratio_max": 1.0489297772121178, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 1803320, + "candidate": 1744101, + "ratio": 0.9671611250360447, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1902563, + "candidate": 1744055, + "ratio": 0.9166871215302725, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1897526, + "candidate": 1865561, + "ratio": 0.9831543810203391, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1919785, + "candidate": 1877113, + "ratio": 0.977772510984303, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1857417, + "candidate": 1948300, + "ratio": 1.0489297772121178, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 71578500, + "candidate_median": 70695629, + "ratio_median": 0.9938776166323969, + "ratio_min": 0.9678130720067601, + "ratio_max": 1.0138554567188534, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 69729495, + "candidate": 70695629, + "ratio": 1.0138554567188534, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 71737570, + "candidate": 69428558, + "ratio": 0.9678130720067601, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 72060286, + "candidate": 70917800, + "ratio": 0.9841454140218094, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 71042268, + "candidate": 70607320, + "ratio": 0.9938776166323969, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 71578500, + "candidate": 71323365, + "ratio": 0.9964355916930363, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 86918961, + "candidate_median": 78667486, + "ratio_median": 0.9693048332687731, + "ratio_min": 0.8630619742589393, + "ratio_max": 1.010764398683735, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 79397299, + "candidate": 78667486, + "ratio": 0.99080808781669, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 92425889, + "candidate": 79999918, + "ratio": 0.8655574630177482, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 88201571, + "candidate": 76123422, + "ratio": 0.8630619742589393, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 74135214, + "candidate": 74933235, + "ratio": 1.010764398683735, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 86918961, + "candidate": 84250969, + "ratio": 0.9693048332687731, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 1265559, + "candidate_median": 1220507, + "ratio_median": 0.9749089198262827, + "ratio_min": 0.8678323175766598, + "ratio_max": 0.9946276484014521, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1166195, + "candidate": 1157263, + "ratio": 0.9923409035367156, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1265559, + "candidate": 1098293, + "ratio": 0.8678323175766598, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1272755, + "candidate": 1223478, + "ratio": 0.9612832006159866, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1251919, + "candidate": 1220507, + "ratio": 0.9749089198262827, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1280631, + "candidate": 1273751, + "ratio": 0.9946276484014521, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 70888406, + "candidate_median": 70259886, + "ratio_median": 0.9912265201731296, + "ratio_min": 0.9711700675411905, + "ratio_max": 1.0111762670297733, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 69230003, + "candidate": 70003736, + "ratio": 1.0111762670297733, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 70907693, + "candidate": 68863429, + "ratio": 0.9711700675411905, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 71020652, + "candidate": 70259886, + "ratio": 0.9892881017200461, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 70888406, + "candidate": 70266468, + "ratio": 0.9912265201731296, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 70524138, + "candidate": 70625235, + "ratio": 1.0014335091908533, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 85995911, + "candidate_median": 78373315, + "ratio_median": 0.9702775984313952, + "ratio_min": 0.8622175999937921, + "ratio_max": 1.0160670160830785, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 78656660, + "candidate": 78373315, + "ratio": 0.996397698554706, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 91872194, + "candidate": 79762520, + "ratio": 0.8681899988150931, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 87629690, + "candidate": 75555861, + "ratio": 0.8622175999937921, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 73401246, + "candidate": 74580585, + "ratio": 1.0160670160830785, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 85995911, + "candidate": 83439906, + "ratio": 0.9702775984313952, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 607572, + "candidate_median": 628927, + "ratio_median": 1.0458380595739392, + "ratio_min": 0.9479428319757621, + "ratio_max": 1.1432729031050097, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 607572, + "candidate": 631170, + "ratio": 1.0388398412040054, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 618792, + "candidate": 668243, + "ratio": 1.0799153835214417, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 619857, + "candidate": 587589, + "ratio": 0.9479428319757621, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 579213, + "candidate": 605763, + "ratio": 1.0458380595739392, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 550111, + "candidate": 628927, + "ratio": 1.1432729031050097, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 1054188, + "candidate_median": 1048007, + "ratio_median": 0.9967760148519589, + "ratio_min": 0.9814255142346526, + "ratio_max": 1.0081457074867288, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1077871, + "candidate": 1072156, + "ratio": 0.9946978812863506, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1054188, + "candidate": 1034607, + "ratio": 0.9814255142346526, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1044703, + "candidate": 1048007, + "ratio": 1.0031626213383134, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1044980, + "candidate": 1041611, + "ratio": 0.9967760148519589, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1070748, + "candidate": 1079470, + "ratio": 1.0081457074867288, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1129300, + "candidate_median": 1120853, + "ratio_median": 0.9892457269747352, + "ratio_min": 0.9167047092996372, + "ratio_max": 1.904330194099708, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1133038, + "candidate": 1120853, + "ratio": 0.9892457269747352, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1114258, + "candidate": 1081080, + "ratio": 0.9702241312155713, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1129300, + "candidate": 1134027, + "ratio": 1.004185778801027, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1189455, + "candidate": 1090379, + "ratio": 0.9167047092996372, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1108657, + "candidate": 2111249, + "ratio": 1.904330194099708, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 1915248, + "candidate_median": 1870419, + "ratio_median": 0.982336976934353, + "ratio_min": 0.907317681976956, + "ratio_max": 1.0076799453647778, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 1863724, + "candidate": 1830805, + "ratio": 0.982336976934353, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1979083, + "candidate": 1795657, + "ratio": 0.907317681976956, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 2005640, + "candidate": 1870419, + "ratio": 0.9325796254562134, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1902189, + "candidate": 1877856, + "ratio": 0.9872078957453755, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1915248, + "candidate": 1929957, + "ratio": 1.0076799453647778, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 93849504, + "candidate_median": 92924963, + "ratio_median": 0.9936941331734401, + "ratio_min": 0.9668650708224135, + "ratio_max": 1.0028901469528286, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 92773610, + "candidate": 92827319, + "ratio": 1.0005789254077748, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 93849504, + "candidate": 92089367, + "ratio": 0.9812451113220588, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 96109546, + "candidate": 92924963, + "ratio": 0.9668650708224135, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 93180037, + "candidate": 93449341, + "ratio": 1.0028901469528286, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 94903368, + "candidate": 94304920, + "ratio": 0.9936941331734401, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 113707512, + "candidate_median": 103095284, + "ratio_median": 0.8996991027713344, + "ratio_min": 0.8494956277629796, + "ratio_max": 1.104926964527217, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 107491397, + "candidate": 118770143, + "ratio": 1.104926964527217, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 109225311, + "candidate": 102387228, + "ratio": 0.9373947033211011, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 125952115, + "candidate": 106995771, + "ratio": 0.8494956277629796, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 113707512, + "candidate": 102005388, + "ratio": 0.8970857439920065, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 114588626, + "candidate": 103095284, + "ratio": 0.8996991027713344, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 1291224, + "candidate_median": 1214937, + "ratio_median": 0.9555945870084801, + "ratio_min": 0.8919807861805726, + "ratio_max": 0.9717857772009069, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1175690, + "candidate": 1123483, + "ratio": 0.9555945870084801, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1255971, + "candidate": 1120302, + "ratio": 0.8919807861805726, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1302756, + "candidate": 1246720, + "ratio": 0.9569865730804541, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1291224, + "candidate": 1214937, + "ratio": 0.9409188490920243, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1311289, + "candidate": 1274292, + "ratio": 0.9717857772009069, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 93368827, + "candidate_median": 92221521, + "ratio_median": 0.9927662185935815, + "ratio_min": 0.9662288102627197, + "ratio_max": 1.0009001875316528, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 92138579, + "candidate": 92221521, + "ratio": 1.0009001875316528, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 93368827, + "candidate": 91733081, + "ratio": 0.9824808123593541, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 95218736, + "candidate": 92003086, + "ratio": 0.9662288102627197, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 92955499, + "candidate": 92830979, + "ratio": 0.998660434279418, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 94287892, + "candidate": 93605834, + "ratio": 0.9927662185935815, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 112824506, + "candidate_median": 102868599, + "ratio_median": 0.8993061046507042, + "ratio_min": 0.8540994607170256, + "ratio_max": 1.10131182928641, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 107120798, + "candidate": 117973402, + "ratio": 1.10131182928641, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 108327966, + "candidate": 101342059, + "ratio": 0.9355115095579289, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 125149510, + "candidate": 106890129, + "ratio": 0.8540994607170256, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 112824506, + "candidate": 101463767, + "ratio": 0.8993061046507042, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 114533900, + "candidate": 102868599, + "ratio": 0.8981497966977463, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 629089, + "candidate_median": 580986, + "ratio_median": 0.9560355617577178, + "ratio_min": 0.901358949210684, + "ratio_max": 1.041177353008519, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 629089, + "candidate": 567035, + "ratio": 0.901358949210684, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 585017, + "candidate": 580986, + "ratio": 0.9931096019431914, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 633285, + "candidate": 659362, + "ratio": 1.041177353008519, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 664470, + "candidate": 609611, + "ratio": 0.9174394630306861, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 604807, + "candidate": 578217, + "ratio": 0.9560355617577178, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 1054865, + "candidate_median": 1053558, + "ratio_median": 0.9820564716812104, + "ratio_min": 0.9732286781353836, + "ratio_max": 1.0169064965688202, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1054865, + "candidate": 1035937, + "ratio": 0.9820564716812104, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1042845, + "candidate": 1022936, + "ratio": 0.9809089557892112, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1082539, + "candidate": 1053558, + "ratio": 0.9732286781353836, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1058528, + "candidate": 1076424, + "ratio": 1.0169064965688202, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1050522, + "candidate": 1060469, + "ratio": 1.009468626073514, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1113981, + "candidate_median": 1146491, + "ratio_median": 1.0073610735837162, + "ratio_min": 0.9667763157894737, + "ratio_max": 1.3863503955632996, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 1113981, + "candidate": 1544368, + "ratio": 1.3863503955632996, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1207276, + "candidate": 1179365, + "ratio": 0.9768810114671376, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1130880, + "candidate": 1093308, + "ratio": 0.9667763157894737, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1099240, + "candidate": 1146491, + "ratio": 1.042985153378698, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1110300, + "candidate": 1118473, + "ratio": 1.0073610735837162, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 3581914, + "candidate_median": 3403318, + "ratio_median": 0.9501395064203105, + "ratio_min": 0.9280527864181849, + "ratio_max": 0.9710113128675949, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 3411065, + "candidate": 3259518, + "ratio": 0.9555719401418619, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 3499385, + "candidate": 3247614, + "ratio": 0.9280527864181849, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 3616782, + "candidate": 3429749, + "ratio": 0.9482874555336761, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 3581914, + "candidate": 3403318, + "ratio": 0.9501395064203105, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 3614203, + "candidate": 3509432, + "ratio": 0.9710113128675949, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 4271712, + "candidate_median": 4189175, + "ratio_median": 0.9679187729916549, + "ratio_min": 0.9261327685488573, + "ratio_max": 1.0044862229840394, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 4169315, + "candidate": 3993036, + "ratio": 0.9577199132231554, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 4268767, + "candidate": 3953445, + "ratio": 0.9261327685488573, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 4393005, + "candidate": 4412713, + "ratio": 1.0044862229840394, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 4440198, + "candidate": 4297751, + "ratio": 0.9679187729916549, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 4271712, + "candidate": 4189175, + "ratio": 0.9806782386078462, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 4942684, + "candidate_median": 5478632, + "ratio_median": 1.0091802437863395, + "ratio_min": 0.9206164100314728, + "ratio_max": 1.155195536997917, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 4781466, + "candidate": 5478632, + "ratio": 1.1458059097356335, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 4959999, + "candidate": 5005533, + "ratio": 1.0091802437863395, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 6082026, + "candidate": 5946853, + "ratio": 0.9777750045790663, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 4942684, + "candidate": 4550316, + "ratio": 0.9206164100314728, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 4827961, + "candidate": 5577239, + "ratio": 1.155195536997917, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 2931292, + "candidate_median": 2774112, + "ratio_median": 0.9463785934666352, + "ratio_min": 0.918214479855551, + "ratio_max": 0.9725607012722973, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 2767199, + "candidate": 2691269, + "ratio": 0.9725607012722973, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 2928648, + "candidate": 2689127, + "ratio": 0.918214479855551, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 2998684, + "candidate": 2774780, + "ratio": 0.9253325792247532, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 2931292, + "candidate": 2774112, + "ratio": 0.9463785934666352, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 2953779, + "candidate": 2863273, + "ratio": 0.9693592513183958, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 3646287, + "candidate_median": 3576395, + "ratio_median": 0.9884989959963245, + "ratio_min": 0.93663910712459, + "ratio_max": 1.0270043082641613, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 3428527, + "candidate": 3521112, + "ratio": 1.0270043082641613, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 3646287, + "candidate": 3415255, + "ratio": 0.93663910712459, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 3758386, + "candidate": 3729663, + "ratio": 0.992357623724652, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 3646899, + "candidate": 3604956, + "ratio": 0.9884989959963245, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 3645754, + "candidate": 3576395, + "ratio": 0.9809754031676301, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 4345660, + "candidate_median": 4406808, + "ratio_median": 1.0807746894936618, + "ratio_min": 0.9277467634375446, + "ratio_max": 1.233146516710864, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 4077453, + "candidate": 4406808, + "ratio": 1.0807746894936618, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 3868481, + "candidate": 4272134, + "ratio": 1.104344056491424, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 5368796, + "candidate": 5231474, + "ratio": 0.9744221981986277, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 4345660, + "candidate": 4031672, + "ratio": 0.9277467634375446, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 4457669, + "candidate": 5496959, + "ratio": 1.233146516710864, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 631856, + "candidate_median": 584849, + "ratio_median": 0.8987100141674744, + "ratio_min": 0.7489417675449007, + "ratio_max": 1.0193997229490535, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 487275, + "candidate": 496728, + "ratio": 1.0193997229490535, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 606672, + "candidate": 454362, + "ratio": 0.7489417675449007, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 631856, + "candidate": 584849, + "ratio": 0.9256048846572636, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 667824, + "candidate": 597281, + "ratio": 0.894368875631903, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 670550, + "candidate": 602630, + "ratio": 0.8987100141674744, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 1037611, + "candidate_median": 1033736, + "ratio_median": 1.0008948729848224, + "ratio_min": 0.9641892177143581, + "ratio_max": 1.023810221630842, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 1021374, + "candidate": 1022288, + "ratio": 1.0008948729848224, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1042228, + "candidate": 1004905, + "ratio": 0.9641892177143581, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1037611, + "candidate": 1033736, + "ratio": 0.9962654597917717, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1023594, + "candidate": 1047966, + "ratio": 1.023810221630842, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1044496, + "candidate": 1054053, + "ratio": 1.0091498674959023, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1129494, + "candidate_median": 1090122, + "ratio_median": 0.9699086493598018, + "ratio_min": 0.8855357938132535, + "ratio_max": 0.9902548031066903, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1126690, + "candidate": 1099063, + "ratio": 0.9754795019038067, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1223745, + "candidate": 1083670, + "ratio": 0.8855357938132535, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1144624, + "candidate": 1080007, + "ratio": 0.9435474007184892, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1129494, + "candidate": 1095506, + "ratio": 0.9699086493598018, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1100850, + "candidate": 1090122, + "ratio": 0.9902548031066903, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 1873547, + "candidate_median": 1772882, + "ratio_median": 0.9490833884896662, + "ratio_min": 0.8801145574056513, + "ratio_max": 1.0264974612896522, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 1657555, + "candidate": 1607987, + "ratio": 0.9700957132644166, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1843268, + "candidate": 1622287, + "ratio": 0.8801145574056513, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1902169, + "candidate": 1805317, + "ratio": 0.9490833884896662, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1873547, + "candidate": 1772882, + "ratio": 0.9462703631133886, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1905692, + "candidate": 1956188, + "ratio": 1.0264974612896522, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 2355418, + "candidate_median": 2264897, + "ratio_median": 0.9790258034879584, + "ratio_min": 0.9265747858480552, + "ratio_max": 1.0192938137867908, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 2139964, + "candidate": 2105604, + "ratio": 0.9839436551269086, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 2366544, + "candidate": 2192780, + "ratio": 0.9265747858480552, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 2431218, + "candidate": 2264897, + "ratio": 0.9315894337735242, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 2355418, + "candidate": 2306015, + "ratio": 0.9790258034879584, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 2329244, + "candidate": 2374184, + "ratio": 1.0192938137867908, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 3095830, + "candidate_median": 2425280, + "ratio_median": 0.780522914852254, + "ratio_min": 0.6135681052251494, + "ratio_max": 0.7834021893967046, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 2956696, + "candidate": 2310040, + "ratio": 0.7812910086123159, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 2966984, + "candidate": 2315799, + "ratio": 0.780522914852254, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 3095830, + "candidate": 2425280, + "ratio": 0.7834021893967046, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 4008652, + "candidate": 2515504, + "ratio": 0.6275186771014296, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 4253888, + "candidate": 2610050, + "ratio": 0.6135681052251494, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 1228907, + "candidate_median": 1184194, + "ratio_median": 0.9636156356827652, + "ratio_min": 0.9110712262711048, + "ratio_max": 1.0288596510597223, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 1103354, + "candidate": 1092555, + "ratio": 0.9902125700364525, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1225194, + "candidate": 1116239, + "ratio": 0.9110712262711048, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1302833, + "candidate": 1217513, + "ratio": 0.9345119443551092, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1228907, + "candidate": 1184194, + "ratio": 0.9636156356827652, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1268068, + "candidate": 1304664, + "ratio": 1.0288596510597223, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 1495681, + "candidate_median": 1442926, + "ratio_median": 0.9150657479603388, + "ratio_min": 0.9041836342889282, + "ratio_max": 1.0339510898380069, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1479288, + "candidate": 1337548, + "ratio": 0.9041836342889282, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1536331, + "candidate": 1390990, + "ratio": 0.9053973395056143, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1576855, + "candidate": 1442926, + "ratio": 0.9150657479603388, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1451629, + "candidate": 1474584, + "ratio": 1.0158132690928605, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1495681, + "candidate": 1546461, + "ratio": 1.0339510898380069, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 2023370, + "candidate_median": 1769285, + "ratio_median": 0.8744248456782496, + "ratio_min": 0.4733960503066152, + "ratio_max": 0.9822315031184916, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1973854, + "candidate": 1727249, + "ratio": 0.8750642144758427, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1810170, + "candidate": 1778006, + "ratio": 0.9822315031184916, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 2023370, + "candidate": 1769285, + "ratio": 0.8744248456782496, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 3618053, + "candidate": 1712772, + "ratio": 0.4733960503066152, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 3227772, + "candidate": 1776811, + "ratio": 0.5504759939673558, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "neighbors", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 564989, + "candidate_median": 605546, + "ratio_median": 1.0114695124970818, + "ratio_min": 0.9447246935094317, + "ratio_max": 1.1379769790932581, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 654180, + "candidate": 618020, + "ratio": 0.9447246935094317, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 565412, + "candidate": 571897, + "ratio": 1.0114695124970818, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 532125, + "candidate": 605546, + "ratio": 1.1379769790932581, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 564902, + "candidate": 611309, + "ratio": 1.0821505323047182, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 564989, + "candidate": 569991, + "ratio": 1.0088532697096757, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "neighbors", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 1049222, + "candidate_median": 1064921, + "ratio_median": 1.0213546237023186, + "ratio_min": 0.9937901913654, + "ratio_max": 1.0421265656299756, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 1068793, + "candidate": 1062156, + "ratio": 0.9937901913654, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1021873, + "candidate": 1064921, + "ratio": 1.0421265656299756, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1042725, + "candidate": 1064992, + "ratio": 1.0213546237023186, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1049222, + "candidate": 1047246, + "ratio": 0.9981166998023298, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1059576, + "candidate": 1085825, + "ratio": 1.0247731167938874, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "neighbors", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1130255, + "candidate_median": 1144465, + "ratio_median": 1.0189725102583156, + "ratio_min": 0.9736109447970303, + "ratio_max": 1.0413632321909658, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 1139807, + "candidate": 1161432, + "ratio": 1.0189725102583156, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1099770, + "candidate": 1108837, + "ratio": 1.0082444511125053, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1135736, + "candidate": 1105765, + "ratio": 0.9736109447970303, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1130255, + "candidate": 1177006, + "ratio": 1.0413632321909658, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1109593, + "candidate": 1144465, + "ratio": 1.0314277397207805, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "neighbors", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 14599247, + "candidate_median": 14217142, + "ratio_median": 0.9837890954238941, + "ratio_min": 0.9656416109544365, + "ratio_max": 0.9884524339112434, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 14381781, + "candidate": 14187429, + "ratio": 0.9864862356060073, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 14641612, + "candidate": 14217142, + "ratio": 0.9710093396819968, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 14599247, + "candidate": 14362580, + "ratio": 0.9837890954238941, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 14597483, + "candidate": 14095937, + "ratio": 0.9656416109544365, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 14638496, + "candidate": 14469457, + "ratio": 0.9884524339112434, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "neighbors", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 16129565, + "candidate_median": 15689984, + "ratio_median": 0.972746878170614, + "ratio_min": 0.8792906477632147, + "ratio_max": 1.0076812825458243, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 16128114, + "candidate": 15678292, + "ratio": 0.9721094481351013, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 16129565, + "candidate": 15689984, + "ratio": 0.972746878170614, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 16409191, + "candidate": 16462710, + "ratio": 1.0032615258119673, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 15545581, + "candidate": 15664991, + "ratio": 1.0076812825458243, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 18359116, + "candidate": 16142999, + "ratio": 0.8792906477632147, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "neighbors", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 25107530, + "candidate_median": 32318721, + "ratio_median": 1.0263827221736563, + "ratio_min": 0.9011128581459655, + "ratio_max": 1.2972245770807127, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 21085886, + "candidate": 19000763, + "ratio": 0.9011128581459655, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 20596897, + "candidate": 26718801, + "ratio": 1.2972245770807127, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 32735553, + "candidate": 33599206, + "ratio": 1.0263827221736563, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 25107530, + "candidate": 32478559, + "ratio": 1.293578420497755, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 35113020, + "candidate": 32318721, + "ratio": 0.920419861350576, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "neighbors", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 14022760, + "candidate_median": 13628627, + "ratio_median": 0.9861783985463632, + "ratio_min": 0.9639506266098112, + "ratio_max": 0.9906639806395453, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 13684655, + "candidate": 13555428, + "ratio": 0.9905567951840949, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 14138304, + "candidate": 13628627, + "ratio": 0.9639506266098112, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 14022760, + "candidate": 13828943, + "ratio": 0.9861783985463632, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 13974826, + "candidate": 13580408, + "ratio": 0.9717765358938995, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 14055455, + "candidate": 13924233, + "ratio": 0.9906639806395453, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "neighbors", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 15440391, + "candidate_median": 15110486, + "ratio_median": 0.9761733356962923, + "ratio_min": 0.8612854335035797, + "ratio_max": 1.0035765627578805, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 15440391, + "candidate": 14884811, + "ratio": 0.9640177505867565, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 15175958, + "candidate": 14966747, + "ratio": 0.9862143134555328, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 15799400, + "candidate": 15422953, + "ratio": 0.9761733356962923, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 15056635, + "candidate": 15110486, + "ratio": 1.0035765627578805, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 17996668, + "candidate": 15500268, + "ratio": 0.8612854335035797, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "neighbors", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 24704512, + "candidate_median": 31403923, + "ratio_median": 1.0264945344598095, + "ratio_min": 0.9116115749524846, + "ratio_max": 1.2812074520721028, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 20489751, + "candidate": 18915927, + "ratio": 0.9231896961558976, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 20248489, + "candidate": 25942515, + "ratio": 1.2812074520721028, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 31871781, + "candidate": 32716209, + "ratio": 1.0264945344598095, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 24704512, + "candidate": 31442365, + "ratio": 1.2727377492823984, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 34448798, + "candidate": 31403923, + "ratio": 0.9116115749524846, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 538287, + "candidate_median": 550114, + "ratio_median": 1.0354425521707844, + "ratio_min": 0.931621655464571, + "ratio_max": 1.1247213082918486, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 539126, + "candidate": 550114, + "ratio": 1.0203811353932104, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 1, + "baseline": 521125, + "candidate": 539595, + "ratio": 1.0354425521707844, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 2, + "baseline": 535538, + "candidate": 602331, + "ratio": 1.1247213082918486, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 3, + "baseline": 538287, + "candidate": 569672, + "ratio": 1.058305327826977, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 4, + "baseline": 567007, + "candidate": 528236, + "ratio": 0.931621655464571, + "n_baseline": 252, + "n_candidate": 252 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 1051095, + "candidate_median": 1042286, + "ratio_median": 0.9692844129217626, + "ratio_min": 0.9522163224338301, + "ratio_max": 1.0070726961078915, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1084973, + "candidate": 1033129, + "ratio": 0.9522163224338301, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 1, + "baseline": 1034966, + "candidate": 1042286, + "ratio": 1.0070726961078915, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 2, + "baseline": 1091940, + "candidate": 1053969, + "ratio": 0.9652261113247981, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 3, + "baseline": 1039980, + "candidate": 1042402, + "ratio": 1.0023288909402104, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 4, + "baseline": 1051095, + "candidate": 1018810, + "ratio": 0.9692844129217626, + "n_baseline": 252, + "n_candidate": 252 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1161413, + "candidate_median": 1114759, + "ratio_median": 0.9773720422830284, + "ratio_min": 0.9078787331282927, + "ratio_max": 1.3663165854129717, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1161413, + "candidate": 1114759, + "ratio": 0.9598299657400081, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 1, + "baseline": 3375026, + "candidate": 4611354, + "ratio": 1.3663165854129717, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 2, + "baseline": 1217645, + "candidate": 1105474, + "ratio": 0.9078787331282927, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 3, + "baseline": 1133545, + "candidate": 1447599, + "ratio": 1.277054726543719, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 4, + "baseline": 1128869, + "candidate": 1103325, + "ratio": 0.9773720422830284, + "n_baseline": 252, + "n_candidate": 252 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 64367574, + "candidate_median": 63606307, + "ratio_median": 0.9856252618232657, + "ratio_min": 0.9796656667931429, + "ratio_max": 1.0163060343932258, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 63473367, + "candidate": 63584586, + "ratio": 1.001752215224379, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 1, + "baseline": 64367574, + "candidate": 63060648, + "ratio": 0.9796958947062383, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 2, + "baseline": 65046263, + "candidate": 64111240, + "ratio": 0.9856252618232657, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 3, + "baseline": 64926545, + "candidate": 63606307, + "ratio": 0.9796656667931429, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 4, + "baseline": 64222298, + "candidate": 65269509, + "ratio": 1.0163060343932258, + "n_baseline": 252, + "n_candidate": 252 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 75467598, + "candidate_median": 73237212, + "ratio_median": 0.970445779922663, + "ratio_min": 0.5976125350354882, + "ratio_max": 1.1845002995161558, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 81377476, + "candidate": 74379471, + "ratio": 0.9140056273065044, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 1, + "baseline": 118423853, + "candidate": 70771579, + "ratio": 0.5976125350354882, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 2, + "baseline": 71460378, + "candidate": 70450628, + "ratio": 0.9858697920685502, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 3, + "baseline": 75467598, + "candidate": 73237212, + "ratio": 0.970445779922663, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 4, + "baseline": 70804194, + "candidate": 83867589, + "ratio": 1.1845002995161558, + "n_baseline": 252, + "n_candidate": 252 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 168700266, + "candidate_median": 159776615, + "ratio_median": 0.7368953191641814, + "ratio_min": 0.44181642914537794, + "ratio_max": 1.1599283939379137, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 326138766, + "candidate": 144093465, + "ratio": 0.44181642914537794, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 1, + "baseline": 168700266, + "candidate": 167794315, + "ratio": 0.9946298187816728, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 2, + "baseline": 133309214, + "candidate": 83133527, + "ratio": 0.6236142612017801, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 3, + "baseline": 216824033, + "candidate": 159776615, + "ratio": 0.7368953191641814, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 4, + "baseline": 143288427, + "candidate": 166204315, + "ratio": 1.1599283939379137, + "n_baseline": 252, + "n_candidate": 252 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 63809893, + "candidate_median": 63140327, + "ratio_median": 0.9896445712453026, + "ratio_min": 0.9786252579987871, + "ratio_max": 1.0191272923329424, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 62849558, + "candidate": 63040943, + "ratio": 1.0030451288137938, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 1, + "baseline": 63809893, + "candidate": 62445973, + "ratio": 0.9786252579987871, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 2, + "baseline": 64305208, + "candidate": 63639300, + "ratio": 0.9896445712453026, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 3, + "baseline": 64356361, + "candidate": 63140327, + "ratio": 0.9811046805458749, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 4, + "baseline": 63566028, + "candidate": 64781874, + "ratio": 1.0191272923329424, + "n_baseline": 252, + "n_candidate": 252 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 74825791, + "candidate_median": 72352140, + "ratio_median": 0.9669411981224495, + "ratio_min": 0.6003408946226503, + "ratio_max": 1.1823777854642088, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 81300518, + "candidate": 74262494, + "ratio": 0.9134319906793213, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 1, + "baseline": 117859882, + "candidate": 70756107, + "ratio": 0.6003408946226503, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 2, + "baseline": 71097970, + "candidate": 69611282, + "ratio": 0.9790895858208047, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 3, + "baseline": 74825791, + "candidate": 72352140, + "ratio": 0.9669411981224495, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 4, + "baseline": 70357752, + "candidate": 83189443, + "ratio": 1.1823777854642088, + "n_baseline": 252, + "n_candidate": 252 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 167827572, + "candidate_median": 159050816, + "ratio_median": 0.7351523536157399, + "ratio_min": 0.4395999444651767, + "ratio_max": 1.1548951269732912, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 325770371, + "candidate": 143208637, + "ratio": 0.4395999444651767, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 1, + "baseline": 167827572, + "candidate": 164432514, + "ratio": 0.9797705587970968, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 2, + "baseline": 133081908, + "candidate": 82748624, + "ratio": 0.6217871778634253, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 3, + "baseline": 216350822, + "candidate": 159050816, + "ratio": 0.7351523536157399, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 4, + "baseline": 143054849, + "candidate": 165213348, + "ratio": 1.1548951269732912, + "n_baseline": 252, + "n_candidate": 252 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 362931, + "candidate_median": 355148, + "ratio_median": 1.067238676222202, + "ratio_min": 0.6017287694296758, + "ratio_max": 1.3196788014135117, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 269117, + "candidate": 355148, + "ratio": 1.3196788014135117, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 1, + "baseline": 408141, + "candidate": 387249, + "ratio": 0.9488118076841092, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 2, + "baseline": 362931, + "candidate": 387334, + "ratio": 1.067238676222202, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 3, + "baseline": 288194, + "candidate": 352088, + "ratio": 1.2217048238339452, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 4, + "baseline": 447833, + "candidate": 269474, + "ratio": 0.6017287694296758, + "n_baseline": 252, + "n_candidate": 252 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 1041463, + "candidate_median": 1034698, + "ratio_median": 0.9942346637210159, + "ratio_min": 0.9415464725127874, + "ratio_max": 1.0188999561794378, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1015505, + "candidate": 1034698, + "ratio": 1.0188999561794378, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 1, + "baseline": 1062134, + "candidate": 1039305, + "ratio": 0.9785064784669354, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 2, + "baseline": 1041648, + "candidate": 980760, + "ratio": 0.9415464725127874, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 3, + "baseline": 1032030, + "candidate": 1026080, + "ratio": 0.9942346637210159, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 4, + "baseline": 1041463, + "candidate": 1046548, + "ratio": 1.0048825546370825, + "n_baseline": 252, + "n_candidate": 252 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1456314, + "candidate_median": 1357163, + "ratio_median": 0.954492643532598, + "ratio_min": 0.6900066297758282, + "ratio_max": 2.6975487411020005, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 1966884, + "candidate": 1357163, + "ratio": 0.6900066297758282, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 1, + "baseline": 3749218, + "candidate": 3578601, + "ratio": 0.954492643532598, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 2, + "baseline": 1144406, + "candidate": 1122044, + "ratio": 0.9804597319482771, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 3, + "baseline": 1456314, + "candidate": 1106807, + "ratio": 0.7600057405202449, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 4, + "baseline": 1121179, + "candidate": 3024435, + "ratio": 2.6975487411020005, + "n_baseline": 252, + "n_candidate": 252 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 3983015, + "candidate_median": 3784876, + "ratio_median": 0.9595655979012079, + "ratio_min": 0.9143754265874607, + "ratio_max": 0.9836521676314778, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 3765331, + "candidate": 3703776, + "ratio": 0.9836521676314778, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 1, + "baseline": 4051045, + "candidate": 3704176, + "ratio": 0.9143754265874607, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 2, + "baseline": 4033147, + "candidate": 3807305, + "ratio": 0.9440035287580641, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 3, + "baseline": 3944364, + "candidate": 3784876, + "ratio": 0.9595655979012079, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 4, + "baseline": 3983015, + "candidate": 3830868, + "ratio": 0.961801047698791, + "n_baseline": 252, + "n_candidate": 252 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 5234398, + "candidate_median": 5081624, + "ratio_median": 0.9557815328461184, + "ratio_min": 0.5450570972194557, + "ratio_max": 1.6479769022016364, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 6462933, + "candidate": 5225810, + "ratio": 0.8085818002445638, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 1, + "baseline": 8866106, + "candidate": 4832534, + "ratio": 0.5450570972194557, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 2, + "baseline": 5234398, + "candidate": 5081624, + "ratio": 0.9708134536196904, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 3, + "baseline": 5136406, + "candidate": 4909282, + "ratio": 0.9557815328461184, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 4, + "baseline": 4900900, + "candidate": 8076570, + "ratio": 1.6479769022016364, + "n_baseline": 252, + "n_candidate": 252 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 26006162, + "candidate_median": 17967760, + "ratio_median": 0.6707751846558145, + "ratio_min": 0.06219684445057316, + "ratio_max": 1.7392411578136142, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 157190756, + "candidate": 9776769, + "ratio": 0.06219684445057316, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 1, + "baseline": 26006162, + "candidate": 17967760, + "ratio": 0.6909039480719993, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 2, + "baseline": 14549772, + "candidate": 9759626, + "ratio": 0.6707751846558145, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 3, + "baseline": 56978564, + "candidate": 18284250, + "ratio": 0.3208969955788988, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 4, + "baseline": 11894937, + "candidate": 20688164, + "ratio": 1.7392411578136142, + "n_baseline": 252, + "n_candidate": 252 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 3506114, + "candidate_median": 3371142, + "ratio_median": 0.955422612374729, + "ratio_min": 0.930209765063807, + "ratio_max": 0.9882127388037947, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 3395937, + "candidate": 3244555, + "ratio": 0.955422612374729, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 1, + "baseline": 3537386, + "candidate": 3290511, + "ratio": 0.930209765063807, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 2, + "baseline": 3611183, + "candidate": 3391931, + "ratio": 0.9392852702286204, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 3, + "baseline": 3506114, + "candidate": 3371142, + "ratio": 0.9615038187577472, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 4, + "baseline": 3485288, + "candidate": 3444206, + "ratio": 0.9882127388037947, + "n_baseline": 252, + "n_candidate": 252 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 4730706, + "candidate_median": 4455509, + "ratio_median": 0.917286131474074, + "ratio_min": 0.549669776813556, + "ratio_max": 1.5853148576079252, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 5933682, + "candidate": 4455509, + "ratio": 0.7508843581438979, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 1, + "baseline": 7710997, + "candidate": 4238502, + "ratio": 0.549669776813556, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 2, + "baseline": 4730706, + "candidate": 4485075, + "ratio": 0.9480773060088705, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 3, + "baseline": 4586486, + "candidate": 4207120, + "ratio": 0.917286131474074, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 4, + "baseline": 4282015, + "candidate": 6788342, + "ratio": 1.5853148576079252, + "n_baseline": 252, + "n_candidate": 252 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 24999970, + "candidate_median": 14389159, + "ratio_median": 0.5755670506804608, + "ratio_min": 0.060411281062999575, + "ratio_max": 1.805256341073363, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 156002320, + "candidate": 9424300, + "ratio": 0.060411281062999575, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 1, + "baseline": 24999970, + "candidate": 14389159, + "ratio": 0.5755670506804608, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 2, + "baseline": 14319720, + "candidate": 9292945, + "ratio": 0.6489613623730073, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 3, + "baseline": 55522250, + "candidate": 17812224, + "ratio": 0.32081235900922606, + "n_baseline": 252, + "n_candidate": 252 + }, + { + "pair": 4, + "baseline": 11183635, + "candidate": 20189328, + "ratio": 1.805256341073363, + "n_baseline": 252, + "n_candidate": 252 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 528307, + "candidate_median": 484195, + "ratio_median": 0.9474267508381377, + "ratio_min": 0.8888013976721868, + "ratio_max": 1.1029373241762623, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 540484, + "candidate": 512069, + "ratio": 0.9474267508381377, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 1, + "baseline": 490388, + "candidate": 481499, + "ratio": 0.9818735368728435, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 2, + "baseline": 439005, + "candidate": 484195, + "ratio": 1.1029373241762623, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 3, + "baseline": 558233, + "candidate": 527877, + "ratio": 0.945621272837686, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 4, + "baseline": 528307, + "candidate": 469560, + "ratio": 0.8888013976721868, + "n_baseline": 216, + "n_candidate": 216 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 1024291, + "candidate_median": 1049508, + "ratio_median": 1.016134086895228, + "ratio_min": 0.9915519594019775, + "ratio_max": 1.0573024335400574, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 1024291, + "candidate": 1040817, + "ratio": 1.016134086895228, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 1, + "baseline": 1012069, + "candidate": 1003519, + "ratio": 0.9915519594019775, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 2, + "baseline": 992628, + "candidate": 1049508, + "ratio": 1.0573024335400574, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 3, + "baseline": 1043038, + "candidate": 1071916, + "ratio": 1.0276864313668341, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 4, + "baseline": 1042053, + "candidate": 1058468, + "ratio": 1.0157525576914035, + "n_baseline": 216, + "n_candidate": 216 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1303005, + "candidate_median": 1219175, + "ratio_median": 0.9322223799521517, + "ratio_min": 0.6815357751355016, + "ratio_max": 1.4234182782872222, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1982634, + "candidate": 1351236, + "ratio": 0.6815357751355016, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 1, + "baseline": 1246857, + "candidate": 1162348, + "ratio": 0.9322223799521517, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 2, + "baseline": 1155444, + "candidate": 1219175, + "ratio": 1.0551571517096459, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 3, + "baseline": 1519847, + "candidate": 2163378, + "ratio": 1.4234182782872222, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 4, + "baseline": 1303005, + "candidate": 1171725, + "ratio": 0.8992482761002452, + "n_baseline": 216, + "n_candidate": 216 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 6078058, + "candidate_median": 5937648, + "ratio_median": 0.9553924906393735, + "ratio_min": 0.9323771373477024, + "ratio_max": 1.0101205372388038, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 5999588, + "candidate": 5832172, + "ratio": 0.972095417218649, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 1, + "baseline": 6078058, + "candidate": 5794161, + "ratio": 0.9532914954085664, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 2, + "baseline": 6222607, + "candidate": 5945032, + "ratio": 0.9553924906393735, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 3, + "baseline": 6368290, + "candidate": 5937648, + "ratio": 0.9323771373477024, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 4, + "baseline": 6029423, + "candidate": 6090444, + "ratio": 1.0101205372388038, + "n_baseline": 216, + "n_candidate": 216 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 12697063, + "candidate_median": 10762424, + "ratio_median": 0.84029621421728, + "ratio_min": 0.7692567633431028, + "ratio_max": 1.3336402262032825, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 12807893, + "candidate": 10762424, + "ratio": 0.84029621421728, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 1, + "baseline": 12736156, + "candidate": 10670884, + "ratio": 0.8378418103547098, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 2, + "baseline": 11233313, + "candidate": 8641302, + "ratio": 0.7692567633431028, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 3, + "baseline": 12697063, + "candidate": 10774398, + "ratio": 0.8485740363736086, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 4, + "baseline": 9151945, + "candidate": 12205402, + "ratio": 1.3336402262032825, + "n_baseline": 216, + "n_candidate": 216 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 17596921, + "candidate_median": 18360401, + "ratio_median": 0.9922781945773355, + "ratio_min": 0.11271968805006702, + "ratio_max": 1.3292335027979791, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 215162989, + "candidate": 24253105, + "ratio": 0.11271968805006702, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 1, + "baseline": 17596921, + "candidate": 17461041, + "ratio": 0.9922781945773355, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 2, + "baseline": 13783024, + "candidate": 14020983, + "ratio": 1.0172646438111115, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 3, + "baseline": 104706577, + "candidate": 18360401, + "ratio": 0.1753509810563285, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 4, + "baseline": 16854843, + "candidate": 22404022, + "ratio": 1.3292335027979791, + "n_baseline": 216, + "n_candidate": 216 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 5521993, + "candidate_median": 5393142, + "ratio_median": 0.9697180424820038, + "ratio_min": 0.9338183820416545, + "ratio_max": 1.0066821888401525, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 5473094, + "candidate": 5307358, + "ratio": 0.9697180424820038, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 1, + "baseline": 5467396, + "candidate": 5311227, + "ratio": 0.9714363108141426, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 2, + "baseline": 5758482, + "candidate": 5505633, + "ratio": 0.9560910323241437, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 3, + "baseline": 5775365, + "candidate": 5393142, + "ratio": 0.9338183820416545, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 4, + "baseline": 5521993, + "candidate": 5558892, + "ratio": 1.0066821888401525, + "n_baseline": 216, + "n_candidate": 216 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 11854918, + "candidate_median": 10422501, + "ratio_median": 0.8598714052682608, + "ratio_min": 0.7505469504549158, + "ratio_max": 1.2805383260092305, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 12311560, + "candidate": 10725103, + "ratio": 0.8711408627338859, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 1, + "baseline": 11854918, + "candidate": 10193705, + "ratio": 0.8598714052682608, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 2, + "baseline": 10356971, + "candidate": 7773393, + "ratio": 0.7505469504549158, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 3, + "baseline": 12325192, + "candidate": 10422501, + "ratio": 0.845625853130726, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 4, + "baseline": 8985856, + "candidate": 11506733, + "ratio": 1.2805383260092305, + "n_baseline": 216, + "n_candidate": 216 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 17398375, + "candidate_median": 17757539, + "ratio_median": 0.9733687197798645, + "ratio_min": 0.11192289167465248, + "ratio_max": 1.3581489255367434, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 214491599, + "candidate": 24006520, + "ratio": 0.11192289167465248, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 1, + "baseline": 17398375, + "candidate": 16935034, + "ratio": 0.9733687197798645, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 2, + "baseline": 13368798, + "candidate": 13688309, + "ratio": 1.0238997552360354, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 3, + "baseline": 104039575, + "candidate": 17757539, + "ratio": 0.17068061840890833, + "n_baseline": 216, + "n_candidate": 216 + }, + { + "pair": 4, + "baseline": 16383436, + "candidate": 22251146, + "ratio": 1.3581489255367434, + "n_baseline": 216, + "n_candidate": 216 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 630749, + "candidate_median": 593030, + "ratio_median": 0.9714636743386027, + "ratio_min": 0.8617367205683161, + "ratio_max": 1.0113200797638422, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 657803, + "candidate": 566853, + "ratio": 0.8617367205683161, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 676497, + "candidate": 684155, + "ratio": 1.0113200797638422, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 630749, + "candidate": 548254, + "ratio": 0.8692110490860866, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 610948, + "candidate": 613844, + "ratio": 1.0047401742865187, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 610450, + "candidate": 593030, + "ratio": 0.9714636743386027, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 1048824, + "candidate_median": 1049295, + "ratio_median": 1.0004490743918903, + "ratio_min": 0.9770027888079681, + "ratio_max": 1.0118604330437182, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 1076087, + "candidate": 1051340, + "ratio": 0.9770027888079681, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1048824, + "candidate": 1049295, + "ratio": 1.0004490743918903, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1053209, + "candidate": 1060301, + "ratio": 1.00673370622545, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1043222, + "candidate": 1035725, + "ratio": 0.9928136101424242, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1036893, + "candidate": 1049191, + "ratio": 1.0118604330437182, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1111983, + "candidate_median": 1102944, + "ratio_median": 0.9951698918310675, + "ratio_min": 0.9593907435052118, + "ratio_max": 1.0126584138007653, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1111186, + "candidate": 1120121, + "ratio": 1.008040958039428, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1089157, + "candidate": 1102944, + "ratio": 1.0126584138007653, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1111983, + "candidate": 1093635, + "ratio": 0.9834997477479422, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1147497, + "candidate": 1100898, + "ratio": 0.9593907435052118, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1145523, + "candidate": 1139990, + "ratio": 0.9951698918310675, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 3193754, + "candidate_median": 3216711, + "ratio_median": 0.9873290089941875, + "ratio_min": 0.9660401968745421, + "ratio_max": 1.059569568982838, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 3132539, + "candidate": 3061467, + "ratio": 0.9773116950818489, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 3221544, + "candidate": 3112141, + "ratio": 0.9660401968745421, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 3257993, + "candidate": 3216711, + "ratio": 0.9873290089941875, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 3193754, + "candidate": 3267392, + "ratio": 1.023056879145983, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 3190337, + "candidate": 3380384, + "ratio": 1.059569568982838, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 4423716, + "candidate_median": 195494581, + "ratio_median": 44.13172748883518, + "ratio_min": 42.21779846394062, + "ratio_max": 45.40425478691322, + "candidate_lower_pairs": 0, + "per_pair": [ + { + "pair": 0, + "baseline": 4317396, + "candidate": 196028148, + "ratio": 45.40425478691322, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 4561933, + "candidate": 192594768, + "ratio": 42.21779846394062, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 4423716, + "candidate": 195226229, + "ratio": 44.13172748883518, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 4488286, + "candidate": 195494581, + "ratio": 43.556622951389464, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 4392135, + "candidate": 195916432, + "ratio": 44.60619539244581, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 21635488, + "candidate_median": 249932003, + "ratio_median": 11.50322548257648, + "ratio_min": 11.14735098326091, + "ratio_max": 11.685888981364664, + "candidate_lower_pairs": 0, + "per_pair": [ + { + "pair": 0, + "baseline": 21437446, + "candidate": 250515614, + "ratio": 11.685888981364664, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 21635488, + "candidate": 248812039, + "ratio": 11.50018150734571, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 21625262, + "candidate": 249093985, + "ratio": 11.51865743869369, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 22420753, + "candidate": 249932003, + "ratio": 11.14735098326091, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 21951909, + "candidate": 252517759, + "ratio": 11.50322548257648, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 2545081, + "candidate_median": 2532818, + "ratio_median": 1.0133017859200464, + "ratio_min": 0.973544653392171, + "ratio_max": 1.0321117980927004, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 2421329, + "candidate": 2453537, + "ratio": 1.0133017859200464, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 2545081, + "candidate": 2477750, + "ratio": 0.973544653392171, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 2554694, + "candidate": 2532818, + "ratio": 0.9914369392185522, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 2518437, + "candidate": 2569132, + "ratio": 1.0201295486049482, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 2583443, + "candidate": 2666402, + "ratio": 1.0321117980927004, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 3653276, + "candidate_median": 194930934, + "ratio_median": 53.35784484938997, + "ratio_min": 50.04875753137849, + "ratio_max": 54.08420457030782, + "candidate_lower_pairs": 0, + "per_pair": [ + { + "pair": 0, + "baseline": 3653276, + "candidate": 194930934, + "ratio": 53.35784484938997, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 3543525, + "candidate": 191648731, + "ratio": 54.08420457030782, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 3890722, + "candidate": 194725802, + "ratio": 50.04875753137849, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 3870045, + "candidate": 195025014, + "ratio": 50.393474494482625, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 3610091, + "candidate": 195234414, + "ratio": 54.0801918843597, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 20884229, + "candidate_median": 249222644, + "ratio_median": 11.91409712630108, + "ratio_min": 11.47331743112745, + "ratio_max": 12.07113001950027, + "candidate_lower_pairs": 0, + "per_pair": [ + { + "pair": 0, + "baseline": 20884229, + "candidate": 250040041, + "ratio": 11.972672824072175, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 20559716, + "candidate": 248179005, + "ratio": 12.07113001950027, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 20841049, + "candidate": 248302282, + "ratio": 11.91409712630108, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 21721934, + "candidate": 249222644, + "ratio": 11.47331743112745, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 21592910, + "candidate": 251904917, + "ratio": 11.666093963249974, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 560844, + "candidate_median": 553744, + "ratio_median": 1.0024993266697177, + "ratio_min": 0.8561847543568111, + "ratio_max": 1.0905135830997568, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 617719, + "candidate": 553744, + "ratio": 0.8964334916037875, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 560844, + "candidate": 611608, + "ratio": 1.0905135830997568, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 550416, + "candidate": 565897, + "ratio": 1.028125999244208, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 530943, + "candidate": 532270, + "ratio": 1.0024993266697177, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 586725, + "candidate": 502345, + "ratio": 0.8561847543568111, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 1052953, + "candidate_median": 1050883, + "ratio_median": 0.9930621267316895, + "ratio_min": 0.9718061456565403, + "ratio_max": 1.0100506907747404, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 1053492, + "candidate": 1023790, + "ratio": 0.9718061456565403, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1065635, + "candidate": 1051938, + "ratio": 0.9871466308820562, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1040426, + "candidate": 1050883, + "ratio": 1.0100506907747404, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1052953, + "candidate": 1052763, + "ratio": 0.9998195550988506, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1034611, + "candidate": 1027433, + "ratio": 0.9930621267316895, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1099030, + "candidate_median": 1110517, + "ratio_median": 1.0132916647657284, + "ratio_min": 0.947676415177178, + "ratio_max": 1.0585532804884195, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 1094582, + "candidate": 1088097, + "ratio": 0.9940753639288787, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1150227, + "candidate": 1090043, + "ratio": 0.947676415177178, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1111176, + "candidate": 1176239, + "ratio": 1.0585532804884195, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1095950, + "candidate": 1110517, + "ratio": 1.0132916647657284, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1099030, + "candidate": 1116639, + "ratio": 1.0160223105829687, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 1869654, + "candidate_median": 1798866, + "ratio_median": 0.9621384491462056, + "ratio_min": 0.9449268969573328, + "ratio_max": 0.9941300423869435, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1849923, + "candidate": 1748042, + "ratio": 0.9449268969573328, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1853165, + "candidate": 1842287, + "ratio": 0.9941300423869435, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1892182, + "candidate": 1843977, + "ratio": 0.9745241208298144, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1869654, + "candidate": 1798866, + "ratio": 0.9621384491462056, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1890068, + "candidate": 1793588, + "ratio": 0.9489542175202162, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 2474725, + "candidate_median": 2296030, + "ratio_median": 0.9488817310418304, + "ratio_min": 0.9274210265787107, + "ratio_max": 0.9962467410595484, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 2317813, + "candidate": 2243731, + "ratio": 0.9680379737278202, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 2419722, + "candidate": 2296030, + "ratio": 0.9488817310418304, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 2498895, + "candidate": 2489516, + "ratio": 0.9962467410595484, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 2543726, + "candidate": 2373456, + "ratio": 0.9330627591179239, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 2474725, + "candidate": 2295112, + "ratio": 0.9274210265787107, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 16493083, + "candidate_median": 6737655, + "ratio_median": 0.3644580769816245, + "ratio_min": 0.0525495831518484, + "ratio_max": 3.732659501506895, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 7528394, + "candidate": 2743784, + "ratio": 0.3644580769816245, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 4015542, + "candidate": 14988651, + "ratio": 3.732659501506895, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 16493083, + "candidate": 6737655, + "ratio": 0.4085139812853667, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 24359917, + "candidate": 6819070, + "ratio": 0.27992993572186636, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 58122364, + "candidate": 3054306, + "ratio": 0.0525495831518484, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 1262843, + "candidate_median": 1238869, + "ratio_median": 0.9817474254977011, + "ratio_min": 0.9503524500436871, + "ratio_max": 1.0161381968937468, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 1202077, + "candidate": 1180136, + "ratio": 0.9817474254977011, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1257950, + "candidate": 1197692, + "ratio": 0.9520982550975794, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1303589, + "candidate": 1238869, + "ratio": 0.9503524500436871, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1262843, + "candidate": 1248695, + "ratio": 0.9887967071124439, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1265383, + "candidate": 1285804, + "ratio": 1.0161381968937468, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 1562872, + "candidate_median": 1581877, + "ratio_median": 0.994435648304734, + "ratio_min": 0.9510290746515171, + "ratio_max": 1.0719155514931251, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1505705, + "candidate": 1458166, + "ratio": 0.9684274144005632, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1562872, + "candidate": 1581877, + "ratio": 1.0121603048746155, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1740611, + "candidate": 1865788, + "ratio": 1.0719155514931251, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1685592, + "candidate": 1603047, + "ratio": 0.9510290746515171, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1558133, + "candidate": 1549463, + "ratio": 0.994435648304734, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 15797802, + "candidate_median": 6080848, + "ratio_median": 0.2645512658306388, + "ratio_min": 0.03534062438653526, + "ratio_max": 4.786419764429173, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 7413788, + "candidate": 1961327, + "ratio": 0.2645512658306388, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 3047576, + "candidate": 14586978, + "ratio": 4.786419764429173, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 15797802, + "candidate": 6621368, + "ratio": 0.4191322311800085, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 24234627, + "candidate": 6080848, + "ratio": 0.2509156835795327, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 57363729, + "candidate": 2027270, + "ratio": 0.03534062438653526, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 670420, + "candidate_median": 596508, + "ratio_median": 0.8897526923421138, + "ratio_min": 0.7975375338563556, + "ratio_max": 1.2021593415104947, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 685248, + "candidate": 546511, + "ratio": 0.7975375338563556, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 1, + "baseline": 574527, + "candidate": 690673, + "ratio": 1.2021593415104947, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 2, + "baseline": 610373, + "candidate": 490791, + "ratio": 0.80408373240625, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 3, + "baseline": 699601, + "candidate": 646052, + "ratio": 0.9234577995171533, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 4, + "baseline": 670420, + "candidate": 596508, + "ratio": 0.8897526923421138, + "n_baseline": 45, + "n_candidate": 45 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 1041178, + "candidate_median": 1065598, + "ratio_median": 1.0198256912503105, + "ratio_min": 0.9791108377026231, + "ratio_max": 1.066832681472462, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 1041178, + "candidate": 1043101, + "ratio": 1.0018469464395137, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 1, + "baseline": 1062712, + "candidate": 1083781, + "ratio": 1.0198256912503105, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 2, + "baseline": 909959, + "candidate": 970774, + "ratio": 1.066832681472462, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 3, + "baseline": 1090757, + "candidate": 1067972, + "ratio": 0.9791108377026231, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 4, + "baseline": 1033764, + "candidate": 1065598, + "ratio": 1.0307942625202657, + "n_baseline": 45, + "n_candidate": 45 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1090641, + "candidate_median": 1093206, + "ratio_median": 1.008713881985841, + "ratio_min": 0.9518450024048658, + "ratio_max": 1.040870810332159, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 1090903, + "candidate": 1100409, + "ratio": 1.008713881985841, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 1, + "baseline": 1090641, + "candidate": 1093206, + "ratio": 1.0023518279617216, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 2, + "baseline": 1030424, + "candidate": 1040676, + "ratio": 1.0099493024230801, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 3, + "baseline": 1126882, + "candidate": 1072617, + "ratio": 0.9518450024048658, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 4, + "baseline": 1061540, + "candidate": 1104926, + "ratio": 1.040870810332159, + "n_baseline": 45, + "n_candidate": 45 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 1232408, + "candidate_median": 1184338, + "ratio_median": 0.9396952104499274, + "ratio_min": 0.8268576640203569, + "ratio_max": 1.0924113989952697, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1232408, + "candidate": 1019026, + "ratio": 0.8268576640203569, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 1, + "baseline": 1084271, + "candidate": 1184470, + "ratio": 1.0924113989952697, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 2, + "baseline": 1185080, + "candidate": 1113614, + "ratio": 0.9396952104499274, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 3, + "baseline": 1358364, + "candidate": 1184338, + "ratio": 0.8718855917854125, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 4, + "baseline": 1267927, + "candidate": 1276624, + "ratio": 1.0068592277000175, + "n_baseline": 45, + "n_candidate": 45 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 1583242, + "candidate_median": 1589698, + "ratio_median": 0.9908429664941066, + "ratio_min": 0.919557560538708, + "ratio_max": 1.0377865226632963, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1559457, + "candidate": 1545177, + "ratio": 0.9908429664941066, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 1, + "baseline": 1583242, + "candidate": 1535879, + "ratio": 0.970084800681134, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 2, + "baseline": 1531816, + "candidate": 1589698, + "ratio": 1.0377865226632963, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 3, + "baseline": 1819910, + "candidate": 1673512, + "ratio": 0.919557560538708, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 4, + "baseline": 1674151, + "candidate": 1685716, + "ratio": 1.0069079790293707, + "n_baseline": 45, + "n_candidate": 45 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1975707, + "candidate_median": 1826726, + "ratio_median": 0.8825083881365, + "ratio_min": 0.1502641038914205, + "ratio_max": 1.095885438247929, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 1975707, + "candidate": 1743578, + "ratio": 0.8825083881365, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 1, + "baseline": 1714925, + "candidate": 1677596, + "ratio": 0.9782328673265595, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 2, + "baseline": 12156769, + "candidate": 1826726, + "ratio": 0.1502641038914205, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 3, + "baseline": 11680254, + "candidate": 9407971, + "ratio": 0.8054594531933981, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 4, + "baseline": 1694178, + "candidate": 1856625, + "ratio": 1.095885438247929, + "n_baseline": 45, + "n_candidate": 45 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 559534, + "candidate_median": 556411, + "ratio_median": 0.9827133492008725, + "ratio_min": 0.8676595797766335, + "ratio_max": 1.087434771000704, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 512997, + "candidate": 504129, + "ratio": 0.9827133492008725, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 1, + "baseline": 543158, + "candidate": 489167, + "ratio": 0.9005979843802356, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 2, + "baseline": 559534, + "candidate": 597827, + "ratio": 1.0684373067588386, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 3, + "baseline": 641278, + "candidate": 556411, + "ratio": 0.8676595797766335, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 4, + "baseline": 618015, + "candidate": 672051, + "ratio": 1.087434771000704, + "n_baseline": 45, + "n_candidate": 45 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 687413, + "candidate_median": 754393, + "ratio_median": 1.0477894020169845, + "ratio_min": 0.9915494760791548, + "ratio_max": 1.1732207753872772, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 658012, + "candidate": 689458, + "ratio": 1.0477894020169845, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 1, + "baseline": 687413, + "candidate": 681604, + "ratio": 0.9915494760791548, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 2, + "baseline": 747866, + "candidate": 754393, + "ratio": 1.008727499311374, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 3, + "baseline": 764727, + "candidate": 811446, + "ratio": 1.0610923898332345, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 4, + "baseline": 684329, + "candidate": 802869, + "ratio": 1.1732207753872772, + "n_baseline": 45, + "n_candidate": 45 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 908168, + "candidate_median": 837087, + "ratio_median": 0.8904570955827505, + "ratio_min": 0.07796683476315097, + "ratio_max": 0.975262479580361, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 884804, + "candidate": 787880, + "ratio": 0.8904570955827505, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 1, + "baseline": 730914, + "candidate": 712833, + "ratio": 0.975262479580361, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 2, + "baseline": 11879487, + "candidate": 926206, + "ratio": 0.07796683476315097, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 3, + "baseline": 10818496, + "candidate": 9174912, + "ratio": 0.8480764793923296, + "n_baseline": 45, + "n_candidate": 45 + }, + { + "pair": 4, + "baseline": 908168, + "candidate": 837087, + "ratio": 0.9217314417596745, + "n_baseline": 45, + "n_candidate": 45 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 650961, + "candidate_median": 598313, + "ratio_median": 0.941627070927723, + "ratio_min": 0.845351718459324, + "ratio_max": 1.0312211139163665, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 729576, + "candidate": 666316, + "ratio": 0.9132921039069267, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 1, + "baseline": 611128, + "candidate": 598313, + "ratio": 0.9790305795185297, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 2, + "baseline": 684838, + "candidate": 644862, + "ratio": 0.941627070927723, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 3, + "baseline": 650961, + "candidate": 550291, + "ratio": 0.845351718459324, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 4, + "baseline": 579864, + "candidate": 597968, + "ratio": 1.0312211139163665, + "n_baseline": 135, + "n_candidate": 135 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 1049006, + "candidate_median": 1050542, + "ratio_median": 0.9870701410399579, + "ratio_min": 0.9644751325928149, + "ratio_max": 1.0304936292070779, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1089237, + "candidate": 1050542, + "ratio": 0.9644751325928149, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 1, + "baseline": 1045023, + "candidate": 1031511, + "ratio": 0.9870701410399579, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 2, + "baseline": 1071204, + "candidate": 1049888, + "ratio": 0.9801008958144294, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 3, + "baseline": 1041314, + "candidate": 1054190, + "ratio": 1.0123651463439463, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 4, + "baseline": 1049006, + "candidate": 1080994, + "ratio": 1.0304936292070779, + "n_baseline": 135, + "n_candidate": 135 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1167253, + "candidate_median": 1108212, + "ratio_median": 0.9370008044528478, + "ratio_min": 0.8937511744624087, + "ratio_max": 1.0190766828205982, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 1239929, + "candidate": 1108188, + "ratio": 0.8937511744624087, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 1, + "baseline": 1167253, + "candidate": 1093717, + "ratio": 0.9370008044528478, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 2, + "baseline": 1116756, + "candidate": 1138060, + "ratio": 1.0190766828205982, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 3, + "baseline": 1228076, + "candidate": 1122152, + "ratio": 0.9137480090808712, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 4, + "baseline": 1108618, + "candidate": 1108212, + "ratio": 0.999633778271686, + "n_baseline": 135, + "n_candidate": 135 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 1629718, + "candidate_median": 1553494, + "ratio_median": 0.912902706048965, + "ratio_min": 0.9058094038066185, + "ratio_max": 0.9965036895953778, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1617103, + "candidate": 1553494, + "ratio": 0.9606648432412778, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 1, + "baseline": 1601482, + "candidate": 1458992, + "ratio": 0.9110261620174314, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 2, + "baseline": 1768815, + "candidate": 1614756, + "ratio": 0.912902706048965, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 3, + "baseline": 1658690, + "candidate": 1502457, + "ratio": 0.9058094038066185, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 4, + "baseline": 1629718, + "candidate": 1624020, + "ratio": 0.9965036895953778, + "n_baseline": 135, + "n_candidate": 135 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 2119537, + "candidate_median": 1974217, + "ratio_median": 0.9505629867973138, + "ratio_min": 0.9146246703452797, + "ratio_max": 0.9982496177231159, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1996415, + "candidate": 1974217, + "ratio": 0.9888810693167502, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 1, + "baseline": 2005198, + "candidate": 1906067, + "ratio": 0.9505629867973138, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 2, + "baseline": 2312647, + "candidate": 2115204, + "ratio": 0.9146246703452797, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 3, + "baseline": 2128791, + "candidate": 1949537, + "ratio": 0.9157953974814813, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 4, + "baseline": 2119537, + "candidate": 2115827, + "ratio": 0.9982496177231159, + "n_baseline": 135, + "n_candidate": 135 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 5306079, + "candidate_median": 8685688, + "ratio_median": 2.717714530823985, + "ratio_min": 0.14544284466069446, + "ratio_max": 3.928434847074833, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 5306079, + "candidate": 14420408, + "ratio": 2.717714530823985, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 1, + "baseline": 2494063, + "candidate": 9797764, + "ratio": 3.928434847074833, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 2, + "baseline": 2652205, + "candidate": 8685688, + "ratio": 3.2748931549408886, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 3, + "baseline": 9837104, + "candidate": 2168945, + "ratio": 0.22048613087754282, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 4, + "baseline": 16185382, + "candidate": 2354048, + "ratio": 0.14544284466069446, + "n_baseline": 135, + "n_candidate": 135 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 1002036, + "candidate_median": 912309, + "ratio_median": 0.9091941010099399, + "ratio_min": 0.8226389475036239, + "ratio_max": 1.0056065849929543, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 885134, + "candidate": 878590, + "ratio": 0.9926067691445589, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 1, + "baseline": 992937, + "candidate": 862983, + "ratio": 0.8691216059024893, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 2, + "baseline": 1144479, + "candidate": 941493, + "ratio": 0.8226389475036239, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 3, + "baseline": 1003426, + "candidate": 912309, + "ratio": 0.9091941010099399, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 4, + "baseline": 1002036, + "candidate": 1007654, + "ratio": 1.0056065849929543, + "n_baseline": 135, + "n_candidate": 135 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 1250487, + "candidate_median": 1169186, + "ratio_median": 0.909270254510893, + "ratio_min": 0.8639309363444161, + "ratio_max": 1.0530522398343485, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 1110283, + "candidate": 1169186, + "ratio": 1.0530522398343485, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 1, + "baseline": 1232602, + "candidate": 1064883, + "ratio": 0.8639309363444161, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 2, + "baseline": 1418289, + "candidate": 1289608, + "ratio": 0.909270254510893, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 3, + "baseline": 1250487, + "candidate": 1126646, + "ratio": 0.9009657837306585, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 4, + "baseline": 1321878, + "candidate": 1258824, + "ratio": 0.9522996827241244, + "n_baseline": 135, + "n_candidate": 135 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 4066150, + "candidate_median": 8167759, + "ratio_median": 3.3676689251503267, + "ratio_min": 0.08916528715739573, + "ratio_max": 6.703443623296025, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 4066150, + "candidate": 13693447, + "ratio": 3.3676689251503267, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 1, + "baseline": 1442289, + "candidate": 9668303, + "ratio": 6.703443623296025, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 2, + "baseline": 1581001, + "candidate": 8167759, + "ratio": 5.166194708289242, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 3, + "baseline": 9580070, + "candidate": 1286402, + "ratio": 0.1342789770847186, + "n_baseline": 135, + "n_candidate": 135 + }, + { + "pair": 4, + "baseline": 15309792, + "candidate": 1365102, + "ratio": 0.08916528715739573, + "n_baseline": 135, + "n_candidate": 135 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 627507, + "candidate_median": 643620, + "ratio_median": 0.9589512613758443, + "ratio_min": 0.9155002102899763, + "ratio_max": 1.104909967056303, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 689612, + "candidate": 661296, + "ratio": 0.9589392295957727, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 625327, + "candidate": 572487, + "ratio": 0.9155002102899763, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 582509, + "candidate": 643620, + "ratio": 1.104909967056303, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 647864, + "candidate": 621270, + "ratio": 0.9589512613758443, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 627507, + "candidate": 669856, + "ratio": 1.0674876933643769, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 1052291, + "candidate_median": 1039854, + "ratio_median": 0.9860823860396153, + "ratio_min": 0.9824835569488297, + "ratio_max": 1.0120153696562761, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1066712, + "candidate": 1048027, + "ratio": 0.9824835569488297, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1055918, + "candidate": 1039854, + "ratio": 0.9847866974518855, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1022274, + "candidate": 1034557, + "ratio": 1.0120153696562761, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1052120, + "candidate": 1037477, + "ratio": 0.9860823860396153, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1052291, + "candidate": 1057663, + "ratio": 1.0051050517394904, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1136553, + "candidate_median": 1095854, + "ratio_median": 0.9744233152946156, + "ratio_min": 0.8576173763137718, + "ratio_max": 1.0322662283640218, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 1124618, + "candidate": 1095854, + "ratio": 0.9744233152946156, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1136553, + "candidate": 1085296, + "ratio": 0.9549013552381631, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1259922, + "candidate": 1080531, + "ratio": 0.8576173763137718, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1161401, + "candidate": 1148460, + "ratio": 0.9888574230605967, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1086616, + "candidate": 1121677, + "ratio": 1.0322662283640218, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 25845949, + "candidate_median": 25427015, + "ratio_median": 0.9837911155825619, + "ratio_min": 0.976712860131509, + "ratio_max": 1.016419028780369, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 24634222, + "candidate": 25038692, + "ratio": 1.016419028780369, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 25560975, + "candidate": 24965733, + "ratio": 0.976712860131509, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 26132611, + "candidate": 25532036, + "ratio": 0.9770181785509301, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 25845949, + "candidate": 25427015, + "ratio": 0.9837911155825619, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 26200288, + "candidate": 25813727, + "ratio": 0.9852459255409711, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 27920082, + "candidate_median": 27826309, + "ratio_median": 1.0393679359537698, + "ratio_min": 0.9694342679820801, + "ratio_max": 1.2097795004295486, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 26567703, + "candidate": 27826309, + "ratio": 1.047373534701137, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 27428813, + "candidate": 27633834, + "ratio": 1.007474658126839, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 29364547, + "candidate": 35524627, + "ratio": 1.2097795004295486, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 28587210, + "candidate": 27713421, + "ratio": 0.9694342679820801, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 27920082, + "candidate": 29019238, + "ratio": 1.0393679359537698, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 100999302, + "candidate_median": 89109566, + "ratio_median": 0.829275887490752, + "ratio_min": 0.772129215942228, + "ratio_max": 1.109518753091068, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 96810299, + "candidate": 77240930, + "ratio": 0.7978586038661031, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 90181213, + "candidate": 100057747, + "ratio": 1.109518753091068, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 107454669, + "candidate": 89109566, + "ratio": 0.829275887490752, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 100999302, + "candidate": 99918522, + "ratio": 0.9892991339682724, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 107878020, + "candidate": 83295771, + "ratio": 0.772129215942228, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 25291738, + "candidate_median": 24903091, + "ratio_median": 0.9846334403748765, + "ratio_min": 0.9785003300676859, + "ratio_max": 1.0203381993805003, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 23968051, + "candidate": 24455518, + "ratio": 1.0203381993805003, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 24911059, + "candidate": 24394969, + "ratio": 0.9792826952880647, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 25502345, + "candidate": 24954053, + "ratio": 0.9785003300676859, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 25291738, + "candidate": 24903091, + "ratio": 0.9846334403748765, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 25617062, + "candidate": 25283310, + "ratio": 0.9869714957944826, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 27363056, + "candidate_median": 26883734, + "ratio_median": 1.031752812989894, + "ratio_min": 0.9565009673120792, + "ratio_max": 1.2166467152022806, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 25633676, + "candidate": 26883734, + "ratio": 1.0487662401600144, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 27069849, + "candidate": 26647502, + "ratio": 0.9843978811998545, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 28829964, + "candidate": 35075881, + "ratio": 1.2166467152022806, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 28073153, + "candidate": 26851998, + "ratio": 0.9565009673120792, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 27363056, + "candidate": 28231910, + "ratio": 1.031752812989894, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 100161322, + "candidate_median": 88393542, + "ratio_median": 0.8229312162272144, + "ratio_min": 0.7685858309561373, + "ratio_max": 1.1047596827810313, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 95836972, + "candidate": 76586512, + "ratio": 0.7991332614306721, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 90112644, + "candidate": 99552816, + "ratio": 1.1047596827810313, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 107413038, + "candidate": 88393542, + "ratio": 0.8229312162272144, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 100161322, + "candidate": 99391117, + "ratio": 0.9923103550889634, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 107346707, + "candidate": 82505158, + "ratio": 0.7685858309561373, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "paths", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 597145, + "candidate_median": 602742, + "ratio_median": 1.0417675044374022, + "ratio_min": 0.960558608043575, + "ratio_max": 1.1720462377674368, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 586447, + "candidate": 687343, + "ratio": 1.1720462377674368, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 609157, + "candidate": 585131, + "ratio": 0.960558608043575, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 597145, + "candidate": 585736, + "ratio": 0.9808940877006422, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 616915, + "candidate": 642682, + "ratio": 1.0417675044374022, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 527857, + "candidate": 602742, + "ratio": 1.1418660735767452, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "paths", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 1059403, + "candidate_median": 1060674, + "ratio_median": 1.0011997323020607, + "ratio_min": 0.9620724995396471, + "ratio_max": 1.0391206460166325, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 1061343, + "candidate": 1056696, + "ratio": 0.9956215851049095, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1075262, + "candidate": 1034480, + "ratio": 0.9620724995396471, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1059403, + "candidate": 1060674, + "ratio": 1.0011997323020607, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1052553, + "candidate": 1074115, + "ratio": 1.0204854292372925, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1037125, + "candidate": 1077698, + "ratio": 1.0391206460166325, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "paths", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1104307, + "candidate_median": 1132927, + "ratio_median": 1.0259167061333487, + "ratio_min": 0.9628045311308275, + "ratio_max": 1.152122312762912, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 1091639, + "candidate": 1170321, + "ratio": 1.0720769411866011, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1135434, + "candidate": 1093201, + "ratio": 0.9628045311308275, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1122761, + "candidate": 1293558, + "ratio": 1.152122312762912, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1104307, + "candidate": 1132927, + "ratio": 1.0259167061333487, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1101879, + "candidate": 1112395, + "ratio": 1.0095436976292316, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "paths", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 43898221, + "candidate_median": 45136656, + "ratio_median": 1.0290618838517396, + "ratio_min": 1.0167511690632633, + "ratio_max": 1.0431071277107686, + "candidate_lower_pairs": 0, + "per_pair": [ + { + "pair": 0, + "baseline": 43408473, + "candidate": 44877717, + "ratio": 1.0338469404348778, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 43655244, + "candidate": 44796864, + "ratio": 1.0261508101981975, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 44393021, + "candidate": 45136656, + "ratio": 1.0167511690632633, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 43898221, + "candidate": 45173986, + "ratio": 1.0290618838517396, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 44183134, + "candidate": 46087742, + "ratio": 1.0431071277107686, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "paths", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 46908956, + "candidate_median": 48609587, + "ratio_median": 1.0336009711381329, + "ratio_min": 0.9688219441858067, + "ratio_max": 1.0383415482536, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 45932196, + "candidate": 47286312, + "ratio": 1.0294807589865722, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 46099737, + "candidate": 47815898, + "ratio": 1.0372271321200812, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 47029355, + "candidate": 48609587, + "ratio": 1.0336009711381329, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 46908956, + "candidate": 48707518, + "ratio": 1.0383415482536, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 50811090, + "candidate": 49226899, + "ratio": 0.9688219441858067, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "paths", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 83088852, + "candidate_median": 88123240, + "ratio_median": 1.1206063513610853, + "ratio_min": 0.8539710894511615, + "ratio_max": 1.5228260679130279, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 48406366, + "candidate": 56327865, + "ratio": 1.1636458105530996, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 62728259, + "candidate": 95524228, + "ratio": 1.5228260679130279, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 88252989, + "candidate": 98896860, + "ratio": 1.1206063513610853, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 83088852, + "candidate": 83518703, + "ratio": 1.0051733895661479, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 103192299, + "candidate": 88123240, + "ratio": 0.8539710894511615, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "paths", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 43142819, + "candidate_median": 44549078, + "ratio_median": 1.0328548303716547, + "ratio_min": 1.0187695538339607, + "ratio_max": 1.0451988726245436, + "candidate_lower_pairs": 0, + "per_pair": [ + { + "pair": 0, + "baseline": 42727301, + "candidate": 44142501, + "ratio": 1.0331216802109733, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 43079851, + "candidate": 44186660, + "ratio": 1.0256920340787623, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 43728317, + "candidate": 44549078, + "ratio": 1.0187695538339607, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 43142819, + "candidate": 44560269, + "ratio": 1.0328548303716547, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 43560998, + "candidate": 45529906, + "ratio": 1.0451988726245436, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "paths", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 46277322, + "candidate_median": 47790790, + "ratio_median": 1.0296741207200182, + "ratio_min": 0.9701286656415792, + "ratio_max": 1.037671845920605, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 45355061, + "candidate": 46453150, + "ratio": 1.0242109474838983, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 45215650, + "candidate": 46919007, + "ratio": 1.037671845920605, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 46277322, + "candidate": 47790790, + "ratio": 1.0327043124924127, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 46479052, + "candidate": 47858277, + "ratio": 1.0296741207200182, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 49864361, + "candidate": 48374846, + "ratio": 0.9701286656415792, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "paths", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 82959552, + "candidate_median": 87095736, + "ratio_median": 1.1071509846719905, + "ratio_min": 0.8529503665721415, + "ratio_max": 1.5289396608047738, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 47629016, + "candidate": 56082319, + "ratio": 1.1774822095841744, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 62289379, + "candidate": 95236702, + "ratio": 1.5289396608047738, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 88157174, + "candidate": 97603302, + "ratio": 1.1071509846719905, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 82959552, + "candidate": 83095937, + "ratio": 1.0016439939309218, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 102111142, + "candidate": 87095736, + "ratio": 0.8529503665721415, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 600184, + "candidate_median": 576249, + "ratio_median": 0.955238726790451, + "ratio_min": 0.8340256941121759, + "ratio_max": 1.1276405202769748, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 536963, + "candidate": 576249, + "ratio": 1.073163327827057, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 600184, + "candidate": 573319, + "ratio": 0.955238726790451, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 558318, + "candidate": 629582, + "ratio": 1.1276405202769748, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 670893, + "candidate": 559542, + "ratio": 0.8340256941121759, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 621049, + "candidate": 585508, + "ratio": 0.9427726314670823, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 1041041, + "candidate_median": 1040210, + "ratio_median": 0.9992017605454541, + "ratio_min": 0.9634185156098453, + "ratio_max": 1.0066994297371228, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1041767, + "candidate": 1023419, + "ratio": 0.982387616424786, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1040809, + "candidate": 1044004, + "ratio": 1.003069727490827, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1041041, + "candidate": 1040210, + "ratio": 0.9992017605454541, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1061958, + "candidate": 1023110, + "ratio": 0.9634185156098453, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1035312, + "candidate": 1042248, + "ratio": 1.0066994297371228, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1091233, + "candidate_median": 1106621, + "ratio_median": 0.9944833046654564, + "ratio_min": 0.9325597902083042, + "ratio_max": 3.1090782912373585, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1091233, + "candidate": 1085213, + "ratio": 0.9944833046654564, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1072896, + "candidate": 1106621, + "ratio": 1.031433615187306, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1072074, + "candidate": 3333162, + "ratio": 3.1090782912373585, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1176405, + "candidate": 1097068, + "ratio": 0.9325597902083042, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1122026, + "candidate": 1108571, + "ratio": 0.9880082992729223, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 1215960, + "candidate_median": 1196891, + "ratio_median": 0.9988257748573042, + "ratio_min": 0.9252454210868488, + "ratio_max": 1.0796448225490827, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1088470, + "candidate": 1175161, + "ratio": 1.0796448225490827, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1215960, + "candidate": 1176272, + "ratio": 0.9673607684463305, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1191506, + "candidate": 1219189, + "ratio": 1.023233621987636, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1293593, + "candidate": 1196891, + "ratio": 0.9252454210868488, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1218676, + "candidate": 1217245, + "ratio": 0.9988257748573042, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 1653442, + "candidate_median": 1656737, + "ratio_median": 1.0043752121829381, + "ratio_min": 0.9490854447476064, + "ratio_max": 1.0408956641825555, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 1589435, + "candidate": 1654436, + "ratio": 1.0408956641825555, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1649520, + "candidate": 1656737, + "ratio": 1.0043752121829381, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1653442, + "candidate": 1662714, + "ratio": 1.0056076959457907, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1675963, + "candidate": 1637601, + "ratio": 0.9771104732025707, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1762168, + "candidate": 1672448, + "ratio": 0.9490854447476064, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 17127144, + "candidate_median": 2202343, + "ratio_median": 0.8408523228357657, + "ratio_min": 0.10381367696688029, + "ratio_max": 1.246536024920442, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 2109060, + "candidate": 1773408, + "ratio": 0.8408523228357657, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 2005007, + "candidate": 1953876, + "ratio": 0.9744983433973049, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 17127144, + "candidate": 21349602, + "ratio": 1.246536024920442, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 21028156, + "candidate": 5547971, + "ratio": 0.2638353548451895, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 21214382, + "candidate": 2202343, + "ratio": 0.10381367696688029, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 597287, + "candidate_median": 589073, + "ratio_median": 0.9919914330766422, + "ratio_min": 0.9854760479694465, + "ratio_max": 1.0275613754969513, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 559914, + "candidate": 575346, + "ratio": 1.0275613754969513, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 597677, + "candidate": 589073, + "ratio": 0.9856042645107642, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 597287, + "candidate": 593317, + "ratio": 0.9933532790768926, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 596463, + "candidate": 587800, + "ratio": 0.9854760479694465, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 610721, + "candidate": 605830, + "ratio": 0.9919914330766422, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 717464, + "candidate_median": 822727, + "ratio_median": 1.0984815962891519, + "ratio_min": 0.946965340201116, + "ratio_max": 1.2029107108836525, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 670441, + "candidate": 724798, + "ratio": 1.0810764854774693, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 717464, + "candidate": 788121, + "ratio": 1.0984815962891519, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 766342, + "candidate": 921841, + "ratio": 1.2029107108836525, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 698714, + "candidate": 822727, + "ratio": 1.1774874984614592, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 959844, + "candidate": 908939, + "ratio": 0.946965340201116, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 16620092, + "candidate_median": 1415907, + "ratio_median": 0.8718615903896193, + "ratio_min": 0.06838757066998671, + "ratio_max": 1.0840156600817854, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 1081081, + "candidate": 942553, + "ratio": 0.8718615903896193, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1198785, + "candidate": 1147445, + "ratio": 0.9571733046376123, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 16620092, + "candidate": 18016440, + "ratio": 1.0840156600817854, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 20512185, + "candidate": 4450903, + "ratio": 0.21698824381702875, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 20704157, + "candidate": 1415907, + "ratio": 0.06838757066998671, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 638708, + "candidate_median": 583051, + "ratio_median": 0.9489625932350934, + "ratio_min": 0.8491289543665878, + "ratio_max": 1.0001752453000425, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 638708, + "candidate": 606110, + "ratio": 0.9489625932350934, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 562040, + "candidate": 557601, + "ratio": 0.992101985623799, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 686646, + "candidate": 583051, + "ratio": 0.8491289543665878, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 576335, + "candidate": 576436, + "ratio": 1.0001752453000425, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 682631, + "candidate": 646135, + "ratio": 0.9465362692289099, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 1046574, + "candidate_median": 1037912, + "ratio_median": 0.9949062369216128, + "ratio_min": 0.9609963992613676, + "ratio_max": 1.0018774729310875, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 1029071, + "candidate": 1007298, + "ratio": 0.9788420818388625, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1035967, + "candidate": 1037912, + "ratio": 1.0018774729310875, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1049703, + "candidate": 1049312, + "ratio": 0.9996275136872048, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1060338, + "candidate": 1018981, + "ratio": 0.9609963992613676, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1046574, + "candidate": 1041243, + "ratio": 0.9949062369216128, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1093459, + "candidate_median": 1081562, + "ratio_median": 0.9965650466073499, + "ratio_min": 0.9755836728262868, + "ratio_max": 1.0306275421907207, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1100336, + "candidate": 1081562, + "ratio": 0.9829379389568277, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1085022, + "candidate": 1081295, + "ratio": 0.9965650466073499, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1104343, + "candidate": 1077379, + "ratio": 0.9755836728262868, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1093459, + "candidate": 1096277, + "ratio": 1.00257714281011, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1088628, + "candidate": 1121970, + "ratio": 1.0306275421907207, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 1969401, + "candidate_median": 1950197, + "ratio_median": 0.9782109099149618, + "ratio_min": 0.924262430176099, + "ratio_max": 1.004771229343777, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 1906321, + "candidate": 1864784, + "ratio": 0.9782109099149618, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1954804, + "candidate": 1844890, + "ratio": 0.9437723679714181, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 2135730, + "candidate": 1973975, + "ratio": 0.924262430176099, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1969401, + "candidate": 1950197, + "ratio": 0.9902488116945203, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 2093800, + "candidate": 2103790, + "ratio": 1.004771229343777, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 2534695, + "candidate_median": 2443240, + "ratio_median": 0.9639187357847788, + "ratio_min": 0.9575252283001146, + "ratio_max": 0.9964585830733467, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 2334094, + "candidate": 2325828, + "ratio": 0.9964585830733467, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 2446596, + "candidate": 2344527, + "ratio": 0.9582812201115346, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 2593516, + "candidate": 2483357, + "ratio": 0.9575252283001146, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 2534695, + "candidate": 2443240, + "ratio": 0.9639187357847788, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 2630863, + "candidate": 2586534, + "ratio": 0.9831503958967076, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 3294492, + "candidate_median": 3123019, + "ratio_median": 1.054983497869048, + "ratio_min": 0.15299383800613256, + "ratio_max": 5.166562193416169, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 2471196, + "candidate": 2607071, + "ratio": 1.054983497869048, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 3026293, + "candidate": 15635531, + "ratio": 5.166562193416169, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 3294492, + "candidate": 2786172, + "ratio": 0.8457061058275449, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 5969640, + "candidate": 11643672, + "ratio": 1.9504814360664964, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 20412711, + "candidate": 3123019, + "ratio": 0.15299383800613256, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 1409565, + "candidate_median": 1355536, + "ratio_median": 0.9616697349891633, + "ratio_min": 0.9333843461249969, + "ratio_max": 1.026371560874911, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1249639, + "candidate": 1274340, + "ratio": 1.0197665085676744, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1345436, + "candidate": 1272529, + "ratio": 0.945811617943923, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1455844, + "candidate": 1358862, + "ratio": 0.9333843461249969, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1409565, + "candidate": 1355536, + "ratio": 0.9616697349891633, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1418384, + "candidate": 1455789, + "ratio": 1.026371560874911, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 1678857, + "candidate_median": 1656086, + "ratio_median": 0.9744403542107833, + "ratio_min": 0.9563186219932525, + "ratio_max": 1.018937345375366, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1497202, + "candidate": 1499671, + "ratio": 1.0016490760765748, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1678857, + "candidate": 1615703, + "ratio": 0.9623827401619077, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 1738287, + "candidate": 1693857, + "ratio": 0.9744403542107833, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 1625307, + "candidate": 1656086, + "ratio": 1.018937345375366, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 1782865, + "candidate": 1704987, + "ratio": 0.9563186219932525, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 2294364, + "candidate_median": 2083715, + "ratio_median": 0.9973782868551696, + "ratio_min": 0.10588602006143753, + "ratio_max": 7.4940482973884714, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 2007466, + "candidate": 2002203, + "ratio": 0.9973782868551696, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 1, + "baseline": 1983130, + "candidate": 14861672, + "ratio": 7.4940482973884714, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 2, + "baseline": 2294364, + "candidate": 2021036, + "ratio": 0.8808698183897585, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 3, + "baseline": 5621399, + "candidate": 10916619, + "ratio": 1.9419754762115267, + "n_baseline": 180, + "n_candidate": 180 + }, + { + "pair": 4, + "baseline": 19678849, + "candidate": 2083715, + "ratio": 0.10588602006143753, + "n_baseline": 180, + "n_candidate": 180 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "metric": "p50", + "pairs": 5, + "baseline_median": 548148, + "candidate_median": 561053, + "ratio_median": 0.9430153900041595, + "ratio_min": 0.8512572785263113, + "ratio_max": 1.5067548345464512, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 548148, + "candidate": 516912, + "ratio": 0.9430153900041595, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 652388, + "candidate": 579703, + "ratio": 0.888586240090253, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 493731, + "candidate": 561053, + "ratio": 1.1363536014550433, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 607781, + "candidate": 517378, + "ratio": 0.8512572785263113, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 432357, + "candidate": 651456, + "ratio": 1.5067548345464512, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "metric": "p95", + "pairs": 5, + "baseline_median": 1050448, + "candidate_median": 1026414, + "ratio_median": 0.9957108593709821, + "ratio_min": 0.9096930353717962, + "ratio_max": 1.0206370915061702, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 1062598, + "candidate": 966638, + "ratio": 0.9096930353717962, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 1050448, + "candidate": 1026414, + "ratio": 0.9771202382221681, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 1011123, + "candidate": 1015489, + "ratio": 1.0043179712062726, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 1013418, + "candidate": 1034332, + "ratio": 1.0206370915061702, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 1069445, + "candidate": 1064858, + "ratio": 0.9957108593709821, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1098649, + "candidate_median": 1074596, + "ratio_median": 0.9775997007791332, + "ratio_min": 0.9657697772445977, + "ratio_max": 8.064346548825455, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 1089856, + "candidate": 1064574, + "ratio": 0.9768024399553703, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 1138958, + "candidate": 1113445, + "ratio": 0.9775997007791332, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 1079990, + "candidate": 1074596, + "ratio": 0.9950055093102714, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 1098649, + "candidate": 1061042, + "ratio": 0.9657697772445977, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 1120604, + "candidate": 9036939, + "ratio": 8.064346548825455, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "scheduled-to-complete", + "metric": "p50", + "pairs": 5, + "baseline_median": 1067807, + "candidate_median": 1006204, + "ratio_median": 0.9423088629312226, + "ratio_min": 0.8772898447579117, + "ratio_max": 1.1952759450248822, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 959118, + "candidate": 936449, + "ratio": 0.9763647434413701, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 1119294, + "candidate": 998911, + "ratio": 0.8924473820104458, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 1067807, + "candidate": 1006204, + "ratio": 0.9423088629312226, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 1147949, + "candidate": 1007084, + "ratio": 0.8772898447579117, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 981826, + "candidate": 1173553, + "ratio": 1.1952759450248822, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "scheduled-to-complete", + "metric": "p95", + "pairs": 5, + "baseline_median": 2723492, + "candidate_median": 2715960, + "ratio_median": 0.9225304200054822, + "ratio_min": 0.8859265971774472, + "ratio_max": 1.0581203545222435, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 2697618, + "candidate": 2663211, + "ratio": 0.9872454142877161, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 2723492, + "candidate": 2412814, + "ratio": 0.8859265971774472, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 3015880, + "candidate": 2742559, + "ratio": 0.9093727204000159, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 2944033, + "candidate": 2715960, + "ratio": 0.9225304200054822, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 2609258, + "candidate": 2760909, + "ratio": 1.0581203545222435, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "scheduled-to-complete", + "metric": "maximum", + "pairs": 5, + "baseline_median": 10081103, + "candidate_median": 3450466, + "ratio_median": 0.3298210523193742, + "ratio_min": 0.012126762457759389, + "ratio_max": 3.1102124905082595, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 429751718, + "candidate": 5211497, + "ratio": 0.012126762457759389, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 3946357, + "candidate": 3031198, + "ratio": 0.7681003011131532, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 204928381, + "candidate": 3450466, + "ratio": 0.016837423802220933, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 10081103, + "candidate": 3324960, + "ratio": 0.3298210523193742, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 3213320, + "candidate": 9994108, + "ratio": 3.1102124905082595, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 445525, + "candidate_median": 376697, + "ratio_median": 0.8455125974973347, + "ratio_min": 0.7748351771147615, + "ratio_max": 0.9873648329299208, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 349105, + "candidate": 344694, + "ratio": 0.9873648329299208, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 399484, + "candidate": 336725, + "ratio": 0.8428998407946251, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 492195, + "candidate": 381370, + "ratio": 0.7748351771147615, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 445525, + "candidate": 376697, + "ratio": 0.8455125974973347, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 451643, + "candidate": 432466, + "ratio": 0.9575394725480081, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 2084379, + "candidate_median": 2099280, + "ratio_median": 1.0071488918282137, + "ratio_min": 0.8936798558786481, + "ratio_max": 1.0372771764801438, + "candidate_lower_pairs": 2, + "per_pair": [ + { + "pair": 0, + "baseline": 2084379, + "candidate": 2099280, + "ratio": 1.0071488918282137, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 2068397, + "candidate": 2145501, + "ratio": 1.0372771764801438, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 2251436, + "candidate": 2012063, + "ratio": 0.8936798558786481, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 2231244, + "candidate": 2007680, + "ratio": 0.8998029798623548, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 2074027, + "candidate": 2124880, + "ratio": 1.0245189672072736, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 10010372, + "candidate_median": 2628354, + "ratio_median": 0.26649808818293663, + "ratio_min": 0.010670717882067998, + "ratio_max": 1.007278512425518, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 429026805, + "candidate": 4578024, + "ratio": 0.010670717882067998, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 1, + "baseline": 2807399, + "candidate": 2473597, + "ratio": 0.8810991953762184, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 2, + "baseline": 204323736, + "candidate": 2628354, + "ratio": 0.012863674340801991, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 3, + "baseline": 10010372, + "candidate": 2667745, + "ratio": 0.26649808818293663, + "n_baseline": 108, + "n_candidate": 108 + }, + { + "pair": 4, + "baseline": 2609187, + "candidate": 2628178, + "ratio": 1.007278512425518, + "n_baseline": 108, + "n_candidate": 108 + } + ] + }, + { + "scenario": "retention", + "event_class": "elapsed_ns", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 11231011113, + "candidate_median": 11091143698, + "ratio_median": 0.9949129675599476, + "ratio_min": 0.9545482745195486, + "ratio_max": 1.0433789644654619, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 11231011113, + "candidate": 11091143698, + "ratio": 0.9875463203096556, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 11454102644, + "candidate": 10933493915, + "ratio": 0.9545482745195486, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 11135927413, + "candidate": 11079278589, + "ratio": 0.9949129675599476, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 11266083649, + "candidate": 11213477754, + "ratio": 0.9953305960936417, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 10941875719, + "candidate": 11416522957, + "ratio": 1.0433789644654619, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "retention", + "event_class": "elapsed_ns", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 11231011113, + "candidate_median": 11091143698, + "ratio_median": 0.9949129675599476, + "ratio_min": 0.9545482745195486, + "ratio_max": 1.0433789644654619, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 11231011113, + "candidate": 11091143698, + "ratio": 0.9875463203096556, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 11454102644, + "candidate": 10933493915, + "ratio": 0.9545482745195486, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 11135927413, + "candidate": 11079278589, + "ratio": 0.9949129675599476, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 11266083649, + "candidate": 11213477754, + "ratio": 0.9953305960936417, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 10941875719, + "candidate": 11416522957, + "ratio": 1.0433789644654619, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "retention", + "event_class": "elapsed_ns", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 11231011113, + "candidate_median": 11091143698, + "ratio_median": 0.9949129675599476, + "ratio_min": 0.9545482745195486, + "ratio_max": 1.0433789644654619, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 11231011113, + "candidate": 11091143698, + "ratio": 0.9875463203096556, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 11454102644, + "candidate": 10933493915, + "ratio": 0.9545482745195486, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 11135927413, + "candidate": 11079278589, + "ratio": 0.9949129675599476, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 11266083649, + "candidate": 11213477754, + "ratio": 0.9953305960936417, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 10941875719, + "candidate": 11416522957, + "ratio": 1.0433789644654619, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "retention-empty", + "event_class": "elapsed_ns", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 298708, + "candidate_median": 267530, + "ratio_median": 0.9039916470401395, + "ratio_min": 0.8018788915996068, + "ratio_max": 0.97655569988082, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 329556, + "candidate": 264264, + "ratio": 0.8018788915996068, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 292207, + "candidate": 261329, + "ratio": 0.8943283357345991, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 295943, + "candidate": 267530, + "ratio": 0.9039916470401395, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 298708, + "candidate": 291705, + "ratio": 0.97655569988082, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 362979, + "candidate": 342349, + "ratio": 0.9431647560878178, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "retention-empty", + "event_class": "elapsed_ns", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 298708, + "candidate_median": 267530, + "ratio_median": 0.9039916470401395, + "ratio_min": 0.8018788915996068, + "ratio_max": 0.97655569988082, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 329556, + "candidate": 264264, + "ratio": 0.8018788915996068, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 292207, + "candidate": 261329, + "ratio": 0.8943283357345991, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 295943, + "candidate": 267530, + "ratio": 0.9039916470401395, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 298708, + "candidate": 291705, + "ratio": 0.97655569988082, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 362979, + "candidate": 342349, + "ratio": 0.9431647560878178, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "retention-empty", + "event_class": "elapsed_ns", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 298708, + "candidate_median": 267530, + "ratio_median": 0.9039916470401395, + "ratio_min": 0.8018788915996068, + "ratio_max": 0.97655569988082, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 329556, + "candidate": 264264, + "ratio": 0.8018788915996068, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 292207, + "candidate": 261329, + "ratio": 0.8943283357345991, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 295943, + "candidate": 267530, + "ratio": 0.9039916470401395, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 298708, + "candidate": 291705, + "ratio": 0.97655569988082, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 362979, + "candidate": 342349, + "ratio": 0.9431647560878178, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 74223869, + "candidate_median": 85746252, + "ratio_median": 1.1552382428353338, + "ratio_min": 0.9339614828856934, + "ratio_max": 1.1878648530312808, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 85076884, + "candidate": 85555694, + "ratio": 1.005627968226951, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 1, + "baseline": 74223869, + "candidate": 85746252, + "ratio": 1.1552382428353338, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 2, + "baseline": 72466679, + "candidate": 86080621, + "ratio": 1.1878648530312808, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 3, + "baseline": 74083704, + "candidate": 87023434, + "ratio": 1.174663648027102, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 4, + "baseline": 77776595, + "candidate": 72640344, + "ratio": 0.9339614828856934, + "n_baseline": 100, + "n_candidate": 100 + } + ] + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 90017141, + "candidate_median": 96890090, + "ratio_median": 1.0748387175984926, + "ratio_min": 1.0203065547260604, + "ratio_max": 1.0824853222582296, + "candidate_lower_pairs": 0, + "per_pair": [ + { + "pair": 0, + "baseline": 90896774, + "candidate": 97699372, + "ratio": 1.0748387175984926, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 1, + "baseline": 89977581, + "candidate": 96890090, + "ratio": 1.076824792611395, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 2, + "baseline": 91204852, + "candidate": 96186277, + "ratio": 1.0546179823854107, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 3, + "baseline": 89864982, + "candidate": 97277524, + "ratio": 1.0824853222582296, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 4, + "baseline": 90017141, + "candidate": 91845079, + "ratio": 1.0203065547260604, + "n_baseline": 100, + "n_candidate": 100 + } + ] + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 94330891, + "candidate_median": 171700778, + "ratio_median": 1.8201967158351129, + "ratio_min": 0.9257597615734515, + "ratio_max": 2.0908514899462345, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 178233843, + "candidate": 165001720, + "ratio": 0.9257597615734515, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 1, + "baseline": 94330891, + "candidate": 171700778, + "ratio": 1.8201967158351129, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 2, + "baseline": 100813816, + "candidate": 183695758, + "ratio": 1.8221288042504016, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 3, + "baseline": 92370615, + "candidate": 193133238, + "ratio": 2.0908514899462345, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 4, + "baseline": 92578051, + "candidate": 96341194, + "ratio": 1.0406483281874233, + "n_baseline": 100, + "n_candidate": 100 + } + ] + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 809623, + "candidate_median": 825573, + "ratio_median": 1.026121734306027, + "ratio_min": 1.005061996676467, + "ratio_max": 1.097823507496179, + "candidate_lower_pairs": 0, + "per_pair": [ + { + "pair": 0, + "baseline": 809623, + "candidate": 887760, + "ratio": 1.096510351114037, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 1, + "baseline": 821415, + "candidate": 825573, + "ratio": 1.005061996676467, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 2, + "baseline": 785859, + "candidate": 806387, + "ratio": 1.026121734306027, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 3, + "baseline": 813281, + "candidate": 892839, + "ratio": 1.097823507496179, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 4, + "baseline": 777303, + "candidate": 792532, + "ratio": 1.0195921024362442, + "n_baseline": 100, + "n_candidate": 100 + } + ] + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 1316671, + "candidate_median": 1324465, + "ratio_median": 1.0559350057835253, + "ratio_min": 0.961860791815878, + "ratio_max": 1.216126926795275, + "candidate_lower_pairs": 1, + "per_pair": [ + { + "pair": 0, + "baseline": 1322372, + "candidate": 1554215, + "ratio": 1.1753235851938788, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 1, + "baseline": 1346803, + "candidate": 1295437, + "ratio": 0.961860791815878, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 2, + "baseline": 1265005, + "candidate": 1324465, + "ratio": 1.0470037667835306, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 3, + "baseline": 1316671, + "candidate": 1390319, + "ratio": 1.0559350057835253, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 4, + "baseline": 1018855, + "candidate": 1239057, + "ratio": 1.216126926795275, + "n_baseline": 100, + "n_candidate": 100 + } + ] + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 1737268, + "candidate_median": 1890525, + "ratio_median": 1.1429692943104752, + "ratio_min": 1.012636768432504, + "ratio_max": 2.2597100724818797, + "candidate_lower_pairs": 0, + "per_pair": [ + { + "pair": 0, + "baseline": 1866933, + "candidate": 1890525, + "ratio": 1.012636768432504, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 1, + "baseline": 1737268, + "candidate": 1810173, + "ratio": 1.0419653156565365, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 2, + "baseline": 2000500, + "candidate": 4520550, + "ratio": 2.2597100724818797, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 3, + "baseline": 1550554, + "candidate": 2616240, + "ratio": 1.687293702766882, + "n_baseline": 100, + "n_candidate": 100 + }, + { + "pair": 4, + "baseline": 1596056, + "candidate": 1824243, + "ratio": 1.1429692943104752, + "n_baseline": 100, + "n_candidate": 100 + } + ] + }, + { + "scenario": "startup", + "event_class": "first_http_ns", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 28255653365, + "candidate_median": 27836115055, + "ratio_median": 0.9891456174097587, + "ratio_min": 0.9781911000945633, + "ratio_max": 0.9963395712519217, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 27657440544, + "candidate": 27535114236, + "ratio": 0.9955770922545999, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 28255653365, + "candidate": 27639428649, + "ratio": 0.9781911000945633, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 28141574471, + "candidate": 27836115055, + "ratio": 0.9891456174097587, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 28446664084, + "candidate": 28342537097, + "ratio": 0.9963395712519217, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 28656349078, + "candidate": 28039934765, + "ratio": 0.9784894331332238, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "startup", + "event_class": "first_http_ns", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 28255653365, + "candidate_median": 27836115055, + "ratio_median": 0.9891456174097587, + "ratio_min": 0.9781911000945633, + "ratio_max": 0.9963395712519217, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 27657440544, + "candidate": 27535114236, + "ratio": 0.9955770922545999, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 28255653365, + "candidate": 27639428649, + "ratio": 0.9781911000945633, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 28141574471, + "candidate": 27836115055, + "ratio": 0.9891456174097587, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 28446664084, + "candidate": 28342537097, + "ratio": 0.9963395712519217, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 28656349078, + "candidate": 28039934765, + "ratio": 0.9784894331332238, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "startup", + "event_class": "first_http_ns", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 28255653365, + "candidate_median": 27836115055, + "ratio_median": 0.9891456174097587, + "ratio_min": 0.9781911000945633, + "ratio_max": 0.9963395712519217, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 27657440544, + "candidate": 27535114236, + "ratio": 0.9955770922545999, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 28255653365, + "candidate": 27639428649, + "ratio": 0.9781911000945633, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 28141574471, + "candidate": 27836115055, + "ratio": 0.9891456174097587, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 28446664084, + "candidate": 28342537097, + "ratio": 0.9963395712519217, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 28656349078, + "candidate": 28039934765, + "ratio": 0.9784894331332238, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "startup", + "event_class": "full_ready_ns", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 28267642321, + "candidate_median": 27847686647, + "ratio_median": 0.9891365320711744, + "ratio_min": 0.9781946750280589, + "ratio_max": 0.9963393850986463, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 27668870381, + "candidate": 27547216614, + "ratio": 0.9956032261048309, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 28267642321, + "candidate": 27651257194, + "ratio": 0.9781946750280589, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 28153531635, + "candidate": 27847686647, + "ratio": 0.9891365320711744, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 28458023804, + "candidate": 28353849938, + "ratio": 0.9963393850986463, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 28667880099, + "candidate": 28050587086, + "ratio": 0.9784674342550521, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "startup", + "event_class": "full_ready_ns", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 28267642321, + "candidate_median": 27847686647, + "ratio_median": 0.9891365320711744, + "ratio_min": 0.9781946750280589, + "ratio_max": 0.9963393850986463, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 27668870381, + "candidate": 27547216614, + "ratio": 0.9956032261048309, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 28267642321, + "candidate": 27651257194, + "ratio": 0.9781946750280589, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 28153531635, + "candidate": 27847686647, + "ratio": 0.9891365320711744, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 28458023804, + "candidate": 28353849938, + "ratio": 0.9963393850986463, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 28667880099, + "candidate": 28050587086, + "ratio": 0.9784674342550521, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "startup", + "event_class": "full_ready_ns", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 28267642321, + "candidate_median": 27847686647, + "ratio_median": 0.9891365320711744, + "ratio_min": 0.9781946750280589, + "ratio_max": 0.9963393850986463, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 27668870381, + "candidate": 27547216614, + "ratio": 0.9956032261048309, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 28267642321, + "candidate": 27651257194, + "ratio": 0.9781946750280589, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 28153531635, + "candidate": 27847686647, + "ratio": 0.9891365320711744, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 28458023804, + "candidate": 28353849938, + "ratio": 0.9963393850986463, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 28667880099, + "candidate": 28050587086, + "ratio": 0.9784674342550521, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "startup", + "event_class": "ready_ns", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 28255655850, + "candidate_median": 27836117559, + "ratio_median": 0.9891456053353236, + "ratio_min": 0.9781911122760224, + "ratio_max": 0.9963395680820688, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 27657443209, + "candidate": 27535116870, + "ratio": 0.9955770915599244, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 28255655850, + "candidate": 27639431424, + "ratio": 0.9781911122760224, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 28141577346, + "candidate": 27836117559, + "ratio": 0.9891456053353236, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 28446666769, + "candidate": 28342539682, + "ratio": 0.9963395680820688, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 28656351984, + "candidate": 28039937100, + "ratio": 0.9784894153888056, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "startup", + "event_class": "ready_ns", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 28255655850, + "candidate_median": 27836117559, + "ratio_median": 0.9891456053353236, + "ratio_min": 0.9781911122760224, + "ratio_max": 0.9963395680820688, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 27657443209, + "candidate": 27535116870, + "ratio": 0.9955770915599244, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 28255655850, + "candidate": 27639431424, + "ratio": 0.9781911122760224, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 28141577346, + "candidate": 27836117559, + "ratio": 0.9891456053353236, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 28446666769, + "candidate": 28342539682, + "ratio": 0.9963395680820688, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 28656351984, + "candidate": 28039937100, + "ratio": 0.9784894153888056, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "startup", + "event_class": "ready_ns", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 28255655850, + "candidate_median": 27836117559, + "ratio_median": 0.9891456053353236, + "ratio_min": 0.9781911122760224, + "ratio_max": 0.9963395680820688, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 27657443209, + "candidate": 27535116870, + "ratio": 0.9955770915599244, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 28255655850, + "candidate": 27639431424, + "ratio": 0.9781911122760224, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 28141577346, + "candidate": 27836117559, + "ratio": 0.9891456053353236, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 28446666769, + "candidate": 28342539682, + "ratio": 0.9963395680820688, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 28656351984, + "candidate": 28039937100, + "ratio": 0.9784894153888056, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "startup-hot1", + "event_class": "first_http_ns", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 9704414902, + "candidate_median": 9656347046, + "ratio_median": 0.9950468053473174, + "ratio_min": 0.9891035479068623, + "ratio_max": 1.000658814080343, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 9461558558, + "candidate": 9467791966, + "ratio": 1.000658814080343, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 9665671826, + "candidate": 9560350296, + "ratio": 0.9891035479068623, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 9861314925, + "candidate": 9759169495, + "ratio": 0.9896418042850407, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 9704414902, + "candidate": 9656347046, + "ratio": 0.9950468053473174, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 9771886468, + "candidate": 9775725081, + "ratio": 1.0003928221037535, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "startup-hot1", + "event_class": "first_http_ns", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 9704414902, + "candidate_median": 9656347046, + "ratio_median": 0.9950468053473174, + "ratio_min": 0.9891035479068623, + "ratio_max": 1.000658814080343, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 9461558558, + "candidate": 9467791966, + "ratio": 1.000658814080343, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 9665671826, + "candidate": 9560350296, + "ratio": 0.9891035479068623, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 9861314925, + "candidate": 9759169495, + "ratio": 0.9896418042850407, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 9704414902, + "candidate": 9656347046, + "ratio": 0.9950468053473174, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 9771886468, + "candidate": 9775725081, + "ratio": 1.0003928221037535, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "startup-hot1", + "event_class": "first_http_ns", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 9704414902, + "candidate_median": 9656347046, + "ratio_median": 0.9950468053473174, + "ratio_min": 0.9891035479068623, + "ratio_max": 1.000658814080343, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 9461558558, + "candidate": 9467791966, + "ratio": 1.000658814080343, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 9665671826, + "candidate": 9560350296, + "ratio": 0.9891035479068623, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 9861314925, + "candidate": 9759169495, + "ratio": 0.9896418042850407, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 9704414902, + "candidate": 9656347046, + "ratio": 0.9950468053473174, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 9771886468, + "candidate": 9775725081, + "ratio": 1.0003928221037535, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "startup-hot1", + "event_class": "full_ready_ns", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 21378797474, + "candidate_median": 21006491908, + "ratio_median": 0.9817185891089658, + "ratio_min": 0.9682400932950227, + "ratio_max": 1.0052476468760272, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 20849118583, + "candidate": 20958527395, + "ratio": 1.0052476468760272, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 21378797474, + "candidate": 20762767605, + "ratio": 0.9711850084295345, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 21695540242, + "candidate": 21006491908, + "ratio": 0.9682400932950227, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 21148167403, + "candidate": 21064304842, + "ratio": 0.9960345234931277, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 21519225641, + "candidate": 21125823835, + "ratio": 0.9817185891089658, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "startup-hot1", + "event_class": "full_ready_ns", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 21378797474, + "candidate_median": 21006491908, + "ratio_median": 0.9817185891089658, + "ratio_min": 0.9682400932950227, + "ratio_max": 1.0052476468760272, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 20849118583, + "candidate": 20958527395, + "ratio": 1.0052476468760272, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 21378797474, + "candidate": 20762767605, + "ratio": 0.9711850084295345, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 21695540242, + "candidate": 21006491908, + "ratio": 0.9682400932950227, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 21148167403, + "candidate": 21064304842, + "ratio": 0.9960345234931277, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 21519225641, + "candidate": 21125823835, + "ratio": 0.9817185891089658, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "startup-hot1", + "event_class": "full_ready_ns", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 21378797474, + "candidate_median": 21006491908, + "ratio_median": 0.9817185891089658, + "ratio_min": 0.9682400932950227, + "ratio_max": 1.0052476468760272, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 20849118583, + "candidate": 20958527395, + "ratio": 1.0052476468760272, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 21378797474, + "candidate": 20762767605, + "ratio": 0.9711850084295345, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 21695540242, + "candidate": 21006491908, + "ratio": 0.9682400932950227, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 21148167403, + "candidate": 21064304842, + "ratio": 0.9960345234931277, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 21519225641, + "candidate": 21125823835, + "ratio": 0.9817185891089658, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "startup-hot1", + "event_class": "ready_ns", + "timing": "service", + "metric": "p50", + "pairs": 5, + "baseline_median": 9704418058, + "candidate_median": 9656349861, + "ratio_median": 0.9950467718195246, + "ratio_min": 0.9891035005432799, + "ratio_max": 1.0006587969535428, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 9461561664, + "candidate": 9467794912, + "ratio": 1.0006587969535428, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 9665674781, + "candidate": 9560352761, + "ratio": 0.9891035005432799, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 9861318161, + "candidate": 9759172351, + "ratio": 0.9896417691496893, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 9704418058, + "candidate": 9656349861, + "ratio": 0.9950467718195246, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 9771889173, + "candidate": 9775727886, + "ratio": 1.0003928322284503, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "startup-hot1", + "event_class": "ready_ns", + "timing": "service", + "metric": "p95", + "pairs": 5, + "baseline_median": 9704418058, + "candidate_median": 9656349861, + "ratio_median": 0.9950467718195246, + "ratio_min": 0.9891035005432799, + "ratio_max": 1.0006587969535428, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 9461561664, + "candidate": 9467794912, + "ratio": 1.0006587969535428, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 9665674781, + "candidate": 9560352761, + "ratio": 0.9891035005432799, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 9861318161, + "candidate": 9759172351, + "ratio": 0.9896417691496893, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 9704418058, + "candidate": 9656349861, + "ratio": 0.9950467718195246, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 9771889173, + "candidate": 9775727886, + "ratio": 1.0003928322284503, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "startup-hot1", + "event_class": "ready_ns", + "timing": "service", + "metric": "maximum", + "pairs": 5, + "baseline_median": 9704418058, + "candidate_median": 9656349861, + "ratio_median": 0.9950467718195246, + "ratio_min": 0.9891035005432799, + "ratio_max": 1.0006587969535428, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 9461561664, + "candidate": 9467794912, + "ratio": 1.0006587969535428, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 1, + "baseline": 9665674781, + "candidate": 9560352761, + "ratio": 0.9891035005432799, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 2, + "baseline": 9861318161, + "candidate": 9759172351, + "ratio": 0.9896417691496893, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 3, + "baseline": 9704418058, + "candidate": 9656349861, + "ratio": 0.9950467718195246, + "n_baseline": 1, + "n_candidate": 1 + }, + { + "pair": 4, + "baseline": 9771889173, + "candidate": 9775727886, + "ratio": 1.0003928322284503, + "n_baseline": 1, + "n_candidate": 1 + } + ] + }, + { + "scenario": "websocket", + "event_class": "new-transmission-sentinels", + "timing": "signed-return-to-receipt", + "metric": "p50", + "pairs": 5, + "baseline_median": 523762740, + "candidate_median": 521439945, + "ratio_median": 0.985949706568562, + "ratio_min": 0.9641930141880654, + "ratio_max": 1.0245282572910732, + "candidate_lower_pairs": 3, + "per_pair": [ + { + "pair": 0, + "baseline": 510088221, + "candidate": 516026499, + "ratio": 1.0116416685497234, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 1, + "baseline": 523762740, + "candidate": 505008375, + "ratio": 0.9641930141880654, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 2, + "baseline": 528870734, + "candidate": 521439945, + "ratio": 0.985949706568562, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 3, + "baseline": 511386759, + "candidate": 523930185, + "ratio": 1.0245282572910732, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 4, + "baseline": 538352698, + "candidate": 527266581, + "ratio": 0.9794073345574652, + "n_baseline": 4500, + "n_candidate": 4500 + } + ] + }, + { + "scenario": "websocket", + "event_class": "new-transmission-sentinels", + "timing": "signed-return-to-receipt", + "metric": "p95", + "pairs": 5, + "baseline_median": 990307917, + "candidate_median": 981719053, + "ratio_median": 0.9888310232313234, + "ratio_min": 0.982959494786089, + "ratio_max": 0.9936505564951524, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 990307917, + "candidate": 977503251, + "ratio": 0.9870700155172041, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 1, + "baseline": 983182587, + "candidate": 966428659, + "ratio": 0.982959494786089, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 2, + "baseline": 987992254, + "candidate": 981719053, + "ratio": 0.9936505564951524, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 3, + "baseline": 990803658, + "candidate": 983327138, + "ratio": 0.9924540851866738, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 4, + "baseline": 997528532, + "candidate": 986387159, + "ratio": 0.9888310232313234, + "n_baseline": 4500, + "n_candidate": 4500 + } + ] + }, + { + "scenario": "websocket", + "event_class": "new-transmission-sentinels", + "timing": "signed-return-to-receipt", + "metric": "p99", + "pairs": 5, + "baseline_median": 1976480272, + "candidate_median": 1317398773, + "ratio_median": 0.5421633481464663, + "ratio_min": 0.4463719675072346, + "ratio_max": 0.9698826721006142, + "candidate_lower_pairs": 5, + "per_pair": [ + { + "pair": 0, + "baseline": 1955465923, + "candidate": 1060181952, + "ratio": 0.5421633481464663, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 1, + "baseline": 2461600967, + "candidate": 1317398773, + "ratio": 0.5351796617977198, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 2, + "baseline": 2336691972, + "candidate": 1043033793, + "ratio": 0.4463719675072346, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 3, + "baseline": 1976480272, + "candidate": 1713094447, + "ratio": 0.8667399676428442, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 4, + "baseline": 1867980260, + "candidate": 1811721686, + "ratio": 0.9698826721006142, + "n_baseline": 4500, + "n_candidate": 4500 + } + ] + }, + { + "scenario": "websocket", + "event_class": "new-transmission-sentinels", + "timing": "signed-return-to-receipt", + "metric": "maximum", + "pairs": 5, + "baseline_median": 2385220598, + "candidate_median": 2253294167, + "ratio_median": 0.9816000306632391, + "ratio_min": 0.8213070089377117, + "ratio_max": 1.3065180017706548, + "candidate_lower_pairs": 4, + "per_pair": [ + { + "pair": 0, + "baseline": 2002829112, + "candidate": 1994489223, + "ratio": 0.99583594578787, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 1, + "baseline": 2520224397, + "candidate": 2253294167, + "ratio": 0.8940847369314631, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 2, + "baseline": 2385220598, + "candidate": 1958998395, + "ratio": 0.8213070089377117, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 3, + "baseline": 2026507903, + "candidate": 2647669056, + "ratio": 1.3065180017706548, + "n_baseline": 4500, + "n_candidate": 4500 + }, + { + "pair": 4, + "baseline": 2781545668, + "candidate": 2730365313, + "ratio": 0.9816000306632391, + "n_baseline": 4500, + "n_candidate": 4500 + } + ] + } + ], + "http_body_comparison": [ + { + "pair": 0, + "by_class": { + "nodes_50": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "nodes_2000": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "nodes_region": { + "requests": 288, + "equal": 0, + "different": 288, + "class_changed": 0 + }, + "channels": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "messages_0": { + "requests": 240, + "equal": 229, + "different": 11, + "class_changed": 0 + }, + "messages_1000": { + "requests": 240, + "equal": 229, + "different": 11, + "class_changed": 0 + }, + "observers": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "metrics_dense_24h": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "metrics_sparse_7d": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "paths": { + "requests": 240, + "equal": 153, + "different": 87, + "class_changed": 0 + }, + "reach": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "neighbors": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "rx_coverage": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "packets_memory": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "packet_detail_memory": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "packet_detail_sql": { + "requests": 195, + "equal": 195, + "different": 0, + "class_changed": 0 + }, + "analytics_rf": { + "requests": 144, + "equal": 144, + "different": 0, + "class_changed": 0 + }, + "analytics_topology": { + "requests": 144, + "equal": 0, + "different": 144, + "class_changed": 0 + }, + "analytics_channels": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "analytics_rf_filtered": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "stats": { + "requests": 144, + "equal": 0, + "different": 144, + "class_changed": 0 + }, + "healthz": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "packet_detail_pruned": { + "requests": 45, + "equal": 45, + "different": 0, + "class_changed": 0 + } + } + }, + { + "pair": 1, + "by_class": { + "nodes_50": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "nodes_2000": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "nodes_region": { + "requests": 288, + "equal": 0, + "different": 288, + "class_changed": 0 + }, + "channels": { + "requests": 240, + "equal": 179, + "different": 61, + "class_changed": 0 + }, + "messages_0": { + "requests": 240, + "equal": 218, + "different": 22, + "class_changed": 0 + }, + "messages_1000": { + "requests": 240, + "equal": 218, + "different": 22, + "class_changed": 0 + }, + "observers": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "metrics_dense_24h": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "metrics_sparse_7d": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "paths": { + "requests": 240, + "equal": 157, + "different": 83, + "class_changed": 0 + }, + "reach": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "neighbors": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "rx_coverage": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "packets_memory": { + "requests": 240, + "equal": 110, + "different": 130, + "class_changed": 0 + }, + "packet_detail_memory": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "packet_detail_sql": { + "requests": 195, + "equal": 195, + "different": 0, + "class_changed": 0 + }, + "analytics_rf": { + "requests": 144, + "equal": 144, + "different": 0, + "class_changed": 0 + }, + "analytics_topology": { + "requests": 144, + "equal": 0, + "different": 144, + "class_changed": 0 + }, + "analytics_channels": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "analytics_rf_filtered": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "stats": { + "requests": 144, + "equal": 0, + "different": 144, + "class_changed": 0 + }, + "healthz": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "packet_detail_pruned": { + "requests": 45, + "equal": 45, + "different": 0, + "class_changed": 0 + } + } + }, + { + "pair": 2, + "by_class": { + "nodes_50": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "nodes_2000": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "nodes_region": { + "requests": 288, + "equal": 0, + "different": 288, + "class_changed": 0 + }, + "channels": { + "requests": 240, + "equal": 179, + "different": 61, + "class_changed": 0 + }, + "messages_0": { + "requests": 240, + "equal": 229, + "different": 11, + "class_changed": 0 + }, + "messages_1000": { + "requests": 240, + "equal": 229, + "different": 11, + "class_changed": 0 + }, + "observers": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "metrics_dense_24h": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "metrics_sparse_7d": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "paths": { + "requests": 240, + "equal": 153, + "different": 87, + "class_changed": 0 + }, + "reach": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "neighbors": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "rx_coverage": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "packets_memory": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "packet_detail_memory": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "packet_detail_sql": { + "requests": 195, + "equal": 195, + "different": 0, + "class_changed": 0 + }, + "analytics_rf": { + "requests": 144, + "equal": 144, + "different": 0, + "class_changed": 0 + }, + "analytics_topology": { + "requests": 144, + "equal": 0, + "different": 144, + "class_changed": 0 + }, + "analytics_channels": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "analytics_rf_filtered": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "stats": { + "requests": 144, + "equal": 0, + "different": 144, + "class_changed": 0 + }, + "healthz": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "packet_detail_pruned": { + "requests": 45, + "equal": 45, + "different": 0, + "class_changed": 0 + } + } + }, + { + "pair": 3, + "by_class": { + "nodes_50": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "nodes_2000": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "nodes_region": { + "requests": 288, + "equal": 0, + "different": 288, + "class_changed": 0 + }, + "channels": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "messages_0": { + "requests": 240, + "equal": 229, + "different": 11, + "class_changed": 0 + }, + "messages_1000": { + "requests": 240, + "equal": 229, + "different": 11, + "class_changed": 0 + }, + "observers": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "metrics_dense_24h": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "metrics_sparse_7d": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "paths": { + "requests": 240, + "equal": 147, + "different": 93, + "class_changed": 0 + }, + "reach": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "neighbors": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "rx_coverage": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "packets_memory": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "packet_detail_memory": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "packet_detail_sql": { + "requests": 195, + "equal": 195, + "different": 0, + "class_changed": 0 + }, + "analytics_rf": { + "requests": 144, + "equal": 144, + "different": 0, + "class_changed": 0 + }, + "analytics_topology": { + "requests": 144, + "equal": 0, + "different": 144, + "class_changed": 0 + }, + "analytics_channels": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "analytics_rf_filtered": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "stats": { + "requests": 144, + "equal": 0, + "different": 144, + "class_changed": 0 + }, + "healthz": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "packet_detail_pruned": { + "requests": 45, + "equal": 45, + "different": 0, + "class_changed": 0 + } + } + }, + { + "pair": 4, + "by_class": { + "nodes_50": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "nodes_2000": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "nodes_region": { + "requests": 288, + "equal": 0, + "different": 288, + "class_changed": 0 + }, + "channels": { + "requests": 240, + "equal": 118, + "different": 122, + "class_changed": 0 + }, + "messages_0": { + "requests": 240, + "equal": 218, + "different": 22, + "class_changed": 0 + }, + "messages_1000": { + "requests": 240, + "equal": 229, + "different": 11, + "class_changed": 0 + }, + "observers": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "metrics_dense_24h": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "metrics_sparse_7d": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "paths": { + "requests": 240, + "equal": 149, + "different": 91, + "class_changed": 0 + }, + "reach": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "neighbors": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "rx_coverage": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "packets_memory": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "packet_detail_memory": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "packet_detail_sql": { + "requests": 195, + "equal": 195, + "different": 0, + "class_changed": 0 + }, + "analytics_rf": { + "requests": 144, + "equal": 144, + "different": 0, + "class_changed": 0 + }, + "analytics_topology": { + "requests": 144, + "equal": 0, + "different": 144, + "class_changed": 0 + }, + "analytics_channels": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "analytics_rf_filtered": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "stats": { + "requests": 144, + "equal": 0, + "different": 144, + "class_changed": 0 + }, + "healthz": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "packet_detail_pruned": { + "requests": 45, + "equal": 45, + "different": 0, + "class_changed": 0 + } + } + } + ], + "raw_file_sha256": { + "corpus/corpus.json": "8fd0bdf64b0cd7aecafae748e6f07c88a4fe18fd584afdc1919d2bea0908a601", + "corpus/validation.json": "a5597ddde5206c1ead87abdd567c8cc6902ccd913191189592a976bfbb4636e5", + "manifest.json": "7fee9718e977a66b314c41ef62400d9107ff2a18e012730fd6e027edf66b0bd4", + "pair-00/baseline/client-validation.json": "37b2e20e25e49f257bd63877607d0cebbc84fa6f8458af408ab50d986c3476f3", + "pair-00/baseline/database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-00/baseline/db-pool.json": "352cd37df26b35c1b459ca25fb02233d18c2b30b326017b17d8188869c1e1101", + "pair-00/baseline/handler.json": "f92d60961eece2d6749a621f72d506aa6fbdeeaba2a2e85ab68f471bcd28d619", + "pair-00/baseline/http-workload.json": "ea45a789e3d5989a8ba6496f82b7dc7afe3a8b84745b9ca0b56be4e712e6feec", + "pair-00/baseline/ingest-validation.json": "de3b0391798f69bee831dcf0e92ab03aee739a49b756b1090409122938764cec", + "pair-00/baseline/ingest.jsonl": "17fccdcb9db1890e91a679a2d0d11127d7c5336e412e18fed3c05e4ee969d31f", + "pair-00/baseline/plans/advert-preservation.json": "6f6b6d6e558851be2940d740effcff5a35830ea2eb5d3bfb348cf4830c884f87", + "pair-00/baseline/plans/retention-empty.json": "97d48c39e5bed367e07a8961acdd0a67859b6404c9d8c0fd3ebdee96def72635", + "pair-00/baseline/processes.csv": "ff56903c8b7003d343f3d7b2a9628bd4f776174487e132466e130ac3e0265b24", + "pair-00/baseline/requests.jsonl": "96e8952912aef597e9e97531209b99ff163c4af9262381455427fe111899542a", + "pair-00/baseline/resource-final.json": "914db5f21d3c2a4a8ebdfc3dc8489d033c6baee5eaf52df822b423992a67d754", + "pair-00/baseline/resources.csv": "42ce6bdde3a983bc329a80394772634f45318a7b19993f1981c2b5a629d50832", + "pair-00/baseline/retention-empty.json": "cc6eda5c8fc33c2406464705eb3c5e4a4a59194b6f971f30f2654130c469ff7d", + "pair-00/baseline/retention.json": "7ef2aa7df0e5f2fb2b6542c026c2fc617c3f1c859c9925a9161ef7497c132b31", + "pair-00/baseline/sql-queries.jsonl": "7645d9493e6cce9e954cbc3c7654554f3a2501ba9cbb86cdbc81f0236f942985", + "pair-00/baseline/startup-hot1.json": "bec3d30c2f642d00fa6391375eb58d4da81200670b896bcabcf5f7a1c790d3c5", + "pair-00/baseline/startup.json": "d0e199fa53e494edcaeb28c67ca9b62ba624725610f060ce549bb6b57669f1b6", + "pair-00/baseline/storage-setup-validation.json": "e06d0f535427a02a9703f9c539d33245df491e15bca0ffdeac3af9885d4dd746", + "pair-00/baseline/validation-after-comparable.json": "1dba641cd05ae341ee1f48dc31be704051bdff5ee0ab431aac140c1272966445", + "pair-00/baseline/validation-after.json": "1dba641cd05ae341ee1f48dc31be704051bdff5ee0ab431aac140c1272966445", + "pair-00/baseline/validation-before.json": "6cc868116a0e1a10e7ad190354a06cb99089a89fd3a9cee64dad83391bec4119", + "pair-00/baseline/visibility.json": "ee09eb4f91e53f71f878d0722590c8a1a233b397689e8fe420415f5bcbe003cd", + "pair-00/baseline/websocket.jsonl": "19ae808f38f490408f06780c5e226859e95387c34944e2e17b856fc8af923ee2", + "pair-00/candidate/client-validation.json": "1e5c89e530667af7fae415d021574726d7dfcab68ddf75719813eefec35d7d3a", + "pair-00/candidate/database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-00/candidate/db-pool.json": "84b44f520adc2546fa05a3d4b11211e71c172c88ddaa774a8febca65419f2886", + "pair-00/candidate/handler.json": "598c19c6bbb9658be48a6a045534a050185521fe2cd6c6ad2c0ca3e0d8f3c21e", + "pair-00/candidate/http-workload.json": "ea45a789e3d5989a8ba6496f82b7dc7afe3a8b84745b9ca0b56be4e712e6feec", + "pair-00/candidate/ingest-validation.json": "eaa64e30815dc4a57472f04dd4ff7029112ce50bfe5ee6c69550a0b53ac6c846", + "pair-00/candidate/ingest.jsonl": "15fceec42fae99725bd2c3190141e2bfcc3abb8a556717b870dda8c4e0c40239", + "pair-00/candidate/plans/advert-preservation.json": "bc3ac435bb3562609141d58ca85d2bb8f7671834deb2f2779b3b8cd414acf89d", + "pair-00/candidate/plans/retention-empty.json": "7043b9c2eed48ae01bb879d8fd5d946408f7ee71fd5f3587a737c70dc36f327f", + "pair-00/candidate/processes.csv": "3be9063bb9b3b81635a9f3640d52da8805743fe8e92240336a6b90d1e9545356", + "pair-00/candidate/requests.jsonl": "7351d0947a2160f06d8287ab1a7eb55a1c6ae081400c286b22faf1f5527c3da5", + "pair-00/candidate/resource-final.json": "d6e2139ee9bc84eb20a9701f297e96f7c44d401ee04bb6ca9ec2ba31a1faa3a8", + "pair-00/candidate/resources.csv": "319fe5a7b0343a77117968b214a295ef4453f3aaa5996f118c5b04fcf6ff5abf", + "pair-00/candidate/retention-empty.json": "8d507d426f2fc1607ba65500c21e6b19c999df66dde4ca6aef8c07c6c9787c5e", + "pair-00/candidate/retention.json": "56ee66daa131b1d7a859f86fdd4097d4bfbfbdfa4ef6c1255d3e53d603e185eb", + "pair-00/candidate/sql-queries.jsonl": "81098e01bc847fa10f70761a0b35687103d05d706f11ce8204351e434e0b4239", + "pair-00/candidate/startup-hot1.json": "d95ead96b7e5f7df6bc0e3764726a3222a6ed61b35fb369c462c692925ef4eb7", + "pair-00/candidate/startup.json": "1d9bfbea7a3eb9b1de3224e0abdda2754f77ea7a5e13e514150b8a44dcf2ddfa", + "pair-00/candidate/storage-setup-validation.json": "294d4eb6af256d6e5d7cf5b6911a596ecc60fb92c1d97974a50545d3f8cf3650", + "pair-00/candidate/validation-after-comparable.json": "1dba641cd05ae341ee1f48dc31be704051bdff5ee0ab431aac140c1272966445", + "pair-00/candidate/validation-after.json": "a2dcd17ee34af81e07ec78349cc2816d515654fd3855077561f9b94a2f8c0c99", + "pair-00/candidate/validation-before.json": "6cc868116a0e1a10e7ad190354a06cb99089a89fd3a9cee64dad83391bec4119", + "pair-00/candidate/visibility.json": "1147e56c5e5e4c9eeb454adf1e919f12577dbdd0e1f98e8468a865ec9636e852", + "pair-00/candidate/websocket.jsonl": "1144e24d02a7e6c0fc10972d305f89dad4517807e360962e2fbfaba3352dcd7e", + "pair-00/corpus-offset.json": "64cf4659e1c6227339e1b6178f176762466d492c8face69f57981092d2c0b999", + "pair-00/events.json": "01da852870ffb22212464b3c1a276444d8b0b7d9f7942e3f2d435bc9f01f2a5e", + "pair-01/baseline/client-validation.json": "b38f7d11d36902f8c80c9e9134604e8cf45cf563fc56fbc2d0f6b0f4d6636062", + "pair-01/baseline/database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-01/baseline/db-pool.json": "0062907febcc52799015ffcbe6310ccaf000b0bcecf26459168408534e068812", + "pair-01/baseline/handler.json": "04c7810cded9cc08af0f83ef502f9ff30c586dc39dd2e65d054a2d3cd95577ff", + "pair-01/baseline/http-workload.json": "c4bccb11e5a5d176f3042f64663b549852b99e1847363cfe7e521de8ff4c5a1b", + "pair-01/baseline/ingest-validation.json": "252a0b12892df959a4f003409141aa80556f4db0c5d88a9aa9338e13d5e34b53", + "pair-01/baseline/ingest.jsonl": "a8096c7bc59080ed791d332a59cdda7035d292fe174ad23b0757ab03536d0ad9", + "pair-01/baseline/plans/advert-preservation.json": "6f6b6d6e558851be2940d740effcff5a35830ea2eb5d3bfb348cf4830c884f87", + "pair-01/baseline/plans/retention-empty.json": "b0c63e56243c04d92da85199ca6443410a4055fe8b04dc8472e5bb53266a2d43", + "pair-01/baseline/processes.csv": "81d316156d79bccec566b9c0fdf73279a9745822ecc31dbdefba7ce4884c696a", + "pair-01/baseline/requests.jsonl": "589b7209d721612487be51452b1fe484eb6e69c07248d38199cadad4ee46b5fa", + "pair-01/baseline/resource-final.json": "b367528aa1d303a990892c9ba1cf6f66e392a044536c5b057c89514d8e8e0ef5", + "pair-01/baseline/resources.csv": "14fbcb23f66f4aabce6c92990276e8de36ed4bad2ee54f35001456aa7ce61372", + "pair-01/baseline/retention-empty.json": "4d68f8e9195018a15e0d2c40e80065b84f9bebcbe11642f2dc9085f20d9f6922", + "pair-01/baseline/retention.json": "e16e2defbaace471d356e3f6cdd7155ba142814eaffb3012d6c8bf9c9a5e188b", + "pair-01/baseline/sql-queries.jsonl": "e181da4941ff51d6e1c9e0cb31cee57e016c5aec46068d9c9804f8859e884b23", + "pair-01/baseline/startup-hot1.json": "6a4115c78d087bf4f08d3ecf099476bc0b511c740e668a99dd6b05d2de7f66b4", + "pair-01/baseline/startup.json": "ddcc012781e2ac41e4e488c0b6896b91a8ebecb7c2428b1e6c7ccbc5f0524ff0", + "pair-01/baseline/storage-setup-validation.json": "15711698812db5449d6f85d4c10b83e9ee40196099eee058c93c8386ba9df40d", + "pair-01/baseline/validation-after-comparable.json": "f6f216bafc655d250a388ea3c3ff1dd414ee13a9221aeeb014fc00d642cfe9c2", + "pair-01/baseline/validation-after.json": "f6f216bafc655d250a388ea3c3ff1dd414ee13a9221aeeb014fc00d642cfe9c2", + "pair-01/baseline/validation-before.json": "6f4a0dfb9e32045f002d1fab1d1ed10966d33f33608a1618a7131c28bf5e297a", + "pair-01/baseline/visibility.json": "e48a369bb9826e663b73c45c6e7495fc8d5fe411dc24eef56ce189a9d340bde1", + "pair-01/baseline/websocket.jsonl": "07579143b53afcffded067daac92322bdc5a8d4d6f606a430d829700fec0a3b9", + "pair-01/candidate/client-validation.json": "614d7f0cbedcf1f3d791c5d8342384831f2b10f8e1b690ca02ccf6ef4c9423f6", + "pair-01/candidate/database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-01/candidate/db-pool.json": "b7734f317f27443589ca5520e80bc23fbb5242881260f972456a763116747c0f", + "pair-01/candidate/handler.json": "043cdc3134600639657ea6133843462f5a0af257ce64806e5fb7579d97f71378", + "pair-01/candidate/http-workload.json": "c4bccb11e5a5d176f3042f64663b549852b99e1847363cfe7e521de8ff4c5a1b", + "pair-01/candidate/ingest-validation.json": "f2b00cca357c63c9e29203df1ac54a971d17dd311d7ee0cde50ded0d3c5bfbb2", + "pair-01/candidate/ingest.jsonl": "d10ce36e7cd9e1f4c837cb7644ae2720f8c8ce78c048924b91e200c273973243", + "pair-01/candidate/plans/advert-preservation.json": "bc3ac435bb3562609141d58ca85d2bb8f7671834deb2f2779b3b8cd414acf89d", + "pair-01/candidate/plans/retention-empty.json": "a6985050f9e38f2e35f043f747e534b1574bb84e1aac21e9e4dc29df0e9b1f10", + "pair-01/candidate/processes.csv": "e869551422870f2ad89b48d72f7420143dbae6783406bfaea60698bf5016d2da", + "pair-01/candidate/requests.jsonl": "125f5487e1c401deb9cf06c60a52c5c8d781f4332d1bb8594bfe861069b53427", + "pair-01/candidate/resource-final.json": "553eb6a883e1f3c66b9fe8756cf82871066c605dadd2ff9b3bd70c361f63a017", + "pair-01/candidate/resources.csv": "913dd1d9e24d67936d4c1d6c0e56d7dd34ed97aa849da2830050a62ac8b72b5f", + "pair-01/candidate/retention-empty.json": "1e04b082d4f2fe0800b1195ad9a2a92cc5d902afdb81b4cefeaec6fbecade093", + "pair-01/candidate/retention.json": "8b94889476dd7bf2acfb40ccc46f85d0c305c5e35731f515295c22eeea1d69d1", + "pair-01/candidate/sql-queries.jsonl": "6779b6681c88b22ad8eddd4391aed0ac795d30ca3bd372a267860c765c71ebf9", + "pair-01/candidate/startup-hot1.json": "31d0ce8f3d8a3086c2467565c658025c859d1f710ad2bf66307fc96a230cca76", + "pair-01/candidate/startup.json": "3d8d1321cf86393cfc1b4ec31eb4fdecf27f9c07b9237cdfa9f74bee5be0ec12", + "pair-01/candidate/storage-setup-validation.json": "67ce11d01ea28d275edc65f568dfa5b3b9344dc6514d0e1121de78156d0f39ac", + "pair-01/candidate/validation-after-comparable.json": "f6f216bafc655d250a388ea3c3ff1dd414ee13a9221aeeb014fc00d642cfe9c2", + "pair-01/candidate/validation-after.json": "c89ae7a8c29e9f1c38c97cb39bee9c998af681bc3ee46bae439809d15f71eba0", + "pair-01/candidate/validation-before.json": "6f4a0dfb9e32045f002d1fab1d1ed10966d33f33608a1618a7131c28bf5e297a", + "pair-01/candidate/visibility.json": "fb16ce0fd16a9a5e7fd040cde25bed9ce48eae4d26e644da061b942226969045", + "pair-01/candidate/websocket.jsonl": "a4454ceee3c61db52d3d5a7498300088a4c8fbf03d76e5a6998506ad28625485", + "pair-01/corpus-offset.json": "489dff6981c5c7342c86b7ba58cd61fd6fc0fefd0a0a2c668289f8d764d7bf21", + "pair-01/events.json": "bee323fc7c38ed2d9c19295832b7f9137e2a13e940105b665370c8f7bbcd0d70", + "pair-02/baseline/client-validation.json": "74c05bf6cfb604da020525cb0d3982ef06837c532166a7bcde8d8b757ca1d8e9", + "pair-02/baseline/database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-02/baseline/db-pool.json": "362ce3721f4db11a6a0f53ddfb6e83ebc91bdcedcd5eeede0c06121166678e96", + "pair-02/baseline/handler.json": "b181ecd5ee68c69395759539cc7a4f7b0b9a4ebc743c1d465af5a87624a1b58f", + "pair-02/baseline/http-workload.json": "716dcd93681aa1a003e31b3bb9083fd3c510bab64d335cf64fa350c50cb2ba14", + "pair-02/baseline/ingest-validation.json": "516c7cd52f2e9db8c46d72bb735b0d4819bf4c9890788629a5520126db1e9555", + "pair-02/baseline/ingest.jsonl": "00db687927bebc6e3ae350c543475f5626a6ae4be41ac45c13a6e4c0be50b366", + "pair-02/baseline/plans/advert-preservation.json": "6f6b6d6e558851be2940d740effcff5a35830ea2eb5d3bfb348cf4830c884f87", + "pair-02/baseline/plans/retention-empty.json": "3bca6ecf625a30f49a8ba60f851cea1148aa477023b77cada3220b2256868f16", + "pair-02/baseline/processes.csv": "37552ec2ee88135024e55766ccc9b809d9dd941b94eb8bc1aa1e2add104e57af", + "pair-02/baseline/requests.jsonl": "84abbb53ff6a677fae836203bb98823b7adaf7c4fa22af05d9a5971d02731bb9", + "pair-02/baseline/resource-final.json": "30ec41b76fa14fe51712fa6965ec4c55b4f89b5d30aa20903296c0b8f77e90a6", + "pair-02/baseline/resources.csv": "7037f2a74c2e1a6551cdc423b820d2be83a12b1eaa4cbcb57f473402003eda7c", + "pair-02/baseline/retention-empty.json": "cbb46d010ddff1e02a642dd2360743d9646bd39bedc04c358ba00bab670429a4", + "pair-02/baseline/retention.json": "976641a5cd72f807a713a5fcc48f02c26ec8adc80f09f3bc599a2bd27c984e1a", + "pair-02/baseline/sql-queries.jsonl": "20ae430883acd8fcd42475a7e63a83bf540ae36b8a945809a155a6bcc830e22d", + "pair-02/baseline/startup-hot1.json": "d58c464d61e5b5a78f5ca15492f499bcc3e87b19ee39e2b9ac836ac24706f90c", + "pair-02/baseline/startup.json": "8032b4c5eb3a754c9c9145c6a510bd21a0b93ce865f4e9992798c97fe94132a7", + "pair-02/baseline/storage-setup-validation.json": "2b4846aea3114a49eafd452e1295ae1860e31a84eacfa613bc277e80a083c2b3", + "pair-02/baseline/validation-after-comparable.json": "d090cc38116737a94f0d7d69abba730c85391d711600a20c016babca7ea2f68f", + "pair-02/baseline/validation-after.json": "d090cc38116737a94f0d7d69abba730c85391d711600a20c016babca7ea2f68f", + "pair-02/baseline/validation-before.json": "ce1993d8ce411a25b9bd70ac31005532054a393edcc47943e56967e022cf2d67", + "pair-02/baseline/visibility.json": "9cc5a1db5d1f655b93115f25c328097ef5fd73e04f660f4e2278629956f40308", + "pair-02/baseline/websocket.jsonl": "5afcda9373d56fc21390ca80708ed0bdf43bb802f610f80854fa30676cba63c9", + "pair-02/candidate/client-validation.json": "61a38c1c7679fa2d0b037d66784e138ea3140af89ef10a72a97ec05e44b43837", + "pair-02/candidate/database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-02/candidate/db-pool.json": "94a4f6cf8d13f63acf7a74f5795eba076827eed941bb9c28d175f1499e5c8954", + "pair-02/candidate/handler.json": "633b3b8a2509adc0f32566f185bcc96a00ac76c8ef094c327c709e96388f5040", + "pair-02/candidate/http-workload.json": "716dcd93681aa1a003e31b3bb9083fd3c510bab64d335cf64fa350c50cb2ba14", + "pair-02/candidate/ingest-validation.json": "2486d11bbf4e39eee8c2d232071cbca306a7bc4b03d2c547c4d82017012a6864", + "pair-02/candidate/ingest.jsonl": "6b1b249441ab8b1fa758a74d83a5adb423c41cb4d7886e5c231ee33160f575a1", + "pair-02/candidate/plans/advert-preservation.json": "bc3ac435bb3562609141d58ca85d2bb8f7671834deb2f2779b3b8cd414acf89d", + "pair-02/candidate/plans/retention-empty.json": "f104ad5ea1cb465bbe8e165b7579826e8b3438c2559524256f4e2fd96f523c0f", + "pair-02/candidate/processes.csv": "489b396fc42e83f1000de5759f1fc17e045ac2048caf48cc8acc9e44fb4a2b6e", + "pair-02/candidate/requests.jsonl": "9c4b5e577ad5344377bfe14d0dacf784d422b9a1efe68a3f44ae7e2c3c278524", + "pair-02/candidate/resource-final.json": "a7e1b07aed2bcc4ab63d983d7610de2491f914677bfeedb759938f4bddbabc8c", + "pair-02/candidate/resources.csv": "3419fe99bdb333cfcf427b10bc769a4f967d81e664c29e95f3d7f97605834c61", + "pair-02/candidate/retention-empty.json": "cbbae274df23e78d4008c7b495f760c81889d3e5d91c0ce15cffb8c6021d0930", + "pair-02/candidate/retention.json": "cfa217f75b5d1357a65a9260fd3bb6da7d77534296f8f81faad2b27a90914b1d", + "pair-02/candidate/sql-queries.jsonl": "b71656dd11bd836937c5a977063e5ecf8fe70994c5cfb2d08b979a8d646d80e7", + "pair-02/candidate/startup-hot1.json": "7b52bdd554acd9978a3aea644d854d29da6a5d06c964e4a739cab958d1640634", + "pair-02/candidate/startup.json": "cdf8be30294d0cd1def01d4dfff6f46ce80e8975c50cd80ef053a7330f1aa9e7", + "pair-02/candidate/storage-setup-validation.json": "d14bc74d20427cfd71c365afe21805af7998d4547e98df8566c57d27f54d41ba", + "pair-02/candidate/validation-after-comparable.json": "d090cc38116737a94f0d7d69abba730c85391d711600a20c016babca7ea2f68f", + "pair-02/candidate/validation-after.json": "7d21f30da57ae935408dd3a48483e732b930b7081cbd2b8e933e730fa74dbf2b", + "pair-02/candidate/validation-before.json": "ce1993d8ce411a25b9bd70ac31005532054a393edcc47943e56967e022cf2d67", + "pair-02/candidate/visibility.json": "eeccfb35d93868e4b47108098f429e36ef44ad059fe70dc3604830fdb473725a", + "pair-02/candidate/websocket.jsonl": "29e49496ce385b025d2ccf46828d5f3c3b10d0699dfc49e9c2e9fba8fa462cd3", + "pair-02/corpus-offset.json": "6bc397ff60df68dcc2a92ec91278a81a11946913bb4633389e26a2ce1e524a95", + "pair-02/events.json": "aa181153a284d7981f5ba6c67898c26a2f423e57035ae3fa6e190d768d0a682c", + "pair-03/baseline/client-validation.json": "3627d4f82b66831491893d714df7e5e99a7f62a56d195eef05bfaf7cc9b7e94a", + "pair-03/baseline/database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-03/baseline/db-pool.json": "ec650ec907f88da98b2695cf9cb01b237881d440151abd9177a211fd909198f5", + "pair-03/baseline/handler.json": "21abc78e0b227ebe045e927fda6dc7527822ecdb3ee6eed44748e8ce59cdfda7", + "pair-03/baseline/http-workload.json": "e8d4c3c8fd0ea5ebf51a919cc4dd09e4effb8155aab7b898119b80bd1c24b610", + "pair-03/baseline/ingest-validation.json": "287b78b0971615556d7613e5e5221ebedb18ec3efa69a92461c1f1a7404cdbba", + "pair-03/baseline/ingest.jsonl": "7b621cd26062171343f65bf0f1abfd21d5ed4eae046afd6d52d8870d3c681b60", + "pair-03/baseline/plans/advert-preservation.json": "6f6b6d6e558851be2940d740effcff5a35830ea2eb5d3bfb348cf4830c884f87", + "pair-03/baseline/plans/retention-empty.json": "05717a2a2e1b47c6b454ffe376bd872d1fb904984e8dace1dfce30cf25444474", + "pair-03/baseline/processes.csv": "8898a7284dac367c64011178080161ebbeda4615c681888c7ffe729d91e2cac4", + "pair-03/baseline/requests.jsonl": "cfec67611d399fb3bfbeccd5a5814320bc847539c0be4716ce436e1c566a8326", + "pair-03/baseline/resource-final.json": "2f69ed73b1d4161a2d8ccaa1d72cf8aa50b7bce57f264144730503622965bf40", + "pair-03/baseline/resources.csv": "79f0eef97ffac3e0851f86e7c7f01e252afe2ad78a062ff221deacc2496d2588", + "pair-03/baseline/retention-empty.json": "497e4bb2de9b74239dc6f4a262630249b23ccfdb3b574c2fa69f6fbafd5135b9", + "pair-03/baseline/retention.json": "6c77c481b36772c7df55c2c54085b50ceb3fb907667f8be8cb6db986a940e050", + "pair-03/baseline/sql-queries.jsonl": "8f43a9cf24a2b3ef61a84464faa69f65aaa698c1a1574845f7e9dd00acdabf58", + "pair-03/baseline/startup-hot1.json": "6b85469a403763f4f48b17d61460641534ce541630e82c9a02bdb268085faa8a", + "pair-03/baseline/startup.json": "6412cfef1ae7f005762631844631fbebe8837917f663e7157e26aa8127f5e875", + "pair-03/baseline/storage-setup-validation.json": "efacc4af7c63baa9b86d3b0225701485025b1386be850efebf6a79f8a0690474", + "pair-03/baseline/validation-after-comparable.json": "0a010ecfb0922e6a7533151a26988e301dfaa959e8fd4ce24f85268ae92e2dff", + "pair-03/baseline/validation-after.json": "0a010ecfb0922e6a7533151a26988e301dfaa959e8fd4ce24f85268ae92e2dff", + "pair-03/baseline/validation-before.json": "7b9e017c1d3ad2f40fdfc724a3c7728f583dda0ce3bfa34f9710c486e48081dc", + "pair-03/baseline/visibility.json": "048523df2f341c6ff71b3481befa80d70c7485820d9f83500554120479643c92", + "pair-03/baseline/websocket.jsonl": "0548d4e7d35a1ce2463895b69f4d6aa453e8c8d82da9d76275aefa540dad49f9", + "pair-03/candidate/client-validation.json": "eb8213a6875c123794b7e36e7046f798f3311900f62ea3600b03f474ec477520", + "pair-03/candidate/database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-03/candidate/db-pool.json": "4f858157e364686659dda8cca27735eb26e9a0000743f5d65421b3edbbaceb9a", + "pair-03/candidate/handler.json": "8b0097c0a81fe320f97786ec799a2667dcceeb830da7d327fad06af7168ef265", + "pair-03/candidate/http-workload.json": "e8d4c3c8fd0ea5ebf51a919cc4dd09e4effb8155aab7b898119b80bd1c24b610", + "pair-03/candidate/ingest-validation.json": "98a19bcbd3ab83d271055e97aa21327f4e460c78dbcdc9c1fd97e20197fc5344", + "pair-03/candidate/ingest.jsonl": "a22e592001c68d7b2420eb85425ac3581bab4ee1d8e5504c138de74f233d8b34", + "pair-03/candidate/plans/advert-preservation.json": "bc3ac435bb3562609141d58ca85d2bb8f7671834deb2f2779b3b8cd414acf89d", + "pair-03/candidate/plans/retention-empty.json": "907df85c0528781c7bd7186fc51ccf0e0a96f704fe5b4367fa93782713abc3a1", + "pair-03/candidate/processes.csv": "95e2d24d701e3574b2b5c0f566a660af7cd4102b734ae5b943ca4443e08572c8", + "pair-03/candidate/requests.jsonl": "0b93e4f88c537473776e09bed33184b33dd132ae7a8119878f59154aba516e54", + "pair-03/candidate/resource-final.json": "3712453a48ce7c36d9d5bd2bda096505186c07c7babb8b1f97385e650b7d8206", + "pair-03/candidate/resources.csv": "ab28ac48f39da0bc978555f460e1c9f3f37b87676f6f1451b216994cd1bddbad", + "pair-03/candidate/retention-empty.json": "c81292bc18f0ecd5dac75466be88ee072516edf9bb4f3404fe622044b9349822", + "pair-03/candidate/retention.json": "6a276eefd3d57d451be835d03ac63e86bcf868b0e8c368e30f79ec46b3e00aba", + "pair-03/candidate/sql-queries.jsonl": "bc2b8e548414decab54a2772268467f192d8b587debd61dbcfc9d7334143c5d7", + "pair-03/candidate/startup-hot1.json": "3b2598f5cc11d1fe7bd8163a0343d9173323e5c34adfc267d4944a36b36fc5fc", + "pair-03/candidate/startup.json": "01f84a50cd6dd0c7621f957a4a1bbf7265ace187daac404e0e9681ebf0f37de8", + "pair-03/candidate/storage-setup-validation.json": "c41e637693eab91e085e2cc469a4ecefcd240c052d908d82b6df827e2ab298d1", + "pair-03/candidate/validation-after-comparable.json": "0a010ecfb0922e6a7533151a26988e301dfaa959e8fd4ce24f85268ae92e2dff", + "pair-03/candidate/validation-after.json": "2accd200a7b8d55e1c72e5fd5472bb027264fa9e377de4f722ad8d4d84dfcd09", + "pair-03/candidate/validation-before.json": "7b9e017c1d3ad2f40fdfc724a3c7728f583dda0ce3bfa34f9710c486e48081dc", + "pair-03/candidate/visibility.json": "5dd3bc866cb1a0fde8db01d200dd5d2e39a47011f0d0820779dcc6b3756d7dc4", + "pair-03/candidate/websocket.jsonl": "4c4ce02e75059445336c59ff6286fc4fac5684f75355ae179250263a5b27c91b", + "pair-03/corpus-offset.json": "be4921a3ea59bf52bff5b9b4eb9ae249659fb23af5433be5f44674eb7156f2df", + "pair-03/events.json": "7d8edb84c6c59eefe4b69396d1f2d8ac975192de38d587a7d1c932ac0d629927", + "pair-04/baseline/client-validation.json": "bf2d083200d0179432c4edfcb423d0a850f73d2429fbb02b5942a37eb6196692", + "pair-04/baseline/database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-04/baseline/db-pool.json": "60569bdadba161ce6b67044eed448a49c2a6459b69e4a2bb760c856f350a47cd", + "pair-04/baseline/handler.json": "8ef20749292b261f20dd0a6e402a6f98d5c979f480bd01a1a215a706ad80a4fd", + "pair-04/baseline/http-workload.json": "9fe0e4abf52b668a1315d38840b71d6031c8a1d455ab1e8ce3cf86c954a722f1", + "pair-04/baseline/ingest-validation.json": "3ec8d10fff5bf7805a20391837ca6c3f6c4fa9d9aeae93aed549a5c47377158b", + "pair-04/baseline/ingest.jsonl": "1221e48a52d79beadb5b316c6fdca3fcab3d23f52c5a3ce043b886add4fddd63", + "pair-04/baseline/plans/advert-preservation.json": "6f6b6d6e558851be2940d740effcff5a35830ea2eb5d3bfb348cf4830c884f87", + "pair-04/baseline/plans/retention-empty.json": "8134944c15d3008c82dc427a54c248a19fab7ff625aeb3ddc8ea37c055c5b0c3", + "pair-04/baseline/processes.csv": "fcd8cf45d7fa100d506fd223ec8f4abf64bfeb62509d35c0637b2edb43520c34", + "pair-04/baseline/requests.jsonl": "ac735de732e167fda28c3696115c811fd4f3a142df48929173de61b69235f260", + "pair-04/baseline/resource-final.json": "f6d46c136cebe51c63f35793524dee814de7b5995baa3a56e14676af93aeb038", + "pair-04/baseline/resources.csv": "8737e0b78db02f468c64d830a6e3d43846a13997e2ea20f8d6b43c7915f61500", + "pair-04/baseline/retention-empty.json": "4c265feb296f333b292679cecc8a15f535f669c25da7457304b7987ba71234aa", + "pair-04/baseline/retention.json": "63d7b346064e008b13b7173cb5263c1e11ece46056a18249bb19b3b0a15869a9", + "pair-04/baseline/sql-queries.jsonl": "2452f9d6c26ef94bef67e10fb7524afe3764ab895363782121869c37cb56b911", + "pair-04/baseline/startup-hot1.json": "c678946ace890d88ebafb89a33ea63f94717b33ad53f450c42df2f14dcccbbec", + "pair-04/baseline/startup.json": "883a1ebbf029a76e82482efb209a1442bc1f8910f0e8a1032bf36abba551c4bd", + "pair-04/baseline/storage-setup-validation.json": "4ec5680ad098869728effe606f11ce2f5b1aa0dea9f8e5a5dbb85051bc35f472", + "pair-04/baseline/validation-after-comparable.json": "d4f2dea1e2ddd015e3810abcaa08d55dc26157954e8d88ec1eae661f82d3fdc5", + "pair-04/baseline/validation-after.json": "d4f2dea1e2ddd015e3810abcaa08d55dc26157954e8d88ec1eae661f82d3fdc5", + "pair-04/baseline/validation-before.json": "b174da776398eda4cfc2e5a864508b163c176af5c4d980ed2adfe6bec7e656f9", + "pair-04/baseline/visibility.json": "a651e386c1000273061fa456657b760c3f5246e5e931cdeba223feb9025bdce1", + "pair-04/baseline/websocket.jsonl": "bc466d1bc188ccb3acf3c679ab63fe86361fd2efeec617efcad668d7a10a0c73", + "pair-04/candidate/client-validation.json": "5156770941d09f456e5a535d30580c0ddb5ae47c0209616c4f90726e53cd6d84", + "pair-04/candidate/database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-04/candidate/db-pool.json": "2bfd9fffb7522ecb8fa3f75674e4bcd82d07c3ba5e5c0ca1af19cdef97c18213", + "pair-04/candidate/handler.json": "a755229d3345dac32888f461015f2c391b932ea54d981e707daa06c6cd7835f7", + "pair-04/candidate/http-workload.json": "9fe0e4abf52b668a1315d38840b71d6031c8a1d455ab1e8ce3cf86c954a722f1", + "pair-04/candidate/ingest-validation.json": "54d47c34175697b1f68b3b54fb8225009163d07b86007533cf45da851dc3bcdb", + "pair-04/candidate/ingest.jsonl": "d21bd18837932466c1c0546aaf93f2fa57158ecbf2e23fcea4b2b0983ef9be78", + "pair-04/candidate/plans/advert-preservation.json": "bc3ac435bb3562609141d58ca85d2bb8f7671834deb2f2779b3b8cd414acf89d", + "pair-04/candidate/plans/retention-empty.json": "8cc68c01c962b590744bde957f62a9e742e092089bef95788e0587641e4e1564", + "pair-04/candidate/processes.csv": "0f0010afbedb0f408d7bfcc91bb0c9b6733b1999a90273df3ae5cc6fd4c8ec26", + "pair-04/candidate/requests.jsonl": "b48ff26cf4d17475d948e08d94805aaecea0919178abaf901ea5491a28e0f669", + "pair-04/candidate/resource-final.json": "a1c8533d30677bf57801e02c30e13b0c5d0ce7e0e83f4cf751c329708c122d09", + "pair-04/candidate/resources.csv": "c974322f1741a821a8de3e9ba85d19a970fcb920df0221557b6f3ba08f8b379d", + "pair-04/candidate/retention-empty.json": "02c5a5948236136dd457f9541b9340a0bc3403dbf8f0dad7095bf58da1740e7d", + "pair-04/candidate/retention.json": "90fe85e13b7bd40a48f0d19dac292c693382067dec64ebd304ff9701ddb953ea", + "pair-04/candidate/sql-queries.jsonl": "fbf81d38644cb08b9b5be9a7890b6ce401c365d3eff3bfc998db809e720fa065", + "pair-04/candidate/startup-hot1.json": "c3edc6242aed6d6336404694b8a1aea703e9a95dd38228d7bedc8c528d4855e9", + "pair-04/candidate/startup.json": "0deca8d39f697c03c81e1a3e597e295107c7e57afe2a0ae6dece2ba6451cc898", + "pair-04/candidate/storage-setup-validation.json": "c463e018908965646656e90cfd6cba42b7515d4f56c51bd6567da204bef1cc1a", + "pair-04/candidate/validation-after-comparable.json": "d4f2dea1e2ddd015e3810abcaa08d55dc26157954e8d88ec1eae661f82d3fdc5", + "pair-04/candidate/validation-after.json": "cc9518ca06ab18dca1a9d22a92e28d4d4916024ab8fb02f58e79ab5f4be18101", + "pair-04/candidate/validation-before.json": "b174da776398eda4cfc2e5a864508b163c176af5c4d980ed2adfe6bec7e656f9", + "pair-04/candidate/visibility.json": "c79f64f0c475c03596edbb274693c7645d597b27d5290ea65e1d7c18ae27de28", + "pair-04/candidate/websocket.jsonl": "0c8b02d658e969c12e2489ceea33852d54ed2ba0bdc0ae4dad0ac7944f62c89d", + "pair-04/corpus-offset.json": "c702f5d2de6085c4245322717a6b6ae6e2ed39fa6139af6decdc48187a2b9f2c", + "pair-04/events.json": "7b135201e6841b2d4d3fd708808f58ac8a4a1736db245c8e996d4b262b0d694f", + "paired-summary.json": "bc2acd310fcb14e1bcd60cb8e82b9c2c07b1970939d843c8d0c5cedad2320174", + "report.md": "5f83ef1371b341b03a7d844f169a7333a162e1e2a4c92e65875ed94ef29b5021", + "resource-preflight.json": "16e6006274973fdff80b18b2afcd4a3ebb02de9fad49588ce88e77bdc443218e", + "summary-resources.csv": "a3e202445451a507425fa650c1416adb1be19e18266b2041f914739c1356341f", + "summary.csv": "d2427bcb23af066624b459ea256dadd510e56cd303559d6885b36cf0d82dfedb" + }, + "average_cpu_cores": { + "baseline_median": 0.3064791847458758, + "candidate_median": 0.29935137929978206, + "ratio_median": 0.9837170737251879, + "per_pair": [ + { + "pair": 0, + "baseline": 0.29689026798636786, + "candidate": 0.29516870607531864, + "ratio": 0.9942013528340772 + }, + { + "pair": 1, + "baseline": 0.3071436059294101, + "candidate": 0.2931456562580985, + "ratio": 0.9544253912466969 + }, + { + "pair": 2, + "baseline": 0.30651638285295957, + "candidate": 0.29978845698557277, + "ratio": 0.9780503547485281 + }, + { + "pair": 3, + "baseline": 0.30430637761138335, + "candidate": 0.29935137929978206, + "ratio": 0.9837170737251879 + }, + { + "pair": 4, + "baseline": 0.3064791847458758, + "candidate": 0.3126241242527696, + "ratio": 1.0200501039311658 + } + ] + } + }, + "postgres_comparison": { + "status": "eligible B public evidence validated", + "analysis_reference": "c03a2585eef3819cf06640911b2f7308d99339db", + "source_bundle": "corescope-bench-37985643046-1/public", + "provenance": { + "baseline_sha": "3e21b179ecccae005aa5f6f45bc095353c94b343", + "candidate_sha": "c03a2585eef3819cf06640911b2f7308d99339db", + "harness_sha256": "7c234ba61c93351688d7068c9d8405f7d2906558d797854738eb0ff059f35d84", + "binary_sha256": { + "migrate": "646d8cee19bbddfa3b64d7ef141be14a44fad0f0b9454c4476f82741467dde67", + "postgres-ingestor.test": "3f2387a9249f4f52610a2d64cc7726144259a6079e5b69cbd858e5e7710cc50f", + "postgres-server": "6e81b75e618b9e7014d7827a0ec6e37d5caa692820d57b7bb2463b99eb09dfb5", + "postgres-server.test": "568e4b5281ade1aaacbe1a789c5ab9e92d10e8edaa5d407fa3a4c688500d3836", + "sqlite-ingestor.test": "d46994f03eb6ec013c364a2e895dd32fb0faf5153c61b8344ecd1d4214e5e7dd", + "sqlite-server": "80edb7b741613974dd88dbec21f756fabf63cf83d5b9bf365b2eb306c0d84f30", + "sqlite-server.test": "c9b9b4171173193f3007a0a4badc6d4a8ef030080dbcb0b1b6d2c4cea36866da" + }, + "canonical_file_sha256": "30ab98f2725a04508033948c1a2f8e0f482c3ec46215524445b0e177e95c46e2", + "firmware_reference": "a366955cb2f67b8e6842d4f00d2b6a554dddd88a", + "host": { + "architecture": "x86_64", + "cpus": 4, + "kernel": "6.17.0-1022-azure", + "ram_bytes": 16766414848, + "system": "Linux" + }, + "go_version": "go version go1.27.2 linux/amd64", + "postgres_version": "postgres (PostgreSQL) 18.6 (Ubuntu 18.6-1.pgdg24.04+2)", + "compiler": "cc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0", + "resource_budget": { + "allowed_cpus": [ + 0, + 1, + 2, + 3 + ], + "cpu_limit": 3.0, + "enforced": true, + "inactive_charge_reset_bytes": 5177344, + "memory_limit_bytes": 6442450944, + "memory_peak_reset": true, + "resource_counters": [ + "cgroup.procs", + "memory.current", + "memory.events", + "cpu.stat", + "io.stat" + ] + }, + "workload": { + "http_concurrency": 64, + "http_rate": 20, + "ingest_rate": 50, + "ingestor_memory_mib": 512, + "measured_seconds": 180, + "packet_store_mib": 2048, + "queue_limit": 1024, + "retention_hours": 168, + "server_memory_mib": 3072, + "sql_reader_pool": 4, + "sql_writer_pool": 1, + "warmup_seconds": 60 + }, + "cache_policy": { + "mixed": "warmup then application-warm measurement", + "sql": "channel application cache explicitly cleared; OS cache warm", + "startup": "new application process after full validation; OS cache warm" + }, + "shape": { + "days": 8, + "nodes": 2000, + "observations": 2048000, + "observers": 128, + "transmissions": 128000 + }, + "order": [ + [ + 0, + "sqlite" + ], + [ + 0, + "postgres" + ], + [ + 1, + "postgres" + ], + [ + 1, + "sqlite" + ], + [ + 2, + "sqlite" + ], + [ + 2, + "postgres" + ], + [ + 3, + "postgres" + ], + [ + 3, + "sqlite" + ], + [ + 4, + "sqlite" + ], + [ + 4, + "postgres" + ] + ] + }, + "input_sha256": { + "corpus\\validation.json": "982c43fdfa3c99b6cdfe82940d86c95fa011a97253864efc927547d74c21dcc1", + "manifest.json": "b7404cd41e2fc7d9b6d054ec0fc791b5daa14775e9d06a38e45fdbf31c5549f4", + "pair-00\\corpus-offset.json": "ae02f6e7f13c8294050860ff815a0c86da81766076559dadafe4f1d27e87e824", + "pair-00\\events.json": "e5366abc060a7a482f7621b9b2e02c50fce40fecec5885536ce48ca689ebe0a0", + "pair-00\\postgres\\client-validation.json": "7512ff388ff6bfefab44334d67c479f169e2cee95736145d3f50a46f8d397f64", + "pair-00\\postgres\\database-settings.json": "999ed1c72e72ba5c038102ccb63b21014bb009c78c16cdf7b4b036ab34a09664", + "pair-00\\postgres\\handler.json": "b5e5f1a57bdab646401f26233438ed82620efc1e87f3c555aca9b75c07738c6e", + "pair-00\\postgres\\ingest-validation.json": "778e8698ce0a6a5e35115fe04be83e5f2d43feec741ce00dfc65ad2163b186fe", + "pair-00\\postgres\\ingest.jsonl": "f810161f5a3dab854a6adbdc821ae6d0bd9a1e9fa6525545f6c3cae2cc9f8242", + "pair-00\\postgres\\migration.json": "459423383c88da65af431c7998783c3af797f8f9ada3a6b4249cee15ac483601", + "pair-00\\postgres\\requests.jsonl": "5b402922e164e51120ae2ff81703082012e7b4f0b8c387a7b0807c6a6a87808f", + "pair-00\\postgres\\resource-final.json": "b44406984880fd0a46ea8356dc9576efdd47f0c5d918951380cf87a080c9a8ed", + "pair-00\\postgres\\resources.csv": "4ede9d032a0b7b6cd18958c906d618a2c197a701ae5d107c43daa4b93e4f5055", + "pair-00\\postgres\\retention-empty.json": "3e1bab4285abffffcec28af2236591281a29ba0e527081a4d99a71a24b111650", + "pair-00\\postgres\\retention.json": "131eee10d5bfba57e84a067a17f85f642f5460363bf96df54960dba251e39863", + "pair-00\\postgres\\sql-queries.jsonl": "5dbf437ab1a9183d4ef70bbed26d05887cb6081a5db96211dda0c2ce115703c3", + "pair-00\\postgres\\startup-hot1.json": "d8fbdaf8125c841bd158eeb862e523a30b24a2f7a76de934ed40ff3e293ea121", + "pair-00\\postgres\\startup.json": "caa1e68dbf7485dd52bd18fb18d00acc123f907a865be564536e63710f225d6c", + "pair-00\\postgres\\validation-after.json": "4e23bff7b3d04a44bed3f1989ff8be8152a0201931af0bfc062dd1e68677eba8", + "pair-00\\postgres\\validation-before.json": "e6f8053a0e6f0efd8a201a530501877cc6f5e6b4a4cf0ac8ed1653bf4bc6754c", + "pair-00\\postgres\\visibility.json": "ccb73e5a797f60e91de7d8e6ba0ca76f8a8920200a6e52f32e25b2d5773b8b75", + "pair-00\\postgres\\websocket.jsonl": "dea51113de1355e850718800c40d8b93a9ce5e610536504aeeaa9e3643226b3a", + "pair-00\\sqlite\\client-validation.json": "0a09c0cc54f2e89daab23a80bb6b9219049848b662cc02cb2fcd45fc6613f34a", + "pair-00\\sqlite\\database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-00\\sqlite\\handler.json": "83882eccf4253ae57841185073a7ac1a7f51d8767e4ba464c4125579e0b5de36", + "pair-00\\sqlite\\ingest-validation.json": "8437e3e393ac27bdfad88c2a767dd5b5bf2a38df92312fe98d3f7fa12c425670", + "pair-00\\sqlite\\ingest.jsonl": "899909b0d81172bb3079f3dd02c607fe0b8701de815f336e08d98fd984147dbd", + "pair-00\\sqlite\\requests.jsonl": "799aadc7bdd0dfbae4735d822dfa21778c2e731ae1824c1fc392c3b9b2cc6022", + "pair-00\\sqlite\\resource-final.json": "840d8c64ea6c23131a517183f13139ae1503778ec72fa6bff47cdfb5d0cb2937", + "pair-00\\sqlite\\resources.csv": "1ad895ce5baee1af7785dc9e504d4437403ade59e5aa6faa81da1a038a8bbc93", + "pair-00\\sqlite\\retention-empty.json": "3f3a0015e80d3cec0b4c70255d497478e8a8670ae92dec7c7a3a17f38f24d164", + "pair-00\\sqlite\\retention.json": "98d9f0d60303d688991c4f590355897774d1094835862b9ba8459943139bf472", + "pair-00\\sqlite\\sql-queries.jsonl": "f2c3ead8c8b9b15b57e6bc9912dd96fa5330cf00298d532ac45df0a3f42fc16f", + "pair-00\\sqlite\\startup-hot1.json": "a01e43a567b6692c886cb55629370c11bb2212d96afd7187c3f22054bf6ff525", + "pair-00\\sqlite\\startup.json": "2388b1304cfd3586f4142c98e68c6273b785e742105dabaf3988beb4823d6b12", + "pair-00\\sqlite\\validation-after.json": "4e23bff7b3d04a44bed3f1989ff8be8152a0201931af0bfc062dd1e68677eba8", + "pair-00\\sqlite\\validation-before.json": "e6f8053a0e6f0efd8a201a530501877cc6f5e6b4a4cf0ac8ed1653bf4bc6754c", + "pair-00\\sqlite\\visibility.json": "0bbf0a9aeafb8da9a176dca870aa26b3ccc139634c8fdb42ef1373253af73d68", + "pair-00\\sqlite\\websocket.jsonl": "5823611d6e3aba964176c15b805cbb9750a9d32b9b0d19933e3fb7e62b8b6eaa", + "pair-01\\corpus-offset.json": "12158a10e58c978253c5e05d15ec088ebcda1809476f28f646e4144f61b519ba", + "pair-01\\events.json": "5c028b88f59de3037503da4cea91f9cbbcb425760a8cbfc92f2a60e53cdeab8e", + "pair-01\\postgres\\client-validation.json": "f176febf0ca3833427622275f9e8a114b54c0d7cd6179d2a5b0c5c7c4cd82ff1", + "pair-01\\postgres\\database-settings.json": "999ed1c72e72ba5c038102ccb63b21014bb009c78c16cdf7b4b036ab34a09664", + "pair-01\\postgres\\handler.json": "ad656602af944ac117c161fcb2021a319c9cec8b4ec6c2114479e1b86255403d", + "pair-01\\postgres\\ingest-validation.json": "29eea1f9d36483386b55fd7518b15d4e1003b1fd9a40a28d0d3778224c5f7626", + "pair-01\\postgres\\ingest.jsonl": "61636b5a47df447e6911a6f525b4376a8f25db9ed2d8596145094eb723d5ff15", + "pair-01\\postgres\\migration.json": "ff68a60b277d7548bd760ae55e9a131b01f727d6203b912f817cc82130e793cb", + "pair-01\\postgres\\requests.jsonl": "76943e80317e96b6b7ba9cc557cba284d5dd4a47ed820d9c927cddfecbc66f08", + "pair-01\\postgres\\resource-final.json": "8c32b2886a1cfea0ade89f7daa425a21a2a98058eb98d93c06ecbdcc05efe961", + "pair-01\\postgres\\resources.csv": "d5f7d7dde906894d56e1aa3688f9a4940d51ca116140164c0b71cab8a93bd1cb", + "pair-01\\postgres\\retention-empty.json": "39d373f08a7d3cafef6f49f8b614edd45d189b7da81425e3da99841460ae5469", + "pair-01\\postgres\\retention.json": "7e9272d8bf484782879267a74afe3ffec5e62d6c10d6916ef12e9cfa5fd299e7", + "pair-01\\postgres\\sql-queries.jsonl": "b4d6beed30b1ebb4b897060b7e033b66a01188a8c053382e9928e8b6abd19221", + "pair-01\\postgres\\startup-hot1.json": "f3844d4e6ba0bb544cd39a22c2504ec831ca5ca08d18e71fe73212a004430e7e", + "pair-01\\postgres\\startup.json": "c3f7a613ba9a450b0398606dc180818cdc3790a35cdd52e2d09f6644bad3982a", + "pair-01\\postgres\\validation-after.json": "2bb797ad4c707e171be54aa6c37764eb58f97b212599531a42717bedeb769a2a", + "pair-01\\postgres\\validation-before.json": "877d32930f005f80bf357e500af0f1b5eb573b08ae23565d62ac9810200268aa", + "pair-01\\postgres\\visibility.json": "74e8d57fe43b23d6be361395bd39ba3b6edee5033252af8df5ebe2b0bb78c9c6", + "pair-01\\postgres\\websocket.jsonl": "05323e934243f7637e3ca134891e7a15a78ce6d3b96add5c00cc85e398706cda", + "pair-01\\sqlite\\client-validation.json": "3793c728b53da9435f2aab8f811b8f847cbeed539620ab9a17c87d93bc0184af", + "pair-01\\sqlite\\database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-01\\sqlite\\handler.json": "d2f982026bc3db8eaa29fd1f347e9131a5eb02b99eaa2bc40c1a4195c0a41162", + "pair-01\\sqlite\\ingest-validation.json": "f219194a1db9e760236d526314cbeef8f86dbff3060183bcfe347c53d3415b6a", + "pair-01\\sqlite\\ingest.jsonl": "3414e9a2dce5a71d67ce49dae3cba3207f1afe0adb3b29ce2dcc4481a381d14a", + "pair-01\\sqlite\\requests.jsonl": "c97c96124e4e43e671ff5225621b7486f508b5545803099f191d66cbbda97229", + "pair-01\\sqlite\\resource-final.json": "6430b202ff14f7c2cead013a312611c9b340d0739efbc921aa273cb0ef35b87a", + "pair-01\\sqlite\\resources.csv": "6b761e6e872ca2ee95b4a8c6c9d966d9e33365aa520bc2f4ad0904cbc0127d1a", + "pair-01\\sqlite\\retention-empty.json": "64aefd505f541fe6fa132188a1d756ea9021822e093114eeee4d0196dc054fa0", + "pair-01\\sqlite\\retention.json": "8612195a68ac99d2d0c305ea69a690e5e18708fde63de304d1419d7996570362", + "pair-01\\sqlite\\sql-queries.jsonl": "06b248cf4d09efdee62e29ad85812d6741979cdb376679bf5c9365631bcfdfbe", + "pair-01\\sqlite\\startup-hot1.json": "8ecafaf365c6f31930375164872e82ec6d2d76ffbed7e42815d98b655d91391c", + "pair-01\\sqlite\\startup.json": "e035409cf284a58aa429e40e2d89a429eb2b780d9d4dd2fe6f578eecd97f4c84", + "pair-01\\sqlite\\validation-after.json": "2bb797ad4c707e171be54aa6c37764eb58f97b212599531a42717bedeb769a2a", + "pair-01\\sqlite\\validation-before.json": "877d32930f005f80bf357e500af0f1b5eb573b08ae23565d62ac9810200268aa", + "pair-01\\sqlite\\visibility.json": "71cafccb8f0b67004e6ad4f21ae63be5b871e46321f78cefc9f7252388bd5fb6", + "pair-01\\sqlite\\websocket.jsonl": "04c2c4f32123081b5abc10ffdd11b00bf4baebc2355714472e682251f967ae97", + "pair-02\\corpus-offset.json": "d141dbf92c207e20123a02272e73071b18ef55ea92bda861d5594f52715a2f7e", + "pair-02\\events.json": "ffb9ca481c7c31c21e32b3fc7f47de94596e20c7bf54fee8f7e7550224c20f70", + "pair-02\\postgres\\client-validation.json": "42480f3c2ffc0e46ecc09110e38dcb93be1aa8d7ddb11de5174ef48b110402ec", + "pair-02\\postgres\\database-settings.json": "999ed1c72e72ba5c038102ccb63b21014bb009c78c16cdf7b4b036ab34a09664", + "pair-02\\postgres\\handler.json": "1a46b8ecb1a7adfff7e36150c7618f2d6d2693a2f23162cd444e0be4b6b4a2b5", + "pair-02\\postgres\\ingest-validation.json": "a115ec64cfc5aea32fceff86f066c33a437706fcd1ef5a863472732172111998", + "pair-02\\postgres\\ingest.jsonl": "15ce0d04b7ecba6a3b387767668454ad948600c20685928f4e8065922d3caca6", + "pair-02\\postgres\\migration.json": "d24bc643f34f314145588b6a41c79864d92a206eda3d3f5eff58fa88298bc6a4", + "pair-02\\postgres\\requests.jsonl": "4e4ccd7ba4386761e1656fde9450e50434455f857353d220f0ccbcc112a34dc3", + "pair-02\\postgres\\resource-final.json": "e0dd9c15e7a5cf55d50e98625fbdda70bb566f3ad5e834697709bcd48f08c94b", + "pair-02\\postgres\\resources.csv": "1a51dbfcc7e61dfefaa6ec74eb66491288f3c42c51969032fe1c491a0006ba4c", + "pair-02\\postgres\\retention-empty.json": "986ce7bfd7fe8d877ad1c6e8507dccca622226d8b575fca14fbbb1fb2a7e6b21", + "pair-02\\postgres\\retention.json": "3a6de0893a496d806966aa5b9e8cbf28fd3fc70188f8feddbb3e200fbf51a0b3", + "pair-02\\postgres\\sql-queries.jsonl": "d94a0c6189f13448e8dde342e0b45f3be2ced96bc46f5dc077ce536d59965ef6", + "pair-02\\postgres\\startup-hot1.json": "d9c9df611dfac41ff8f100e3ca58efa422b2acac5478d87a3eea9b97d3a25c8d", + "pair-02\\postgres\\startup.json": "f8e1bcb3fcb0b296955964034796c48f19ea56e2f51c9b4ec2f640eeac5a4e05", + "pair-02\\postgres\\validation-after.json": "f61176cb63616b0a5bfd2a627919edd4b1c8168d9d4d008754fbc1cb1643387a", + "pair-02\\postgres\\validation-before.json": "108cfbd317d2f83bc57bac1d55b67a65629332fb57164e512ec852902873e729", + "pair-02\\postgres\\visibility.json": "b16a9a041bbd39221ba4aa7a26d01e7400e54120b45a7bff5c494cb2494ed96b", + "pair-02\\postgres\\websocket.jsonl": "03a47ec400325a2ad955b604cd1cb1d87c986352b723e4a3c171dd49490129c5", + "pair-02\\sqlite\\client-validation.json": "c970080dd992a0d0003148d28c6b58705ed517e06aff4d2c9b2ab02a56d82478", + "pair-02\\sqlite\\database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-02\\sqlite\\handler.json": "9c6bf50dae42087fb0cdd39cc006fcd119660568f25102a0776cdac35fad4601", + "pair-02\\sqlite\\ingest-validation.json": "e2198143a3c3b5bf198440f62704f00082150c7285822755d422138854af93dd", + "pair-02\\sqlite\\ingest.jsonl": "7ec642a934c661df4960c15baec1564a12697a5aba4fc6bf35dd55b304176c8d", + "pair-02\\sqlite\\requests.jsonl": "a1fd7a21a264d57d69533dc37e6a1ce3fc8128e14851f6d2c2d81eb087285a90", + "pair-02\\sqlite\\resource-final.json": "04e524872062c83b6e90f942391546a6285b34cd3f029921ebdae12ec355c1f3", + "pair-02\\sqlite\\resources.csv": "d5e930f71b159fb845fca766080979da4835f732f7e6a55618b74522f1a9e3fe", + "pair-02\\sqlite\\retention-empty.json": "64ab659c3d7f4c00bc8c1595cc1f60397f72f145e7611ce932be0a42139ad7eb", + "pair-02\\sqlite\\retention.json": "fcce7d1d4523f09eb5675ebf67aca18b08bb6a36b3a513b41da7238793fdbfd6", + "pair-02\\sqlite\\sql-queries.jsonl": "d957eb46cf1b8303d47ef788307c863373637cd2ba471bd2b1548a6be415d309", + "pair-02\\sqlite\\startup-hot1.json": "482ecda906ea563bd43192166ddca81739223b9959cda4903f9c141c58bb3607", + "pair-02\\sqlite\\startup.json": "9b5eab62d9f595a81a73d21dc1d473f7254cc1e901878b51e06ffdfdd54caa71", + "pair-02\\sqlite\\validation-after.json": "f61176cb63616b0a5bfd2a627919edd4b1c8168d9d4d008754fbc1cb1643387a", + "pair-02\\sqlite\\validation-before.json": "108cfbd317d2f83bc57bac1d55b67a65629332fb57164e512ec852902873e729", + "pair-02\\sqlite\\visibility.json": "3ee0bc5124d9da5359b06cc6ad876b80cf3c09942e6929f02e226ebbfb03f1c8", + "pair-02\\sqlite\\websocket.jsonl": "e2663885d378b344289069443ee37164a21c41f06e53265279948a45acaefec1", + "pair-03\\corpus-offset.json": "4999ed14bbc7e0bc0d8e5501be711dfd4174d98be1f7275822fc4ea762110fa7", + "pair-03\\events.json": "2d397fafc2172c46c1fdcadb50b066c51560bde241eb4e269aa876b343b7f486", + "pair-03\\postgres\\client-validation.json": "60520ca0cd72688e817efb8fab95b556f488248146e079bd5843e341beb4d9f4", + "pair-03\\postgres\\database-settings.json": "999ed1c72e72ba5c038102ccb63b21014bb009c78c16cdf7b4b036ab34a09664", + "pair-03\\postgres\\handler.json": "09baad87b3ed38b33132a4ec01dc96754cae24dfbd608b97c0e30d0719b24627", + "pair-03\\postgres\\ingest-validation.json": "fe296b8e425d4da92b722b8d60d2c9f69433a823833ec8bb3570822bafeade4f", + "pair-03\\postgres\\ingest.jsonl": "5822924874004261ccdb32512a985e226e50374feaff4419b866dd23f36d5126", + "pair-03\\postgres\\migration.json": "65b0632398569c3177c6d4ad9f2bb5e51e45ec62bdea17499a3d6d17b5853f14", + "pair-03\\postgres\\requests.jsonl": "8dca668c3a359977dacc0d793c3bc18a9fdfca3fc8894b02841ca290c4d64eda", + "pair-03\\postgres\\resource-final.json": "6d1fb666ad190480b0b90d3a9bd13438d93b58d98073889bb38906a0e513d82d", + "pair-03\\postgres\\resources.csv": "0ce0fb686f532f1026d83e256e9555345b25163c363add3468514fd2f144148b", + "pair-03\\postgres\\retention-empty.json": "f799d68420852495f926d4611bdd588fbdda15e5bf0a153f8d7e2ad7b4fc79c2", + "pair-03\\postgres\\retention.json": "024b0288ee92fa19de5d66aee3662602c31d69bf1acc85dc4ed21f70299f7b0b", + "pair-03\\postgres\\sql-queries.jsonl": "c617cc81a6cf73198bfac580ba54e8c68887d7fae998b987c14078e715c0d2d2", + "pair-03\\postgres\\startup-hot1.json": "8925a58dab558383583bd23b9448efa87ca3695777cc67e21530413f06cf6dc6", + "pair-03\\postgres\\startup.json": "2738fec95fb6e1aaa4cda8b55b54c04bb1c13cd6ffd9fc4c0fd7346739cf09c2", + "pair-03\\postgres\\validation-after.json": "81783421a01a1cecafb1f014cc314bdef9ceb71b3df691f0068bb959d32f9be8", + "pair-03\\postgres\\validation-before.json": "a7299d0640e1bae78bda2ce4630ce2f48b7828a492d21e2b89a7f8ad2c948965", + "pair-03\\postgres\\visibility.json": "6c94d6123bbb284e81cf5eee4b0e7398e109e3765d1ee2b72a4be789c9060248", + "pair-03\\postgres\\websocket.jsonl": "09c052514aa289599dcdbf03388db743f7972fcced85c855ae97e4fbb2ed2b96", + "pair-03\\sqlite\\client-validation.json": "04aa6fc778587f20e453258bd7c1c741ab9d7574c7438d71f4ed4f41c1465eff", + "pair-03\\sqlite\\database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-03\\sqlite\\handler.json": "0d5eb6ba78e49024f9aee8d1868f81d678ec58d7a2c470c0ea00fa62f409143e", + "pair-03\\sqlite\\ingest-validation.json": "378a915a810db739671af7f66921cc87273ddc148e360990c958a709d2a81450", + "pair-03\\sqlite\\ingest.jsonl": "5833afb18f3be42ef4b2e7ed84173ca71c8d66e9c2b75345e17571d575638f2c", + "pair-03\\sqlite\\requests.jsonl": "66df8806baebc45c433b2933356a4c7430f525cbfa345d5d11325e69a1e2df75", + "pair-03\\sqlite\\resource-final.json": "8dbf793c2d9db0638a4a1c2c0da07cafe3fd16b2e47dffab1d8775a8a8d9011e", + "pair-03\\sqlite\\resources.csv": "efc7b09cdbeea6551da0911bf09803fed693ff047b7d011caacb86abe4338860", + "pair-03\\sqlite\\retention-empty.json": "478a3287ba0389b83aeff8875504d8e5acaeef449a4808013395f24bbc78b437", + "pair-03\\sqlite\\retention.json": "c7c4b178d866825a60a0c3d9567b3fef9a4d21502d60b8c4af5d96ce10d148b8", + "pair-03\\sqlite\\sql-queries.jsonl": "7cde9514ba474df767bd732f39f9242592f8cb044f8fe4bc55611b415ada7ec7", + "pair-03\\sqlite\\startup-hot1.json": "b255d8ae00fb11ba086df57e0bec0a02baa45ccea1f469e861a918c0afa9f9c4", + "pair-03\\sqlite\\startup.json": "4a738e9039d78e4d0c252e9c3b69c3f6bd00fd820635b9f212aa41457025406d", + "pair-03\\sqlite\\validation-after.json": "81783421a01a1cecafb1f014cc314bdef9ceb71b3df691f0068bb959d32f9be8", + "pair-03\\sqlite\\validation-before.json": "a7299d0640e1bae78bda2ce4630ce2f48b7828a492d21e2b89a7f8ad2c948965", + "pair-03\\sqlite\\visibility.json": "2ead7da59f9f620dd11271451e9998933ebd48a2eb78b5492c29689e332df800", + "pair-03\\sqlite\\websocket.jsonl": "76526b57a8cfb3e1824254270d2c2b231b87add4866e56855cd6f78b8b99b9ce", + "pair-04\\corpus-offset.json": "a4f3d21b79417937889f1555454ea19f43c78dd2795c46b261a633a6985c52e0", + "pair-04\\events.json": "6fe3e3ae3ec316890c799801978d28a827ba418ef8b468c9c0d231459b085a7c", + "pair-04\\postgres\\client-validation.json": "f5945b02ad871b45fd721e8e36cc336f7312ad138cb427c5ef0e060ed5f5f6da", + "pair-04\\postgres\\database-settings.json": "999ed1c72e72ba5c038102ccb63b21014bb009c78c16cdf7b4b036ab34a09664", + "pair-04\\postgres\\handler.json": "3855863d418b0cfb5a7e28cb71dde71d98990f8608ed89037960b695f445d724", + "pair-04\\postgres\\ingest-validation.json": "067b18bf75fc4311be0b6e129d794a70eae904fbe43857ca7bb90ab8e2759c0f", + "pair-04\\postgres\\ingest.jsonl": "aa104b8121a9a25afb78048fef85e331ebed14a2c9e120b8df37aef495465eaf", + "pair-04\\postgres\\migration.json": "2b96bebe5e42fae298508cd025179fccc955ce46bc8fd3d42d2b4a5d1fbfb3c9", + "pair-04\\postgres\\requests.jsonl": "7e841567d6158de9e15175035b676ec77328d41efdd81e855e4fc5a9c87bdcc3", + "pair-04\\postgres\\resource-final.json": "a7f4739c1017278d622812216f994b0e5cce6f783ac61344e7de3647053f9858", + "pair-04\\postgres\\resources.csv": "985b85e22df8621ee4b3606c1ed48d41291a4fe19331ecb650646be117576fdc", + "pair-04\\postgres\\retention-empty.json": "8a97bd06ac181a9e04eb536ebe6790a7026272042fdde161753389ccc6415295", + "pair-04\\postgres\\retention.json": "30227c4f50470347b479b116f3b2815e7d29e15b47d5c09e8325229a11a53be4", + "pair-04\\postgres\\sql-queries.jsonl": "37c2de1685284bf89dfb8c689c158b2c700a4a8a221af8d5749b81e2b0bb9b14", + "pair-04\\postgres\\startup-hot1.json": "e4a7898254c92b25b056c86b4aa3519a777507f4fcf1f2b10200e6e46cb90352", + "pair-04\\postgres\\startup.json": "74264c21b03b78ce292982ded70677e0fcebc758686ecd508aa4038c07ad29a1", + "pair-04\\postgres\\validation-after.json": "0259715944229d5e6a5638b4918795885eccd2dad185b3cf9e1329ce2e334c09", + "pair-04\\postgres\\validation-before.json": "f7a6aa8342280bd2f1c036462854ef165a97308735ae165e9dc87307c273bac4", + "pair-04\\postgres\\visibility.json": "d6327f09993bdb500554f2255d1bfd8f20094252bbf332a875d01c3bd3084238", + "pair-04\\postgres\\websocket.jsonl": "5c40b92584014c3ebab054277a502d24a5ff73a6c38e570143406ff05f888adf", + "pair-04\\sqlite\\client-validation.json": "5df7ed7ac7fbd16a67d626b7b37116bf2d6ad5c41f31362cf6de0d8f6c30d573", + "pair-04\\sqlite\\database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-04\\sqlite\\handler.json": "73bc91d3e4e478badd36926b9213d6f3f09646b99ba1ce8b17a46d2ed1da1a29", + "pair-04\\sqlite\\ingest-validation.json": "463f519107a2128b262583b815e29e569a4c94b48542715f13fab1687f180d27", + "pair-04\\sqlite\\ingest.jsonl": "491977c8c175cb807d9011170783b68fc2f28b7e83d138febfd99ba11fd93fbd", + "pair-04\\sqlite\\requests.jsonl": "de0ceebe2883916ba3566ea0b613a77a50da73c9effe8f5e6d275604bf17d4fc", + "pair-04\\sqlite\\resource-final.json": "972ea9717de8d0b38ca1d6f10819d8e9cc4a806f526259de8ee99ce4557602da", + "pair-04\\sqlite\\resources.csv": "5916f3bc51e730e4f620462695f39193785861a81246614665beb0c5dbcba632", + "pair-04\\sqlite\\retention-empty.json": "c05294d9257e603df407210914bb2005b2c6410d6858ebec95d43cc67f01e68c", + "pair-04\\sqlite\\retention.json": "c7ad6f8fd1dbc52d80d507db797b7d762e7711cae21ea678699ed7ac11b5846c", + "pair-04\\sqlite\\sql-queries.jsonl": "82f34e4466c7339e92590e7244312a607247420c3fd362e27338e009913f718e", + "pair-04\\sqlite\\startup-hot1.json": "a33d4f2cc070380bc70fe3829c6c8f99f150768fd26c906ea05bef803ba53207", + "pair-04\\sqlite\\startup.json": "c85a68ac6f560b8bf18194b97fc51dd485fa99b5bd436b889d0aacc63e5a0e68", + "pair-04\\sqlite\\validation-after.json": "0259715944229d5e6a5638b4918795885eccd2dad185b3cf9e1329ce2e334c09", + "pair-04\\sqlite\\validation-before.json": "f7a6aa8342280bd2f1c036462854ef165a97308735ae165e9dc87307c273bac4", + "pair-04\\sqlite\\visibility.json": "874cd2bf5b579477bd50432e56c7319bc83f649fb317488383e5c24599f07fd7", + "pair-04\\sqlite\\websocket.jsonl": "f9df1e4d8ab6c3551d884c20b1d9f1e18f63e1fb20a92c21f644d8bd28c0a0f6", + "postgres-preflight.json": "e926795faf9c07ff13185a7914625c659ad3a825a513fb7dd1006af59ca700ed", + "resource-preflight.json": "16e6006274973fdff80b18b2afcd4a3ebb02de9fad49588ce88e77bdc443218e" + }, + "latency": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1431285, + "minimum": 1396577, + "maximum": 1552503, + "runs": 5 + }, + "postgres": { + "median": 2077878, + "minimum": 1990859, + "maximum": 2098396, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.4241966681756708, + "minimum": 1.3384051431784674, + "maximum": 1.4639138955553925, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2261999432, + "minimum": 2068421459, + "maximum": 2407753090, + "runs": 5 + }, + "postgres": { + "median": 3044122, + "minimum": 2855209, + "maximum": 3117397, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.00134576603200491, + "minimum": 0.001185839616137716, + "maximum": 0.0014565529411367474, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": { + "median": 8074723038, + "minimum": 7891463420, + "maximum": 8322791838, + "runs": 5 + }, + "postgres": { + "median": 5570235, + "minimum": 5064645, + "maximum": 6074051, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.0006807210981935891, + "minimum": 0.0006152463198845574, + "maximum": 0.0007696989362715693, + "runs": 5 + }, + "note": "" + }, + "samples_per_run": { + "sqlite": [ + 2700, + 2700, + 2700, + 2700, + 2700 + ], + "postgres": [ + 2700, + 2700, + 2700, + 2700, + 2700 + ] + } + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 648517, + "minimum": 640395, + "maximum": 654117, + "runs": 5 + }, + "postgres": { + "median": 617157, + "minimum": 610233, + "maximum": 630862, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9529009439486567, + "minimum": 0.9434963469837965, + "maximum": 0.9767737120720649, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2261611474, + "minimum": 2048429324, + "maximum": 2407422741, + "runs": 5 + }, + "postgres": { + "median": 1074396, + "minimum": 1070487, + "maximum": 1077388, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.0004761206831408214, + "minimum": 0.0004446609985728302, + "maximum": 0.0005243954416266695, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": { + "median": 8074256022, + "minimum": 7891136968, + "maximum": 8283303554, + "runs": 5 + }, + "postgres": { + "median": 1129961, + "minimum": 1118967, + "maximum": 1136500, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.00014029565038729212, + "minimum": 0.00013720371257566494, + "maximum": 0.00014232672009555604, + "runs": 5 + }, + "note": "" + }, + "samples_per_run": { + "sqlite": [ + 2700, + 2700, + 2700, + 2700, + 2700 + ], + "postgres": [ + 2700, + 2700, + 2700, + 2700, + 2700 + ] + } + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 797721, + "minimum": 730751, + "maximum": 800720, + "runs": 5 + }, + "postgres": { + "median": 1382488, + "minimum": 1275829, + "maximum": 1413317, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.7459148191381195, + "minimum": 1.7226308639569983, + "maximum": 1.7691887263842871, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1626878, + "minimum": 1526136, + "maximum": 2240945, + "runs": 5 + }, + "postgres": { + "median": 2238979, + "minimum": 2077817, + "maximum": 2329768, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.3728723038963204, + "minimum": 0.955954296067061, + "maximum": 1.513253982430216, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": { + "median": 4737857, + "minimum": 3958335, + "maximum": 6089155, + "runs": 5 + }, + "postgres": { + "median": 4421953, + "minimum": 4182259, + "maximum": 4830973, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.8852578285921251, + "minimum": 0.7933733005646925, + "maximum": 1.147871011422732, + "runs": 5 + }, + "note": "" + }, + "samples_per_run": { + "sqlite": [ + 2700, + 2700, + 2700, + 2700, + 2700 + ], + "postgres": [ + 2700, + 2700, + 2700, + 2700, + 2700 + ] + } + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1295980, + "minimum": 1262384, + "maximum": 1439975, + "runs": 5 + }, + "postgres": { + "median": 1939823, + "minimum": 1858390, + "maximum": 1959562, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.4921806676890708, + "minimum": 1.3471226931023108, + "maximum": 1.507181437985154, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2183934179, + "minimum": 1990077090, + "maximum": 2329276946, + "runs": 5 + }, + "postgres": { + "median": 2686557, + "minimum": 2533888, + "maximum": 2744298, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.0012495317057813215, + "minimum": 0.0010878431628112632, + "maximum": 0.0013429848727865767, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": { + "median": 8153439399, + "minimum": 7895613951, + "maximum": 8462577431, + "runs": 5 + }, + "postgres": { + "median": 4656678, + "minimum": 4348628, + "maximum": 6215643, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.000589780355131246, + "minimum": 0.0005287755569732841, + "maximum": 0.0007773369786622005, + "runs": 5 + }, + "note": "" + }, + "samples_per_run": { + "sqlite": [ + 1350, + 1350, + 1350, + 1350, + 1350 + ], + "postgres": [ + 1350, + 1350, + 1350, + 1350, + 1350 + ] + } + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 652590, + "minimum": 641317, + "maximum": 660370, + "runs": 5 + }, + "postgres": { + "median": 613057, + "minimum": 605557, + "maximum": 615031, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9374127384005969, + "minimum": 0.928353801656647, + "maximum": 0.9503553222623806, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2183610642, + "minimum": 1989797647, + "maximum": 2329010005, + "runs": 5 + }, + "postgres": { + "median": 1067078, + "minimum": 1063306, + "maximum": 1078020, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.000488675947751678, + "minimum": 0.0004570555719875493, + "maximum": 0.0005370887846868582, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": { + "median": 7995796260, + "minimum": 7832232039, + "maximum": 8461983848, + "runs": 5 + }, + "postgres": { + "median": 1113332, + "minimum": 1111594, + "maximum": 1171401, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.00013923966591915138, + "minimum": 0.00013659939486593793, + "maximum": 0.00014205490777850535, + "runs": 5 + }, + "note": "" + }, + "samples_per_run": { + "sqlite": [ + 1350, + 1350, + 1350, + 1350, + 1350 + ], + "postgres": [ + 1350, + 1350, + 1350, + 1350, + 1350 + ] + } + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 676305, + "minimum": 614232, + "maximum": 683442, + "runs": 5 + }, + "postgres": { + "median": 1285879, + "minimum": 1190019, + "maximum": 1302281, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.9178010753369938, + "minimum": 1.8814749459354267, + "maximum": 1.9374096432618295, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1049436, + "minimum": 990831, + "maximum": 1134258, + "runs": 5 + }, + "postgres": { + "median": 1873817, + "minimum": 1743644, + "maximum": 1921850, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.7128419455783868, + "minimum": 1.626397037562145, + "maximum": 1.9396345088112907, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": { + "median": 3322592, + "minimum": 3204335, + "maximum": 3726371, + "runs": 5 + }, + "postgres": { + "median": 3904657, + "minimum": 3460660, + "maximum": 4862061, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.130622110359872, + "minimum": 1.0022491278330583, + "maximum": 1.463333746665254, + "runs": 5 + }, + "note": "" + }, + "samples_per_run": { + "sqlite": [ + 1350, + 1350, + 1350, + 1350, + 1350 + ], + "postgres": [ + 1350, + 1350, + 1350, + 1350, + 1350 + ] + } + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1705457, + "minimum": 1541612, + "maximum": 1788740, + "runs": 5 + }, + "postgres": { + "median": 2251854, + "minimum": 2155810, + "maximum": 2299519, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.3483300956869626, + "minimum": 1.2755749857441552, + "maximum": 1.3984128302063035, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2145071261, + "minimum": 1951205083, + "maximum": 2290146785, + "runs": 5 + }, + "postgres": { + "median": 3048149, + "minimum": 2911444, + "maximum": 3227421, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.0015045751899614864, + "minimum": 0.0012712914382035996, + "maximum": 0.0015258089321568459, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 450, + 450, + 450, + 450, + 450 + ], + "postgres": [ + 450, + 450, + 450, + 450, + 450 + ] + } + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 629515, + "minimum": 618449, + "maximum": 673139, + "runs": 5 + }, + "postgres": { + "median": 632488, + "minimum": 620802, + "maximum": 666357, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9899248149342113, + "minimum": 0.9377167262454646, + "maximum": 1.0321077404927488, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2144307389, + "minimum": 1950409764, + "maximum": 2289550899, + "runs": 5 + }, + "postgres": { + "median": 1072949, + "minimum": 1056680, + "maximum": 1079311, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.0005033378169271421, + "minimum": 0.0004654310155172488, + "maximum": 0.0005501146578550455, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 450, + 450, + 450, + 450, + 450 + ], + "postgres": [ + 450, + 450, + 450, + 450, + 450 + ] + } + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1003492, + "minimum": 936905, + "maximum": 1020925, + "runs": 5 + }, + "postgres": { + "median": 1580121, + "minimum": 1476057, + "maximum": 1605017, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.5691482915441481, + "minimum": 1.555140773751234, + "maximum": 1.5999263532588683, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2336338, + "minimum": 1589693, + "maximum": 3364572, + "runs": 5 + }, + "postgres": { + "median": 2221719, + "minimum": 2095094, + "maximum": 2363344, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.001652158206561, + "minimum": 0.6312172841003254, + "maximum": 1.3975773938741631, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 450, + 450, + 450, + 450, + 450 + ], + "postgres": [ + 450, + 450, + 450, + 450, + 450 + ] + } + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1792553, + "minimum": 1636669, + "maximum": 1840812, + "runs": 5 + }, + "postgres": { + "median": 2215991, + "minimum": 2107279, + "maximum": 2268504, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.2367400231790973, + "minimum": 1.232338772237469, + "maximum": 1.296948946950213, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2069693373, + "minimum": 1873867961, + "maximum": 2251485053, + "runs": 5 + }, + "postgres": { + "median": 3039894, + "minimum": 2913401, + "maximum": 3088245, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.0014831274236292392, + "minimum": 0.0012939908244640698, + "maximum": 0.0016096271616195962, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": { + "median": 8210935496, + "minimum": 8047117650, + "maximum": 8348964133, + "runs": 5 + }, + "postgres": { + "median": 4890648, + "minimum": 4353968, + "maximum": 5480239, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.0006077515220620641, + "minimum": 0.000521497988330141, + "maximum": 0.0006596103953986107, + "runs": 5 + }, + "note": "" + }, + "samples_per_run": { + "sqlite": [ + 4500, + 4500, + 4500, + 4500, + 4500 + ], + "postgres": [ + 4500, + 4500, + 4500, + 4500, + 4500 + ] + } + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 648594, + "minimum": 633777, + "maximum": 657646, + "runs": 5 + }, + "postgres": { + "median": 611364, + "minimum": 597807, + "maximum": 625383, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.945137069539356, + "minimum": 0.9412297986105329, + "maximum": 0.9575393829662602, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2049720113, + "minimum": 1853879693, + "maximum": 2231492778, + "runs": 5 + }, + "postgres": { + "median": 1069241, + "minimum": 1061088, + "maximum": 1075483, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.0005246974907349216, + "minimum": 0.0004755059081799995, + "maximum": 0.0005775558166168446, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": { + "median": 8194834912, + "minimum": 8027129843, + "maximum": 8328976175, + "runs": 5 + }, + "postgres": { + "median": 1121437, + "minimum": 1112348, + "maximum": 1125281, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.00013717787021601442, + "minimum": 0.00013355158865009, + "maximum": 0.00013887828175249313, + "runs": 5 + }, + "note": "" + }, + "samples_per_run": { + "sqlite": [ + 4500, + 4500, + 4500, + 4500, + 4500 + ], + "postgres": [ + 4500, + 4500, + 4500, + 4500, + 4500 + ] + } + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1073680, + "minimum": 1016586, + "maximum": 1083380, + "runs": 5 + }, + "postgres": { + "median": 1554692, + "minimum": 1443354, + "maximum": 1588396, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.4350384906496336, + "minimum": 1.4198051124056401, + "maximum": 1.4768564189581832, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2688791, + "minimum": 2504751, + "maximum": 3097267, + "runs": 5 + }, + "postgres": { + "median": 2237734, + "minimum": 2092845, + "maximum": 2311314, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.8562465435208613, + "minimum": 0.6948619541034079, + "maximum": 0.9194948597234809, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": { + "median": 5492336, + "minimum": 4973305, + "maximum": 6501449, + "runs": 5 + }, + "postgres": { + "median": 4041005, + "minimum": 3715538, + "maximum": 4563639, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.7721613695520383, + "minimum": 0.6215545180774317, + "maximum": 0.8536905903688535, + "runs": 5 + }, + "note": "" + }, + "samples_per_run": { + "sqlite": [ + 4500, + 4500, + 4500, + 4500, + 4500 + ], + "postgres": [ + 4500, + 4500, + 4500, + 4500, + 4500 + ] + } + }, + { + "scenario": "handler-control", + "event_class": "200_valid_envelopes", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1180371, + "minimum": 1126030, + "maximum": 1206984, + "runs": 5 + }, + "postgres": { + "median": 1971488, + "minimum": 1952226, + "maximum": 2052401, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.6861649504527425, + "minimum": 1.633400277054211, + "maximum": 1.8226876726197347, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2917607, + "minimum": 2563010, + "maximum": 3331540, + "runs": 5 + }, + "postgres": { + "median": 2733058, + "minimum": 2691655, + "maximum": 2843413, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.936746450087349, + "minimum": 0.8101670698835973, + "maximum": 1.0704718280459304, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 200, + 200, + 200, + 200, + 200 + ], + "postgres": [ + 200, + 200, + 200, + 200, + 200 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 33848143, + "minimum": 33062198, + "maximum": 34237276, + "runs": 5 + }, + "postgres": { + "median": 33661987, + "minimum": 32738811, + "maximum": 34813321, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9847587210166929, + "minimum": 0.9672262079488378, + "maximum": 1.029274853414162, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 43084785, + "minimum": 40074044, + "maximum": 45855185, + "runs": 5 + }, + "postgres": { + "median": 43871911, + "minimum": 42400747, + "maximum": 65193092, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.093216946111054, + "minimum": 0.9567491876872811, + "maximum": 1.559361433149514, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 72, + 72, + 72, + 72, + 72 + ], + "postgres": [ + 72, + 72, + 72, + 72, + 72 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 501917, + "minimum": 375627, + "maximum": 557313, + "runs": 5 + }, + "postgres": { + "median": 490091, + "minimum": 375906, + "maximum": 522258, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9370999779298168, + "minimum": 0.7629851546040213, + "maximum": 1.3448341040447038, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1037465, + "minimum": 1008672, + "maximum": 1071839, + "runs": 5 + }, + "postgres": { + "median": 1042244, + "minimum": 974119, + "maximum": 1084633, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.012298246205896, + "minimum": 0.9088295910113366, + "maximum": 1.068035724836048, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 72, + 72, + 72, + 72, + 72 + ], + "postgres": [ + 72, + 72, + 72, + 72, + 72 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 33280952, + "minimum": 32724667, + "maximum": 33404928, + "runs": 5 + }, + "postgres": { + "median": 33063740, + "minimum": 31969470, + "maximum": 34020848, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9976793332113817, + "minimum": 0.9586310413835317, + "maximum": 1.0184379981300962, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 42477711, + "minimum": 39714943, + "maximum": 44891357, + "runs": 5 + }, + "postgres": { + "median": 43788604, + "minimum": 41901684, + "maximum": 61696459, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0956572189968898, + "minimum": 0.9668357069042055, + "maximum": 1.5040132276901266, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 72, + 72, + 72, + 72, + 72 + ], + "postgres": [ + 72, + 72, + 72, + 72, + 72 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 2453295, + "minimum": 2367044, + "maximum": 2528664, + "runs": 5 + }, + "postgres": { + "median": 2374199, + "minimum": 2370085, + "maximum": 2412098, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9660823504715087, + "minimum": 0.9448558606441979, + "maximum": 1.0190338667130818, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 3185257, + "minimum": 3132581, + "maximum": 3631674, + "runs": 5 + }, + "postgres": { + "median": 3333970, + "minimum": 3218135, + "maximum": 3353599, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0103219300671813, + "minimum": 0.9096879290376835, + "maximum": 1.0642885211906732, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 108, + 108, + 108, + 108, + 108 + ], + "postgres": [ + 108, + 108, + 108, + 108, + 108 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 587481, + "minimum": 544034, + "maximum": 665586, + "runs": 5 + }, + "postgres": { + "median": 515539, + "minimum": 491168, + "maximum": 619536, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9308128476260017, + "minimum": 0.8109330441545631, + "maximum": 0.9854242094637954, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1080471, + "minimum": 1052580, + "maximum": 1091637, + "runs": 5 + }, + "postgres": { + "median": 1046137, + "minimum": 1028661, + "maximum": 1076439, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9586955999504762, + "minimum": 0.9561965970864039, + "maximum": 1.022667160690874, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 108, + 108, + 108, + 108, + 108 + ], + "postgres": [ + 108, + 108, + 108, + 108, + 108 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1773917, + "minimum": 1684541, + "maximum": 1827935, + "runs": 5 + }, + "postgres": { + "median": 1738288, + "minimum": 1683473, + "maximum": 1762772, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.988786695798239, + "minimum": 0.959930194454398, + "maximum": 0.9993659994028047, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2540853, + "minimum": 2481120, + "maximum": 2714123, + "runs": 5 + }, + "postgres": { + "median": 2557171, + "minimum": 2501726, + "maximum": 2705144, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0149831985272812, + "minimum": 0.924827283067127, + "maximum": 1.0668500515896047, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 108, + 108, + 108, + 108, + 108 + ], + "postgres": [ + 108, + 108, + 108, + 108, + 108 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 2254734, + "minimum": 2167140, + "maximum": 2346257, + "runs": 5 + }, + "postgres": { + "median": 2494416, + "minimum": 2461849, + "maximum": 2544249, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1044242936208182, + "minimum": 1.0681135953989696, + "maximum": 1.1383588508356637, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 536034954, + "minimum": 523425952, + "maximum": 559179086, + "runs": 5 + }, + "postgres": { + "median": 555544077, + "minimum": 512663358, + "maximum": 561210139, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.018289426429567, + "minimum": 0.9563991194499604, + "maximum": 1.0721863080262402, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 72, + 72, + 72, + 72, + 72 + ], + "postgres": [ + 72, + 72, + 72, + 72, + 72 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 181229, + "minimum": 122926, + "maximum": 221454, + "runs": 5 + }, + "postgres": { + "median": 155176, + "minimum": 147183, + "maximum": 176372, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.8121382339471056, + "minimum": 0.7007143695756229, + "maximum": 1.3334845460593963, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1052859, + "minimum": 991297, + "maximum": 1078508, + "runs": 5 + }, + "postgres": { + "median": 1047252, + "minimum": 1008881, + "maximum": 1076340, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0095648618093473, + "minimum": 0.9354413690023625, + "maximum": 1.0529316642741782, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 72, + 72, + 72, + 72, + 72 + ], + "postgres": [ + 72, + 72, + 72, + 72, + 72 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1926528, + "minimum": 1870467, + "maximum": 1965236, + "runs": 5 + }, + "postgres": { + "median": 2179547, + "minimum": 2112116, + "maximum": 2247440, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.129295810909574, + "minimum": 1.0830919757138169, + "maximum": 1.1681214650063567, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 535697290, + "minimum": 523272043, + "maximum": 558823354, + "runs": 5 + }, + "postgres": { + "median": 555026701, + "minimum": 511729119, + "maximum": 560740967, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0182369200445323, + "minimum": 0.9552579946783005, + "maximum": 1.0716050561103643, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 72, + 72, + 72, + 72, + 72 + ], + "postgres": [ + 72, + 72, + 72, + 72, + 72 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 30577776, + "minimum": 30232848, + "maximum": 30807730, + "runs": 5 + }, + "postgres": { + "median": 31890510, + "minimum": 31641605, + "maximum": 32084806, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0465969001663356, + "minimum": 1.034824257463475, + "maximum": 1.0492851409468105, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 36085140, + "minimum": 33737423, + "maximum": 37266963, + "runs": 5 + }, + "postgres": { + "median": 38718420, + "minimum": 35683926, + "maximum": 43615827, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1476401146584314, + "minimum": 1.0086806363488032, + "maximum": 1.178598669701711, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 108, + 108, + 108, + 108, + 108 + ], + "postgres": [ + 108, + 108, + 108, + 108, + 108 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 680023, + "minimum": 572853, + "maximum": 693147, + "runs": 5 + }, + "postgres": { + "median": 687531, + "minimum": 591114, + "maximum": 722076, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0318772878906106, + "minimum": 1.0068725810034962, + "maximum": 1.1707010559113726, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1062544, + "minimum": 1031180, + "maximum": 1080653, + "runs": 5 + }, + "postgres": { + "median": 1072554, + "minimum": 1025508, + "maximum": 1099648, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.014289138531892, + "minimum": 0.9721459373527691, + "maximum": 1.034919965667304, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 108, + 108, + 108, + 108, + 108 + ], + "postgres": [ + 108, + 108, + 108, + 108, + 108 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 30116595, + "minimum": 29557543, + "maximum": 30122866, + "runs": 5 + }, + "postgres": { + "median": 31231050, + "minimum": 30975017, + "maximum": 31366518, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0415027993702475, + "minimum": 1.035543860982954, + "maximum": 1.0479564218176052, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 35396834, + "minimum": 32674879, + "maximum": 36689737, + "runs": 5 + }, + "postgres": { + "median": 38079272, + "minimum": 35376337, + "maximum": 43504270, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1653990210644698, + "minimum": 1.0112335363209728, + "maximum": 1.1857340378318875, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 108, + 108, + 108, + 108, + 108 + ], + "postgres": [ + 108, + 108, + 108, + 108, + 108 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1217296, + "minimum": 1160416, + "maximum": 1330680, + "runs": 5 + }, + "postgres": { + "median": 1278497, + "minimum": 1166645, + "maximum": 1305333, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.984416254335417, + "minimum": 0.9583905640041535, + "maximum": 1.0746962442874468, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1779334, + "minimum": 1638934, + "maximum": 1822727, + "runs": 5 + }, + "postgres": { + "median": 1856776, + "minimum": 1703442, + "maximum": 2959713, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0441910676062267, + "minimum": 1.0186802521716087, + "maximum": 1.6633824790623908, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 617424, + "minimum": 542346, + "maximum": 645323, + "runs": 5 + }, + "postgres": { + "median": 582248, + "minimum": 567348, + "maximum": 603125, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9346094901312986, + "minimum": 0.9167046753003212, + "maximum": 1.0945724684979699, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1060620, + "minimum": 1020445, + "maximum": 1068893, + "runs": 5 + }, + "postgres": { + "median": 1046744, + "minimum": 1032863, + "maximum": 1084810, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9980710682162959, + "minimum": 0.9762563406309517, + "maximum": 1.0148911069676758, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 656952, + "minimum": 550664, + "maximum": 671371, + "runs": 5 + }, + "postgres": { + "median": 654476, + "minimum": 565316, + "maximum": 689581, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0047708346056055, + "minimum": 0.9492398572066642, + "maximum": 1.0522133279095782, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 936897, + "minimum": 872738, + "maximum": 980696, + "runs": 5 + }, + "postgres": { + "median": 1177886, + "minimum": 907011, + "maximum": 2150776, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.2010714839257017, + "minimum": 1.0392706631314323, + "maximum": 2.3211658192971454, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1949504, + "minimum": 1806321, + "maximum": 1988432, + "runs": 5 + }, + "postgres": { + "median": 1977368, + "minimum": 1896404, + "maximum": 2148358, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0498709808500262, + "minimum": 0.9874106833927436, + "maximum": 1.0867147985587748, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 69708296, + "minimum": 68660510, + "maximum": 71038643, + "runs": 5 + }, + "postgres": { + "median": 115740755, + "minimum": 111238306, + "maximum": 178502469, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.6539582012017504, + "minimum": 1.6105287269406134, + "maximum": 2.5127516723538763, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 579543, + "minimum": 505132, + "maximum": 599032, + "runs": 5 + }, + "postgres": { + "median": 558990, + "minimum": 480468, + "maximum": 663883, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9511731586991123, + "minimum": 0.9056110524980302, + "maximum": 1.1455284594930832, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1042355, + "minimum": 1022211, + "maximum": 1058939, + "runs": 5 + }, + "postgres": { + "median": 1052654, + "minimum": 1029934, + "maximum": 1064345, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9972970515700589, + "minimum": 0.9914412918441509, + "maximum": 1.0412184959856625, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1323509, + "minimum": 1234951, + "maximum": 1339914, + "runs": 5 + }, + "postgres": { + "median": 1373795, + "minimum": 1307766, + "maximum": 1477247, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0589618535472258, + "minimum": 1.035622284850854, + "maximum": 1.1024938913990003, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 69114273, + "minimum": 68286431, + "maximum": 70444454, + "runs": 5 + }, + "postgres": { + "median": 114901575, + "minimum": 111198714, + "maximum": 177473720, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.6541510286274193, + "minimum": 1.6255058440461425, + "maximum": 2.5193426866506767, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1953773, + "minimum": 1827712, + "maximum": 2052117, + "runs": 5 + }, + "postgres": { + "median": 2024723, + "minimum": 1949635, + "maximum": 2121542, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0363143517696274, + "minimum": 0.9861918204468848, + "maximum": 1.0859920800674874, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 94614918, + "minimum": 93906524, + "maximum": 95110782, + "runs": 5 + }, + "postgres": { + "median": 117220797, + "minimum": 114091242, + "maximum": 176622903, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.2324659153785529, + "minimum": 1.2058483420130428, + "maximum": 1.8581334240327123, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 216, + 216, + 216, + 216, + 216 + ], + "postgres": [ + 216, + 216, + 216, + 216, + 216 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 594805, + "minimum": 545391, + "maximum": 627218, + "runs": 5 + }, + "postgres": { + "median": 575452, + "minimum": 534792, + "maximum": 620659, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9805662359664901, + "minimum": 0.9183481262327416, + "maximum": 1.0550795638518475, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1058374, + "minimum": 1022059, + "maximum": 1067482, + "runs": 5 + }, + "postgres": { + "median": 1067833, + "minimum": 1032312, + "maximum": 1069802, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0089372943779797, + "minimum": 0.9960236327009231, + "maximum": 1.0176146502752823, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 216, + 216, + 216, + 216, + 216 + ], + "postgres": [ + 216, + 216, + 216, + 216, + 216 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1330628, + "minimum": 1248795, + "maximum": 1367852, + "runs": 5 + }, + "postgres": { + "median": 1376783, + "minimum": 1315600, + "maximum": 1497157, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0711165563603313, + "minimum": 1.0065292151490073, + "maximum": 1.099725134274331, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 94304613, + "minimum": 93448017, + "maximum": 94835336, + "runs": 5 + }, + "postgres": { + "median": 116602516, + "minimum": 113234613, + "maximum": 175509176, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.2295260492354876, + "minimum": 1.2007324922694926, + "maximum": 1.8507965026773403, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 216, + 216, + 216, + 216, + 216 + ], + "postgres": [ + 216, + 216, + 216, + 216, + 216 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 3621204, + "minimum": 3485341, + "maximum": 3710083, + "runs": 5 + }, + "postgres": { + "median": 4137480, + "minimum": 4014418, + "maximum": 4242533, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1366298573693925, + "minimum": 1.1151987704857276, + "maximum": 1.1715807781058454, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 4447823, + "minimum": 4304452, + "maximum": 4639102, + "runs": 5 + }, + "postgres": { + "median": 4997094, + "minimum": 4784876, + "maximum": 5141636, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0974091658481013, + "minimum": 1.0855816492071095, + "maximum": 1.194492585815802, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 611559, + "minimum": 591965, + "maximum": 665554, + "runs": 5 + }, + "postgres": { + "median": 595414, + "minimum": 552909, + "maximum": 613651, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9558506470476646, + "minimum": 0.8946141109511775, + "maximum": 1.036633922613668, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1055773, + "minimum": 1031274, + "maximum": 1086233, + "runs": 5 + }, + "postgres": { + "median": 1058110, + "minimum": 1005144, + "maximum": 1072913, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9914444657948179, + "minimum": 0.9741096063183498, + "maximum": 1.0279868889904504, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 3032689, + "minimum": 2877634, + "maximum": 3077804, + "runs": 5 + }, + "postgres": { + "median": 3550258, + "minimum": 3360008, + "maximum": 3629651, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1617692354678855, + "minimum": 1.135823859328986, + "maximum": 1.196842472142709, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 3712770, + "minimum": 3580873, + "maximum": 3900559, + "runs": 5 + }, + "postgres": { + "median": 4423986, + "minimum": 4102282, + "maximum": 4503215, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1508583542927018, + "minimum": 1.1002397348687714, + "maximum": 1.247765323810537, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1978674, + "minimum": 1761261, + "maximum": 1982957, + "runs": 5 + }, + "postgres": { + "median": 2274317, + "minimum": 2155434, + "maximum": 2390276, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.2078829772342876, + "minimum": 1.1494147090425204, + "maximum": 1.2692070811199463, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2446799, + "minimum": 2287669, + "maximum": 2551945, + "runs": 5 + }, + "postgres": { + "median": 2849199, + "minimum": 2743285, + "maximum": 2908058, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1790875876746447, + "minimum": 1.1204159964262552, + "maximum": 1.2115326965541553, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 595669, + "minimum": 568597, + "maximum": 674563, + "runs": 5 + }, + "postgres": { + "median": 569247, + "minimum": 462390, + "maximum": 598684, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.887926580138723, + "minimum": 0.7762532547438259, + "maximum": 1.0529144543499174, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1043301, + "minimum": 973475, + "maximum": 1053304, + "runs": 5 + }, + "postgres": { + "median": 1045578, + "minimum": 1037826, + "maximum": 1050698, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9987557117015381, + "minimum": 0.9853052869826755, + "maximum": 1.079007678676905, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1323771, + "minimum": 1170162, + "maximum": 1370050, + "runs": 5 + }, + "postgres": { + "median": 1744629, + "minimum": 1583383, + "maximum": 1772797, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.305162510454742, + "minimum": 1.2789598919747454, + "maximum": 1.3531314467569449, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1581667, + "minimum": 1473307, + "maximum": 1719904, + "runs": 5 + }, + "postgres": { + "median": 2001077, + "minimum": 1895839, + "maximum": 2067350, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.2906732859853018, + "minimum": 1.1634817989841293, + "maximum": 1.3186504917169335, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 65598933, + "minimum": 65015030, + "maximum": 66181845, + "runs": 5 + }, + "postgres": { + "median": 68611765, + "minimum": 68547942, + "maximum": 69025769, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.044955136694068, + "minimum": 1.0381977961205993, + "maximum": 1.0554654208419192, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 81916871, + "minimum": 78314685, + "maximum": 95711344, + "runs": 5 + }, + "postgres": { + "median": 100399062, + "minimum": 81794656, + "maximum": 101233767, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.167308176207029, + "minimum": 0.9985080606899646, + "maximum": 1.2926536957915364, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 547065, + "minimum": 479163, + "maximum": 622282, + "runs": 5 + }, + "postgres": { + "median": 588937, + "minimum": 494600, + "maximum": 643262, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0877573962874614, + "minimum": 0.7948164979864435, + "maximum": 1.3424700989016263, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1056408, + "minimum": 1011057, + "maximum": 1089672, + "runs": 5 + }, + "postgres": { + "median": 1055771, + "minimum": 1041072, + "maximum": 1078740, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0191194726131836, + "minimum": 0.9688888032362032, + "maximum": 1.0368459234297482, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 65013849, + "minimum": 64485794, + "maximum": 65894630, + "runs": 5 + }, + "postgres": { + "median": 68074162, + "minimum": 67981040, + "maximum": 68507164, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0458687348290978, + "minimum": 1.0353624319234485, + "maximum": 1.0557096652946538, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 80781371, + "minimum": 78081945, + "maximum": 94659115, + "runs": 5 + }, + "postgres": { + "median": 94743795, + "minimum": 81517582, + "maximum": 100878123, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1672928652083503, + "minimum": 1.0008945784037808, + "maximum": 1.2919519743008452, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 4114718, + "minimum": 3956656, + "maximum": 4189279, + "runs": 5 + }, + "postgres": { + "median": 6026968, + "minimum": 5764592, + "maximum": 6091277, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.4569353514685128, + "minimum": 1.4271949772785364, + "maximum": 1.4661813957720247, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 5957833, + "minimum": 5520800, + "maximum": 8324411, + "runs": 5 + }, + "postgres": { + "median": 11030216, + "minimum": 9929255, + "maximum": 13257679, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.7293833177264284, + "minimum": 1.3850767145978602, + "maximum": 1.9979379800028982, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 322332, + "minimum": 280072, + "maximum": 403257, + "runs": 5 + }, + "postgres": { + "median": 332146, + "minimum": 317003, + "maximum": 398597, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.029438591266148, + "minimum": 0.8513677381917735, + "maximum": 1.4231947499214488, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1029296, + "minimum": 999342, + "maximum": 1069901, + "runs": 5 + }, + "postgres": { + "median": 1038788, + "minimum": 985874, + "maximum": 1668082, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0092230687792567, + "minimum": 0.9578138844414046, + "maximum": 1.5590993933083528, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 3735186, + "minimum": 3486101, + "maximum": 3767730, + "runs": 5 + }, + "postgres": { + "median": 5517813, + "minimum": 5340250, + "maximum": 5613696, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.4797352782710014, + "minimum": 1.4688163056151942, + "maximum": 1.5318689848630318, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 5240543, + "minimum": 4893358, + "maximum": 7466353, + "runs": 5 + }, + "postgres": { + "median": 10550240, + "minimum": 9327223, + "maximum": 12367177, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.7836648179374142, + "minimum": 1.6563879312965781, + "maximum": 2.1560327284453744, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 252, + 252, + 252, + 252, + 252 + ], + "postgres": [ + 252, + 252, + 252, + 252, + 252 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 6214424, + "minimum": 6107216, + "maximum": 6327265, + "runs": 5 + }, + "postgres": { + "median": 8376940, + "minimum": 8215721, + "maximum": 8658349, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.3588737018664574, + "minimum": 1.3129341671238495, + "maximum": 1.3778317321673248, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 13036877, + "minimum": 11361960, + "maximum": 13717888, + "runs": 5 + }, + "postgres": { + "median": 17667061, + "minimum": 15243733, + "maximum": 18184385, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.3416464236804213, + "minimum": 1.235364702867062, + "maximum": 1.4267927115671002, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 216, + 216, + 216, + 216, + 216 + ], + "postgres": [ + 216, + 216, + 216, + 216, + 216 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 478862, + "minimum": 434298, + "maximum": 564086, + "runs": 5 + }, + "postgres": { + "median": 494577, + "minimum": 440445, + "maximum": 527650, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.950348952307763, + "minimum": 0.8767758816917988, + "maximum": 1.170109003495296, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1039736, + "minimum": 1026283, + "maximum": 1053300, + "runs": 5 + }, + "postgres": { + "median": 1028520, + "minimum": 1006389, + "maximum": 1060890, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.979180094231756, + "minimum": 0.9715668208083591, + "maximum": 1.0337207183593609, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 216, + 216, + 216, + 216, + 216 + ], + "postgres": [ + 216, + 216, + 216, + 216, + 216 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 5703947, + "minimum": 5534435, + "maximum": 5767479, + "runs": 5 + }, + "postgres": { + "median": 7864980, + "minimum": 7613812, + "maximum": 8016992, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.3900340165954657, + "minimum": 1.3213876752825677, + "maximum": 1.421098992038031, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 12103249, + "minimum": 11080171, + "maximum": 13101166, + "runs": 5 + }, + "postgres": { + "median": 16872037, + "minimum": 15143200, + "maximum": 17435104, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.3731769121613737, + "minimum": 1.251168178065245, + "maximum": 1.4888258447316531, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 216, + 216, + 216, + 216, + 216 + ], + "postgres": [ + 216, + 216, + 216, + 216, + 216 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 3303521, + "minimum": 3216765, + "maximum": 3393701, + "runs": 5 + }, + "postgres": { + "median": 3471552, + "minimum": 3327265, + "maximum": 3503985, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0343512814893223, + "minimum": 1.022939852391239, + "maximum": 1.059166870741854, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 4419893, + "minimum": 4254801, + "maximum": 4576523, + "runs": 5 + }, + "postgres": { + "median": 4696072, + "minimum": 4522869, + "maximum": 5217161, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0624854493083882, + "minimum": 1.0335904159239466, + "maximum": 1.1913551434948353, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 360, + 360, + 360, + 360, + 360 + ], + "postgres": [ + 360, + 360, + 360, + 360, + 360 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 595802, + "minimum": 581888, + "maximum": 609413, + "runs": 5 + }, + "postgres": { + "median": 572235, + "minimum": 559074, + "maximum": 613455, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9834108969423669, + "minimum": 0.9212858063314773, + "maximum": 1.0066326120381417, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1049613, + "minimum": 1043068, + "maximum": 1076361, + "runs": 5 + }, + "postgres": { + "median": 1045481, + "minimum": 1044006, + "maximum": 1112245, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9986350153477076, + "minimum": 0.9855075744499434, + "maximum": 1.0333382573318803, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 360, + 360, + 360, + 360, + 360 + ], + "postgres": [ + 360, + 360, + 360, + 360, + 360 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 2654894, + "minimum": 2572515, + "maximum": 2710245, + "runs": 5 + }, + "postgres": { + "median": 2839061, + "minimum": 2701803, + "maximum": 2922084, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.057943423042325, + "minimum": 1.050257432901266, + "maximum": 1.0860855461649317, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 3761869, + "minimum": 3556429, + "maximum": 3951227, + "runs": 5 + }, + "postgres": { + "median": 4059676, + "minimum": 3927441, + "maximum": 4232679, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0791646386410585, + "minimum": 1.0712315440241726, + "maximum": 1.1544305404174888, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 360, + 360, + 360, + 360, + 360 + ], + "postgres": [ + 360, + 360, + 360, + 360, + 360 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1935530, + "minimum": 1806772, + "maximum": 1946465, + "runs": 5 + }, + "postgres": { + "median": 4065974, + "minimum": 3907691, + "maximum": 4136807, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 2.1224791142477772, + "minimum": 2.054725321759779, + "maximum": 2.1964503545549743, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2393852, + "minimum": 2341272, + "maximum": 2536977, + "runs": 5 + }, + "postgres": { + "median": 4818157, + "minimum": 4639106, + "maximum": 5216864, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 2.056330822076826, + "minimum": 1.9070188424284096, + "maximum": 2.0888793459244765, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 574619, + "minimum": 565662, + "maximum": 620187, + "runs": 5 + }, + "postgres": { + "median": 569722, + "minimum": 558604, + "maximum": 652365, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9940673733378117, + "minimum": 0.9007025300433579, + "maximum": 1.140869233701868, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1045338, + "minimum": 1029716, + "maximum": 1049240, + "runs": 5 + }, + "postgres": { + "median": 1047278, + "minimum": 1038649, + "maximum": 1443510, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0033529824803078, + "minimum": 0.9910167699682462, + "maximum": 1.3895643334469234, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1318433, + "minimum": 1221779, + "maximum": 1365405, + "runs": 5 + }, + "postgres": { + "median": 3455241, + "minimum": 3326121, + "maximum": 3566886, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 2.6211269647886546, + "minimum": 2.53056126204313, + "maximum": 2.7604722294293813, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1643928, + "minimum": 1529953, + "maximum": 1746421, + "runs": 5 + }, + "postgres": { + "median": 4013238, + "minimum": 3837048, + "maximum": 4121300, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 2.4069207043401186, + "minimum": 2.2979785515634545, + "maximum": 2.5532856238067443, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1313863, + "minimum": 1126866, + "maximum": 1420374, + "runs": 5 + }, + "postgres": { + "median": 1330544, + "minimum": 1250025, + "maximum": 1346280, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0157039227742717, + "minimum": 0.9347784456769836, + "maximum": 1.109293385371464, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1777662, + "minimum": 1646191, + "maximum": 1884087, + "runs": 5 + }, + "postgres": { + "median": 1929635, + "minimum": 1703208, + "maximum": 1988883, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0854903800610014, + "minimum": 0.9833494950073961, + "maximum": 1.1899951954632728, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 45, + 45, + 45, + 45, + 45 + ], + "postgres": [ + 45, + 45, + 45, + 45, + 45 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 633164, + "minimum": 495364, + "maximum": 754955, + "runs": 5 + }, + "postgres": { + "median": 589844, + "minimum": 505181, + "maximum": 695785, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9093160439329444, + "minimum": 0.7812968984906385, + "maximum": 1.4045933899112573, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1023902, + "minimum": 1013947, + "maximum": 1081470, + "runs": 5 + }, + "postgres": { + "median": 1017178, + "minimum": 980369, + "maximum": 1043670, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9574832357002916, + "minimum": 0.925531796092562, + "maximum": 1.0287287437261219, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 45, + 45, + 45, + 45, + 45 + ], + "postgres": [ + 45, + 45, + 45, + 45, + 45 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 686431, + "minimum": 580420, + "maximum": 712958, + "runs": 5 + }, + "postgres": { + "median": 706158, + "minimum": 590007, + "maximum": 742811, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.02863562790515, + "minimum": 1.0047797957842812, + "maximum": 1.0702495771222411, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 844782, + "minimum": 762845, + "maximum": 1165325, + "runs": 5 + }, + "postgres": { + "median": 1115371, + "minimum": 838290, + "maximum": 1323650, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1358633857507563, + "minimum": 0.9923151771699681, + "maximum": 1.503816849066718, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 45, + 45, + 45, + 45, + 45 + ], + "postgres": [ + 45, + 45, + 45, + 45, + 45 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1689418, + "minimum": 1601670, + "maximum": 1792109, + "runs": 5 + }, + "postgres": { + "median": 2104728, + "minimum": 2095063, + "maximum": 2309270, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.2982220779108113, + "minimum": 1.196018971688089, + "maximum": 1.3669026848299237, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2174378, + "minimum": 2076777, + "maximum": 2263324, + "runs": 5 + }, + "postgres": { + "median": 2817773, + "minimum": 2754264, + "maximum": 3114445, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.3050826489230483, + "minimum": 1.2449711132829413, + "maximum": 1.3866345569057803, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 135, + 135, + 135, + 135, + 135 + ], + "postgres": [ + 135, + 135, + 135, + 135, + 135 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 587610, + "minimum": 581017, + "maximum": 681901, + "runs": 5 + }, + "postgres": { + "median": 605695, + "minimum": 469480, + "maximum": 668517, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9907996565022988, + "minimum": 0.7407687913988267, + "maximum": 1.1505980031565342, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1066939, + "minimum": 1039154, + "maximum": 1073574, + "runs": 5 + }, + "postgres": { + "median": 1062000, + "minimum": 1047300, + "maximum": 1076218, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0078390690888934, + "minimum": 0.9896215123844608, + "maximum": 1.0093569863755816, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 135, + 135, + 135, + 135, + 135 + ], + "postgres": [ + 135, + 135, + 135, + 135, + 135 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1095235, + "minimum": 931217, + "maximum": 1162979, + "runs": 5 + }, + "postgres": { + "median": 1591962, + "minimum": 1435772, + "maximum": 1615196, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.4284958746684566, + "minimum": 1.3688656458973034, + "maximum": 1.5418232270244208, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1326226, + "minimum": 1201301, + "maximum": 1410245, + "runs": 5 + }, + "postgres": { + "median": 2047726, + "minimum": 1968266, + "maximum": 2285591, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.6207049129761142, + "minimum": 1.4948409136132135, + "maximum": 1.704590273378612, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 135, + 135, + 135, + 135, + 135 + ], + "postgres": [ + 135, + 135, + 135, + 135, + 135 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 26540811, + "minimum": 25976110, + "maximum": 27594701, + "runs": 5 + }, + "postgres": { + "median": 34160080, + "minimum": 33524303, + "maximum": 37112054, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.303182308667464, + "minimum": 1.2795701547271987, + "maximum": 1.38490826215249, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 30617926, + "minimum": 28145003, + "maximum": 33443503, + "runs": 5 + }, + "postgres": { + "median": 37589285, + "minimum": 37115620, + "maximum": 43283084, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.2951851721045782, + "minimum": 1.137012654177654, + "maximum": 1.3745029627415, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 639982, + "minimum": 544571, + "maximum": 656353, + "runs": 5 + }, + "postgres": { + "median": 584599, + "minimum": 525931, + "maximum": 697363, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9150147972911739, + "minimum": 0.8012929018378829, + "maximum": 1.1712945389315694, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1067151, + "minimum": 1038899, + "maximum": 1074011, + "runs": 5 + }, + "postgres": { + "median": 1057421, + "minimum": 1033204, + "maximum": 1074282, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.999408705984439, + "minimum": 0.9620050446410698, + "maximum": 1.0178284895836842, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 25896441, + "minimum": 25377023, + "maximum": 26982418, + "runs": 5 + }, + "postgres": { + "median": 33631958, + "minimum": 33015043, + "maximum": 36484020, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.3091961574846664, + "minimum": 1.2861990341618148, + "maximum": 1.3931598168257948, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 30280915, + "minimum": 27689755, + "maximum": 32591535, + "runs": 5 + }, + "postgres": { + "median": 36968978, + "minimum": 36453181, + "maximum": 42700273, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.3101645258500405, + "minimum": 1.1347772131206064, + "maximum": 1.3601862427208689, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1364229, + "minimum": 1276500, + "maximum": 1385008, + "runs": 5 + }, + "postgres": { + "median": 1383107, + "minimum": 1291276, + "maximum": 1528221, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.011575401488445, + "minimum": 0.9986274447512217, + "maximum": 1.110750040338817, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1841010, + "minimum": 1767405, + "maximum": 1852645, + "runs": 5 + }, + "postgres": { + "median": 1930079, + "minimum": 1802579, + "maximum": 2481867, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0510742434225193, + "minimum": 1.019901493998263, + "maximum": 1.3396344145802352, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 587085, + "minimum": 583027, + "maximum": 617120, + "runs": 5 + }, + "postgres": { + "median": 574633, + "minimum": 565569, + "maximum": 687279, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9787901240876534, + "minimum": 0.9249918978480685, + "maximum": 1.1152600405679514, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1058406, + "minimum": 1041996, + "maximum": 1083818, + "runs": 5 + }, + "postgres": { + "median": 1072149, + "minimum": 1052004, + "maximum": 1718224, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0162237154068352, + "minimum": 0.9975784339846901, + "maximum": 1.5853436647112338, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 772411, + "minimum": 658939, + "maximum": 782000, + "runs": 5 + }, + "postgres": { + "median": 804338, + "minimum": 710671, + "maximum": 814914, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0472339208012316, + "minimum": 1.0285652173913042, + "maximum": 1.0785080257808386, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 901521, + "minimum": 897776, + "maximum": 1046312, + "runs": 5 + }, + "postgres": { + "median": 1015812, + "minimum": 886601, + "maximum": 1193621, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.126790747513336, + "minimum": 0.929476102730352, + "maximum": 1.3240079820658643, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 2145047, + "minimum": 1938965, + "maximum": 2157413, + "runs": 5 + }, + "postgres": { + "median": 2272966, + "minimum": 2090949, + "maximum": 2319932, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0753305000016222, + "minimum": 1.0538984625742214, + "maximum": 1.0783840863553493, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2652220, + "minimum": 2455765, + "maximum": 2673650, + "runs": 5 + }, + "postgres": { + "median": 2773720, + "minimum": 2568425, + "maximum": 2805055, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0458757250795576, + "minimum": 1.0244376261354333, + "maximum": 1.0531501911606127, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 617118, + "minimum": 549551, + "maximum": 671592, + "runs": 5 + }, + "postgres": { + "median": 599414, + "minimum": 587542, + "maximum": 648280, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9694483064827148, + "minimum": 0.8748496110733898, + "maximum": 1.1063431783401358, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1051346, + "minimum": 1031640, + "maximum": 1065467, + "runs": 5 + }, + "postgres": { + "median": 1049585, + "minimum": 1035952, + "maximum": 1065189, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9997390815482788, + "minimum": 0.9856161007147868, + "maximum": 1.0123722721092099, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1498731, + "minimum": 1387484, + "maximum": 1538313, + "runs": 5 + }, + "postgres": { + "median": 1683737, + "minimum": 1506707, + "maximum": 1714497, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1048148198708585, + "minimum": 1.0859274773619012, + "maximum": 1.143965795062623, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1714378, + "minimum": 1613502, + "maximum": 1796929, + "runs": 5 + }, + "postgres": { + "median": 1936598, + "minimum": 1779121, + "maximum": 1951423, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.102325741464251, + "minimum": 1.0812313675164684, + "maximum": 1.1310655540947874, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "end_to_end", + "unit": "ns", + "p50": { + "sqlite": { + "median": 1082535, + "minimum": 956817, + "maximum": 1124581, + "runs": 5 + }, + "postgres": { + "median": 1052673, + "minimum": 960577, + "maximum": 1131775, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.003929696065183, + "minimum": 0.9148415524671257, + "maximum": 1.0330140963491374, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 3003745, + "minimum": 2722768, + "maximum": 3081010, + "runs": 5 + }, + "postgres": { + "median": 4717236, + "minimum": 4084346, + "maximum": 5071998, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.5979591169925178, + "minimum": 1.4819999789547706, + "maximum": 1.6885581166177555, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "unit": "ns", + "p50": { + "sqlite": { + "median": 573625, + "minimum": 545515, + "maximum": 605442, + "runs": 5 + }, + "postgres": { + "median": 588494, + "minimum": 551164, + "maximum": 642932, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0298726891102903, + "minimum": 0.9608437568097625, + "maximum": 1.0666193949815197, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1064329, + "minimum": 1041025, + "maximum": 1079856, + "runs": 5 + }, + "postgres": { + "median": 1046491, + "minimum": 1027026, + "maximum": 1076667, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9870563867294716, + "minimum": 0.9601658137269383, + "maximum": 1.0202627218366513, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 463802, + "minimum": 364624, + "maximum": 477312, + "runs": 5 + }, + "postgres": { + "median": 446719, + "minimum": 370837, + "maximum": 455210, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9541678761077057, + "minimum": 0.863140278886595, + "maximum": 1.024183816753697, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 2276763, + "minimum": 2149496, + "maximum": 2350755, + "runs": 5 + }, + "postgres": { + "median": 3854922, + "minimum": 3660450, + "maximum": 4649564, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.733408668822831, + "minimum": 1.6077430984252643, + "maximum": 2.013027457238638, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 180, + 180, + 180, + 180, + 180 + ], + "postgres": [ + 180, + 180, + 180, + 180, + 180 + ] + } + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 83202054, + "minimum": 78486818, + "maximum": 89086196, + "runs": 5 + }, + "postgres": { + "median": 107964669, + "minimum": 103400400, + "maximum": 122143362, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.261058206567833, + "minimum": 1.2121703119976073, + "maximum": 1.5562277222144487, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 95288558, + "minimum": 92099760, + "maximum": 99277759, + "runs": 5 + }, + "postgres": { + "median": 122645583, + "minimum": 117746508, + "maximum": 146742113, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.2784670448652635, + "minimum": 1.219284542877521, + "maximum": 1.5399762162420383, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 100, + 100, + 100, + 100, + 100 + ], + "postgres": [ + 100, + 100, + 100, + 100, + 100 + ] + } + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "unit": "ns", + "p50": { + "sqlite": { + "median": 854921, + "minimum": 817541, + "maximum": 945308, + "runs": 5 + }, + "postgres": { + "median": 797084, + "minimum": 769543, + "maximum": 828791, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9323481350908447, + "minimum": 0.8605007637096451, + "maximum": 0.9627842619082448, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 1341007, + "minimum": 1235474, + "maximum": 1509437, + "runs": 5 + }, + "postgres": { + "median": 1401317, + "minimum": 1322560, + "maximum": 1567553, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0704879260915243, + "minimum": 0.9300984406768882, + "maximum": 1.168937224041336, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": null, + "postgres": null, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null + }, + "note": "omitted: at least one run has fewer than 1000 successful samples" + }, + "samples_per_run": { + "sqlite": [ + 100, + 100, + 100, + 100, + 100 + ], + "postgres": [ + 100, + 100, + 100, + 100, + 100 + ] + } + }, + { + "scenario": "websocket", + "event_class": "new_transmission", + "timing": "receipt_after_ingest_return", + "unit": "ns", + "p50": { + "sqlite": { + "median": 512546498, + "minimum": 504260023, + "maximum": 525747562, + "runs": 5 + }, + "postgres": { + "median": 505394149, + "minimum": 503054792, + "maximum": 518333693, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.990595652915937, + "minimum": 0.9612867191954758, + "maximum": 1.0279095493556505, + "runs": 5 + }, + "note": "" + }, + "p95": { + "sqlite": { + "median": 991098321, + "minimum": 985123611, + "maximum": 1091642087, + "runs": 5 + }, + "postgres": { + "median": 958879301, + "minimum": 945775962, + "maximum": 964605246, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9660684694066796, + "minimum": 0.8783824958921724, + "maximum": 0.9791717863921952, + "runs": 5 + }, + "note": "" + }, + "p99": { + "sqlite": { + "median": 2035356108, + "minimum": 1827343470, + "maximum": 2448201481, + "runs": 5 + }, + "postgres": { + "median": 998007405, + "minimum": 985737995, + "maximum": 1451344825, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.5317570903885952, + "minimum": 0.4524160112869924, + "maximum": 0.5928208263345953, + "runs": 5 + }, + "note": "" + }, + "samples_per_run": { + "sqlite": [ + 4500, + 4500, + 4500, + 4500, + 4500 + ], + "postgres": [ + 4500, + 4500, + 4500, + 4500, + 4500 + ] + } + } + ], + "scalar_metrics": [ + { + "metric": "durable-ingest/additional/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 15.024930271060285, + "minimum": 15.024886476391238, + "maximum": 15.024947060749692, + "runs": 5 + }, + "postgres": { + "median": 15.024845084666719, + "minimum": 15.024828072386086, + "maximum": 15.024934291224621, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999958399701213, + "minimum": 0.9999935320045223, + "maximum": 0.9999991501118094, + "runs": 5 + } + }, + { + "metric": "durable-ingest/duplicate/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 7.514980647741974, + "minimum": 7.514904501268186, + "maximum": 7.514986694455287, + "runs": 5 + }, + "postgres": { + "median": 7.514965849832371, + "minimum": 7.514939544673058, + "maximum": 7.51497555165654, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999981355641467, + "minimum": 0.9999945305156669, + "maximum": 1.0000094545963083, + "runs": 5 + } + }, + { + "metric": "durable-ingest/late/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 2.5055413593198366, + "minimum": 2.5055391980361836, + "maximum": 2.505550092004002, + "runs": 5 + }, + "postgres": { + "median": 2.5055373102414924, + "minimum": 2.5055252649504234, + "maximum": 2.5055433088099575, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999983839507063, + "minimum": 0.99999225361186, + "maximum": 1.0000016406743015, + "runs": 5 + } + }, + { + "metric": "durable-ingest/new/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 25.03032008887913, + "minimum": 25.03030412370661, + "maximum": 25.030408895462084, + "runs": 5 + }, + "postgres": { + "median": 25.030257308838635, + "minimum": 25.030244157068257, + "maximum": 25.030340899004116, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999975098286968, + "minimum": 0.9999961094441225, + "maximum": 0.9999996167854808, + "runs": 5 + } + }, + { + "metric": "migration/elapsed_ns", + "unit": "ns", + "sqlite": null, + "postgres": { + "median": 148886831843, + "minimum": 147219062637, + "maximum": 153642982703, + "runs": 5 + }, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null, + "runs": 0 + } + }, + { + "metric": "migration/harness_validation_ns", + "unit": "ns", + "sqlite": null, + "postgres": { + "median": 58164959829, + "minimum": 56907451138, + "maximum": 59945115482, + "runs": 5 + }, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null, + "runs": 0 + } + }, + { + "metric": "migration/migration_tool_ns", + "unit": "ns", + "sqlite": null, + "postgres": { + "median": 90001959193, + "minimum": 89485454621, + "maximum": 92961857345, + "runs": 5 + }, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null, + "runs": 0 + } + }, + { + "metric": "migration/role_grants_ns", + "unit": "ns", + "sqlite": null, + "postgres": { + "median": 731288250, + "minimum": 688529717, + "maximum": 748674192, + "runs": 5 + }, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null, + "runs": 0 + } + }, + { + "metric": "mixed-http/analytics_channels/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 0.41088183093092123, + "minimum": 0.41087495323156137, + "maximum": 0.41088448788658594, + "runs": 5 + }, + "postgres": { + "median": 0.4108827104864554, + "minimum": 0.41087384832861445, + "maximum": 0.41088932704955566, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0000116430714416, + "minimum": 0.9999741057200134, + "maximum": 1.0000228174376282, + "runs": 5 + } + }, + { + "metric": "mixed-http/analytics_rf/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 0.615728247327062, + "minimum": 0.6157278777121242, + "maximum": 0.6157293605765729, + "runs": 5 + }, + "postgres": { + "median": 0.6157286119190807, + "minimum": 0.6157283776079031, + "maximum": 0.615729273290617, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.000000918866931, + "minimum": 0.9999987841127285, + "maximum": 1.0000009597394988, + "runs": 5 + } + }, + { + "metric": "mixed-http/analytics_rf_filtered/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 0.410953637961553, + "minimum": 0.4109527654088309, + "maximum": 0.41095411837061285, + "runs": 5 + }, + "postgres": { + "median": 0.410953129540234, + "minimum": 0.4109510955157029, + "maximum": 0.41095377337987593, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999991940410053, + "minimum": 0.9999931939135709, + "maximum": 1.000002452766181, + "runs": 5 + } + }, + { + "metric": "mixed-http/analytics_topology/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 0.6156280388644793, + "minimum": 0.6156209166888316, + "maximum": 0.6156327644428657, + "runs": 5 + }, + "postgres": { + "median": 0.6156254122729737, + "minimum": 0.6155975641427371, + "maximum": 0.6156291610652173, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999951077502538, + "minimum": 0.99995408776467, + "maximum": 1.0000055365318643, + "runs": 5 + } + }, + { + "metric": "mixed-http/channels/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.4326212474181543, + "minimum": 1.432618326422076, + "maximum": 1.4326243274555144, + "runs": 5 + }, + "postgres": { + "median": 1.4326185940968201, + "minimum": 1.4326159213039131, + "maximum": 1.4326239215259087, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999996024241672, + "minimum": 0.9999958331955497, + "maximum": 1.0000018665838994, + "runs": 5 + } + }, + { + "metric": "mixed-http/messages_0/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.4326182308961137, + "minimum": 1.4326133511726002, + "maximum": 1.4326208426305178, + "runs": 5 + }, + "postgres": { + "median": 1.4326133211931782, + "minimum": 1.4316812043604843, + "maximum": 1.4326176490830909, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999973277051453, + "minimum": 0.9993459342374532, + "maximum": 1.0000016934769183, + "runs": 5 + } + }, + { + "metric": "mixed-http/messages_1000/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.2290056624062093, + "minimum": 1.2290032098728656, + "maximum": 1.2290092649426563, + "runs": 5 + }, + "postgres": { + "median": 1.228199949692814, + "minimum": 1.2279194621068021, + "maximum": 1.2290065601065858, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9993464132773626, + "minimum": 0.9991142744844707, + "maximum": 0.9999977991735721, + "runs": 5 + } + }, + { + "metric": "mixed-http/metrics_dense_24h/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.0262044936711134, + "minimum": 1.0262022581352577, + "maximum": 1.0262078860310873, + "runs": 5 + }, + "postgres": { + "median": 1.0262026278288632, + "minimum": 1.0262015863630505, + "maximum": 1.0262058941966479, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.999997926729153, + "minimum": 0.9999951787104583, + "maximum": 1.0000019247973844, + "runs": 5 + } + }, + { + "metric": "mixed-http/metrics_sparse_7d/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.0262149859108898, + "minimum": 1.0262134891527042, + "maximum": 1.0262160380989125, + "runs": 5 + }, + "postgres": { + "median": 1.0262138283550255, + "minimum": 1.0262101284951664, + "maximum": 1.0262149466062171, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999989363909209, + "minimum": 0.9999952666684953, + "maximum": 1.0000007037492002, + "runs": 5 + } + }, + { + "metric": "mixed-http/nodes_2000/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.4321105228925104, + "minimum": 1.4320569916909363, + "maximum": 1.4321144857845218, + "runs": 5 + }, + "postgres": { + "median": 1.4321072565085593, + "minimum": 1.4320219231522593, + "maximum": 1.4321117542880417, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999980059038978, + "minimum": 0.9999381334479184, + "maximum": 1.0000350997327025, + "runs": 5 + } + }, + { + "metric": "mixed-http/nodes_50/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.432598216162731, + "minimum": 1.4325558457486738, + "maximum": 1.4326021056626153, + "runs": 5 + }, + "postgres": { + "median": 1.4325822781627682, + "minimum": 1.4325739527466708, + "maximum": 1.4325856953918086, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999885451300526, + "minimum": 0.9999831311972592, + "maximum": 1.000020587456504, + "runs": 5 + } + }, + { + "metric": "mixed-http/nodes_region/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.2289677728687, + "minimum": 1.228911930142778, + "maximum": 1.2289728046456092, + "runs": 5 + }, + "postgres": { + "median": 1.228957240520968, + "minimum": 1.228944937461886, + "maximum": 1.2289680997851973, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999914299235794, + "minimum": 0.9999885235512442, + "maximum": 1.000027743693128, + "runs": 5 + } + }, + { + "metric": "mixed-http/observers/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 2.0518240460051866, + "minimum": 2.0518178811133025, + "maximum": 2.0518314499282937, + "runs": 5 + }, + "postgres": { + "median": 2.0518273505773417, + "minimum": 2.0518240927827645, + "maximum": 2.051836578636568, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0000033694709785, + "minimum": 0.9999990252125706, + "maximum": 1.0000045099103423, + "runs": 5 + } + }, + { + "metric": "mixed-http/packet_detail_memory/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.0250458382186123, + "minimum": 1.025043048316334, + "maximum": 1.0250490387711255, + "runs": 5 + }, + "postgres": { + "median": 1.0250341703682644, + "minimum": 1.0250306195449674, + "maximum": 1.0250378874022743, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999878748786337, + "minimum": 0.9999854949350726, + "maximum": 0.9999922434528862, + "runs": 5 + } + }, + { + "metric": "mixed-http/packet_detail_pruned/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.1083439301938458, + "minimum": 1.1083299179582484, + "maximum": 1.1083504799557249, + "runs": 5 + }, + "postgres": { + "median": 1.1083452704904195, + "minimum": 1.1083406429622016, + "maximum": 1.108350205631316, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0000007344384874, + "minimum": 0.9999940399105073, + "maximum": 1.0000171710345194, + "runs": 5 + } + }, + { + "metric": "mixed-http/packet_detail_sql/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.0336772822007385, + "minimum": 1.0336728762943936, + "maximum": 1.0336795072501905, + "runs": 5 + }, + "postgres": { + "median": 1.0336725572069057, + "minimum": 1.0336698381567269, + "maximum": 1.033674054330128, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999968772936523, + "minimum": 0.9999918885247916, + "maximum": 0.9999996913070902, + "runs": 5 + } + }, + { + "metric": "mixed-http/packets_memory/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.0248997384007508, + "minimum": 1.0248720946093346, + "maximum": 1.0249036991510676, + "runs": 5 + }, + "postgres": { + "median": 1.0248531419798628, + "minimum": 1.0248405324823453, + "maximum": 1.0248628867039926, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999572583841526, + "minimum": 0.9999399866820976, + "maximum": 0.9999910155565847, + "runs": 5 + } + }, + { + "metric": "mixed-http/reach/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.026218231973548, + "minimum": 1.0262134540664531, + "maximum": 1.0262199749198988, + "runs": 5 + }, + "postgres": { + "median": 1.0262195466424024, + "minimum": 1.026214496158971, + "maximum": 1.0262218067096294, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0000027158216156, + "minimum": 0.9999963596295012, + "maximum": 1.0000052081345194, + "runs": 5 + } + }, + { + "metric": "mixed-http/rx_coverage/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.0262118842560923, + "minimum": 1.0262097713690779, + "maximum": 1.0262143254020595, + "runs": 5 + }, + "postgres": { + "median": 1.0262117959995616, + "minimum": 1.026210594717649, + "maximum": 1.0262145751017369, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.0000008023199487, + "minimum": 0.9999975033031433, + "maximum": 1.0000035765742181, + "runs": 5 + } + }, + { + "metric": "mixed-http/stats/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 1.0273914408344735, + "minimum": 1.0273895804840816, + "maximum": 1.027392178494361, + "runs": 5 + }, + "postgres": { + "median": 1.0273905231312968, + "minimum": 1.0273884582523665, + "maximum": 1.0273908412919814, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9999989056808595, + "minimum": 0.9999970969369721, + "maximum": 1.0000012271955292, + "runs": 5 + } + }, + { + "metric": "resource/aggregate_accounted_io_dbytes", + "unit": "bytes", + "sqlite": { + "median": 0, + "minimum": 0, + "maximum": 0, + "runs": 5 + }, + "postgres": { + "median": 4096, + "minimum": 0, + "maximum": 4096, + "runs": 5 + }, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null, + "runs": 0 + } + }, + { + "metric": "resource/aggregate_accounted_io_dios", + "unit": "operations", + "sqlite": { + "median": 0, + "minimum": 0, + "maximum": 0, + "runs": 5 + }, + "postgres": { + "median": 1, + "minimum": 0, + "maximum": 1, + "runs": 5 + }, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null, + "runs": 0 + } + }, + { + "metric": "resource/aggregate_accounted_io_rbytes", + "unit": "bytes", + "sqlite": { + "median": 0, + "minimum": 0, + "maximum": 0, + "runs": 5 + }, + "postgres": { + "median": 8294400, + "minimum": 6127616, + "maximum": 8859648, + "runs": 5 + }, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null, + "runs": 0 + } + }, + { + "metric": "resource/aggregate_accounted_io_rios", + "unit": "operations", + "sqlite": { + "median": 0, + "minimum": 0, + "maximum": 0, + "runs": 5 + }, + "postgres": { + "median": 851, + "minimum": 668, + "maximum": 864, + "runs": 5 + }, + "paired": { + "pairs": 0, + "median": null, + "minimum": null, + "maximum": null, + "runs": 0 + } + }, + { + "metric": "resource/aggregate_accounted_io_wbytes", + "unit": "bytes", + "sqlite": { + "median": 2485182464, + "minimum": 2454315008, + "maximum": 2502598656, + "runs": 5 + }, + "postgres": { + "median": 419500032, + "minimum": 404680704, + "maximum": 538468352, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.1702392197570753, + "minimum": 0.1619090383194065, + "maximum": 0.2151636862383099, + "runs": 5 + } + }, + { + "metric": "resource/aggregate_accounted_io_wios", + "unit": "operations", + "sqlite": { + "median": 222200, + "minimum": 214108, + "maximum": 224715, + "runs": 5 + }, + "postgres": { + "median": 18377, + "minimum": 18146, + "maximum": 27649, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.08579782166009678, + "minimum": 0.08166516651665166, + "maximum": 0.12304029548539261, + "runs": 5 + } + }, + { + "metric": "resource/average_cores_over_sampled_span", + "unit": "cores", + "sqlite": { + "median": 0.2505865322028969, + "minimum": 0.24582361721318563, + "maximum": 0.25309924764113856, + "runs": 5 + }, + "postgres": { + "median": 0.33112132572206215, + "minimum": 0.3251323001096783, + "maximum": 0.38583603339623906, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.3226243425899709, + "minimum": 1.3082667325489195, + "maximum": 1.5247636375440012, + "runs": 5 + } + }, + { + "metric": "resource/cpu_delta_usec", + "unit": "us", + "sqlite": { + "median": 45022426, + "minimum": 43925702, + "maximum": 45465614, + "runs": 5 + }, + "postgres": { + "median": 59226706, + "minimum": 58383925, + "maximum": 69041731, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.3291517799761061, + "minimum": 1.3097204485821992, + "maximum": 1.5185483033397504, + "runs": 5 + } + }, + { + "metric": "resource/observed_peak_cgroup_memory_bytes", + "unit": "bytes", + "sqlite": { + "median": 4246102016, + "minimum": 4181094400, + "maximum": 4280754176, + "runs": 5 + }, + "postgres": { + "median": 5944741888, + "minimum": 5869744128, + "maximum": 6442430464, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.42181479758027, + "minimum": 1.3711939267404454, + "maximum": 1.5271335667445352, + "runs": 5 + } + }, + { + "metric": "resource/observed_peak_pss_bytes", + "unit": "bytes", + "sqlite": { + "median": 3080015872, + "minimum": 3009097728, + "maximum": 3091423232, + "runs": 5 + }, + "postgres": { + "median": 3429304320, + "minimum": 3364481024, + "maximum": 3509343232, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1103260826320018, + "minimum": 1.0883275344422332, + "maximum": 1.1476019888102633, + "runs": 5 + } + }, + { + "metric": "resource/observed_peak_rss_bytes", + "unit": "bytes", + "sqlite": { + "median": 3083268096, + "minimum": 3012349952, + "maximum": 3094675456, + "runs": 5 + }, + "postgres": { + "median": 5582184448, + "minimum": 5356032000, + "maximum": 6220292096, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.853099585688509, + "minimum": 1.749633047040894, + "maximum": 2.011883501936865, + "runs": 5 + } + }, + { + "metric": "resource/observed_peak_storage_bytes", + "unit": "bytes", + "sqlite": { + "median": 1151541616, + "minimum": 1137166624, + "maximum": 1162624192, + "runs": 5 + }, + "postgres": { + "median": 2440179516, + "minimum": 2439917372, + "maximum": 2440343356, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 2.11914704435658, + "minimum": 2.098995851619093, + "maximum": 2.14561113605107, + "runs": 5 + } + }, + { + "metric": "resource/observed_peak_wal_bytes", + "unit": "bytes", + "sqlite": { + "median": 32197832, + "minimum": 26322712, + "maximum": 42040512, + "runs": 5 + }, + "postgres": { + "median": 1073741824, + "minimum": 1073741824, + "maximum": 1073741824, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 33.34826469061644, + "minimum": 25.540645746655034, + "maximum": 40.79145887399444, + "runs": 5 + } + }, + { + "metric": "retention-empty/elapsed_ns", + "unit": "ns", + "sqlite": { + "median": 297698, + "minimum": 274432, + "maximum": 323997, + "runs": 5 + }, + "postgres": { + "median": 1900989, + "minimum": 1728440, + "maximum": 2140589, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 6.167896900459756, + "minimum": 5.762426812593944, + "maximum": 7.190471551706763, + "runs": 5 + } + }, + { + "metric": "retention/elapsed_ns", + "unit": "ns", + "sqlite": { + "median": 11267969655, + "minimum": 11079614774, + "maximum": 11340422266, + "runs": 5 + }, + "postgres": { + "median": 973707150, + "minimum": 889359099, + "maximum": 1009484608, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.0870833938031326, + "minimum": 0.0784238080504647, + "maximum": 0.09111188688336973, + "runs": 5 + } + }, + { + "metric": "sql-miss/channel_messages_sql/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 12.18961064472026, + "minimum": 11.717865722706463, + "maximum": 12.35984317747424, + "runs": 5 + }, + "postgres": { + "median": 8.905742149372086, + "minimum": 7.709219902584099, + "maximum": 9.258089083946725, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.7448165171078089, + "minimum": 0.632441849643756, + "maximum": 0.7589863972925845, + "runs": 5 + } + }, + { + "metric": "sql-miss/observer_metrics_sql/completed_per_second_including_drain", + "unit": "per_second", + "sqlite": { + "median": 12.295960435980614, + "minimum": 11.832089757410333, + "maximum": 12.467865317696237, + "runs": 5 + }, + "postgres": { + "median": 9.05684106446128, + "minimum": 7.859378748000186, + "maximum": 9.421000955023542, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.7494008738751256, + "minimum": 0.6391838025927572, + "maximum": 0.7652400906667209, + "runs": 5 + } + }, + { + "metric": "startup-hot1/first_http_ns", + "unit": "ns", + "sqlite": { + "median": 9767586615, + "minimum": 9665927094, + "maximum": 9876754329, + "runs": 5 + }, + "postgres": { + "median": 11271217649, + "minimum": 10885141683, + "maximum": 11576476851, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1427042860637666, + "minimum": 1.1231501696505848, + "maximum": 1.1720932267201682, + "runs": 5 + } + }, + { + "metric": "startup-hot1/full_ready_ns", + "unit": "ns", + "sqlite": { + "median": 21549346472, + "minimum": 21399464749, + "maximum": 21883091242, + "runs": 5 + }, + "postgres": { + "median": 20913380649, + "minimum": 20765466482, + "maximum": 21116822964, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9701924932265464, + "minimum": 0.9628810002200902, + "maximum": 0.9772852215837448, + "runs": 5 + } + }, + { + "metric": "startup-hot1/ready_ns", + "unit": "ns", + "sqlite": { + "median": 9767590402, + "minimum": 9665929900, + "maximum": 9876758828, + "runs": 5 + }, + "postgres": { + "median": 11271221036, + "minimum": 10885145460, + "maximum": 11576480628, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 1.1427043358818336, + "minimum": 1.1231502120405814, + "maximum": 1.1720930752284235, + "runs": 5 + } + }, + { + "metric": "startup/first_http_ns", + "unit": "ns", + "sqlite": { + "median": 28951575010, + "minimum": 28351764144, + "maximum": 29083449285, + "runs": 5 + }, + "postgres": { + "median": 27549217386, + "minimum": 27343648617, + "maximum": 28856225086, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9648042431020765, + "minimum": 0.9411968825274457, + "maximum": 0.992187164707551, + "runs": 5 + } + }, + { + "metric": "startup/full_ready_ns", + "unit": "ns", + "sqlite": { + "median": 28964685757, + "minimum": 28369145905, + "maximum": 29096733146, + "runs": 5 + }, + "postgres": { + "median": 27712102966, + "minimum": 27504488994, + "maximum": 29419540593, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9700964361474097, + "minimum": 0.9463392585507668, + "maximum": 1.0110942848937794, + "runs": 5 + } + }, + { + "metric": "startup/ready_ns", + "unit": "ns", + "sqlite": { + "median": 28951578015, + "minimum": 28351766949, + "maximum": 29083452731, + "runs": 5 + }, + "postgres": { + "median": 27549219950, + "minimum": 27343651192, + "maximum": 28856227611, + "runs": 5 + }, + "paired": { + "pairs": 5, + "median": 0.9648042387951868, + "minimum": 0.9411968938948369, + "maximum": 0.9921871339657756, + "runs": 5 + } + } + ], + "runs": [ + { + "pair": 0, + "backend": "sqlite", + "metrics": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "samples": 2700, + "p50": 800490, + "p95": 1539575, + "p99": 4424662, + "maximum": 404046959 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "samples": 2700, + "p50": 654117, + "p95": 2336468521, + "p99": 8283303554, + "maximum": 9759883730 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "end_to_end", + "samples": 2700, + "p50": 1431285, + "p95": 2336917573, + "p99": 8322791838, + "maximum": 9760321751 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "samples": 1350, + "p50": 683442, + "p95": 1134258, + "p99": 3212546, + "maximum": 353703839 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "samples": 1350, + "p50": 655357, + "p95": 2258048994, + "p99": 8461983848, + "maximum": 9642483905 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "end_to_end", + "samples": 1350, + "p50": 1310644, + "p95": 2258352233, + "p99": 8462577431, + "maximum": 9643023397 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "samples": 450, + "p50": 1003492, + "p95": 1589693, + "p99": null, + "maximum": 392272011 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "samples": 450, + "p50": 629515, + "p95": 2218785365, + "p99": null, + "maximum": 9583804622 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "end_to_end", + "samples": 450, + "p50": 1705457, + "p95": 2219273801, + "p99": null, + "maximum": 9584327112 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "samples": 4500, + "p50": 1075525, + "p95": 2513678, + "p99": 5492336, + "maximum": 459020210 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "samples": 4500, + "p50": 648594, + "p95": 2160839937, + "p99": 8307795986, + "maximum": 9817232879 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "end_to_end", + "samples": 4500, + "p50": 1819743, + "p95": 2180824539, + "p99": 8308296895, + "maximum": 9837206199 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "samples": 72, + "p50": 33280952, + "p95": 43338991, + "p99": null, + "maximum": 90099029 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "samples": 72, + "p50": 520719, + "p95": 1037465, + "p99": null, + "maximum": 1083766 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "end_to_end", + "samples": 72, + "p50": 34237276, + "p95": 43465342, + "p99": null, + "maximum": 90731044 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "samples": 108, + "p50": 1773917, + "p95": 2556925, + "p99": null, + "maximum": 4002144 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "samples": 108, + "p50": 559427, + "p95": 1052580, + "p99": null, + "maximum": 1134924 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "end_to_end", + "samples": 108, + "p50": 2399496, + "p95": 3393879, + "p99": null, + "maximum": 4631122 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "samples": 72, + "p50": 1950080, + "p95": 545086011, + "p99": null, + "maximum": 912742765 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "samples": 72, + "p50": 211293, + "p95": 1052960, + "p99": null, + "maximum": 1083997 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "end_to_end", + "samples": 72, + "p50": 2297689, + "p95": 545565988, + "p99": null, + "maximum": 913325023 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "samples": 108, + "p50": 30122866, + "p95": 36159673, + "p99": null, + "maximum": 93399445 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "samples": 108, + "p50": 577700, + "p95": 1068014, + "p99": null, + "maximum": 1128508 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "end_to_end", + "samples": 108, + "p50": 30766399, + "p95": 36938421, + "p99": null, + "maximum": 93945626 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "samples": 252, + "p50": 671371, + "p95": 954600, + "p99": null, + "maximum": 22090247 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "samples": 252, + "p50": 542346, + "p95": 1068776, + "p99": null, + "maximum": 1131704 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "end_to_end", + "samples": 252, + "p50": 1212899, + "p95": 1822727, + "p99": null, + "maximum": 22484741 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "samples": 252, + "p50": 1339914, + "p95": 70444454, + "p99": null, + "maximum": 83419135 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "samples": 252, + "p50": 579543, + "p95": 1055884, + "p99": null, + "maximum": 1107495 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "end_to_end", + "samples": 252, + "p50": 1976929, + "p95": 71038643, + "p99": null, + "maximum": 83881914 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "samples": 216, + "p50": 1361392, + "p95": 94828997, + "p99": null, + "maximum": 102676796 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "samples": 216, + "p50": 545411, + "p95": 1067482, + "p99": null, + "maximum": 1104461 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "end_to_end", + "samples": 216, + "p50": 1953552, + "p95": 95053940, + "p99": null, + "maximum": 103487011 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "samples": 180, + "p50": 3077804, + "p95": 3609024, + "p99": null, + "maximum": 5291017 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "samples": 180, + "p50": 606215, + "p95": 1061652, + "p99": null, + "maximum": 1100544 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "end_to_end", + "samples": 180, + "p50": 3690372, + "p95": 4304452, + "p99": null, + "maximum": 5984714 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "samples": 180, + "p50": 1370050, + "p95": 1581667, + "p99": null, + "maximum": 3549646 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "samples": 180, + "p50": 595669, + "p95": 1036735, + "p99": null, + "maximum": 1091734 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "end_to_end", + "samples": 180, + "p50": 1978674, + "p95": 2446799, + "p99": null, + "maximum": 4260865 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "samples": 252, + "p50": 65894630, + "p95": 78081945, + "p99": null, + "maximum": 179671339 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "samples": 252, + "p50": 479163, + "p95": 1011057, + "p99": null, + "maximum": 1111235 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "end_to_end", + "samples": 252, + "p50": 66181845, + "p95": 78314685, + "p99": null, + "maximum": 180007426 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "samples": 252, + "p50": 3767730, + "p95": 4893358, + "p99": null, + "maximum": 22474230 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "samples": 252, + "p50": 303726, + "p95": 1023183, + "p99": null, + "maximum": 5427107 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "end_to_end", + "samples": 252, + "p50": 4114718, + "p95": 5520800, + "p99": null, + "maximum": 25451950 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "samples": 216, + "p50": 5767479, + "p95": 13101166, + "p99": null, + "maximum": 75030172 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "samples": 216, + "p50": 564086, + "p95": 1053300, + "p99": null, + "maximum": 1319009 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "end_to_end", + "samples": 216, + "p50": 6327265, + "p95": 13717888, + "p99": null, + "maximum": 75969216 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "samples": 360, + "p50": 2654894, + "p95": 3761869, + "p99": null, + "maximum": 21860294 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "samples": 360, + "p50": 581888, + "p95": 1043068, + "p99": null, + "maximum": 1106283 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "end_to_end", + "samples": 360, + "p50": 3303521, + "p95": 4419893, + "p99": null, + "maximum": 22797000 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "samples": 180, + "p50": 1319493, + "p95": 1676918, + "p99": null, + "maximum": 16170451 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "samples": 180, + "p50": 574619, + "p95": 1048064, + "p99": null, + "maximum": 1096509 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 1946465, + "p95": 2510186, + "p99": null, + "maximum": 16242571 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "samples": 45, + "p50": 694054, + "p95": 844782, + "p99": null, + "maximum": 7970155 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "samples": 45, + "p50": 673360, + "p95": 1081470, + "p99": null, + "maximum": 1631227 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "end_to_end", + "samples": 45, + "p50": 1325465, + "p95": 1884087, + "p99": null, + "maximum": 9601382 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "samples": 135, + "p50": 1162979, + "p95": 1410245, + "p99": null, + "maximum": 1596622 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "samples": 135, + "p50": 585739, + "p95": 1066939, + "p99": null, + "maximum": 1136243 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "end_to_end", + "samples": 135, + "p50": 1792109, + "p95": 2246046, + "p99": null, + "maximum": 2603186 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "samples": 180, + "p50": 26982418, + "p95": 32591535, + "p99": null, + "maximum": 106027946 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "samples": 180, + "p50": 651540, + "p95": 1074011, + "p99": null, + "maximum": 1112957 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 27594701, + "p95": 33443503, + "p99": null, + "maximum": 107034549 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "samples": 180, + "p50": 774596, + "p95": 901521, + "p99": null, + "maximum": 1870026 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "samples": 180, + "p50": 583027, + "p95": 1054137, + "p99": null, + "maximum": 1150617 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "end_to_end", + "samples": 180, + "p50": 1364229, + "p95": 1848529, + "p99": null, + "maximum": 2104978 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "samples": 180, + "p50": 1520241, + "p95": 1712189, + "p99": null, + "maximum": 2027339 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "samples": 180, + "p50": 671592, + "p95": 1051346, + "p99": null, + "maximum": 1084648 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "end_to_end", + "samples": 180, + "p50": 2156722, + "p95": 2652220, + "p99": null, + "maximum": 2952753 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "samples": 180, + "p50": 477312, + "p95": 2350755, + "p99": null, + "maximum": 413441892 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "samples": 180, + "p50": 605442, + "p95": 1064329, + "p99": null, + "maximum": 1127466 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "end_to_end", + "samples": 180, + "p50": 1118828, + "p95": 3081010, + "p99": null, + "maximum": 414170243 + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "samples": 100, + "p50": 78486818, + "p95": 95288558, + "p99": null, + "maximum": 99263313 + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "samples": 100, + "p50": 834002, + "p95": 1236095, + "p99": null, + "maximum": 1928081 + }, + { + "scenario": "websocket", + "event_class": "new_transmission", + "timing": "receipt_after_ingest_return", + "samples": 4500, + "p50": 508878496, + "p95": 986277329, + "p99": 1827343470, + "maximum": 1880530128 + }, + { + "scenario": "handler-control", + "event_class": "200_valid_envelopes", + "timing": "service", + "samples": 200, + "p50": 1126030, + "p95": 3060096, + "p99": null, + "maximum": 10290208 + } + ], + "counts": { + "durable-ingest": { + "total": 12000, + "measured": 9000, + "successful": 12000, + "errors": 0, + "drops": 0 + }, + "mixed-http": { + "total": 4800, + "measured": 3600, + "successful": 4800, + "errors": 0, + "drops": 0 + }, + "sql-miss": { + "total": 200, + "measured": 200, + "successful": 200, + "errors": 0, + "drops": 0 + } + }, + "throughput_including_drain": { + "durable-ingest/new": 25.03032008887913, + "durable-ingest/additional": 15.024930271060285, + "durable-ingest/duplicate": 7.514986694455287, + "durable-ingest/late": 2.5055393895766143, + "mixed-http/nodes_50": 1.4326021056626153, + "mixed-http/nodes_region": 1.228911930142778, + "mixed-http/nodes_2000": 1.4321105228925104, + "mixed-http/channels": 1.4326243274555144, + "mixed-http/messages_0": 1.4326136504212166, + "mixed-http/messages_1000": 1.2290080258741092, + "mixed-http/observers": 2.051826092877047, + "mixed-http/metrics_dense_24h": 1.0262078860310873, + "mixed-http/metrics_sparse_7d": 1.0262134891527042, + "mixed-http/reach": 1.0262134540664531, + "mixed-http/rx_coverage": 1.0262134013873507, + "mixed-http/packets_memory": 1.0248939846320004, + "mixed-http/packet_detail_memory": 1.025043048316334, + "mixed-http/packet_detail_sql": 1.0336728762943936, + "mixed-http/analytics_rf": 0.6157293605765729, + "mixed-http/analytics_topology": 0.6156209166888316, + "mixed-http/analytics_channels": 0.41087495323156137, + "mixed-http/analytics_rf_filtered": 0.41095411837061285, + "mixed-http/stats": 1.027392178494361, + "mixed-http/packet_detail_pruned": 1.1083472488109936, + "sql-miss/channel_messages_sql": 12.18961064472026, + "sql-miss/observer_metrics_sql": 12.295960435980614 + }, + "scalars_ns": { + "startup/first_http_ns": 29083449285, + "startup/ready_ns": 29083452731, + "startup/full_ready_ns": 29096733146, + "startup-hot1/first_http_ns": 9876754329, + "startup-hot1/ready_ns": 9876758828, + "startup-hot1/full_ready_ns": 21883091242, + "retention/elapsed_ns": 11340422266, + "retention-empty/elapsed_ns": 297698 + }, + "resources": { + "sample_count": 180, + "window_start_ns": 882737670614, + "window_end_ns": 1062737670614, + "sampled_start_ns": 882873676911, + "sampled_end_ns": 1062546670062, + "sampled_span_ns": 179672993151, + "sampled_span_fraction": 0.9981832952833334, + "first_boundary_gap_ns": 136006297, + "last_boundary_gap_ns": 191000552, + "sampling_interval_ns": { + "median": 1003548245, + "minimum": 1003261198, + "maximum": 1009564707, + "runs": 179 + }, + "phases": [ + "mixed_with_retention" + ], + "cpu_delta_usec": 45465614, + "average_cores_over_sampled_span": 0.2530464551330204, + "per_device_io": { + "8:0": { + "wios": 224715, + "rios": 0, + "dios": 0, + "rbytes": 0, + "wbytes": 2502598656, + "dbytes": 0 + } + }, + "aggregate_accounted_io": { + "dbytes": 0, + "dios": 0, + "rbytes": 0, + "rios": 0, + "wbytes": 2502598656, + "wios": 224715 + }, + "observed_peaks": { + "cgroup_memory_bytes": 4246102016, + "rss_bytes": 3091775488, + "pss_bytes": 3088556032, + "storage_bytes": 1137166624, + "wal_bytes": 32037152 + }, + "memory_event_deltas": { + "oom_group_kill": 0, + "oom_kill": 0, + "oom": 0, + "high": 0, + "max": 0, + "low": 0 + }, + "minimum_free_bytes": 85432963072 + }, + "websocket": { + "full_stream_sentinels": 6000, + "full_stream_received": 6000, + "measured_sentinels": 4500, + "measured_missing": 0, + "full_stream_missing": 0, + "negative_measured_lags": 0, + "negative_full_stream_lags": 0, + "duplicate_measured_receipts": 0, + "duplicate_full_stream_receipts": 0 + }, + "retention": { + "retention": { + "elapsed_ns": 11340422266, + "transmissions_deleted": 16000, + "observations_deleted": 256000 + }, + "retention-empty": { + "elapsed_ns": 297698, + "transmissions_deleted": 0, + "observations_deleted": 0 + } + }, + "startup_counts": [ + { + "transmissions": 112000, + "observations": 1792000 + }, + { + "transmissions": 112000, + "observations": 1792000 + } + ], + "database_settings": { + "busy_timeout": "5000", + "cache_size": "-2000", + "cache_spill": "483", + "foreign_keys": "1", + "journal_mode": "wal", + "journal_size_limit": "-1", + "mmap_size": "0", + "page_size": "4096", + "sqlite_version": "3.53.4", + "synchronous": "2", + "wal_autocheckpoint": "1000" + }, + "durable_counts": { + "scheduled": 12000, + "completed": 12000, + "dropped_schedules": 0, + "write_errors": 0, + "actual_transmissions": 118000, + "actual_observations": 1802200 + }, + "full_leg_kernel_peak_bytes": 4454326272, + "full_leg_memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "ingest_drain_seconds": 0, + "http_drain_seconds": 0 + }, + { + "pair": 0, + "backend": "postgres", + "metrics": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "samples": 2700, + "p50": 1413317, + "p95": 2329768, + "p99": 4421953, + "maximum": 94169069 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "samples": 2700, + "p50": 617157, + "p95": 1077388, + "p99": 1136500, + "maximum": 76366924 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "end_to_end", + "samples": 2700, + "p50": 2095278, + "p95": 3117397, + "p99": 5665500, + "maximum": 96343159 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "samples": 1350, + "p50": 1285879, + "p95": 1873817, + "p99": 4590910, + "maximum": 54069585 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "samples": 1350, + "p50": 614340, + "p95": 1078020, + "p99": 1171401, + "maximum": 46304116 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "end_to_end", + "samples": 1350, + "p50": 1959562, + "p95": 2686557, + "p99": 5432307, + "maximum": 66277937 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "samples": 450, + "p50": 1605017, + "p95": 2221719, + "p99": null, + "maximum": 10005902 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "samples": 450, + "p50": 632488, + "p95": 1056680, + "p99": null, + "maximum": 8562847 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "end_to_end", + "samples": 450, + "p50": 2299519, + "p95": 3048149, + "p99": null, + "maximum": 10398691 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "samples": 4500, + "p50": 1588396, + "p95": 2311314, + "p99": 4563639, + "maximum": 39362517 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "samples": 4500, + "p50": 610476, + "p95": 1066745, + "p99": 1125281, + "maximum": 13019112 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "end_to_end", + "samples": 4500, + "p50": 2250549, + "p95": 3088245, + "p99": 5480239, + "maximum": 40151999 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "samples": 72, + "p50": 33203718, + "p95": 41901684, + "p99": null, + "maximum": 51878356 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "samples": 72, + "p50": 455160, + "p95": 1050224, + "p99": null, + "maximum": 1104041 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "end_to_end", + "samples": 72, + "p50": 33661987, + "p95": 42400747, + "p99": null, + "maximum": 52850122 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "samples": 108, + "p50": 1762772, + "p95": 2705144, + "p99": null, + "maximum": 3167791 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "samples": 108, + "p50": 491168, + "p95": 1076439, + "p99": null, + "maximum": 1104469 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "end_to_end", + "samples": 108, + "p50": 2374199, + "p95": 3350044, + "p99": null, + "maximum": 3865155 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "samples": 72, + "p50": 2112116, + "p95": 555026701, + "p99": null, + "maximum": 1021442704 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "samples": 72, + "p50": 165266, + "p95": 1076340, + "p99": null, + "maximum": 1173997 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "end_to_end", + "samples": 72, + "p50": 2494416, + "p95": 555544077, + "p99": null, + "maximum": 1022483039 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "samples": 108, + "p50": 31264489, + "p95": 36565874, + "p99": null, + "maximum": 90067409 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "samples": 108, + "p50": 676314, + "p95": 1083275, + "p99": null, + "maximum": 1131164 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "end_to_end", + "samples": 108, + "p50": 31837816, + "p95": 37259070, + "p99": null, + "maximum": 90864462 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "samples": 252, + "p50": 674574, + "p95": 1186002, + "p99": null, + "maximum": 28319029 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "samples": 252, + "p50": 593637, + "p95": 1053393, + "p99": null, + "maximum": 1135375 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "end_to_end", + "samples": 252, + "p50": 1303498, + "p95": 1856776, + "p99": null, + "maximum": 28661426 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "samples": 252, + "p50": 1477247, + "p95": 177473720, + "p99": null, + "maximum": 261112580 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "samples": 252, + "p50": 663883, + "p95": 1053030, + "p99": null, + "maximum": 3519358 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "end_to_end", + "samples": 252, + "p50": 2148358, + "p95": 178502469, + "p99": null, + "maximum": 261707232 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "samples": 216, + "p50": 1497157, + "p95": 175509176, + "p99": null, + "maximum": 223049459 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "samples": 216, + "p50": 575452, + "p95": 1068155, + "p99": null, + "maximum": 1135690 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "end_to_end", + "samples": 216, + "p50": 2121542, + "p95": 176622903, + "p99": null, + "maximum": 223975359 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "samples": 180, + "p50": 3575698, + "p95": 4503215, + "p99": null, + "maximum": 11615835 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "samples": 180, + "p50": 579451, + "p95": 1052569, + "p99": null, + "maximum": 1124188 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "end_to_end", + "samples": 180, + "p50": 4194587, + "p95": 5141636, + "p99": null, + "maximum": 11933037 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "samples": 180, + "p50": 1752239, + "p95": 2067350, + "p99": null, + "maximum": 3221329 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "samples": 180, + "p50": 462390, + "p95": 1045578, + "p99": null, + "maximum": 1081785 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "end_to_end", + "samples": 180, + "p50": 2274317, + "p95": 2794642, + "p99": null, + "maximum": 3594746 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "samples": 252, + "p50": 68507164, + "p95": 100878123, + "p99": null, + "maximum": 287402400 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "samples": 252, + "p50": 643262, + "p95": 1046416, + "p99": null, + "maximum": 4505528 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "end_to_end", + "samples": 252, + "p50": 69025769, + "p95": 101233767, + "p99": null, + "maximum": 287509515 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "samples": 252, + "p50": 5575243, + "p95": 10550240, + "p99": null, + "maximum": 58801153 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "samples": 252, + "p50": 317003, + "p95": 1056539, + "p99": null, + "maximum": 3669763 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "end_to_end", + "samples": 252, + "p50": 6026968, + "p95": 11030216, + "p99": null, + "maximum": 59096904 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "samples": 216, + "p50": 8016992, + "p95": 17342410, + "p99": null, + "maximum": 93884887 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "samples": 216, + "p50": 494577, + "p95": 1060511, + "p99": null, + "maximum": 1203449 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "end_to_end", + "samples": 216, + "p50": 8658349, + "p95": 18184385, + "p99": null, + "maximum": 94557864 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "samples": 360, + "p50": 2883442, + "p95": 4059676, + "p99": null, + "maximum": 102482392 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "samples": 360, + "p50": 572235, + "p95": 1049922, + "p99": null, + "maximum": 1143164 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "end_to_end", + "samples": 360, + "p50": 3498980, + "p95": 4696072, + "p99": null, + "maximum": 103537206 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "samples": 180, + "p50": 3455803, + "p95": 4026397, + "p99": null, + "maximum": 14924842 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "samples": 180, + "p50": 571210, + "p95": 1038649, + "p99": null, + "maximum": 1130213 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 4065974, + "p95": 4786972, + "p99": null, + "maximum": 14973876 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "samples": 45, + "p50": 742811, + "p95": 838290, + "p99": null, + "maximum": 1329209 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "samples": 45, + "p50": 686268, + "p95": 1018593, + "p99": null, + "maximum": 1146467 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "end_to_end", + "samples": 45, + "p50": 1346280, + "p95": 1852716, + "p99": null, + "maximum": 2332385 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "samples": 135, + "p50": 1591962, + "p95": 2285591, + "p99": null, + "maximum": 20137144 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "samples": 135, + "p50": 580350, + "p95": 1076218, + "p99": null, + "maximum": 1184975 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "end_to_end", + "samples": 135, + "p50": 2187851, + "p95": 3114445, + "p99": null, + "maximum": 21020373 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "samples": 180, + "p50": 36484020, + "p95": 42700273, + "p99": null, + "maximum": 115401240 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "samples": 180, + "p50": 584599, + "p95": 1033204, + "p99": null, + "maximum": 1078991 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 37112054, + "p95": 43283084, + "p99": null, + "maximum": 116328957 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "samples": 180, + "p50": 814914, + "p95": 1193621, + "p99": null, + "maximum": 1879968 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "samples": 180, + "p50": 649314, + "p95": 1072149, + "p99": null, + "maximum": 1112525 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "end_to_end", + "samples": 180, + "p50": 1447074, + "p95": 1996247, + "p99": null, + "maximum": 2653487 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "samples": 180, + "p50": 1683737, + "p95": 1936598, + "p99": null, + "maximum": 2223710 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "samples": 180, + "p50": 587542, + "p95": 1049585, + "p99": null, + "maximum": 1102590 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "end_to_end", + "samples": 180, + "p50": 2272966, + "p95": 2793186, + "p99": null, + "maximum": 3039997 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "samples": 180, + "p50": 446719, + "p95": 3854922, + "p99": null, + "maximum": 17602895 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "samples": 180, + "p50": 642932, + "p95": 1076667, + "p99": null, + "maximum": 1890876 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "end_to_end", + "samples": 180, + "p50": 1131775, + "p95": 4717236, + "p99": null, + "maximum": 18223438 + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "samples": 100, + "p50": 122143362, + "p95": 146742113, + "p99": null, + "maximum": 269008163 + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "samples": 100, + "p50": 802964, + "p95": 1394923, + "p99": null, + "maximum": 3561830 + }, + { + "scenario": "websocket", + "event_class": "new_transmission", + "timing": "receipt_after_ingest_return", + "samples": 4500, + "p50": 504092826, + "p95": 962889358, + "p99": 1002731690, + "maximum": 1561818612 + }, + { + "scenario": "handler-control", + "event_class": "200_valid_envelopes", + "timing": "service", + "samples": 200, + "p50": 2052401, + "p95": 2843413, + "p99": null, + "maximum": 6740752 + } + ], + "counts": { + "durable-ingest": { + "total": 12000, + "measured": 9000, + "successful": 12000, + "errors": 0, + "drops": 0 + }, + "mixed-http": { + "total": 4800, + "measured": 3600, + "successful": 4800, + "errors": 0, + "drops": 0 + }, + "sql-miss": { + "total": 200, + "measured": 200, + "successful": 200, + "errors": 0, + "drops": 0 + } + }, + "throughput_including_drain": { + "durable-ingest/new": 25.030244157068257, + "durable-ingest/additional": 15.024843481368846, + "durable-ingest/duplicate": 7.514964265026795, + "durable-ingest/late": 2.505536504196812, + "mixed-http/nodes_50": 1.4325856953918086, + "mixed-http/nodes_region": 1.228944937461886, + "mixed-http/nodes_2000": 1.4320219231522593, + "mixed-http/channels": 1.4326237578787042, + "mixed-http/messages_0": 1.4326098220551298, + "mixed-http/messages_1000": 1.2279194621068021, + "mixed-http/observers": 2.0518240927827645, + "mixed-http/metrics_dense_24h": 1.0262029383857387, + "mixed-http/metrics_sparse_7d": 1.0262142113496264, + "mixed-http/reach": 1.0262187987241675, + "mixed-http/rx_coverage": 1.0262108392435771, + "mixed-http/packets_memory": 1.024850179007025, + "mixed-http/packet_detail_memory": 1.0250306195449674, + "mixed-http/packet_detail_sql": 1.0336725572069057, + "mixed-http/analytics_rf": 0.6157286119190807, + "mixed-http/analytics_topology": 0.6156243250936532, + "mixed-http/analytics_channels": 0.4108827104864554, + "mixed-http/analytics_rf_filtered": 0.41095141642466254, + "mixed-http/stats": 1.0273906926731056, + "mixed-http/packet_detail_pruned": 1.1083406429622016, + "sql-miss/channel_messages_sql": 7.709219902584099, + "sql-miss/observer_metrics_sql": 7.859378748000186 + }, + "scalars_ns": { + "startup/first_http_ns": 28856225086, + "startup/ready_ns": 28856227611, + "startup/full_ready_ns": 29419540593, + "startup-hot1/first_http_ns": 11576476851, + "startup-hot1/ready_ns": 11576480628, + "startup-hot1/full_ready_ns": 21116822964, + "retention/elapsed_ns": 889359099, + "retention-empty/elapsed_ns": 2140589, + "migration/elapsed_ns": 147219062637, + "migration/migration_tool_ns": 89562938449, + "migration/role_grants_ns": 748674192, + "migration/harness_validation_ns": 56907451138 + }, + "resources": { + "sample_count": 169, + "window_start_ns": 1444980932298, + "window_end_ns": 1624980932298, + "sampled_start_ns": 1445778566540, + "sampled_end_ns": 1624719165717, + "sampled_span_ns": 178940599177, + "sampled_span_fraction": 0.9941144398722223, + "first_boundary_gap_ns": 797634242, + "last_boundary_gap_ns": 261766581, + "sampling_interval_ns": { + "median": 1062735972.5, + "minimum": 1057746558, + "maximum": 1088793218, + "runs": 168 + }, + "phases": [ + "mixed_with_retention" + ], + "cpu_delta_usec": 69041731, + "average_cores_over_sampled_span": 0.38583603339623906, + "per_device_io": { + "8:0": { + "wios": 27649, + "rios": 668, + "dios": 1, + "rbytes": 6127616, + "wbytes": 538468352, + "dbytes": 4096 + } + }, + "aggregate_accounted_io": { + "dbytes": 4096, + "dios": 1, + "rbytes": 6127616, + "rios": 668, + "wbytes": 538468352, + "wios": 27649 + }, + "observed_peaks": { + "cgroup_memory_bytes": 6442385408, + "rss_bytes": 6220292096, + "pss_bytes": 3429304320, + "storage_bytes": 2439917372, + "wal_bytes": 1073741824 + }, + "memory_event_deltas": { + "oom_group_kill": 0, + "oom_kill": 0, + "oom": 0, + "high": 0, + "max": 338, + "low": 0 + }, + "minimum_free_bytes": 79748284416 + }, + "websocket": { + "full_stream_sentinels": 6000, + "full_stream_received": 6000, + "measured_sentinels": 4500, + "measured_missing": 0, + "full_stream_missing": 0, + "negative_measured_lags": 0, + "negative_full_stream_lags": 0, + "duplicate_measured_receipts": 0, + "duplicate_full_stream_receipts": 0 + }, + "retention": { + "retention": { + "elapsed_ns": 889359099, + "transmissions_deleted": 16000, + "observations_deleted": 256000 + }, + "retention-empty": { + "elapsed_ns": 2140589, + "transmissions_deleted": 0, + "observations_deleted": 0 + } + }, + "startup_counts": [ + { + "transmissions": 112000, + "observations": 1792000 + }, + { + "transmissions": 112000, + "observations": 1792000 + } + ], + "database_settings": { + "autovacuum": "on", + "checkpoint_timeout": "5min", + "effective_cache_size": "4GB", + "fsync": "on", + "full_page_writes": "on", + "jit": "on", + "maintenance_work_mem": "64MB", + "max_connections": "32", + "max_parallel_workers_per_gather": "2", + "max_wal_size": "1GB", + "server_version": "18.6 (Ubuntu 18.6-1.pgdg24.04+2)", + "shared_buffers": "256MB", + "synchronous_commit": "on", + "work_mem": "4MB" + }, + "durable_counts": { + "scheduled": 12000, + "completed": 12000, + "dropped_schedules": 0, + "write_errors": 0, + "actual_transmissions": 118000, + "actual_observations": 1802200 + }, + "full_leg_kernel_peak_bytes": 6442561536, + "full_leg_memory_events": { + "low": 0, + "high": 0, + "max": 3474, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "ingest_drain_seconds": 0, + "http_drain_seconds": 0 + }, + { + "pair": 1, + "backend": "sqlite", + "metrics": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "samples": 2700, + "p50": 800720, + "p95": 1526136, + "p99": 4855494, + "maximum": 459531674 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "samples": 2700, + "p50": 648517, + "p95": 2082205837, + "p99": 7932860388, + "maximum": 9544973424 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "end_to_end", + "samples": 2700, + "p50": 1552503, + "p95": 2102193414, + "p99": 7933301965, + "maximum": 9545571304 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "samples": 1350, + "p50": 676305, + "p95": 990831, + "p99": 3204335, + "maximum": 437635875 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "samples": 1350, + "p50": 660370, + "p95": 2043004843, + "p99": 7961026926, + "maximum": 9427611886 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "end_to_end", + "samples": 1350, + "p50": 1439975, + "p95": 2043431803, + "p99": 8223958053, + "maximum": 9427915504 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "samples": 450, + "p50": 1016063, + "p95": 2336338, + "p99": null, + "maximum": 348858321 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "samples": 450, + "p50": 627230, + "p95": 2003720674, + "p99": null, + "maximum": 9368690636 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "end_to_end", + "samples": 450, + "p50": 1710814, + "p95": 2004318454, + "p99": null, + "maximum": 9369330474 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "samples": 4500, + "p50": 1083380, + "p95": 2504751, + "p99": 4973305, + "maximum": 488275727 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "samples": 4500, + "p50": 654996, + "p95": 1887821084, + "p99": 8088406405, + "maximum": 9641437127 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "end_to_end", + "samples": 4500, + "p50": 1792553, + "p95": 1888570268, + "p99": 8088911401, + "maximum": 9661413172 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "samples": 72, + "p50": 33404928, + "p95": 41021221, + "p99": null, + "maximum": 43513567 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "samples": 72, + "p50": 557313, + "p95": 1008672, + "p99": null, + "maximum": 1055147 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "end_to_end", + "samples": 72, + "p50": 33974823, + "p95": 41807557, + "p99": null, + "maximum": 44301964 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "samples": 108, + "p50": 1758001, + "p95": 2481120, + "p99": null, + "maximum": 2982649 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "samples": 108, + "p50": 587481, + "p95": 1082306, + "p99": null, + "maximum": 1127461 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "end_to_end", + "samples": 108, + "p50": 2453295, + "p95": 3155945, + "p99": null, + "maximum": 3737946 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "samples": 72, + "p50": 1965236, + "p95": 523272043, + "p99": null, + "maximum": 949101223 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "samples": 72, + "p50": 181229, + "p95": 991297, + "p99": null, + "maximum": 1093060 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "end_to_end", + "samples": 72, + "p50": 2346257, + "p95": 523425952, + "p99": null, + "maximum": 949695208 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "samples": 108, + "p50": 30120335, + "p95": 35396834, + "p99": null, + "maximum": 43016661 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "samples": 108, + "p50": 684168, + "p95": 1080653, + "p99": null, + "maximum": 1159763 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "end_to_end", + "samples": 108, + "p50": 30807730, + "p95": 36085140, + "p99": null, + "maximum": 43871857 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "samples": 252, + "p50": 660655, + "p95": 926593, + "p99": null, + "maximum": 21002759 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "samples": 252, + "p50": 645323, + "p95": 1068893, + "p99": null, + "maximum": 1152485 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "end_to_end", + "samples": 252, + "p50": 1325997, + "p95": 1779334, + "p99": null, + "maximum": 21750947 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "samples": 252, + "p50": 1323509, + "p95": 69462566, + "p99": null, + "maximum": 75071186 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "samples": 252, + "p50": 599032, + "p95": 1042355, + "p99": null, + "maximum": 1159159 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "end_to_end", + "samples": 252, + "p50": 1988432, + "p95": 70097422, + "p99": null, + "maximum": 75483729 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "samples": 216, + "p50": 1367852, + "p95": 94835336, + "p99": null, + "maximum": 105018980 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "samples": 216, + "p50": 627218, + "p95": 1064288, + "p99": null, + "maximum": 1160023 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "end_to_end", + "samples": 216, + "p50": 2052117, + "p95": 95110782, + "p99": null, + "maximum": 105371558 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "samples": 180, + "p50": 3061331, + "p95": 3869789, + "p99": null, + "maximum": 5397622 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "samples": 180, + "p50": 665554, + "p95": 1086233, + "p99": null, + "maximum": 1141891 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "end_to_end", + "samples": 180, + "p50": 3710083, + "p95": 4639102, + "p99": null, + "maximum": 5561555 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "samples": 180, + "p50": 1336714, + "p95": 1719904, + "p99": null, + "maximum": 3965490 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "samples": 180, + "p50": 651186, + "p95": 1053304, + "p99": null, + "maximum": 1101486 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "end_to_end", + "samples": 180, + "p50": 1982957, + "p95": 2551945, + "p99": null, + "maximum": 4074778 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "samples": 252, + "p50": 65013849, + "p95": 80781371, + "p99": null, + "maximum": 474229828 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "samples": 252, + "p50": 547065, + "p95": 1089672, + "p99": null, + "maximum": 1197872 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "end_to_end", + "samples": 252, + "p50": 65598933, + "p95": 81916871, + "p99": null, + "maximum": 475188808 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "samples": 252, + "p50": 3735186, + "p95": 5240543, + "p99": null, + "maximum": 254379376 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "samples": 252, + "p50": 280072, + "p95": 1069901, + "p99": null, + "maximum": 2054618 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "end_to_end", + "samples": 252, + "p50": 4154518, + "p95": 5957833, + "p99": null, + "maximum": 255309208 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "samples": 216, + "p50": 5703947, + "p95": 12548480, + "p99": null, + "maximum": 330398404 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "samples": 216, + "p50": 478862, + "p95": 1039736, + "p99": null, + "maximum": 1271060 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "end_to_end", + "samples": 216, + "p50": 6214424, + "p95": 13036877, + "p99": null, + "maximum": 330750165 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "samples": 360, + "p50": 2683566, + "p95": 3624050, + "p99": null, + "maximum": 21755868 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "samples": 360, + "p50": 609413, + "p95": 1076361, + "p99": null, + "maximum": 1253911 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "end_to_end", + "samples": 360, + "p50": 3359280, + "p95": 4379182, + "p99": null, + "maximum": 22631758 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "samples": 180, + "p50": 1365405, + "p95": 1746421, + "p99": null, + "maximum": 18239264 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "samples": 180, + "p50": 571814, + "p95": 1038822, + "p99": null, + "maximum": 1132743 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 1936243, + "p95": 2536977, + "p99": null, + "maximum": 18822443 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "samples": 45, + "p50": 712958, + "p95": 830135, + "p99": null, + "maximum": 907981 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "samples": 45, + "p50": 754955, + "p95": 1074566, + "p99": null, + "maximum": 1105091 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "end_to_end", + "samples": 45, + "p50": 1420374, + "p95": 1796594, + "p99": null, + "maximum": 1932856 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "samples": 135, + "p50": 1130697, + "p95": 1367341, + "p99": null, + "maximum": 1575654 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "samples": 135, + "p50": 633774, + "p95": 1073574, + "p99": null, + "maximum": 1137806 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "end_to_end", + "samples": 135, + "p50": 1757988, + "p95": 2263324, + "p99": null, + "maximum": 2339196 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "samples": 180, + "p50": 25972936, + "p95": 32326006, + "p99": null, + "maximum": 96287855 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "samples": 180, + "p50": 595378, + "p95": 1073394, + "p99": null, + "maximum": 1143031 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 26540811, + "p95": 32803554, + "p99": null, + "maximum": 96582545 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "samples": 180, + "p50": 782000, + "p95": 923579, + "p99": null, + "maximum": 1172213 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "samples": 180, + "p50": 616250, + "p95": 1083818, + "p99": null, + "maximum": 1345366 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "end_to_end", + "samples": 180, + "p50": 1375846, + "p95": 1852645, + "p99": null, + "maximum": 2264748 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "samples": 180, + "p50": 1538313, + "p95": 1796929, + "p99": null, + "maximum": 2018298 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "samples": 180, + "p50": 659254, + "p95": 1065467, + "p99": null, + "maximum": 1092024 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "end_to_end", + "samples": 180, + "p50": 2157413, + "p95": 2670583, + "p99": null, + "maximum": 2991181 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "samples": 180, + "p50": 468536, + "p95": 2149496, + "p99": null, + "maximum": 655471280 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "samples": 180, + "p50": 564115, + "p95": 1060214, + "p99": null, + "maximum": 1110994 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "end_to_end", + "samples": 180, + "p50": 1124581, + "p95": 2722768, + "p99": null, + "maximum": 655574886 + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "samples": 100, + "p50": 89086196, + "p95": 98947453, + "p99": null, + "maximum": 196344593 + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "samples": 100, + "p50": 854921, + "p95": 1509437, + "p99": null, + "maximum": 1956204 + }, + { + "scenario": "websocket", + "event_class": "new_transmission", + "timing": "receipt_after_ingest_return", + "samples": 4500, + "p50": 504260023, + "p95": 1091642087, + "p99": 2035356108, + "maximum": 2135386176 + }, + { + "scenario": "handler-control", + "event_class": "200_valid_envelopes", + "timing": "service", + "samples": 200, + "p50": 1180371, + "p95": 2563010, + "p99": null, + "maximum": 4813456 + } + ], + "counts": { + "durable-ingest": { + "total": 12000, + "measured": 9000, + "successful": 12000, + "errors": 0, + "drops": 0 + }, + "mixed-http": { + "total": 4800, + "measured": 3600, + "successful": 4800, + "errors": 0, + "drops": 0 + }, + "sql-miss": { + "total": 200, + "measured": 200, + "successful": 200, + "errors": 0, + "drops": 0 + } + }, + "throughput_including_drain": { + "durable-ingest/new": 25.030408895462084, + "durable-ingest/additional": 15.024890576379807, + "durable-ingest/duplicate": 7.514959950781447, + "durable-ingest/late": 2.5055446738719693, + "mixed-http/nodes_50": 1.4325981189617463, + "mixed-http/nodes_region": 1.2289677728687, + "mixed-http/nodes_2000": 1.4321140482889434, + "mixed-http/channels": 1.4326223199383141, + "mixed-http/messages_0": 1.4326208426305178, + "mixed-http/messages_1000": 1.2290056624062093, + "mixed-http/observers": 2.0518314499282937, + "mixed-http/metrics_dense_24h": 1.0262044936711134, + "mixed-http/metrics_sparse_7d": 1.0262160380989125, + "mixed-http/reach": 1.026218231973548, + "mixed-http/rx_coverage": 1.0262097713690779, + "mixed-http/packets_memory": 1.0249020402543059, + "mixed-http/packet_detail_memory": 1.0250458382186123, + "mixed-http/packet_detail_sql": 1.0336795072501905, + "mixed-http/analytics_rf": 0.6157278777121242, + "mixed-http/analytics_topology": 0.6156258289006689, + "mixed-http/analytics_channels": 0.41088448788658594, + "mixed-http/analytics_rf_filtered": 0.4109527654088309, + "mixed-http/stats": 1.0273895804840816, + "mixed-http/packet_detail_pruned": 1.108330782476172, + "sql-miss/channel_messages_sql": 11.733730908933508, + "sql-miss/observer_metrics_sql": 11.835293491445857 + }, + "scalars_ns": { + "startup/first_http_ns": 28951575010, + "startup/ready_ns": 28951578015, + "startup/full_ready_ns": 28964685757, + "startup-hot1/first_http_ns": 9767586615, + "startup-hot1/ready_ns": 9767590402, + "startup-hot1/full_ready_ns": 21549346472, + "retention/elapsed_ns": 11121674371, + "retention-empty/elapsed_ns": 323997 + }, + "resources": { + "sample_count": 179, + "window_start_ns": 2519965065964, + "window_end_ns": 2699965065964, + "sampled_start_ns": 2520319206674, + "sampled_end_ns": 2698987758325, + "sampled_span_ns": 178668551651, + "sampled_span_fraction": 0.9926030647277778, + "first_boundary_gap_ns": 354140710, + "last_boundary_gap_ns": 977307639, + "sampling_interval_ns": { + "median": 1003557602.0, + "minimum": 1003299641, + "maximum": 1007550294, + "runs": 178 + }, + "phases": [ + "mixed_with_retention" + ], + "cpu_delta_usec": 45220876, + "average_cores_over_sampled_span": 0.25309924764113856, + "per_device_io": { + "8:0": { + "wios": 214108, + "rios": 0, + "dios": 0, + "rbytes": 0, + "wbytes": 2454315008, + "dbytes": 0 + } + }, + "aggregate_accounted_io": { + "dbytes": 0, + "dios": 0, + "rbytes": 0, + "rios": 0, + "wbytes": 2454315008, + "wios": 214108 + }, + "observed_peaks": { + "cgroup_memory_bytes": 4181094400, + "rss_bytes": 3012349952, + "pss_bytes": 3009097728, + "storage_bytes": 1151541616, + "wal_bytes": 40137072 + }, + "memory_event_deltas": { + "oom_group_kill": 0, + "oom_kill": 0, + "oom": 0, + "high": 0, + "max": 0, + "low": 0 + }, + "minimum_free_bytes": 83228786688 + }, + "websocket": { + "full_stream_sentinels": 6000, + "full_stream_received": 6000, + "measured_sentinels": 4500, + "measured_missing": 0, + "full_stream_missing": 0, + "negative_measured_lags": 0, + "negative_full_stream_lags": 0, + "duplicate_measured_receipts": 0, + "duplicate_full_stream_receipts": 0 + }, + "retention": { + "retention": { + "elapsed_ns": 11121674371, + "transmissions_deleted": 16000, + "observations_deleted": 256000 + }, + "retention-empty": { + "elapsed_ns": 323997, + "transmissions_deleted": 0, + "observations_deleted": 0 + } + }, + "startup_counts": [ + { + "transmissions": 112000, + "observations": 1792000 + }, + { + "transmissions": 112000, + "observations": 1792000 + } + ], + "database_settings": { + "busy_timeout": "5000", + "cache_size": "-2000", + "cache_spill": "483", + "foreign_keys": "1", + "journal_mode": "wal", + "journal_size_limit": "-1", + "mmap_size": "0", + "page_size": "4096", + "sqlite_version": "3.53.4", + "synchronous": "2", + "wal_autocheckpoint": "1000" + }, + "durable_counts": { + "scheduled": 12000, + "completed": 12000, + "dropped_schedules": 0, + "write_errors": 0, + "actual_transmissions": 118000, + "actual_observations": 1802200 + }, + "full_leg_kernel_peak_bytes": 4402548736, + "full_leg_memory_events": { + "low": 0, + "high": 0, + "max": 6237, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "ingest_drain_seconds": 0, + "http_drain_seconds": 0 + }, + { + "pair": 1, + "backend": "postgres", + "metrics": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "samples": 2700, + "p50": 1382488, + "p95": 2255434, + "p99": 4182259, + "maximum": 25791309 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "samples": 2700, + "p50": 616179, + "p95": 1074396, + "p99": 1129058, + "maximum": 19121203 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "end_to_end", + "samples": 2700, + "p50": 2077878, + "p95": 3061956, + "p99": 5253933, + "maximum": 26181923 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "samples": 1350, + "p50": 1288225, + "p95": 1921850, + "p99": 3622892, + "maximum": 15224979 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "samples": 1350, + "p50": 613057, + "p95": 1063306, + "p99": 1111594, + "maximum": 2874198 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "end_to_end", + "samples": 1350, + "p50": 1939823, + "p95": 2744298, + "p99": 4348628, + "maximum": 15743791 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "samples": 450, + "p50": 1580121, + "p95": 2340198, + "p99": null, + "maximum": 15561236 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "samples": 450, + "p50": 620802, + "p95": 1073675, + "p99": null, + "maximum": 1158364 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "end_to_end", + "samples": 450, + "p50": 2251854, + "p95": 3058207, + "p99": null, + "maximum": 15646688 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "samples": 4500, + "p50": 1554692, + "p95": 2237734, + "p99": 3840194, + "maximum": 38261319 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "samples": 4500, + "p50": 619061, + "p95": 1069241, + "p99": 1121437, + "maximum": 2531026 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "end_to_end", + "samples": 4500, + "p50": 2215991, + "p95": 3039894, + "p99": 4713459, + "maximum": 39081879 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "samples": 72, + "p50": 34020848, + "p95": 61696459, + "p99": null, + "maximum": 86791315 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "samples": 72, + "p50": 522258, + "p95": 1042244, + "p99": null, + "maximum": 3496633 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "end_to_end", + "samples": 72, + "p50": 34813321, + "p95": 65193092, + "p99": null, + "maximum": 87205801 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "samples": 108, + "p50": 1738288, + "p95": 2646983, + "p99": null, + "maximum": 3855992 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "samples": 108, + "p50": 578918, + "p95": 1037602, + "p99": null, + "maximum": 1085866 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "end_to_end", + "samples": 108, + "p50": 2370085, + "p95": 3353599, + "p99": null, + "maximum": 3963571 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "samples": 72, + "p50": 2199747, + "p95": 560740967, + "p99": null, + "maximum": 567373014 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "samples": 72, + "p50": 147183, + "p95": 1043768, + "p99": null, + "maximum": 4733194 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "end_to_end", + "samples": 72, + "p50": 2506069, + "p95": 561210139, + "p99": null, + "maximum": 567960286 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "samples": 108, + "p50": 31190928, + "p95": 41673102, + "p99": null, + "maximum": 88976811 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "samples": 108, + "p50": 688870, + "p95": 1072554, + "p99": null, + "maximum": 1124662 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "end_to_end", + "samples": 108, + "p50": 31890510, + "p95": 42529898, + "p99": null, + "maximum": 89721169 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "samples": 252, + "p50": 654476, + "p95": 2150776, + "p99": null, + "maximum": 25091485 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "samples": 252, + "p50": 603125, + "p95": 1084810, + "p99": null, + "maximum": 3773065 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "end_to_end", + "samples": 252, + "p50": 1305333, + "p95": 2959713, + "p99": null, + "maximum": 25638455 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "samples": 252, + "p50": 1373795, + "p95": 114901575, + "p99": null, + "maximum": 121820179 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "samples": 252, + "p50": 542490, + "p95": 1049641, + "p99": null, + "maximum": 1140194 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "end_to_end", + "samples": 252, + "p50": 1963399, + "p95": 115938206, + "p99": null, + "maximum": 122389072 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "samples": 216, + "p50": 1376783, + "p95": 116602516, + "p99": null, + "maximum": 178523251 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "samples": 216, + "p50": 605993, + "p95": 1060056, + "p99": null, + "maximum": 1132884 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "end_to_end", + "samples": 216, + "p50": 2023781, + "p95": 117220797, + "p99": null, + "maximum": 179207793 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "samples": 180, + "p50": 3550258, + "p95": 4453579, + "p99": null, + "maximum": 8326107 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "samples": 180, + "p50": 595414, + "p95": 1058110, + "p99": null, + "maximum": 1134131 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "end_to_end", + "samples": 180, + "p50": 4137480, + "p95": 5036124, + "p99": null, + "maximum": 8481128 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "samples": 180, + "p50": 1744629, + "p95": 2001077, + "p99": null, + "maximum": 2854288 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "samples": 180, + "p50": 595063, + "p95": 1037826, + "p99": null, + "maximum": 1088671 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "end_to_end", + "samples": 180, + "p50": 2343528, + "p95": 2859240, + "p99": null, + "maximum": 3629842 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "samples": 252, + "p50": 67995952, + "p95": 81517582, + "p99": null, + "maximum": 168408898 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "samples": 252, + "p50": 595074, + "p95": 1055771, + "p99": null, + "maximum": 2925940 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "end_to_end", + "samples": 252, + "p50": 68547942, + "p95": 81794656, + "p99": null, + "maximum": 169541723 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "samples": 252, + "p50": 5517813, + "p95": 9327223, + "p99": null, + "maximum": 43183906 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "samples": 252, + "p50": 398597, + "p95": 1668082, + "p99": null, + "maximum": 2810072 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "end_to_end", + "samples": 252, + "p50": 6091277, + "p95": 10303377, + "p99": null, + "maximum": 44746133 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "samples": 216, + "p50": 7725093, + "p95": 17435104, + "p99": null, + "maximum": 27594930 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "samples": 216, + "p50": 455086, + "p95": 1010173, + "p99": null, + "maximum": 1086817 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "end_to_end", + "samples": 216, + "p50": 8259494, + "p95": 17888554, + "p99": null, + "maximum": 28239567 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "samples": 360, + "p50": 2839061, + "p95": 4183714, + "p99": null, + "maximum": 96340559 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "samples": 360, + "p50": 613455, + "p95": 1112245, + "p99": null, + "maximum": 2250088 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "end_to_end", + "samples": 360, + "p50": 3503985, + "p95": 5217161, + "p99": null, + "maximum": 96719894 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "samples": 180, + "p50": 3455241, + "p95": 4013238, + "p99": null, + "maximum": 5520251 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "samples": 180, + "p50": 652365, + "p95": 1443510, + "p99": null, + "maximum": 2521494 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 4136807, + "p95": 5216864, + "p99": null, + "maximum": 6208153 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "samples": 45, + "p50": 733374, + "p95": 1248371, + "p99": null, + "maximum": 1893949 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "samples": 45, + "p50": 589844, + "p95": 994545, + "p99": null, + "maximum": 1038216 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "end_to_end", + "samples": 45, + "p50": 1327735, + "p95": 1980035, + "p99": null, + "maximum": 2608422 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "samples": 135, + "p50": 1615196, + "p95": 2238229, + "p99": null, + "maximum": 3051458 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "samples": 135, + "p50": 469480, + "p95": 1072480, + "p99": null, + "maximum": 1163784 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "end_to_end", + "samples": 135, + "p50": 2102587, + "p95": 2817773, + "p99": null, + "maximum": 3669389 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "samples": 180, + "p50": 33631958, + "p95": 36682815, + "p99": null, + "maximum": 39612274 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "samples": 180, + "p50": 697363, + "p95": 1074282, + "p99": null, + "maximum": 1137656 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 34160080, + "p95": 37298056, + "p99": null, + "maximum": 40507493 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "samples": 180, + "p50": 804338, + "p95": 1131978, + "p99": null, + "maximum": 1989711 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "samples": 180, + "p50": 687279, + "p95": 1718224, + "p99": null, + "maximum": 2064023 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "end_to_end", + "samples": 180, + "p50": 1528221, + "p95": 2481867, + "p99": null, + "maximum": 2992627 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "samples": 180, + "p50": 1699551, + "p95": 1942896, + "p99": null, + "maximum": 2347310 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "samples": 180, + "p50": 599414, + "p95": 1065189, + "p99": null, + "maximum": 1132016 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "end_to_end", + "samples": 180, + "p50": 2319932, + "p95": 2805055, + "p99": null, + "maximum": 3178055 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "samples": 180, + "p50": 447062, + "p95": 3725955, + "p99": null, + "maximum": 5918493 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "samples": 180, + "p50": 601696, + "p95": 1046491, + "p99": null, + "maximum": 1122738 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "end_to_end", + "samples": 180, + "p50": 1052673, + "p95": 4429745, + "p99": null, + "maximum": 6317250 + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "samples": 100, + "p50": 107987642, + "p95": 120645100, + "p99": null, + "maximum": 188102127 + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "samples": 100, + "p50": 797084, + "p95": 1403925, + "p99": null, + "maximum": 3677210 + }, + { + "scenario": "websocket", + "event_class": "new_transmission", + "timing": "receipt_after_ingest_return", + "samples": 4500, + "p50": 518333693, + "p95": 958879301, + "p99": 998007405, + "maximum": 1103677066 + }, + { + "scenario": "handler-control", + "event_class": "200_valid_envelopes", + "timing": "service", + "samples": 200, + "p50": 2037157, + "p95": 2743630, + "p99": null, + "maximum": 5573830 + } + ], + "counts": { + "durable-ingest": { + "total": 12000, + "measured": 9000, + "successful": 12000, + "errors": 0, + "drops": 0 + }, + "mixed-http": { + "total": 4800, + "measured": 3600, + "successful": 4800, + "errors": 0, + "drops": 0 + }, + "sql-miss": { + "total": 200, + "measured": 200, + "successful": 200, + "errors": 0, + "drops": 0 + } + }, + "throughput_including_drain": { + "durable-ingest/new": 25.03031151325764, + "durable-ingest/additional": 15.024828072386086, + "durable-ingest/duplicate": 7.514965849832371, + "durable-ingest/late": 2.5055252649504234, + "mixed-http/nodes_50": 1.4325739527466708, + "mixed-http/nodes_region": 1.228957240520968, + "mixed-http/nodes_2000": 1.4320640923246393, + "mixed-http/channels": 1.432616350481256, + "mixed-http/messages_0": 1.4326176490830909, + "mixed-http/messages_1000": 1.2282079954210312, + "mixed-http/observers": 2.051836578636568, + "mixed-http/metrics_dense_24h": 1.0262023660712536, + "mixed-http/metrics_sparse_7d": 1.0262149466062171, + "mixed-http/reach": 1.026214496158971, + "mixed-http/rx_coverage": 1.026210594717649, + "mixed-http/packets_memory": 1.0248405324823453, + "mixed-http/packet_detail_memory": 1.0250378874022743, + "mixed-http/packet_detail_sql": 1.0336713000123516, + "mixed-http/analytics_rf": 0.6157284537411133, + "mixed-http/analytics_topology": 0.6155975641427371, + "mixed-http/analytics_channels": 0.41087384832861445, + "mixed-http/analytics_rf_filtered": 0.41095377337987593, + "mixed-http/stats": 1.0273908412919814, + "mixed-http/packet_detail_pruned": 1.1083452704904195, + "sql-miss/channel_messages_sql": 8.905742149372086, + "sql-miss/observer_metrics_sql": 9.05684106446128 + }, + "scalars_ns": { + "startup/first_http_ns": 27446512673, + "startup/ready_ns": 27446515859, + "startup/full_ready_ns": 27549230486, + "startup-hot1/first_http_ns": 11290777432, + "startup-hot1/ready_ns": 11290781599, + "startup-hot1/full_ready_ns": 20957487683, + "retention/elapsed_ns": 968513149, + "retention-empty/elapsed_ns": 1867009, + "migration/elapsed_ns": 153642982703, + "migration/migration_tool_ns": 92961857345, + "migration/role_grants_ns": 736011148, + "migration/harness_validation_ns": 59945115482 + }, + "resources": { + "sample_count": 171, + "window_start_ns": 2113084555619, + "window_end_ns": 2293084555619, + "sampled_start_ns": 2113895503011, + "sampled_end_ns": 2292762587065, + "sampled_span_ns": 178867084054, + "sampled_span_fraction": 0.9937060225222222, + "first_boundary_gap_ns": 810947392, + "last_boundary_gap_ns": 321968554, + "sampling_interval_ns": { + "median": 1050827899.0, + "minimum": 1047766062, + "maximum": 1077535327, + "runs": 170 + }, + "phases": [ + "mixed_with_retention" + ], + "cpu_delta_usec": 59226706, + "average_cores_over_sampled_span": 0.33112132572206215, + "per_device_io": { + "8:0": { + "wios": 18370, + "rios": 832, + "dios": 1, + "rbytes": 7946240, + "wbytes": 417820672, + "dbytes": 4096 + } + }, + "aggregate_accounted_io": { + "dbytes": 4096, + "dios": 1, + "rbytes": 7946240, + "rios": 832, + "wbytes": 417820672, + "wios": 18370 + }, + "observed_peaks": { + "cgroup_memory_bytes": 5944741888, + "rss_bytes": 5582184448, + "pss_bytes": 3444285440, + "storage_bytes": 2440286012, + "wal_bytes": 1073741824 + }, + "memory_event_deltas": { + "oom_group_kill": 0, + "oom_kill": 0, + "oom": 0, + "high": 0, + "max": 0, + "low": 0 + }, + "minimum_free_bytes": 79712296960 + }, + "websocket": { + "full_stream_sentinels": 6000, + "full_stream_received": 6000, + "measured_sentinels": 4500, + "measured_missing": 0, + "full_stream_missing": 0, + "negative_measured_lags": 0, + "negative_full_stream_lags": 0, + "duplicate_measured_receipts": 0, + "duplicate_full_stream_receipts": 0 + }, + "retention": { + "retention": { + "elapsed_ns": 968513149, + "transmissions_deleted": 16000, + "observations_deleted": 256000 + }, + "retention-empty": { + "elapsed_ns": 1867009, + "transmissions_deleted": 0, + "observations_deleted": 0 + } + }, + "startup_counts": [ + { + "transmissions": 112000, + "observations": 1792000 + }, + { + "transmissions": 112000, + "observations": 1792000 + } + ], + "database_settings": { + "autovacuum": "on", + "checkpoint_timeout": "5min", + "effective_cache_size": "4GB", + "fsync": "on", + "full_page_writes": "on", + "jit": "on", + "maintenance_work_mem": "64MB", + "max_connections": "32", + "max_parallel_workers_per_gather": "2", + "max_wal_size": "1GB", + "server_version": "18.6 (Ubuntu 18.6-1.pgdg24.04+2)", + "shared_buffers": "256MB", + "synchronous_commit": "on", + "work_mem": "4MB" + }, + "durable_counts": { + "scheduled": 12000, + "completed": 12000, + "dropped_schedules": 0, + "write_errors": 0, + "actual_transmissions": 118000, + "actual_observations": 1802200 + }, + "full_leg_kernel_peak_bytes": 6442569728, + "full_leg_memory_events": { + "low": 0, + "high": 0, + "max": 6237, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "ingest_drain_seconds": 0, + "http_drain_seconds": 0 + }, + { + "pair": 2, + "backend": "sqlite", + "metrics": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "samples": 2700, + "p50": 797721, + "p95": 1630872, + "p99": 3958335, + "maximum": 451765820 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "samples": 2700, + "p50": 640395, + "p95": 2261611474, + "p99": 8074256022, + "maximum": 9666794924 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "end_to_end", + "samples": 2700, + "p50": 1493472, + "p95": 2261999432, + "p99": 8074723038, + "maximum": 9686768735 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "samples": 1350, + "p50": 678503, + "p95": 1021305, + "p99": 3322592, + "maximum": 395344821 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "samples": 1350, + "p50": 652590, + "p95": 2183610642, + "p99": 7995796260, + "maximum": 9569400569 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "end_to_end", + "samples": 1350, + "p50": 1295980, + "p95": 2183934179, + "p99": 7996072708, + "maximum": 9569846515 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "samples": 450, + "p50": 1020925, + "p95": 3240915, + "p99": null, + "maximum": 344392043 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "samples": 450, + "p50": 672219, + "p95": 2144307389, + "p99": null, + "maximum": 9510557078 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "end_to_end", + "samples": 450, + "p50": 1788740, + "p95": 2145071261, + "p99": null, + "maximum": 9511082342 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "samples": 4500, + "p50": 1073680, + "p95": 2688791, + "p99": 5159012, + "maximum": 441938062 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "samples": 4500, + "p50": 657646, + "p95": 2049720113, + "p99": 8194834912, + "maximum": 9491090948 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "end_to_end", + "samples": 4500, + "p50": 1840812, + "p95": 2069693373, + "p99": 8210935496, + "maximum": 9492428797 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "samples": 72, + "p50": 32940401, + "p95": 39714943, + "p99": null, + "maximum": 41618326 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "samples": 72, + "p50": 375627, + "p95": 1057456, + "p99": null, + "maximum": 1091169 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "end_to_end", + "samples": 72, + "p50": 33420358, + "p95": 40074044, + "p99": null, + "maximum": 42681911 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "samples": 108, + "p50": 1827935, + "p95": 2714123, + "p99": null, + "maximum": 4468500 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "samples": 108, + "p50": 544034, + "p95": 1075784, + "p99": null, + "maximum": 1116270 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "end_to_end", + "samples": 108, + "p50": 2472268, + "p95": 3631674, + "p99": null, + "maximum": 4566398 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "samples": 72, + "p50": 1923978, + "p95": 558823354, + "p99": null, + "maximum": 896609025 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "samples": 72, + "p50": 132264, + "p95": 1052859, + "p99": null, + "maximum": 1119473 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "end_to_end", + "samples": 72, + "p50": 2254734, + "p95": 559179086, + "p99": null, + "maximum": 897484943 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "samples": 108, + "p50": 29927262, + "p95": 32674879, + "p99": null, + "maximum": 48873750 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "samples": 108, + "p50": 572853, + "p95": 1062544, + "p99": null, + "maximum": 1146875 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "end_to_end", + "samples": 108, + "p50": 30462552, + "p95": 33737423, + "p99": null, + "maximum": 49344184 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "samples": 252, + "p50": 656952, + "p95": 980696, + "p99": null, + "maximum": 20923716 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "samples": 252, + "p50": 629243, + "p95": 1060620, + "p99": null, + "maximum": 1117274 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "end_to_end", + "samples": 252, + "p50": 1330680, + "p95": 1797893, + "p99": null, + "maximum": 21453360 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "samples": 252, + "p50": 1324915, + "p95": 69114273, + "p99": null, + "maximum": 73114185 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "samples": 252, + "p50": 594023, + "p95": 1022211, + "p99": null, + "maximum": 1095787 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "end_to_end", + "samples": 252, + "p50": 1949504, + "p95": 69708296, + "p99": null, + "maximum": 73769851 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "samples": 216, + "p50": 1330628, + "p95": 93615795, + "p99": null, + "maximum": 105381431 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "samples": 216, + "p50": 608400, + "p95": 1058374, + "p99": null, + "maximum": 2985216 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "end_to_end", + "samples": 216, + "p50": 1990114, + "p95": 94610315, + "p99": null, + "maximum": 106418150 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "samples": 180, + "p50": 3032689, + "p95": 3712770, + "p99": null, + "maximum": 14299286 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "samples": 180, + "p50": 611559, + "p95": 1043703, + "p99": null, + "maximum": 1138539 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "end_to_end", + "samples": 180, + "p50": 3621204, + "p95": 4447823, + "p99": null, + "maximum": 14747305 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "samples": 180, + "p50": 1323771, + "p95": 1587676, + "p99": null, + "maximum": 12607062 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "samples": 180, + "p50": 674563, + "p95": 1043301, + "p99": null, + "maximum": 1107614 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "end_to_end", + "samples": 180, + "p50": 1978897, + "p95": 2466363, + "p99": null, + "maximum": 13143594 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "samples": 252, + "p50": 65659172, + "p95": 79137840, + "p99": null, + "maximum": 260858454 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "samples": 252, + "p50": 534789, + "p95": 1025568, + "p99": null, + "maximum": 1268339 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "end_to_end", + "samples": 252, + "p50": 66087373, + "p95": 80034923, + "p99": null, + "maximum": 261400528 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "samples": 252, + "p50": 3752825, + "p95": 5151472, + "p99": null, + "maximum": 94837513 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "samples": 252, + "p50": 322332, + "p95": 1029296, + "p99": null, + "maximum": 2272237 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "end_to_end", + "samples": 252, + "p50": 4189279, + "p95": 5577154, + "p99": null, + "maximum": 95847144 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "samples": 216, + "p50": 5676176, + "p95": 11080171, + "p99": null, + "maximum": 138795716 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "samples": 216, + "p50": 465342, + "p95": 1029887, + "p99": null, + "maximum": 1123278 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "end_to_end", + "samples": 216, + "p50": 6164620, + "p95": 11361960, + "p99": null, + "maximum": 139522910 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "samples": 360, + "p50": 2710245, + "p95": 3951227, + "p99": null, + "maximum": 24923368 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "samples": 360, + "p50": 582164, + "p95": 1060209, + "p99": null, + "maximum": 1102039 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "end_to_end", + "samples": 360, + "p50": 3393701, + "p95": 4576523, + "p99": null, + "maximum": 25576160 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "samples": 180, + "p50": 1318433, + "p95": 1643928, + "p99": null, + "maximum": 1983282 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "samples": 180, + "p50": 574991, + "p95": 1049240, + "p99": null, + "maximum": 1102350 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 1935530, + "p95": 2393852, + "p99": null, + "maximum": 2889381 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "samples": 45, + "p50": 686431, + "p95": 877573, + "p99": null, + "maximum": 926544 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "samples": 45, + "p50": 633164, + "p95": 1013947, + "p99": null, + "maximum": 1081018 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "end_to_end", + "samples": 45, + "p50": 1309041, + "p95": 1671337, + "p99": null, + "maximum": 1786631 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "samples": 135, + "p50": 1095235, + "p95": 1326226, + "p99": null, + "maximum": 1509201 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "samples": 135, + "p50": 587610, + "p95": 1072796, + "p99": null, + "maximum": 1098865 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "end_to_end", + "samples": 135, + "p50": 1689418, + "p95": 2174378, + "p99": null, + "maximum": 2297639 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "samples": 180, + "p50": 25668689, + "p95": 28395855, + "p99": null, + "maximum": 42467487 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "samples": 180, + "p50": 544571, + "p95": 1038899, + "p99": null, + "maximum": 1114833 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 26199660, + "p95": 29022325, + "p99": null, + "maximum": 43213949 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "samples": 180, + "p50": 772411, + "p95": 1046312, + "p99": null, + "maximum": 2226838 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "samples": 180, + "p50": 587085, + "p95": 1041996, + "p99": null, + "maximum": 1112227 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "end_to_end", + "samples": 180, + "p50": 1385008, + "p95": 1841010, + "p99": null, + "maximum": 2900066 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "samples": 180, + "p50": 1498731, + "p95": 1783356, + "p99": null, + "maximum": 4160723 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "samples": 180, + "p50": 617118, + "p95": 1054860, + "p99": null, + "maximum": 1093608 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "end_to_end", + "samples": 180, + "p50": 2145047, + "p95": 2673650, + "p99": null, + "maximum": 4377350 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "samples": 180, + "p50": 463802, + "p95": 2309737, + "p99": null, + "maximum": 406759838 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "samples": 180, + "p50": 579322, + "p95": 1041025, + "p99": null, + "maximum": 1097426 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "end_to_end", + "samples": 180, + "p50": 1065666, + "p95": 3003745, + "p99": null, + "maximum": 407219243 + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "samples": 100, + "p50": 87554528, + "p95": 99277759, + "p99": null, + "maximum": 186710922 + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "samples": 100, + "p50": 945308, + "p95": 1341007, + "p99": null, + "maximum": 6358469 + }, + { + "scenario": "websocket", + "event_class": "new_transmission", + "timing": "receipt_after_ingest_return", + "samples": 4500, + "p50": 512546498, + "p95": 991098321, + "p99": 1874582094, + "maximum": 1931171250 + }, + { + "scenario": "handler-control", + "event_class": "200_valid_envelopes", + "timing": "service", + "samples": 200, + "p50": 1182105, + "p95": 2917607, + "p99": null, + "maximum": 10390261 + } + ], + "counts": { + "durable-ingest": { + "total": 12000, + "measured": 9000, + "successful": 12000, + "errors": 0, + "drops": 0 + }, + "mixed-http": { + "total": 4800, + "measured": 3600, + "successful": 4800, + "errors": 0, + "drops": 0 + }, + "sql-miss": { + "total": 200, + "measured": 200, + "successful": 200, + "errors": 0, + "drops": 0 + } + }, + "throughput_including_drain": { + "durable-ingest/new": 25.03030412370661, + "durable-ingest/additional": 15.024886476391238, + "durable-ingest/duplicate": 7.514982517522926, + "durable-ingest/late": 2.5055413593198366, + "mixed-http/nodes_50": 1.4325558457486738, + "mixed-http/nodes_region": 1.2289701555459547, + "mixed-http/nodes_2000": 1.4320790766580669, + "mixed-http/channels": 1.4326205845979636, + "mixed-http/messages_0": 1.4326184380821607, + "mixed-http/messages_1000": 1.2290048337963742, + "mixed-http/observers": 2.0518178811133025, + "mixed-http/metrics_dense_24h": 1.026203918962029, + "mixed-http/metrics_sparse_7d": 1.0262149859108898, + "mixed-http/reach": 1.0262167596207443, + "mixed-http/rx_coverage": 1.0262143254020595, + "mixed-http/packets_memory": 1.0249036991510676, + "mixed-http/packet_detail_memory": 1.0250490387711255, + "mixed-http/packet_detail_sql": 1.0336751246890665, + "mixed-http/analytics_rf": 0.61572868235148, + "mixed-http/analytics_topology": 0.6156280388644793, + "mixed-http/analytics_channels": 0.41088183093092123, + "mixed-http/analytics_rf_filtered": 0.4109538925034137, + "mixed-http/stats": 1.027391490280341, + "mixed-http/packet_detail_pruned": 1.1083504799557249, + "sql-miss/channel_messages_sql": 11.717865722706463, + "sql-miss/observer_metrics_sql": 11.832089757410333 + }, + "scalars_ns": { + "startup/first_http_ns": 29051996585, + "startup/ready_ns": 29051998970, + "startup/full_ready_ns": 29064089591, + "startup-hot1/first_http_ns": 9863634701, + "startup-hot1/ready_ns": 9863637235, + "startup-hot1/full_ready_ns": 21565973861, + "retention/elapsed_ns": 11267969655, + "retention-empty/elapsed_ns": 293269 + }, + "resources": { + "sample_count": 180, + "window_start_ns": 2996675779446, + "window_end_ns": 3176675779446, + "sampled_start_ns": 2996976927002, + "sampled_end_ns": 3176645106310, + "sampled_span_ns": 179668179308, + "sampled_span_fraction": 0.9981565517111111, + "first_boundary_gap_ns": 301147556, + "last_boundary_gap_ns": 30673136, + "sampling_interval_ns": { + "median": 1003561913, + "minimum": 1003331967, + "maximum": 1006459589, + "runs": 179 + }, + "phases": [ + "mixed_with_retention" + ], + "cpu_delta_usec": 45022426, + "average_cores_over_sampled_span": 0.2505865322028969, + "per_device_io": { + "8:0": { + "wios": 224267, + "rios": 0, + "dios": 0, + "rbytes": 0, + "wbytes": 2499432448, + "dbytes": 0 + } + }, + "aggregate_accounted_io": { + "dbytes": 0, + "dios": 0, + "rbytes": 0, + "rios": 0, + "wbytes": 2499432448, + "wios": 224267 + }, + "observed_peaks": { + "cgroup_memory_bytes": 4218642432, + "rss_bytes": 3061231616, + "pss_bytes": 3057979392, + "storage_bytes": 1140987672, + "wal_bytes": 26322712 + }, + "memory_event_deltas": { + "oom_group_kill": 0, + "oom_kill": 0, + "oom": 0, + "high": 0, + "max": 0, + "low": 0 + }, + "minimum_free_bytes": 82117414912 + }, + "websocket": { + "full_stream_sentinels": 6000, + "full_stream_received": 6000, + "measured_sentinels": 4500, + "measured_missing": 0, + "full_stream_missing": 0, + "negative_measured_lags": 0, + "negative_full_stream_lags": 0, + "duplicate_measured_receipts": 0, + "duplicate_full_stream_receipts": 0 + }, + "retention": { + "retention": { + "elapsed_ns": 11267969655, + "transmissions_deleted": 16000, + "observations_deleted": 256000 + }, + "retention-empty": { + "elapsed_ns": 293269, + "transmissions_deleted": 0, + "observations_deleted": 0 + } + }, + "startup_counts": [ + { + "transmissions": 112000, + "observations": 1792000 + }, + { + "transmissions": 112000, + "observations": 1792000 + } + ], + "database_settings": { + "busy_timeout": "5000", + "cache_size": "-2000", + "cache_spill": "483", + "foreign_keys": "1", + "journal_mode": "wal", + "journal_size_limit": "-1", + "mmap_size": "0", + "page_size": "4096", + "sqlite_version": "3.53.4", + "synchronous": "2", + "wal_autocheckpoint": "1000" + }, + "durable_counts": { + "scheduled": 12000, + "completed": 12000, + "dropped_schedules": 0, + "write_errors": 0, + "actual_transmissions": 118000, + "actual_observations": 1802200 + }, + "full_leg_kernel_peak_bytes": 4527357952, + "full_leg_memory_events": { + "low": 0, + "high": 0, + "max": 6237, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "ingest_drain_seconds": 0, + "http_drain_seconds": 0 + }, + { + "pair": 2, + "backend": "postgres", + "metrics": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "samples": 2700, + "p50": 1411319, + "p95": 2238979, + "p99": 4543658, + "maximum": 32273025 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "samples": 2700, + "p50": 610233, + "p95": 1076800, + "p99": 1132783, + "maximum": 15782019 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "end_to_end", + "samples": 2700, + "p50": 2098396, + "p95": 3044122, + "p99": 5570235, + "maximum": 32979624 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "samples": 1350, + "p50": 1302281, + "p95": 1912573, + "p99": 4862061, + "maximum": 22982006 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "samples": 1350, + "p50": 607451, + "p95": 1067078, + "p99": 1113332, + "maximum": 22838678 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "end_to_end", + "samples": 1350, + "p50": 1953277, + "p95": 2728895, + "p99": 6215643, + "maximum": 28704930 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "samples": 450, + "p50": 1599613, + "p95": 2363344, + "p99": null, + "maximum": 14775189 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "samples": 450, + "p50": 630351, + "p95": 1079311, + "p99": null, + "maximum": 8730048 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "end_to_end", + "samples": 450, + "p50": 2281672, + "p95": 3227421, + "p99": null, + "maximum": 15042338 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "samples": 4500, + "p50": 1577716, + "p95": 2302268, + "p99": 4404200, + "maximum": 23508138 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "samples": 4500, + "p50": 625383, + "p95": 1075483, + "p99": 1124150, + "maximum": 35089781 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "end_to_end", + "samples": 4500, + "p50": 2268504, + "p95": 3069619, + "p99": 5240617, + "maximum": 39121940 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "samples": 72, + "p50": 32545477, + "p95": 43513964, + "p99": null, + "maximum": 70858697 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "samples": 72, + "p50": 505156, + "p95": 1028074, + "p99": null, + "maximum": 1085832 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "end_to_end", + "samples": 72, + "p50": 32910989, + "p95": 43809624, + "p99": null, + "maximum": 71677167 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "samples": 108, + "p50": 1754690, + "p95": 2510095, + "p99": null, + "maximum": 24254910 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "samples": 108, + "p50": 515539, + "p95": 1028661, + "p99": null, + "maximum": 1101981 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "end_to_end", + "samples": 108, + "p50": 2371516, + "p95": 3303690, + "p99": null, + "maximum": 25035582 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "samples": 72, + "p50": 2247440, + "p95": 559361321, + "p99": null, + "maximum": 970534947 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "samples": 72, + "p50": 176372, + "p95": 1047252, + "p99": null, + "maximum": 2869284 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "end_to_end", + "samples": 72, + "p50": 2544249, + "p95": 559479910, + "p99": null, + "maximum": 970656356 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "samples": 108, + "p50": 31231050, + "p95": 38079272, + "p99": null, + "maximum": 186997374 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "samples": 108, + "p50": 591114, + "p95": 1099648, + "p99": null, + "maximum": 1177340 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "end_to_end", + "samples": 108, + "p50": 31909560, + "p95": 38718420, + "p99": null, + "maximum": 187390491 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "samples": 252, + "p50": 689581, + "p95": 1177886, + "p99": null, + "maximum": 31099144 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "samples": 252, + "p50": 576830, + "p95": 1035437, + "p99": null, + "maximum": 1310901 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "end_to_end", + "samples": 252, + "p50": 1278497, + "p95": 1921677, + "p99": null, + "maximum": 31952530 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "samples": 252, + "p50": 1414410, + "p95": 115241466, + "p99": null, + "maximum": 394993600 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "samples": 252, + "p50": 558990, + "p95": 1064345, + "p99": null, + "maximum": 2922255 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "end_to_end", + "samples": 252, + "p50": 2035544, + "p95": 115740755, + "p99": null, + "maximum": 395894448 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "samples": 216, + "p50": 1443448, + "p95": 117862956, + "p99": null, + "maximum": 177443852 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "samples": 216, + "p50": 558723, + "p95": 1067833, + "p99": null, + "maximum": 1118256 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "end_to_end", + "samples": 216, + "p50": 2045280, + "p95": 118451324, + "p99": null, + "maximum": 177799502 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "samples": 180, + "p50": 3629651, + "p95": 4423986, + "p99": null, + "maximum": 11367486 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "samples": 180, + "p50": 552909, + "p95": 1072913, + "p99": null, + "maximum": 1145396 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "end_to_end", + "samples": 180, + "p50": 4242533, + "p95": 4997094, + "p99": null, + "maximum": 12335247 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "samples": 180, + "p50": 1772797, + "p95": 2049171, + "p99": null, + "maximum": 5073384 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "samples": 180, + "p50": 569247, + "p95": 1041349, + "p99": null, + "maximum": 1098908 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "end_to_end", + "samples": 180, + "p50": 2390276, + "p95": 2908058, + "p99": null, + "maximum": 5551389 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "samples": 252, + "p50": 67981040, + "p95": 92377036, + "p99": null, + "maximum": 208689413 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "samples": 252, + "p50": 588937, + "p95": 1063356, + "p99": null, + "maximum": 3424867 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "end_to_end", + "samples": 252, + "p50": 68611765, + "p95": 93425420, + "p99": null, + "maximum": 209613215 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "samples": 252, + "p50": 5613696, + "p95": 9329955, + "p99": null, + "maximum": 42234249 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "samples": 252, + "p50": 331821, + "p95": 985874, + "p99": null, + "maximum": 3061777 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "end_to_end", + "samples": 252, + "p50": 6071229, + "p95": 9929255, + "p99": null, + "maximum": 43005490 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "samples": 216, + "p50": 7982792, + "p95": 15215035, + "p99": null, + "maximum": 48043767 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "samples": 216, + "p50": 440445, + "p95": 1006389, + "p99": null, + "maximum": 1101269 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "end_to_end", + "samples": 216, + "p50": 8376940, + "p95": 15243733, + "p99": null, + "maximum": 48812159 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "samples": 360, + "p50": 2922084, + "p95": 4232679, + "p99": null, + "maximum": 90321993 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "samples": 360, + "p50": 566636, + "p95": 1044844, + "p99": null, + "maximum": 1141068 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "end_to_end", + "samples": 360, + "p50": 3471552, + "p95": 4792331, + "p99": null, + "maximum": 90923556 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "samples": 180, + "p50": 3566886, + "p95": 4121300, + "p99": null, + "maximum": 5021346 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "samples": 180, + "p50": 569722, + "p95": 1047278, + "p99": null, + "maximum": 1109009 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 4108122, + "p95": 5000468, + "p99": null, + "maximum": 5301373 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "samples": 45, + "p50": 689712, + "p95": 1115371, + "p99": null, + "maximum": 1517946 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "samples": 45, + "p50": 505181, + "p95": 1017178, + "p99": null, + "maximum": 3075995 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "end_to_end", + "samples": 45, + "p50": 1331375, + "p95": 1988883, + "p99": null, + "maximum": 3754513 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "samples": 135, + "p50": 1607747, + "p95": 2029594, + "p99": null, + "maximum": 2272145 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "samples": 135, + "p50": 643138, + "p95": 1061662, + "p99": null, + "maximum": 2664515 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "end_to_end", + "samples": 135, + "p50": 2309270, + "p95": 2837743, + "p99": null, + "maximum": 4715196 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "samples": 180, + "p50": 33015043, + "p95": 36968978, + "p99": null, + "maximum": 41113383 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "samples": 180, + "p50": 572430, + "p95": 1057421, + "p99": null, + "maximum": 1117596 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 33524303, + "p95": 37589285, + "p99": null, + "maximum": 41942527 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "samples": 180, + "p50": 808895, + "p95": 972522, + "p99": null, + "maximum": 2492759 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "samples": 180, + "p50": 574633, + "p95": 1052004, + "p99": null, + "maximum": 1351231 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "end_to_end", + "samples": 180, + "p50": 1383107, + "p95": 1930079, + "p99": null, + "maximum": 3404338 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "samples": 180, + "p50": 1714497, + "p95": 1951423, + "p99": null, + "maximum": 2570011 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "samples": 180, + "p50": 598264, + "p95": 1039687, + "p99": null, + "maximum": 1085782 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "end_to_end", + "samples": 180, + "p50": 2310134, + "p95": 2773720, + "p99": null, + "maximum": 3197652 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "samples": 180, + "p50": 455210, + "p95": 4649564, + "p99": null, + "maximum": 73336252 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "samples": 180, + "p50": 588494, + "p95": 1062119, + "p99": null, + "maximum": 1092865 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "end_to_end", + "samples": 180, + "p50": 1100848, + "p95": 5071998, + "p99": null, + "maximum": 74153507 + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "samples": 100, + "p50": 107964669, + "p95": 122771882, + "p99": null, + "maximum": 247462183 + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "samples": 100, + "p50": 828791, + "p95": 1567553, + "p99": null, + "maximum": 3389623 + }, + { + "scenario": "websocket", + "event_class": "new_transmission", + "timing": "receipt_after_ingest_return", + "samples": 4500, + "p50": 516991231, + "p95": 957468838, + "p99": 996822320, + "maximum": 1330102690 + }, + { + "scenario": "handler-control", + "event_class": "200_valid_envelopes", + "timing": "service", + "samples": 200, + "p50": 1952226, + "p95": 2733058, + "p99": null, + "maximum": 5311818 + } + ], + "counts": { + "durable-ingest": { + "total": 12000, + "measured": 9000, + "successful": 12000, + "errors": 0, + "drops": 0 + }, + "mixed-http": { + "total": 4800, + "measured": 3600, + "successful": 4800, + "errors": 0, + "drops": 0 + }, + "sql-miss": { + "total": 200, + "measured": 200, + "successful": 200, + "errors": 0, + "drops": 0 + } + }, + "throughput_including_drain": { + "durable-ingest/new": 25.030244334580306, + "durable-ingest/additional": 15.024850200741014, + "durable-ingest/duplicate": 7.514968506320083, + "durable-ingest/late": 2.5055373102414924, + "mixed-http/nodes_50": 1.4325853384298377, + "mixed-http/nodes_region": 1.228956737446466, + "mixed-http/nodes_2000": 1.4321117542880417, + "mixed-http/channels": 1.4326159213039131, + "mixed-http/messages_0": 1.4326133211931782, + "mixed-http/messages_1000": 1.2281574065882535, + "mixed-http/observers": 2.051827134627985, + "mixed-http/metrics_dense_24h": 1.0262058941966479, + "mixed-http/metrics_sparse_7d": 1.0262101284951664, + "mixed-http/reach": 1.0262195466424024, + "mixed-http/rx_coverage": 1.0262117959995616, + "mixed-http/packets_memory": 1.0248612883806114, + "mixed-http/packet_detail_memory": 1.0250341703682644, + "mixed-http/packet_detail_sql": 1.0336733483884766, + "mixed-http/analytics_rf": 0.615729273290617, + "mixed-http/analytics_topology": 0.6156291610652173, + "mixed-http/analytics_channels": 0.41088661485743283, + "mixed-http/analytics_rf_filtered": 0.4109510955157029, + "mixed-http/stats": 1.0273903659861683, + "mixed-http/packet_detail_pruned": 1.108350205631316, + "sql-miss/channel_messages_sql": 8.727659935523207, + "sql-miss/observer_metrics_sql": 8.866978403972226 + }, + "scalars_ns": { + "startup/first_http_ns": 27343648617, + "startup/ready_ns": 27343651192, + "startup/full_ready_ns": 27504488994, + "startup-hot1/first_http_ns": 11271217649, + "startup-hot1/ready_ns": 11271221036, + "startup-hot1/full_ready_ns": 20765466482, + "retention/elapsed_ns": 999440814, + "retention-empty/elapsed_ns": 1728440, + "migration/elapsed_ns": 152245472865, + "migration/migration_tool_ns": 92752430553, + "migration/role_grants_ns": 731288250, + "migration/harness_validation_ns": 58761755054 + }, + "resources": { + "sample_count": 170, + "window_start_ns": 3560155386744, + "window_end_ns": 3740155386744, + "sampled_start_ns": 3561053236760, + "sampled_end_ns": 3739155114716, + "sampled_span_ns": 178101877956, + "sampled_span_fraction": 0.9894548775333334, + "first_boundary_gap_ns": 897850016, + "last_boundary_gap_ns": 1000272028, + "sampling_interval_ns": { + "median": 1051825013, + "minimum": 1047127886, + "maximum": 1101741324, + "runs": 169 + }, + "phases": [ + "mixed_with_retention" + ], + "cpu_delta_usec": 60473083, + "average_cores_over_sampled_span": 0.3395420850921058, + "per_device_io": { + "8:0": { + "wios": 18377, + "rios": 853, + "dios": 0, + "rbytes": 8859648, + "wbytes": 404680704, + "dbytes": 0 + } + }, + "aggregate_accounted_io": { + "dbytes": 0, + "dios": 0, + "rbytes": 8859648, + "rios": 853, + "wbytes": 404680704, + "wios": 18377 + }, + "observed_peaks": { + "cgroup_memory_bytes": 6442430464, + "rss_bytes": 5356032000, + "pss_bytes": 3509343232, + "storage_bytes": 2440179516, + "wal_bytes": 1073741824 + }, + "memory_event_deltas": { + "oom_group_kill": 0, + "oom_kill": 0, + "oom": 0, + "high": 0, + "max": 522, + "low": 0 + }, + "minimum_free_bytes": 77456941056 + }, + "websocket": { + "full_stream_sentinels": 6000, + "full_stream_received": 6000, + "measured_sentinels": 4500, + "measured_missing": 0, + "full_stream_missing": 0, + "negative_measured_lags": 0, + "negative_full_stream_lags": 0, + "duplicate_measured_receipts": 0, + "duplicate_full_stream_receipts": 0 + }, + "retention": { + "retention": { + "elapsed_ns": 999440814, + "transmissions_deleted": 16000, + "observations_deleted": 256000 + }, + "retention-empty": { + "elapsed_ns": 1728440, + "transmissions_deleted": 0, + "observations_deleted": 0 + } + }, + "startup_counts": [ + { + "transmissions": 112000, + "observations": 1792000 + }, + { + "transmissions": 112000, + "observations": 1792000 + } + ], + "database_settings": { + "autovacuum": "on", + "checkpoint_timeout": "5min", + "effective_cache_size": "4GB", + "fsync": "on", + "full_page_writes": "on", + "jit": "on", + "maintenance_work_mem": "64MB", + "max_connections": "32", + "max_parallel_workers_per_gather": "2", + "max_wal_size": "1GB", + "server_version": "18.6 (Ubuntu 18.6-1.pgdg24.04+2)", + "shared_buffers": "256MB", + "synchronous_commit": "on", + "work_mem": "4MB" + }, + "durable_counts": { + "scheduled": 12000, + "completed": 12000, + "dropped_schedules": 0, + "write_errors": 0, + "actual_transmissions": 118000, + "actual_observations": 1802200 + }, + "full_leg_kernel_peak_bytes": 6442561536, + "full_leg_memory_events": { + "low": 0, + "high": 0, + "max": 10427, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "ingest_drain_seconds": 0, + "http_drain_seconds": 0 + }, + { + "pair": 3, + "backend": "sqlite", + "metrics": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "samples": 2700, + "p50": 749366, + "p95": 2240945, + "p99": 6089155, + "maximum": 455119158 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "samples": 2700, + "p50": 650130, + "p95": 2048429324, + "p99": 7891136968, + "maximum": 9500482190 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "end_to_end", + "samples": 2700, + "p50": 1409318, + "p95": 2068421459, + "p99": 7891463420, + "maximum": 9501245034 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "samples": 1350, + "p50": 630872, + "p95": 1049436, + "p99": 3726371, + "maximum": 393685035 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "samples": 1350, + "p50": 647159, + "p95": 1989797647, + "p99": 7832232039, + "maximum": 9383439734 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "end_to_end", + "samples": 1350, + "p50": 1262384, + "p95": 1990077090, + "p99": 7895613951, + "maximum": 9383858088 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "samples": 450, + "p50": 936905, + "p95": 3364572, + "p99": null, + "maximum": 157590962 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "samples": 450, + "p50": 673139, + "p95": 1950409764, + "p99": null, + "maximum": 9324480634 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "end_to_end", + "samples": 450, + "p50": 1591211, + "p95": 1951205083, + "p99": null, + "maximum": 9325183981 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "samples": 4500, + "p50": 1029330, + "p95": 3097267, + "p99": 6501449, + "maximum": 457766975 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "samples": 4500, + "p50": 638474, + "p95": 1853879693, + "p99": 8027129843, + "maximum": 9339193282 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "end_to_end", + "samples": 4500, + "p50": 1658706, + "p95": 1873867961, + "p99": 8047117650, + "maximum": 9520455280 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "samples": 72, + "p50": 32724667, + "p95": 42477711, + "p99": null, + "maximum": 59345206 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "samples": 72, + "p50": 492678, + "p95": 1015540, + "p99": null, + "maximum": 1041883 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "end_to_end", + "samples": 72, + "p50": 33062198, + "p95": 43084785, + "p99": null, + "maximum": 59874881 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "samples": 108, + "p50": 1778766, + "p95": 2519422, + "p99": null, + "maximum": 2962939 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "samples": 108, + "p50": 632596, + "p95": 1091637, + "p99": null, + "maximum": 1146041 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "end_to_end", + "samples": 108, + "p50": 2528664, + "p95": 3185257, + "p99": null, + "maximum": 3882787 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "samples": 72, + "p50": 1926528, + "p95": 526797662, + "p99": null, + "maximum": 784499610 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "samples": 72, + "p50": 221454, + "p95": 1078508, + "p99": null, + "maximum": 1105901 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "end_to_end", + "samples": 72, + "p50": 2229079, + "p95": 527044345, + "p99": null, + "maximum": 785328169 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "samples": 108, + "p50": 30116595, + "p95": 36689737, + "p99": null, + "maximum": 48480999 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "samples": 108, + "p50": 680023, + "p95": 1031180, + "p99": null, + "maximum": 1083051 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "end_to_end", + "samples": 108, + "p50": 30577776, + "p95": 37266963, + "p99": null, + "maximum": 48983578 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "samples": 252, + "p50": 595546, + "p95": 936897, + "p99": null, + "maximum": 21116273 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "samples": 252, + "p50": 617424, + "p95": 1048767, + "p99": null, + "maximum": 1093510 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "end_to_end", + "samples": 252, + "p50": 1217296, + "p95": 1724353, + "p99": null, + "maximum": 21988497 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "samples": 252, + "p50": 1271339, + "p95": 68408683, + "p99": null, + "maximum": 72244566 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "samples": 252, + "p50": 569876, + "p95": 1038825, + "p99": null, + "maximum": 1195352 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "end_to_end", + "samples": 252, + "p50": 1865177, + "p95": 69069433, + "p99": null, + "maximum": 72673842 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "samples": 216, + "p50": 1266795, + "p95": 94304613, + "p99": null, + "maximum": 103596774 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "samples": 216, + "p50": 594805, + "p95": 1051284, + "p99": null, + "maximum": 1174227 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "end_to_end", + "samples": 216, + "p50": 1953773, + "p95": 94614918, + "p99": null, + "maximum": 103721795 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "samples": 180, + "p50": 2994334, + "p95": 3900559, + "p99": null, + "maximum": 5790736 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "samples": 180, + "p50": 622551, + "p95": 1031274, + "p99": null, + "maximum": 1094418 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "end_to_end", + "samples": 180, + "p50": 3583052, + "p95": 4510941, + "p99": null, + "maximum": 6795986 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "samples": 180, + "p50": 1246377, + "p95": 1473307, + "p99": null, + "maximum": 1839322 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "samples": 180, + "p50": 568597, + "p95": 1052007, + "p99": null, + "maximum": 1778192 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "end_to_end", + "samples": 180, + "p50": 1784407, + "p95": 2351731, + "p99": null, + "maximum": 3224638 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "samples": 252, + "p50": 64768150, + "p95": 94659115, + "p99": null, + "maximum": 281016948 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "samples": 252, + "p50": 622282, + "p95": 1058502, + "p99": null, + "maximum": 2682310 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "end_to_end", + "samples": 252, + "p50": 65407716, + "p95": 95711344, + "p99": null, + "maximum": 281738668 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "samples": 252, + "p50": 3649295, + "p95": 7466353, + "p99": null, + "maximum": 250121208 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "samples": 252, + "p50": 403257, + "p95": 1055668, + "p99": null, + "maximum": 1560351 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "end_to_end", + "samples": 252, + "p50": 4100088, + "p95": 8324411, + "p99": null, + "maximum": 250947854 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "samples": 216, + "p50": 5761982, + "p95": 12103249, + "p99": null, + "maximum": 158916511 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "samples": 216, + "p50": 434298, + "p95": 1026283, + "p99": null, + "maximum": 1195001 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "end_to_end", + "samples": 216, + "p50": 6257527, + "p95": 13047224, + "p99": null, + "maximum": 159048176 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "samples": 360, + "p50": 2638662, + "p95": 3773031, + "p99": null, + "maximum": 21104879 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "samples": 360, + "p50": 606841, + "p95": 1045433, + "p99": null, + "maximum": 1199453 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "end_to_end", + "samples": 360, + "p50": 3278621, + "p95": 4459397, + "p99": null, + "maximum": 21750724 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "samples": 180, + "p50": 1268966, + "p95": 1594173, + "p99": null, + "maximum": 2013474 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "samples": 180, + "p50": 620187, + "p95": 1045338, + "p99": null, + "maximum": 1103319 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 1901807, + "p95": 2386535, + "p99": null, + "maximum": 2677681 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "samples": 45, + "p50": 678531, + "p95": 1165325, + "p99": null, + "maximum": 1654684 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "samples": 45, + "p50": 608928, + "p95": 1023902, + "p99": null, + "maximum": 1079832 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "end_to_end", + "samples": 45, + "p50": 1313863, + "p95": 1777662, + "p99": null, + "maximum": 2309646 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "samples": 135, + "p50": 1010946, + "p95": 1316706, + "p99": null, + "maximum": 1435687 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "samples": 135, + "p50": 581017, + "p95": 1039154, + "p99": null, + "maximum": 1085729 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "end_to_end", + "samples": 135, + "p50": 1601670, + "p95": 2129333, + "p99": null, + "maximum": 2374743 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "samples": 180, + "p50": 25896441, + "p95": 30280915, + "p99": null, + "maximum": 34903693 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "samples": 180, + "p50": 656353, + "p95": 1067151, + "p99": null, + "maximum": 1124046 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 26565317, + "p95": 30617926, + "p99": null, + "maximum": 35827790 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "samples": 180, + "p50": 704434, + "p95": 901509, + "p99": null, + "maximum": 1118688 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "samples": 180, + "p50": 585723, + "p95": 1058406, + "p99": null, + "maximum": 1114519 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "end_to_end", + "samples": 180, + "p50": 1296770, + "p95": 1816767, + "p99": null, + "maximum": 2055386 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "samples": 180, + "p50": 1434843, + "p95": 1714378, + "p99": null, + "maximum": 2703714 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "samples": 180, + "p50": 595656, + "p95": 1039906, + "p99": null, + "maximum": 1082814 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "end_to_end", + "samples": 180, + "p50": 2060478, + "p95": 2622677, + "p99": null, + "maximum": 3089661 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "samples": 180, + "p50": 429637, + "p95": 2276763, + "p99": null, + "maximum": 14954681 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "samples": 180, + "p50": 573625, + "p95": 1069634, + "p99": null, + "maximum": 1138576 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "end_to_end", + "samples": 180, + "p50": 1082535, + "p95": 2755969, + "p99": null, + "maximum": 15662052 + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "samples": 100, + "p50": 79510422, + "p95": 92099760, + "p99": null, + "maximum": 98025354 + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "samples": 100, + "p50": 817541, + "p95": 1235474, + "p99": null, + "maximum": 1438049 + }, + { + "scenario": "websocket", + "event_class": "new_transmission", + "timing": "receipt_after_ingest_return", + "samples": 4500, + "p50": 525747562, + "p95": 985123611, + "p99": 2448201481, + "maximum": 2590308718 + }, + { + "scenario": "handler-control", + "event_class": "200_valid_envelopes", + "timing": "service", + "samples": 200, + "p50": 1165453, + "p95": 3331540, + "p99": null, + "maximum": 4639110 + } + ], + "counts": { + "durable-ingest": { + "total": 12000, + "measured": 9000, + "successful": 12000, + "errors": 0, + "drops": 0 + }, + "mixed-http": { + "total": 4800, + "measured": 3600, + "successful": 4800, + "errors": 0, + "drops": 0 + }, + "sql-miss": { + "total": 200, + "measured": 200, + "successful": 200, + "errors": 0, + "drops": 0 + } + }, + "throughput_including_drain": { + "durable-ingest/new": 25.03031963862231, + "durable-ingest/additional": 15.024947060749692, + "durable-ingest/duplicate": 7.514904501268186, + "durable-ingest/late": 2.505550092004002, + "mixed-http/nodes_50": 1.4326001132500645, + "mixed-http/nodes_region": 1.2289728046456092, + "mixed-http/nodes_2000": 1.4320569916909363, + "mixed-http/channels": 1.432618326422076, + "mixed-http/messages_0": 1.4326133511726002, + "mixed-http/messages_1000": 1.2290032098728656, + "mixed-http/observers": 2.0518240460051866, + "mixed-http/metrics_dense_24h": 1.0262022581352577, + "mixed-http/metrics_sparse_7d": 1.0262137862889407, + "mixed-http/reach": 1.026218915030138, + "mixed-http/rx_coverage": 1.0262108412152307, + "mixed-http/packets_memory": 1.0248720946093346, + "mixed-http/packet_detail_memory": 1.0250450143038312, + "mixed-http/packet_detail_sql": 1.0336782228120047, + "mixed-http/analytics_rf": 0.6157281059604941, + "mixed-http/analytics_topology": 0.6156284240809752, + "mixed-http/analytics_channels": 0.4108827517992465, + "mixed-http/analytics_rf_filtered": 0.41095346075187206, + "mixed-http/stats": 1.0273910248255624, + "mixed-http/packet_detail_pruned": 1.1083299179582484, + "sql-miss/channel_messages_sql": 12.35984317747424, + "sql-miss/observer_metrics_sql": 12.467865317696237 + }, + "scalars_ns": { + "startup/first_http_ns": 28554204216, + "startup/ready_ns": 28554207001, + "startup/full_ready_ns": 28566338287, + "startup-hot1/first_http_ns": 9763817804, + "startup-hot1/ready_ns": 9763820379, + "startup-hot1/full_ready_ns": 21441677892, + "retention/elapsed_ns": 11079614774, + "retention-empty/elapsed_ns": 308207 + }, + "resources": { + "sample_count": 179, + "window_start_ns": 4632771524972, + "window_end_ns": 4812771524972, + "sampled_start_ns": 4633355162981, + "sampled_end_ns": 4812084930022, + "sampled_span_ns": 178729767041, + "sampled_span_fraction": 0.9929431502277778, + "first_boundary_gap_ns": 583638009, + "last_boundary_gap_ns": 686594950, + "sampling_interval_ns": { + "median": 1003838598.5, + "minimum": 1003620899, + "maximum": 1008956946, + "runs": 178 + }, + "phases": [ + "mixed_with_retention" + ], + "cpu_delta_usec": 44689472, + "average_cores_over_sampled_span": 0.2500393344649097, + "per_device_io": { + "8:0": { + "wios": 218469, + "rios": 0, + "dios": 0, + "rbytes": 0, + "wbytes": 2468835328, + "dbytes": 0 + } + }, + "aggregate_accounted_io": { + "dbytes": 0, + "dios": 0, + "rbytes": 0, + "rios": 0, + "wbytes": 2468835328, + "wios": 218469 + }, + "observed_peaks": { + "cgroup_memory_bytes": 4280754176, + "rss_bytes": 3094675456, + "pss_bytes": 3091423232, + "storage_bytes": 1162624192, + "wal_bytes": 42040512 + }, + "memory_event_deltas": { + "oom_group_kill": 0, + "oom_kill": 0, + "oom": 0, + "high": 0, + "max": 0, + "low": 0 + }, + "minimum_free_bytes": 80958263296 + }, + "websocket": { + "full_stream_sentinels": 6000, + "full_stream_received": 6000, + "measured_sentinels": 4500, + "measured_missing": 0, + "full_stream_missing": 0, + "negative_measured_lags": 0, + "negative_full_stream_lags": 0, + "duplicate_measured_receipts": 0, + "duplicate_full_stream_receipts": 0 + }, + "retention": { + "retention": { + "elapsed_ns": 11079614774, + "transmissions_deleted": 16000, + "observations_deleted": 256000 + }, + "retention-empty": { + "elapsed_ns": 308207, + "transmissions_deleted": 0, + "observations_deleted": 0 + } + }, + "startup_counts": [ + { + "transmissions": 112000, + "observations": 1792000 + }, + { + "transmissions": 112000, + "observations": 1792000 + } + ], + "database_settings": { + "busy_timeout": "5000", + "cache_size": "-2000", + "cache_spill": "483", + "foreign_keys": "1", + "journal_mode": "wal", + "journal_size_limit": "-1", + "mmap_size": "0", + "page_size": "4096", + "sqlite_version": "3.53.4", + "synchronous": "2", + "wal_autocheckpoint": "1000" + }, + "durable_counts": { + "scheduled": 12000, + "completed": 12000, + "dropped_schedules": 0, + "write_errors": 0, + "actual_transmissions": 118000, + "actual_observations": 1802200 + }, + "full_leg_kernel_peak_bytes": 4482707456, + "full_leg_memory_events": { + "low": 0, + "high": 0, + "max": 13342, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "ingest_drain_seconds": 0, + "http_drain_seconds": 0 + }, + { + "pair": 3, + "backend": "postgres", + "metrics": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "samples": 2700, + "p50": 1290881, + "p95": 2142241, + "p99": 4830973, + "maximum": 48157637 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "samples": 2700, + "p50": 628657, + "p95": 1074187, + "p99": 1118967, + "maximum": 28680726 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "end_to_end", + "samples": 2700, + "p50": 2007146, + "p95": 2937185, + "p99": 6074051, + "maximum": 48652142 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "samples": 1350, + "p50": 1209887, + "p95": 1797518, + "p99": 3904657, + "maximum": 14065680 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "samples": 1350, + "p50": 615031, + "p95": 1068698, + "p99": 1112607, + "maximum": 1491951 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "end_to_end", + "samples": 1350, + "p50": 1883705, + "p95": 2634651, + "p99": 4656678, + "maximum": 14976473 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "samples": 450, + "p50": 1498979, + "p95": 2123776, + "p99": null, + "maximum": 9474818 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "samples": 450, + "p50": 666357, + "p95": 1072949, + "p99": null, + "maximum": 1149018 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "end_to_end", + "samples": 450, + "p50": 2217720, + "p95": 2940751, + "p99": null, + "maximum": 10147622 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "samples": 4500, + "p50": 1472038, + "p95": 2152173, + "p99": 4041005, + "maximum": 29248390 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "samples": 4500, + "p50": 611364, + "p95": 1070719, + "p99": 1114794, + "maximum": 9925216 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "end_to_end", + "samples": 4500, + "p50": 2151257, + "p95": 2936511, + "p99": 4890648, + "maximum": 29904568 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "samples": 72, + "p50": 33063740, + "p95": 54355592, + "p99": null, + "maximum": 99832491 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "samples": 72, + "p50": 375906, + "p95": 1084633, + "p99": null, + "maximum": 1810626 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "end_to_end", + "samples": 72, + "p50": 34030089, + "p95": 54408893, + "p99": null, + "maximum": 99867992 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "samples": 108, + "p50": 1722016, + "p95": 2557171, + "p99": null, + "maximum": 2842800 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "samples": 108, + "p50": 512993, + "p95": 1046137, + "p99": null, + "maximum": 1118192 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "end_to_end", + "samples": 108, + "p50": 2389223, + "p95": 3218135, + "p99": null, + "maximum": 3868962 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "samples": 72, + "p50": 2175620, + "p95": 546993532, + "p99": null, + "maximum": 1133136977 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "samples": 72, + "p50": 155176, + "p95": 1008881, + "p99": null, + "maximum": 1086456 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "end_to_end", + "samples": 72, + "p50": 2461849, + "p95": 547274024, + "p99": null, + "maximum": 1133609223 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "samples": 108, + "p50": 31366518, + "p95": 43504270, + "p99": null, + "maximum": 66652313 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "samples": 108, + "p50": 687531, + "p95": 1050854, + "p99": null, + "maximum": 1115824 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "end_to_end", + "samples": 108, + "p50": 32084806, + "p95": 43615827, + "p99": null, + "maximum": 67619441 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "samples": 252, + "p50": 565316, + "p95": 987492, + "p99": null, + "maximum": 25487340 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "samples": 252, + "p50": 567348, + "p95": 1046744, + "p99": null, + "maximum": 1127962 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "end_to_end", + "samples": 252, + "p50": 1166645, + "p95": 1800554, + "p99": null, + "maximum": 26553567 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "samples": 252, + "p50": 1316627, + "p95": 111198714, + "p99": null, + "maximum": 121984843 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "samples": 252, + "p50": 561308, + "p95": 1029934, + "p99": null, + "maximum": 1117485 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "end_to_end", + "samples": 252, + "p50": 1977368, + "p95": 111238306, + "p99": null, + "maximum": 123054350 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "samples": 216, + "p50": 1315600, + "p95": 113234613, + "p99": null, + "maximum": 182714601 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "samples": 216, + "p50": 620659, + "p95": 1069802, + "p99": null, + "maximum": 1125697 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "end_to_end", + "samples": 216, + "p50": 2024723, + "p95": 114091242, + "p99": null, + "maximum": 183144280 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "samples": 180, + "p50": 3401036, + "p95": 4291550, + "p99": null, + "maximum": 6545058 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "samples": 180, + "p50": 603397, + "p95": 1005144, + "p99": null, + "maximum": 1141374 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "end_to_end", + "samples": 180, + "p50": 4047831, + "p95": 4950348, + "p99": null, + "maximum": 7679817 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "samples": 180, + "p50": 1610178, + "p95": 1942777, + "p99": null, + "maximum": 2691974 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "samples": 180, + "p50": 598684, + "p95": 1050698, + "p99": null, + "maximum": 1122831 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "end_to_end", + "samples": 180, + "p50": 2264782, + "p95": 2849199, + "p99": null, + "maximum": 3295975 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "samples": 252, + "p50": 68074162, + "p95": 94743795, + "p99": null, + "maximum": 174462724 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "samples": 252, + "p50": 494600, + "p95": 1078740, + "p99": null, + "maximum": 5655267 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "end_to_end", + "samples": 252, + "p50": 68565532, + "p95": 100399062, + "p99": null, + "maximum": 176941254 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "samples": 252, + "p50": 5360144, + "p95": 12367177, + "p99": null, + "maximum": 25877376 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "samples": 252, + "p50": 343320, + "p95": 1038788, + "p99": null, + "maximum": 2777864 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "end_to_end", + "samples": 252, + "p50": 5851625, + "p95": 13257679, + "p99": null, + "maximum": 27555906 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "samples": 216, + "p50": 7613812, + "p95": 15143200, + "p99": null, + "maximum": 46464482 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "samples": 216, + "p50": 508176, + "p95": 1060890, + "p99": null, + "maximum": 1867280 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "end_to_end", + "samples": 216, + "p50": 8215721, + "p95": 16118080, + "p99": null, + "maximum": 47179845 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "samples": 360, + "p50": 2784966, + "p95": 4046807, + "p99": null, + "maximum": 83777748 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "samples": 360, + "p50": 559074, + "p95": 1044006, + "p99": null, + "maximum": 1135393 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "end_to_end", + "samples": 360, + "p50": 3378452, + "p95": 4609190, + "p99": null, + "maximum": 83986050 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "samples": 180, + "p50": 3326121, + "p95": 3837048, + "p99": null, + "maximum": 15938490 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "samples": 180, + "p50": 558604, + "p95": 1048843, + "p99": null, + "maximum": 1086900 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 3907691, + "p95": 4639106, + "p99": null, + "maximum": 17018978 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "samples": 45, + "p50": 706158, + "p95": 1323650, + "p99": null, + "maximum": 1555755 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "samples": 45, + "p50": 553708, + "p95": 980369, + "p99": null, + "maximum": 1026225 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "end_to_end", + "samples": 45, + "p50": 1330544, + "p95": 1929635, + "p99": null, + "maximum": 2536124 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "samples": 135, + "p50": 1441890, + "p95": 1968266, + "p99": null, + "maximum": 2355041 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "samples": 135, + "p50": 668517, + "p95": 1047300, + "p99": null, + "maximum": 1170450 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "end_to_end", + "samples": 135, + "p50": 2104728, + "p95": 2754264, + "p99": null, + "maximum": 3147958 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "samples": 180, + "p50": 36077881, + "p95": 41187684, + "p99": null, + "maximum": 125306183 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "samples": 180, + "p50": 525931, + "p95": 1066520, + "p99": null, + "maximum": 1139198 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 36790527, + "p95": 42084430, + "p99": null, + "maximum": 126160094 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "samples": 180, + "p50": 729549, + "p95": 1015812, + "p99": null, + "maximum": 1245966 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "samples": 180, + "p50": 565569, + "p95": 1055843, + "p99": null, + "maximum": 1110467 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "end_to_end", + "samples": 180, + "p50": 1296722, + "p95": 1909557, + "p99": null, + "maximum": 2241436 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "samples": 180, + "p50": 1565606, + "p95": 1889803, + "p99": null, + "maximum": 3082307 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "samples": 180, + "p50": 648280, + "p95": 1052772, + "p99": null, + "maximum": 1088952 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "end_to_end", + "samples": 180, + "p50": 2177902, + "p95": 2686769, + "p99": null, + "maximum": 3888693 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "samples": 180, + "p50": 370837, + "p95": 3660450, + "p99": null, + "maximum": 5787043 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "samples": 180, + "p50": 551164, + "p95": 1027026, + "p99": null, + "maximum": 1125036 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "end_to_end", + "samples": 180, + "p50": 990348, + "p95": 4084346, + "p99": null, + "maximum": 6573244 + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "samples": 100, + "p50": 103400400, + "p95": 117746508, + "p99": null, + "maximum": 187622515 + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "samples": 100, + "p50": 769543, + "p95": 1322560, + "p99": null, + "maximum": 4223567 + }, + { + "scenario": "websocket", + "event_class": "new_transmission", + "timing": "receipt_after_ingest_return", + "samples": 4500, + "p50": 505394149, + "p95": 964605246, + "p99": 1451344825, + "maximum": 2190890626 + }, + { + "scenario": "handler-control", + "event_class": "200_valid_envelopes", + "timing": "service", + "samples": 200, + "p50": 1965146, + "p95": 2699104, + "p99": null, + "maximum": 6469420 + } + ], + "counts": { + "durable-ingest": { + "total": 12000, + "measured": 9000, + "successful": 12000, + "errors": 0, + "drops": 0 + }, + "mixed-http": { + "total": 4800, + "measured": 3600, + "successful": 4800, + "errors": 0, + "drops": 0 + }, + "sql-miss": { + "total": 200, + "measured": 200, + "successful": 200, + "errors": 0, + "drops": 0 + } + }, + "throughput_including_drain": { + "durable-ingest/new": 25.030257308838635, + "durable-ingest/additional": 15.024934291224621, + "durable-ingest/duplicate": 7.51497555165654, + "durable-ingest/late": 2.5055404609619716, + "mixed-http/nodes_50": 1.4325782056126484, + "mixed-http/nodes_region": 1.2289587004021945, + "mixed-http/nodes_2000": 1.4321072565085593, + "mixed-http/channels": 1.4326185940968201, + "mixed-http/messages_0": 1.4326157772702433, + "mixed-http/messages_1000": 1.228199949692814, + "mixed-http/observers": 2.051830985493875, + "mixed-http/metrics_dense_24h": 1.0262015863630505, + "mixed-http/metrics_sparse_7d": 1.0262138283550255, + "mixed-http/reach": 1.0262218067096294, + "mixed-http/rx_coverage": 1.0262145115344676, + "mixed-http/packets_memory": 1.0248628867039926, + "mixed-http/packet_detail_memory": 1.0250319623315352, + "mixed-http/packet_detail_sql": 1.0336698381567269, + "mixed-http/analytics_rf": 0.6157283776079031, + "mixed-http/analytics_topology": 0.6156254122729737, + "mixed-http/analytics_channels": 0.4108769814636148, + "mixed-http/analytics_rf_filtered": 0.410953129540234, + "mixed-http/stats": 1.0273905231312968, + "mixed-http/packet_detail_pruned": 1.1083489491295286, + "sql-miss/channel_messages_sql": 9.258089083946725, + "sql-miss/observer_metrics_sql": 9.421000955023542 + }, + "scalars_ns": { + "startup/first_http_ns": 27549217386, + "startup/ready_ns": 27549219950, + "startup/full_ready_ns": 27712102966, + "startup-hot1/first_http_ns": 10966233623, + "startup-hot1/ready_ns": 10966236929, + "startup-hot1/full_ready_ns": 20802554933, + "retention/elapsed_ns": 1009484608, + "retention-empty/elapsed_ns": 1900989, + "migration/elapsed_ns": 148886831843, + "migration/migration_tool_ns": 90001959193, + "migration/role_grants_ns": 719914023, + "migration/harness_validation_ns": 58164959829 + }, + "resources": { + "sample_count": 171, + "window_start_ns": 4222632280409, + "window_end_ns": 4402632280409, + "sampled_start_ns": 4222775218057, + "sampled_end_ns": 4401969728232, + "sampled_span_ns": 179194510175, + "sampled_span_fraction": 0.9955250565277778, + "first_boundary_gap_ns": 142937648, + "last_boundary_gap_ns": 662552177, + "sampling_interval_ns": { + "median": 1051840630.5, + "minimum": 1048567421, + "maximum": 1103034968, + "runs": 170 + }, + "phases": [ + "mixed_with_retention" + ], + "cpu_delta_usec": 59082390, + "average_cores_over_sampled_span": 0.32971093780886807, + "per_device_io": { + "8:0": { + "wios": 18894, + "rios": 864, + "dios": 1, + "rbytes": 8552448, + "wbytes": 424558592, + "dbytes": 4096 + } + }, + "aggregate_accounted_io": { + "dbytes": 4096, + "dios": 1, + "rbytes": 8552448, + "rios": 864, + "wbytes": 424558592, + "wios": 18894 + }, + "observed_peaks": { + "cgroup_memory_bytes": 5869744128, + "rss_bytes": 5487980544, + "pss_bytes": 3364481024, + "storage_bytes": 2440343356, + "wal_bytes": 1073741824 + }, + "memory_event_deltas": { + "oom_group_kill": 0, + "oom_kill": 0, + "oom": 0, + "high": 0, + "max": 0, + "low": 0 + }, + "minimum_free_bytes": 77427404800 + }, + "websocket": { + "full_stream_sentinels": 6000, + "full_stream_received": 6000, + "measured_sentinels": 4500, + "measured_missing": 0, + "full_stream_missing": 0, + "negative_measured_lags": 0, + "negative_full_stream_lags": 0, + "duplicate_measured_receipts": 0, + "duplicate_full_stream_receipts": 0 + }, + "retention": { + "retention": { + "elapsed_ns": 1009484608, + "transmissions_deleted": 16000, + "observations_deleted": 256000 + }, + "retention-empty": { + "elapsed_ns": 1900989, + "transmissions_deleted": 0, + "observations_deleted": 0 + } + }, + "startup_counts": [ + { + "transmissions": 112000, + "observations": 1792000 + }, + { + "transmissions": 112000, + "observations": 1792000 + } + ], + "database_settings": { + "autovacuum": "on", + "checkpoint_timeout": "5min", + "effective_cache_size": "4GB", + "fsync": "on", + "full_page_writes": "on", + "jit": "on", + "maintenance_work_mem": "64MB", + "max_connections": "32", + "max_parallel_workers_per_gather": "2", + "max_wal_size": "1GB", + "server_version": "18.6 (Ubuntu 18.6-1.pgdg24.04+2)", + "shared_buffers": "256MB", + "synchronous_commit": "on", + "work_mem": "4MB" + }, + "durable_counts": { + "scheduled": 12000, + "completed": 12000, + "dropped_schedules": 0, + "write_errors": 0, + "actual_transmissions": 118000, + "actual_observations": 1802200 + }, + "full_leg_kernel_peak_bytes": 6442508288, + "full_leg_memory_events": { + "low": 0, + "high": 0, + "max": 13342, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "ingest_drain_seconds": 0, + "http_drain_seconds": 0 + }, + { + "pair": 4, + "backend": "sqlite", + "metrics": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "samples": 2700, + "p50": 730751, + "p95": 1626878, + "p99": 4737857, + "maximum": 460169127 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "samples": 2700, + "p50": 645863, + "p95": 2407422741, + "p99": 8231436376, + "maximum": 9659880030 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "end_to_end", + "samples": 2700, + "p50": 1396577, + "p95": 2407753090, + "p99": 8231898081, + "maximum": 9679860794 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "samples": 1350, + "p50": 614232, + "p95": 1072090, + "p99": 3452894, + "maximum": 240406855 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "samples": 1350, + "p50": 641317, + "p95": 2329010005, + "p99": 8153147392, + "maximum": 9600992915 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "end_to_end", + "samples": 1350, + "p50": 1264426, + "p95": 2329276946, + "p99": 8153439399, + "maximum": 9601251009 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "samples": 450, + "p50": 940674, + "p95": 1738170, + "p99": null, + "maximum": 407811296 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "samples": 450, + "p50": 618449, + "p95": 2289550899, + "p99": null, + "maximum": 9541916556 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "end_to_end", + "samples": 450, + "p50": 1541612, + "p95": 2290146785, + "p99": null, + "maximum": 9542541517 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "samples": 4500, + "p50": 1016586, + "p95": 2745275, + "p99": 5855864, + "maximum": 504108393 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "samples": 4500, + "p50": 633777, + "p95": 2231492778, + "p99": 8328976175, + "maximum": 9522550063 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "end_to_end", + "samples": 4500, + "p50": 1636669, + "p95": 2251485053, + "p99": 8348964133, + "maximum": 9523753649 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "samples": 72, + "p50": 33349087, + "p95": 44891357, + "p99": null, + "maximum": 85110009 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "samples": 72, + "p50": 501917, + "p95": 1071839, + "p99": null, + "maximum": 1654611 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "end_to_end", + "samples": 72, + "p50": 33848143, + "p95": 45855185, + "p99": null, + "maximum": 86764620 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "samples": 108, + "p50": 1684541, + "p95": 2540853, + "p99": null, + "maximum": 4542343 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "samples": 108, + "p50": 665586, + "p95": 1080471, + "p99": null, + "maximum": 1125474 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "end_to_end", + "samples": 108, + "p50": 2367044, + "p95": 3132581, + "p99": null, + "maximum": 5214513 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "samples": 72, + "p50": 1870467, + "p95": 535697290, + "p99": null, + "maximum": 772331446 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "samples": 72, + "p50": 122926, + "p95": 1039325, + "p99": null, + "maximum": 1302305 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "end_to_end", + "samples": 72, + "p50": 2167140, + "p95": 536034954, + "p99": null, + "maximum": 772451749 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "samples": 108, + "p50": 29557543, + "p95": 33359331, + "p99": null, + "maximum": 135160746 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "samples": 108, + "p50": 693147, + "p95": 1054891, + "p99": null, + "maximum": 1090553 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "end_to_end", + "samples": 108, + "p50": 30232848, + "p95": 33742678, + "p99": null, + "maximum": 135290746 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "samples": 252, + "p50": 550664, + "p95": 872738, + "p99": null, + "maximum": 20557804 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "samples": 252, + "p50": 547283, + "p95": 1020445, + "p99": null, + "maximum": 1128720 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "end_to_end", + "samples": 252, + "p50": 1160416, + "p95": 1638934, + "p99": null, + "maximum": 20910343 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "samples": 252, + "p50": 1234951, + "p95": 68286431, + "p99": null, + "maximum": 71896021 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "samples": 252, + "p50": 505132, + "p95": 1058939, + "p99": null, + "maximum": 1749165 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "end_to_end", + "samples": 252, + "p50": 1806321, + "p95": 68660510, + "p99": null, + "maximum": 72322569 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "samples": 216, + "p50": 1248795, + "p95": 93448017, + "p99": null, + "maximum": 102973327 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "samples": 216, + "p50": 545391, + "p95": 1022059, + "p99": null, + "maximum": 1307991 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "end_to_end", + "samples": 216, + "p50": 1827712, + "p95": 93906524, + "p99": null, + "maximum": 103051489 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "samples": 180, + "p50": 2877634, + "p95": 3580873, + "p99": null, + "maximum": 4361280 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "samples": 180, + "p50": 591965, + "p95": 1055773, + "p99": null, + "maximum": 2488819 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "end_to_end", + "samples": 180, + "p50": 3485341, + "p95": 4371246, + "p99": null, + "maximum": 5096857 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "samples": 180, + "p50": 1170162, + "p95": 1519077, + "p99": null, + "maximum": 1823176 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "samples": 180, + "p50": 584474, + "p95": 973475, + "p99": null, + "maximum": 1103826 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "end_to_end", + "samples": 180, + "p50": 1761261, + "p95": 2287669, + "p99": null, + "maximum": 2529380 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "samples": 252, + "p50": 64485794, + "p95": 81661579, + "p99": null, + "maximum": 206520136 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "samples": 252, + "p50": 582068, + "p95": 1056408, + "p99": null, + "maximum": 1239555 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "end_to_end", + "samples": 252, + "p50": 65015030, + "p95": 82319858, + "p99": null, + "maximum": 207137546 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "samples": 252, + "p50": 3486101, + "p95": 6349767, + "p99": null, + "maximum": 199588201 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "samples": 252, + "p50": 335694, + "p95": 999342, + "p99": null, + "maximum": 2699494 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "end_to_end", + "samples": 252, + "p50": 3956656, + "p95": 8290078, + "p99": null, + "maximum": 200529421 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "samples": 216, + "p50": 5534435, + "p95": 11332445, + "p99": null, + "maximum": 96064334 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "samples": 216, + "p50": 517357, + "p95": 1050389, + "p99": null, + "maximum": 3078634 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "end_to_end", + "samples": 216, + "p50": 6107216, + "p95": 12382360, + "p99": null, + "maximum": 96197164 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "samples": 360, + "p50": 2572515, + "p95": 3556429, + "p99": null, + "maximum": 21793235 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "samples": 360, + "p50": 595802, + "p95": 1049613, + "p99": null, + "maximum": 1105624 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "end_to_end", + "samples": 360, + "p50": 3216765, + "p95": 4254801, + "p99": null, + "maximum": 21861924 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "samples": 180, + "p50": 1221779, + "p95": 1529953, + "p99": null, + "maximum": 2139691 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "samples": 180, + "p50": 565662, + "p95": 1029716, + "p99": null, + "maximum": 1135171 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 1806772, + "p95": 2341272, + "p99": null, + "maximum": 2610446 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "samples": 45, + "p50": 580420, + "p95": 762845, + "p99": null, + "maximum": 11122892 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "samples": 45, + "p50": 495364, + "p95": 1014524, + "p99": null, + "maximum": 1087565 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "end_to_end", + "samples": 45, + "p50": 1126866, + "p95": 1646191, + "p99": null, + "maximum": 11486954 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "samples": 135, + "p50": 931217, + "p95": 1201301, + "p99": null, + "maximum": 1397542 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "samples": 135, + "p50": 681901, + "p95": 1052155, + "p99": null, + "maximum": 1094019 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "end_to_end", + "samples": 135, + "p50": 1613794, + "p95": 2076777, + "p99": null, + "maximum": 2345519 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "samples": 180, + "p50": 25377023, + "p95": 27689755, + "p99": null, + "maximum": 90739252 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "samples": 180, + "p50": 639982, + "p95": 1058599, + "p99": null, + "maximum": 1128327 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 25976110, + "p95": 28145003, + "p99": null, + "maximum": 90950141 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "samples": 180, + "p50": 658939, + "p95": 897776, + "p99": null, + "maximum": 1457514 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "samples": 180, + "p50": 617120, + "p95": 1062457, + "p99": null, + "maximum": 1122561 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "end_to_end", + "samples": 180, + "p50": 1276500, + "p95": 1767405, + "p99": null, + "maximum": 2145726 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "samples": 180, + "p50": 1387484, + "p95": 1613502, + "p99": null, + "maximum": 2300132 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "samples": 180, + "p50": 549551, + "p95": 1031640, + "p99": null, + "maximum": 1083276 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "end_to_end", + "samples": 180, + "p50": 1938965, + "p95": 2455765, + "p99": null, + "maximum": 2747767 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "samples": 180, + "p50": 364624, + "p95": 2242682, + "p99": null, + "maximum": 14370013 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "samples": 180, + "p50": 545515, + "p95": 1079856, + "p99": null, + "maximum": 1178306 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "end_to_end", + "samples": 180, + "p50": 956817, + "p95": 3029963, + "p99": null, + "maximum": 14757265 + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "samples": 100, + "p50": 83202054, + "p95": 92954077, + "p99": null, + "maximum": 105376588 + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "samples": 100, + "p50": 922471, + "p95": 1367096, + "p99": null, + "maximum": 1679715 + }, + { + "scenario": "websocket", + "event_class": "new_transmission", + "timing": "receipt_after_ingest_return", + "samples": 4500, + "p50": 516706753, + "p95": 995604334, + "p99": 2178830922, + "maximum": 2227491145 + }, + { + "scenario": "handler-control", + "event_class": "200_valid_envelopes", + "timing": "service", + "samples": 200, + "p50": 1206984, + "p95": 2569860, + "p99": null, + "maximum": 5158955 + } + ], + "counts": { + "durable-ingest": { + "total": 12000, + "measured": 9000, + "successful": 12000, + "errors": 0, + "drops": 0 + }, + "mixed-http": { + "total": 4800, + "measured": 3600, + "successful": 4800, + "errors": 0, + "drops": 0 + }, + "sql-miss": { + "total": 200, + "measured": 200, + "successful": 200, + "errors": 0, + "drops": 0 + } + }, + "throughput_including_drain": { + "durable-ingest/new": 25.030350490997847, + "durable-ingest/additional": 15.024942265925347, + "durable-ingest/duplicate": 7.514980647741974, + "durable-ingest/late": 2.5055391980361836, + "mixed-http/nodes_50": 1.432598216162731, + "mixed-http/nodes_region": 1.2289340046172985, + "mixed-http/nodes_2000": 1.4321144857845218, + "mixed-http/channels": 1.4326212474181543, + "mixed-http/messages_0": 1.4326182308961137, + "mixed-http/messages_1000": 1.2290092649426563, + "mixed-http/observers": 2.051820437027926, + "mixed-http/metrics_dense_24h": 1.0262068579348287, + "mixed-http/metrics_sparse_7d": 1.0262154104401595, + "mixed-http/reach": 1.0262199749198988, + "mixed-http/rx_coverage": 1.0262118842560923, + "mixed-http/packets_memory": 1.0248997384007508, + "mixed-http/packet_detail_memory": 1.0250481196789774, + "mixed-http/packet_detail_sql": 1.0336772822007385, + "mixed-http/analytics_rf": 0.615728247327062, + "mixed-http/analytics_topology": 0.6156327644428657, + "mixed-http/analytics_channels": 0.4108799518218823, + "mixed-http/analytics_rf_filtered": 0.410953637961553, + "mixed-http/stats": 1.0273914408344735, + "mixed-http/packet_detail_pruned": 1.1083439301938458, + "sql-miss/channel_messages_sql": 12.286491539349301, + "sql-miss/observer_metrics_sql": 12.41244480822242 + }, + "scalars_ns": { + "startup/first_http_ns": 28351764144, + "startup/ready_ns": 28351766949, + "startup/full_ready_ns": 28369145905, + "startup-hot1/first_http_ns": 9665927094, + "startup-hot1/ready_ns": 9665929900, + "startup-hot1/full_ready_ns": 21399464749, + "retention/elapsed_ns": 11300768983, + "retention-empty/elapsed_ns": 274432 + }, + "resources": { + "sample_count": 179, + "window_start_ns": 5108619433168, + "window_end_ns": 5288619433168, + "sampled_start_ns": 5109336254148, + "sampled_end_ns": 5288024138160, + "sampled_span_ns": 178687884012, + "sampled_span_fraction": 0.9927104667333333, + "first_boundary_gap_ns": 716820980, + "last_boundary_gap_ns": 595295008, + "sampling_interval_ns": { + "median": 1003649878.5, + "minimum": 1003366556, + "maximum": 1013776747, + "runs": 178 + }, + "phases": [ + "mixed_with_retention" + ], + "cpu_delta_usec": 43925702, + "average_cores_over_sampled_span": 0.24582361721318563, + "per_device_io": { + "8:0": { + "wios": 222200, + "rios": 0, + "dios": 0, + "rbytes": 0, + "wbytes": 2485182464, + "dbytes": 0 + } + }, + "aggregate_accounted_io": { + "dbytes": 0, + "dios": 0, + "rbytes": 0, + "rios": 0, + "wbytes": 2485182464, + "wios": 222200 + }, + "observed_peaks": { + "cgroup_memory_bytes": 4257333248, + "rss_bytes": 3083268096, + "pss_bytes": 3080015872, + "storage_bytes": 1153572040, + "wal_bytes": 32197832 + }, + "memory_event_deltas": { + "oom_group_kill": 0, + "oom_kill": 0, + "oom": 0, + "high": 0, + "max": 0, + "low": 0 + }, + "minimum_free_bytes": 84293013504 + }, + "websocket": { + "full_stream_sentinels": 6000, + "full_stream_received": 6000, + "measured_sentinels": 4500, + "measured_missing": 0, + "full_stream_missing": 0, + "negative_measured_lags": 0, + "negative_full_stream_lags": 0, + "duplicate_measured_receipts": 0, + "duplicate_full_stream_receipts": 0 + }, + "retention": { + "retention": { + "elapsed_ns": 11300768983, + "transmissions_deleted": 16000, + "observations_deleted": 256000 + }, + "retention-empty": { + "elapsed_ns": 274432, + "transmissions_deleted": 0, + "observations_deleted": 0 + } + }, + "startup_counts": [ + { + "transmissions": 112000, + "observations": 1792000 + }, + { + "transmissions": 112000, + "observations": 1792000 + } + ], + "database_settings": { + "busy_timeout": "5000", + "cache_size": "-2000", + "cache_spill": "483", + "foreign_keys": "1", + "journal_mode": "wal", + "journal_size_limit": "-1", + "mmap_size": "0", + "page_size": "4096", + "sqlite_version": "3.53.4", + "synchronous": "2", + "wal_autocheckpoint": "1000" + }, + "durable_counts": { + "scheduled": 12000, + "completed": 12000, + "dropped_schedules": 0, + "write_errors": 0, + "actual_transmissions": 118000, + "actual_observations": 1802200 + }, + "full_leg_kernel_peak_bytes": 4496756736, + "full_leg_memory_events": { + "low": 0, + "high": 0, + "max": 13342, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "ingest_drain_seconds": 0, + "http_drain_seconds": 0 + }, + { + "pair": 4, + "backend": "postgres", + "metrics": [ + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "service", + "samples": 2700, + "p50": 1275829, + "p95": 2077817, + "p99": 4194225, + "maximum": 45456246 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "queue", + "samples": 2700, + "p50": 630862, + "p95": 1070487, + "p99": 1129961, + "maximum": 25854078 + }, + { + "scenario": "durable-ingest", + "event_class": "additional", + "timing": "end_to_end", + "samples": 2700, + "p50": 1990859, + "p95": 2855209, + "p99": 5064645, + "maximum": 45828210 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "service", + "samples": 1350, + "p50": 1190019, + "p95": 1743644, + "p99": 3460660, + "maximum": 12277467 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "queue", + "samples": 1350, + "p50": 605557, + "p95": 1064487, + "p99": 1113715, + "maximum": 1692727 + }, + { + "scenario": "durable-ingest", + "event_class": "duplicate", + "timing": "end_to_end", + "samples": 1350, + "p50": 1858390, + "p95": 2533888, + "p99": 4367166, + "maximum": 13188749 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "service", + "samples": 450, + "p50": 1476057, + "p95": 2095094, + "p99": null, + "maximum": 19711444 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "queue", + "samples": 450, + "p50": 638306, + "p95": 1065628, + "p99": null, + "maximum": 1145160 + }, + { + "scenario": "durable-ingest", + "event_class": "late", + "timing": "end_to_end", + "samples": 450, + "p50": 2155810, + "p95": 2911444, + "p99": null, + "maximum": 20432994 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "service", + "samples": 4500, + "p50": 1443354, + "p95": 2092845, + "p99": 3715538, + "maximum": 18054651 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "queue", + "samples": 4500, + "p50": 597807, + "p95": 1061088, + "p99": 1112348, + "maximum": 2599158 + }, + { + "scenario": "durable-ingest", + "event_class": "new", + "timing": "end_to_end", + "samples": 4500, + "p50": 2107279, + "p95": 2913401, + "p99": 4353968, + "maximum": 18735143 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "service", + "samples": 72, + "p50": 31969470, + "p95": 43788604, + "p99": null, + "maximum": 48529713 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "queue", + "samples": 72, + "p50": 490091, + "p95": 974119, + "p99": null, + "maximum": 1080089 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_channels", + "timing": "end_to_end", + "samples": 72, + "p50": 32738811, + "p95": 43871911, + "p99": null, + "maximum": 48918363 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "service", + "samples": 108, + "p50": 1683473, + "p95": 2501726, + "p99": null, + "maximum": 2808853 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "queue", + "samples": 108, + "p50": 619536, + "p95": 1054223, + "p99": null, + "maximum": 1069624 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf", + "timing": "end_to_end", + "samples": 108, + "p50": 2412098, + "p95": 3333970, + "p99": null, + "maximum": 3809920 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "service", + "samples": 72, + "p50": 2179547, + "p95": 511729119, + "p99": null, + "maximum": 835267710 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "queue", + "samples": 72, + "p50": 154397, + "p95": 1049266, + "p99": null, + "maximum": 1099474 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_rf_filtered", + "timing": "end_to_end", + "samples": 72, + "p50": 2466983, + "p95": 512663358, + "p99": null, + "maximum": 835374255 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "service", + "samples": 108, + "p50": 30975017, + "p95": 35376337, + "p99": null, + "maximum": 64003743 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "queue", + "samples": 108, + "p50": 722076, + "p95": 1025508, + "p99": null, + "maximum": 1077715 + }, + { + "scenario": "mixed-http", + "event_class": "analytics_topology", + "timing": "end_to_end", + "samples": 108, + "p50": 31641605, + "p95": 35683926, + "p99": null, + "maximum": 64235427 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "service", + "samples": 252, + "p50": 579416, + "p95": 907011, + "p99": null, + "maximum": 25756795 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "queue", + "samples": 252, + "p50": 582248, + "p95": 1032863, + "p99": null, + "maximum": 1121198 + }, + { + "scenario": "mixed-http", + "event_class": "channels", + "timing": "end_to_end", + "samples": 252, + "p50": 1190345, + "p95": 1703442, + "p99": null, + "maximum": 26303130 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "service", + "samples": 252, + "p50": 1307766, + "p95": 111802030, + "p99": null, + "maximum": 179298901 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "queue", + "samples": 252, + "p50": 480468, + "p95": 1052654, + "p99": null, + "maximum": 6815876 + }, + { + "scenario": "mixed-http", + "event_class": "messages_0", + "timing": "end_to_end", + "samples": 252, + "p50": 1896404, + "p95": 111921886, + "p99": null, + "maximum": 180190713 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "service", + "samples": 216, + "p50": 1337605, + "p95": 113737629, + "p99": null, + "maximum": 121673514 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "queue", + "samples": 216, + "p50": 534792, + "p95": 1032312, + "p99": null, + "maximum": 1092510 + }, + { + "scenario": "mixed-http", + "event_class": "messages_1000", + "timing": "end_to_end", + "samples": 216, + "p50": 1949635, + "p95": 114234778, + "p99": null, + "maximum": 121964847 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "service", + "samples": 180, + "p50": 3360008, + "p95": 4102282, + "p99": null, + "maximum": 5052961 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "queue", + "samples": 180, + "p50": 613651, + "p95": 1065748, + "p99": null, + "maximum": 1125381 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_dense_24h", + "timing": "end_to_end", + "samples": 180, + "p50": 4014418, + "p95": 4784876, + "p99": null, + "maximum": 5303719 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "service", + "samples": 180, + "p50": 1583383, + "p95": 1895839, + "p99": null, + "maximum": 2297272 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "queue", + "samples": 180, + "p50": 518970, + "p95": 1050387, + "p99": null, + "maximum": 1082642 + }, + { + "scenario": "mixed-http", + "event_class": "metrics_sparse_7d", + "timing": "end_to_end", + "samples": 180, + "p50": 2155434, + "p95": 2743285, + "p99": null, + "maximum": 3020775 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "service", + "samples": 252, + "p50": 68078276, + "p95": 97670536, + "p99": null, + "maximum": 200606713 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "queue", + "samples": 252, + "p50": 497148, + "p95": 1041072, + "p99": null, + "maximum": 4626456 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_2000", + "timing": "end_to_end", + "samples": 252, + "p50": 68621116, + "p95": 101208898, + "p99": null, + "maximum": 203732843 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "service", + "samples": 252, + "p50": 5340250, + "p95": 11325856, + "p99": null, + "maximum": 65736611 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "queue", + "samples": 252, + "p50": 332146, + "p95": 1008559, + "p99": null, + "maximum": 4695730 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_50", + "timing": "end_to_end", + "samples": 252, + "p50": 5764592, + "p95": 11482394, + "p99": null, + "maximum": 66679352 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "service", + "samples": 216, + "p50": 7864980, + "p95": 16872037, + "p99": null, + "maximum": 51642573 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "queue", + "samples": 216, + "p50": 527650, + "p95": 1028520, + "p99": null, + "maximum": 4609547 + }, + { + "scenario": "mixed-http", + "event_class": "nodes_region", + "timing": "end_to_end", + "samples": 216, + "p50": 8414716, + "p95": 17667061, + "p99": null, + "maximum": 52653735 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "service", + "samples": 360, + "p50": 2701803, + "p95": 3927441, + "p99": null, + "maximum": 96730297 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "queue", + "samples": 360, + "p50": 592031, + "p95": 1045481, + "p99": null, + "maximum": 1127295 + }, + { + "scenario": "mixed-http", + "event_class": "observers", + "timing": "end_to_end", + "samples": 360, + "p50": 3327265, + "p95": 4522869, + "p99": null, + "maximum": 97681422 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "service", + "samples": 180, + "p50": 3372687, + "p95": 3906407, + "p99": null, + "maximum": 13595745 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "queue", + "samples": 180, + "p50": 565628, + "p95": 1045654, + "p99": null, + "maximum": 1154346 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 3968485, + "p95": 4818157, + "p99": null, + "maximum": 14450420 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "service", + "samples": 45, + "p50": 590007, + "p95": 864808, + "p99": null, + "maximum": 1445607 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "queue", + "samples": 45, + "p50": 695785, + "p95": 1043670, + "p99": null, + "maximum": 1093541 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_pruned", + "timing": "end_to_end", + "samples": 45, + "p50": 1250025, + "p95": 1703208, + "p99": null, + "maximum": 2166736 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "service", + "samples": 135, + "p50": 1435772, + "p95": 2047726, + "p99": null, + "maximum": 4782458 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "queue", + "samples": 135, + "p50": 605695, + "p95": 1062000, + "p99": null, + "maximum": 1121702 + }, + { + "scenario": "mixed-http", + "event_class": "packet_detail_sql", + "timing": "end_to_end", + "samples": 135, + "p50": 2095063, + "p95": 2783660, + "p99": null, + "maximum": 4796560 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "service", + "samples": 180, + "p50": 33223501, + "p95": 36453181, + "p99": null, + "maximum": 97637987 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "queue", + "samples": 180, + "p50": 585593, + "p95": 1048907, + "p99": null, + "maximum": 1138177 + }, + { + "scenario": "mixed-http", + "event_class": "packets_memory", + "timing": "end_to_end", + "samples": 180, + "p50": 33851607, + "p95": 37115620, + "p99": null, + "maximum": 98258023 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "service", + "samples": 180, + "p50": 710671, + "p95": 886601, + "p99": null, + "maximum": 1147112 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "queue", + "samples": 180, + "p50": 570831, + "p95": 1079694, + "p99": null, + "maximum": 1156302 + }, + { + "scenario": "mixed-http", + "event_class": "reach", + "timing": "end_to_end", + "samples": 180, + "p50": 1291276, + "p95": 1802579, + "p99": null, + "maximum": 2035177 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "service", + "samples": 180, + "p50": 1506707, + "p95": 1779121, + "p99": null, + "maximum": 2103613 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "queue", + "samples": 180, + "p50": 607992, + "p95": 1035952, + "p99": null, + "maximum": 1088846 + }, + { + "scenario": "mixed-http", + "event_class": "rx_coverage", + "timing": "end_to_end", + "samples": 180, + "p50": 2090949, + "p95": 2568425, + "p99": null, + "maximum": 2910696 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "service", + "samples": 180, + "p50": 373442, + "p95": 4452893, + "p99": null, + "maximum": 32581727 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "queue", + "samples": 180, + "p50": 561811, + "p95": 1041864, + "p99": null, + "maximum": 1549869 + }, + { + "scenario": "mixed-http", + "event_class": "stats", + "timing": "end_to_end", + "samples": 180, + "p50": 960577, + "p95": 4841757, + "p99": null, + "maximum": 33544155 + }, + { + "scenario": "sql-miss", + "event_class": "channel_messages_sql", + "timing": "service", + "samples": 100, + "p50": 104922633, + "p95": 122645583, + "p99": null, + "maximum": 193861220 + }, + { + "scenario": "sql-miss", + "event_class": "observer_metrics_sql", + "timing": "service", + "samples": 100, + "p50": 793787, + "p95": 1401317, + "p99": null, + "maximum": 4381694 + }, + { + "scenario": "websocket", + "event_class": "new_transmission", + "timing": "receipt_after_ingest_return", + "samples": 4500, + "p50": 503054792, + "p95": 945775962, + "p99": 985737995, + "maximum": 1210957684 + }, + { + "scenario": "handler-control", + "event_class": "200_valid_envelopes", + "timing": "service", + "samples": 200, + "p50": 1971488, + "p95": 2691655, + "p99": null, + "maximum": 5279854 + } + ], + "counts": { + "durable-ingest": { + "total": 12000, + "measured": 9000, + "successful": 12000, + "errors": 0, + "drops": 0 + }, + "mixed-http": { + "total": 4800, + "measured": 3600, + "successful": 4800, + "errors": 0, + "drops": 0 + }, + "sql-miss": { + "total": 200, + "measured": 200, + "successful": 200, + "errors": 0, + "drops": 0 + } + }, + "throughput_including_drain": { + "durable-ingest/new": 25.030340899004116, + "durable-ingest/additional": 15.024845084666719, + "durable-ingest/duplicate": 7.514939544673058, + "durable-ingest/late": 2.5055433088099575, + "mixed-http/nodes_50": 1.4325822781627682, + "mixed-http/nodes_region": 1.2289680997851973, + "mixed-http/nodes_2000": 1.4321116300106078, + "mixed-http/channels": 1.4326239215259087, + "mixed-http/messages_0": 1.4316812043604843, + "mixed-http/messages_1000": 1.2290065601065858, + "mixed-http/observers": 2.0518273505773417, + "mixed-http/metrics_dense_24h": 1.0262026278288632, + "mixed-http/metrics_sparse_7d": 1.0262111482140577, + "mixed-http/reach": 1.0262209119701449, + "mixed-http/rx_coverage": 1.0262145751017369, + "mixed-http/packets_memory": 1.0248531419798628, + "mixed-http/packet_detail_memory": 1.0250374446888952, + "mixed-http/packet_detail_sql": 1.033674054330128, + "mixed-http/analytics_rf": 0.6157288130993869, + "mixed-http/analytics_topology": 0.6156261041144447, + "mixed-http/analytics_channels": 0.41088932704955566, + "mixed-http/analytics_rf_filtered": 0.4109534409761724, + "mixed-http/stats": 1.0273884582523665, + "mixed-http/packet_detail_pruned": 1.1083447442042853, + "sql-miss/channel_messages_sql": 9.125080188489985, + "sql-miss/observer_metrics_sql": 9.289008067919283 + }, + "scalars_ns": { + "startup/first_http_ns": 27647495666, + "startup/ready_ns": 27647498311, + "startup/full_ready_ns": 27752671120, + "startup-hot1/first_http_ns": 10885141683, + "startup-hot1/ready_ns": 10885145460, + "startup-hot1/full_ready_ns": 20913380649, + "retention/elapsed_ns": 973707150, + "retention-empty/elapsed_ns": 1941244, + "migration/elapsed_ns": 147494851772, + "migration/migration_tool_ns": 89485454621, + "migration/role_grants_ns": 688529717, + "migration/harness_validation_ns": 57320868406 + }, + "resources": { + "sample_count": 171, + "window_start_ns": 5667840429199, + "window_end_ns": 5847840429199, + "sampled_start_ns": 5668175875073, + "sampled_end_ns": 5847745622466, + "sampled_span_ns": 179569747393, + "sampled_span_fraction": 0.9976097077388889, + "first_boundary_gap_ns": 335445874, + "last_boundary_gap_ns": 94806733, + "sampling_interval_ns": { + "median": 1054325627.0, + "minimum": 1051044950, + "maximum": 1119388778, + "runs": 170 + }, + "phases": [ + "mixed_with_retention" + ], + "cpu_delta_usec": 58383925, + "average_cores_over_sampled_span": 0.3251323001096783, + "per_device_io": { + "8:0": { + "wios": 18146, + "rios": 851, + "dios": 1, + "rbytes": 8294400, + "wbytes": 419500032, + "dbytes": 4096 + } + }, + "aggregate_accounted_io": { + "dbytes": 4096, + "dios": 1, + "rbytes": 8294400, + "rios": 851, + "wbytes": 419500032, + "wios": 18146 + }, + "observed_peaks": { + "cgroup_memory_bytes": 5898973184, + "rss_bytes": 5796024320, + "pss_bytes": 3377251328, + "storage_bytes": 2440056636, + "wal_bytes": 1073741824 + }, + "memory_event_deltas": { + "oom_group_kill": 0, + "oom_kill": 0, + "oom": 0, + "high": 0, + "max": 0, + "low": 0 + }, + "minimum_free_bytes": 79618281472 + }, + "websocket": { + "full_stream_sentinels": 6000, + "full_stream_received": 6000, + "measured_sentinels": 4500, + "measured_missing": 0, + "full_stream_missing": 0, + "negative_measured_lags": 0, + "negative_full_stream_lags": 0, + "duplicate_measured_receipts": 0, + "duplicate_full_stream_receipts": 0 + }, + "retention": { + "retention": { + "elapsed_ns": 973707150, + "transmissions_deleted": 16000, + "observations_deleted": 256000 + }, + "retention-empty": { + "elapsed_ns": 1941244, + "transmissions_deleted": 0, + "observations_deleted": 0 + } + }, + "startup_counts": [ + { + "transmissions": 112000, + "observations": 1792000 + }, + { + "transmissions": 112000, + "observations": 1792000 + } + ], + "database_settings": { + "autovacuum": "on", + "checkpoint_timeout": "5min", + "effective_cache_size": "4GB", + "fsync": "on", + "full_page_writes": "on", + "jit": "on", + "maintenance_work_mem": "64MB", + "max_connections": "32", + "max_parallel_workers_per_gather": "2", + "max_wal_size": "1GB", + "server_version": "18.6 (Ubuntu 18.6-1.pgdg24.04+2)", + "shared_buffers": "256MB", + "synchronous_commit": "on", + "work_mem": "4MB" + }, + "durable_counts": { + "scheduled": 12000, + "completed": 12000, + "dropped_schedules": 0, + "write_errors": 0, + "actual_transmissions": 118000, + "actual_observations": 1802200 + }, + "full_leg_kernel_peak_bytes": 6442455040, + "full_leg_memory_events": { + "low": 0, + "high": 0, + "max": 16106, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "ingest_drain_seconds": 0, + "http_drain_seconds": 0 + } + ], + "limits": [ + "Five paired runs are the independent repetitions; request records are never pooled across runs.", + "Absolute entries are medians of per-run quantiles, with observed min/max, not confidence intervals.", + "p99 is omitted unless every contributing run has at least 1000 successful samples.", + "Ingest service excludes replay queue wait; HTTP service includes response reading and validation. Queue and schedule-to-complete distributions are separate.", + "Resource deltas use the first and last samples strictly inside the measured window; no boundary interpolation. Coverage and sampling gaps are reported.", + "Memory/RSS/PSS/storage/WAL are observed sampled peaks, not proof against shorter spikes. Storage includes selected database files and WAL.", + "I/O is per-device cgroup accounting. The aggregate can count layered devices twice and is not physical storage bytes.", + "WebSocket lag is signed receipt minus ingest return; negative lags remain. Nonpositive baseline denominators have no ordinary speed ratio.", + "Accounts are disabled by the reviewed harness. The OS cache is warm; startup uses a new process and approximately 100ms controller probes.", + "No capacity, OS-cold, 65-minute soak or engine-wide causal claim follows from this report.", + "Validation uses public digests and runtime evidence; it does not reopen private databases or payload files." + ], + "checks": { + "pairs": 5, + "legs": 10, + "production_hashes_recomputed_from_git": true, + "harness_hash_recomputed_from_git": true, + "full_retained_loads": true, + "replay_identity_parity": true, + "all_18_table_digests_pair_equal": true, + "isolated_sql_body_parity": true, + "complete_ingest_events": 120000, + "complete_http_requests": 48000, + "errors": 0, + "drops": 0 + }, + "http_body_comparison": [ + { + "pair": 0, + "by_class": { + "nodes_50": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "nodes_2000": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "nodes_region": { + "requests": 288, + "equal": 0, + "different": 288, + "class_changed": 0 + }, + "channels": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "messages_0": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "messages_1000": { + "requests": 288, + "equal": 31, + "different": 257, + "class_changed": 0 + }, + "observers": { + "requests": 480, + "equal": 0, + "different": 480, + "class_changed": 0 + }, + "metrics_dense_24h": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "metrics_sparse_7d": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "reach": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "rx_coverage": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "packets_memory": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "packet_detail_memory": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "packet_detail_sql": { + "requests": 195, + "equal": 195, + "different": 0, + "class_changed": 0 + }, + "analytics_rf": { + "requests": 144, + "equal": 144, + "different": 0, + "class_changed": 0 + }, + "analytics_topology": { + "requests": 144, + "equal": 0, + "different": 144, + "class_changed": 0 + }, + "analytics_channels": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "analytics_rf_filtered": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "stats": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "packet_detail_pruned": { + "requests": 45, + "equal": 45, + "different": 0, + "class_changed": 0 + } + } + }, + { + "pair": 1, + "by_class": { + "nodes_50": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "nodes_2000": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "nodes_region": { + "requests": 288, + "equal": 0, + "different": 288, + "class_changed": 0 + }, + "channels": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "messages_0": { + "requests": 336, + "equal": 291, + "different": 45, + "class_changed": 0 + }, + "messages_1000": { + "requests": 288, + "equal": 246, + "different": 42, + "class_changed": 0 + }, + "observers": { + "requests": 480, + "equal": 0, + "different": 480, + "class_changed": 0 + }, + "metrics_dense_24h": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "metrics_sparse_7d": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "reach": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "rx_coverage": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "packets_memory": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "packet_detail_memory": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "packet_detail_sql": { + "requests": 195, + "equal": 195, + "different": 0, + "class_changed": 0 + }, + "analytics_rf": { + "requests": 144, + "equal": 144, + "different": 0, + "class_changed": 0 + }, + "analytics_topology": { + "requests": 144, + "equal": 0, + "different": 144, + "class_changed": 0 + }, + "analytics_channels": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "analytics_rf_filtered": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "stats": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "packet_detail_pruned": { + "requests": 45, + "equal": 45, + "different": 0, + "class_changed": 0 + } + } + }, + { + "pair": 2, + "by_class": { + "nodes_50": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "nodes_2000": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "nodes_region": { + "requests": 288, + "equal": 0, + "different": 288, + "class_changed": 0 + }, + "channels": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "messages_0": { + "requests": 336, + "equal": 285, + "different": 51, + "class_changed": 0 + }, + "messages_1000": { + "requests": 288, + "equal": 233, + "different": 55, + "class_changed": 0 + }, + "observers": { + "requests": 480, + "equal": 0, + "different": 480, + "class_changed": 0 + }, + "metrics_dense_24h": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "metrics_sparse_7d": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "reach": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "rx_coverage": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "packets_memory": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "packet_detail_memory": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "packet_detail_sql": { + "requests": 195, + "equal": 195, + "different": 0, + "class_changed": 0 + }, + "analytics_rf": { + "requests": 144, + "equal": 144, + "different": 0, + "class_changed": 0 + }, + "analytics_topology": { + "requests": 144, + "equal": 0, + "different": 144, + "class_changed": 0 + }, + "analytics_channels": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "analytics_rf_filtered": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "stats": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "packet_detail_pruned": { + "requests": 45, + "equal": 45, + "different": 0, + "class_changed": 0 + } + } + }, + { + "pair": 3, + "by_class": { + "nodes_50": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "nodes_2000": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "nodes_region": { + "requests": 288, + "equal": 0, + "different": 288, + "class_changed": 0 + }, + "channels": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "messages_0": { + "requests": 336, + "equal": 306, + "different": 30, + "class_changed": 0 + }, + "messages_1000": { + "requests": 288, + "equal": 246, + "different": 42, + "class_changed": 0 + }, + "observers": { + "requests": 480, + "equal": 0, + "different": 480, + "class_changed": 0 + }, + "metrics_dense_24h": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "metrics_sparse_7d": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "reach": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "rx_coverage": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "packets_memory": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "packet_detail_memory": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "packet_detail_sql": { + "requests": 195, + "equal": 195, + "different": 0, + "class_changed": 0 + }, + "analytics_rf": { + "requests": 144, + "equal": 144, + "different": 0, + "class_changed": 0 + }, + "analytics_topology": { + "requests": 144, + "equal": 0, + "different": 144, + "class_changed": 0 + }, + "analytics_channels": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "analytics_rf_filtered": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "stats": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "packet_detail_pruned": { + "requests": 45, + "equal": 45, + "different": 0, + "class_changed": 0 + } + } + }, + { + "pair": 4, + "by_class": { + "nodes_50": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "nodes_2000": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "nodes_region": { + "requests": 288, + "equal": 0, + "different": 288, + "class_changed": 0 + }, + "channels": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "messages_0": { + "requests": 336, + "equal": 0, + "different": 336, + "class_changed": 0 + }, + "messages_1000": { + "requests": 288, + "equal": 1, + "different": 287, + "class_changed": 0 + }, + "observers": { + "requests": 480, + "equal": 0, + "different": 480, + "class_changed": 0 + }, + "metrics_dense_24h": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "metrics_sparse_7d": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "reach": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "rx_coverage": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "packets_memory": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "packet_detail_memory": { + "requests": 240, + "equal": 240, + "different": 0, + "class_changed": 0 + }, + "packet_detail_sql": { + "requests": 195, + "equal": 195, + "different": 0, + "class_changed": 0 + }, + "analytics_rf": { + "requests": 144, + "equal": 144, + "different": 0, + "class_changed": 0 + }, + "analytics_topology": { + "requests": 144, + "equal": 0, + "different": 144, + "class_changed": 0 + }, + "analytics_channels": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "analytics_rf_filtered": { + "requests": 96, + "equal": 0, + "different": 96, + "class_changed": 0 + }, + "stats": { + "requests": 240, + "equal": 0, + "different": 240, + "class_changed": 0 + }, + "packet_detail_pruned": { + "requests": 45, + "equal": 45, + "different": 0, + "class_changed": 0 + } + } + } + ], + "manifest": { + "baseline_sha": "3e21b179ecccae005aa5f6f45bc095353c94b343", + "binary_sha256": { + "migrate": "646d8cee19bbddfa3b64d7ef141be14a44fad0f0b9454c4476f82741467dde67", + "postgres-ingestor.test": "3f2387a9249f4f52610a2d64cc7726144259a6079e5b69cbd858e5e7710cc50f", + "postgres-server": "6e81b75e618b9e7014d7827a0ec6e37d5caa692820d57b7bb2463b99eb09dfb5", + "postgres-server.test": "568e4b5281ade1aaacbe1a789c5ab9e92d10e8edaa5d407fa3a4c688500d3836", + "sqlite-ingestor.test": "d46994f03eb6ec013c364a2e895dd32fb0faf5153c61b8344ecd1d4214e5e7dd", + "sqlite-server": "80edb7b741613974dd88dbec21f756fabf63cf83d5b9bf365b2eb306c0d84f30", + "sqlite-server.test": "c9b9b4171173193f3007a0a4badc6d4a8ef030080dbcb0b1b6d2c4cea36866da" + }, + "cache_policy": { + "mixed": "warmup then application-warm measurement", + "sql": "channel application cache explicitly cleared; OS cache warm", + "startup": "new application process after full validation; OS cache warm" + }, + "candidate_sha": "c03a2585eef3819cf06640911b2f7308d99339db", + "canonical_file_sha256": "30ab98f2725a04508033948c1a2f8e0f482c3ec46215524445b0e177e95c46e2", + "comparison_eligible": true, + "compiler": "cc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0", + "completed_pairs": 5, + "corpus": "B", + "epoch": 1791577272, + "extended_120s_600s_profile": "not run unless explicitly requested by duration flags", + "firmware_reference": "a366955cb2f67b8e6842d4f00d2b6a554dddd88a", + "go_version": "go version go1.27.2 linux/amd64", + "harness_sha256": "7c234ba61c93351688d7068c9d8405f7d2906558d797854738eb0ff059f35d84", + "host": { + "architecture": "x86_64", + "cpus": 4, + "kernel": "6.17.0-1022-azure", + "ram_bytes": 16766414848, + "system": "Linux" + }, + "measured_rows": 365, + "order": [ + [ + 0, + "sqlite" + ], + [ + 0, + "postgres" + ], + [ + 1, + "postgres" + ], + [ + 1, + "sqlite" + ], + [ + 2, + "sqlite" + ], + [ + 2, + "postgres" + ], + [ + 3, + "postgres" + ], + [ + 3, + "sqlite" + ], + [ + 4, + "sqlite" + ], + [ + 4, + "postgres" + ] + ], + "os_cold": "not measured", + "postgres_production_hash_after": "74bcf9bd678e4b320e3da198c02a4f2696453360c81dfa1fa5483e3e9c100ada", + "postgres_production_hash_before": "74bcf9bd678e4b320e3da198c02a4f2696453360c81dfa1fa5483e3e9c100ada", + "postgres_version": "postgres (PostgreSQL) 18.6 (Ubuntu 18.6-1.pgdg24.04+2)", + "resource_budget": { + "allowed_cpus": [ + 0, + 1, + 2, + 3 + ], + "cpu_limit": 3.0, + "enforced": true, + "inactive_charge_reset_bytes": 5177344, + "memory_limit_bytes": 6442450944, + "memory_peak_reset": true, + "resource_counters": [ + "cgroup.procs", + "memory.current", + "memory.events", + "cpu.stat", + "io.stat" + ] + }, + "seed": 20261008, + "shape": { + "days": 8, + "nodes": 2000, + "observations": 2048000, + "observers": 128, + "transmissions": 128000 + }, + "sqlite_production_hash_after": "8b8fd97e7e8b834dc7829e7615f2c192faef9def8e20987874fa06606e182f7f", + "sqlite_production_hash_before": "8b8fd97e7e8b834dc7829e7615f2c192faef9def8e20987874fa06606e182f7f", + "status": "completed", + "table_manifest": [ + { + "columns": [ + "public_key", + "name", + "role", + "lat", + "lon", + "last_seen", + "first_seen", + "advert_count", + "battery_mv", + "temperature_c", + "foreign_advert", + "default_scope", + "configured_scope", + "configured_scope_at", + "multibyte_sup", + "multibyte_evidence" + ], + "name": "nodes" + }, + { + "columns": [ + "public_key", + "name", + "role", + "lat", + "lon", + "last_seen", + "first_seen", + "advert_count", + "battery_mv", + "temperature_c", + "foreign_advert", + "default_scope", + "configured_scope", + "configured_scope_at", + "multibyte_sup", + "multibyte_evidence" + ], + "name": "inactive_nodes" + }, + { + "columns": [ + "rowid", + "id", + "name", + "iata", + "last_seen", + "first_seen", + "packet_count", + "model", + "firmware", + "client_version", + "radio", + "battery_mv", + "uptime_secs", + "noise_floor", + "inactive", + "last_packet_at", + "clock_skew_seconds", + "clock_skew_count_24h", + "clock_last_naive_at", + "can_relay", + "can_relay_seen" + ], + "name": "observers" + }, + { + "columns": [ + "id", + "raw_hex", + "hash", + "first_seen", + "route_type", + "payload_type", + "payload_version", + "decoded_json", + "from_pubkey", + "last_seen", + "created_at", + "code1", + "code2", + "channel_hash", + "scope_name" + ], + "name": "transmissions" + }, + { + "columns": [ + "id", + "transmission_id", + "observer_idx", + "direction", + "snr", + "rssi", + "score", + "path_json", + "timestamp", + "resolved_path", + "raw_hex" + ], + "name": "observations" + }, + { + "columns": [ + "observer_id", + "timestamp", + "noise_floor", + "tx_air_secs", + "rx_air_secs", + "recv_errors", + "battery_mv", + "packets_sent", + "packets_recv" + ], + "name": "observer_metrics" + }, + { + "columns": [ + "node_a", + "node_b", + "count", + "last_seen" + ], + "name": "neighbor_edges" + }, + { + "columns": [ + "id", + "hash", + "raw_hex", + "reason", + "observer_id", + "observer_name", + "node_pubkey", + "node_name", + "dropped_at" + ], + "name": "dropped_packets" + }, + { + "columns": [ + "id", + "rx_pubkey", + "heard_key", + "heard_keylen", + "rssi", + "snr", + "lat", + "lon", + "pos_acc_m", + "rx_at", + "ingested_at", + "src" + ], + "name": "client_receptions" + }, + { + "columns": [ + "pubkey", + "name", + "last_seen" + ], + "name": "client_observers" + }, + { + "columns": [ + "id", + "rx_pubkey", + "rx_at", + "ingested_at", + "pkt_hash", + "route_type", + "payload_type", + "code1", + "code2", + "scope_name", + "hash_size", + "hop_count", + "path_json", + "forwarder", + "snr", + "rssi", + "lat", + "lon", + "pos_acc_m" + ], + "name": "client_rx_observations" + }, + { + "columns": [ + "id", + "rx_pubkey", + "sampled_at", + "ingested_at", + "lat", + "lon", + "pos_acc_m", + "stationary", + "uptime_secs", + "battery_mv", + "queue_len", + "errors", + "noise_floor", + "last_rssi", + "last_snr", + "tx_air_secs", + "rx_air_secs", + "recv", + "sent", + "flood_rx", + "direct_rx", + "flood_tx", + "direct_tx", + "recv_errors" + ], + "name": "client_rf_samples" + }, + { + "columns": [ + "id", + "target", + "rx_pubkey", + "observed_at", + "ingested_at", + "regions_csv", + "truncated", + "lat", + "lon", + "pos_acc_m", + "repeater_clock" + ], + "name": "node_declared_regions" + }, + { + "columns": [ + "id", + "since_unix", + "unique_matches", + "explicit_over_derived", + "ambiguous", + "none_matches", + "updated_unix" + ], + "name": "scope_match_totals" + }, + { + "columns": [ + "name" + ], + "name": "_migrations" + }, + { + "columns": [ + "name", + "status", + "started_at", + "ended_at", + "error" + ], + "name": "_async_migrations" + }, + { + "columns": [ + "id", + "tx_id", + "bit" + ], + "name": "advert_route_evidence" + }, + { + "columns": [ + "id", + "tx_cursor", + "obs_cursor" + ], + "name": "advert_evidence_backfill" + } + ], + "workload": { + "http_concurrency": 64, + "http_rate": 20, + "ingest_rate": 50, + "ingestor_memory_mib": 512, + "measured_seconds": 180, + "packet_store_mib": 2048, + "queue_limit": 1024, + "retention_hours": 168, + "server_memory_mib": 3072, + "sql_reader_pool": 4, + "sql_writer_pool": 1, + "warmup_seconds": 60 + } + }, + "run_id": 37985643046, + "job_id": 114006882924, + "analysis_helper_sha256": "edc15e8eb3ed419e46fa01635510877a07a5b730e084762d2fe12af91ed8465c", + "raw_file_sha256": { + "corpus/corpus.json": "99cb2583f49eec5a598d1091cf0dcdd914446dfda46ddbb0c25e0f0086a90473", + "corpus/validation.json": "982c43fdfa3c99b6cdfe82940d86c95fa011a97253864efc927547d74c21dcc1", + "manifest.json": "b7404cd41e2fc7d9b6d054ec0fc791b5daa14775e9d06a38e45fdbf31c5549f4", + "pair-00/corpus-offset.json": "ae02f6e7f13c8294050860ff815a0c86da81766076559dadafe4f1d27e87e824", + "pair-00/events.json": "e5366abc060a7a482f7621b9b2e02c50fce40fecec5885536ce48ca689ebe0a0", + "pair-00/postgres/client-validation.json": "7512ff388ff6bfefab44334d67c479f169e2cee95736145d3f50a46f8d397f64", + "pair-00/postgres/database-settings.json": "999ed1c72e72ba5c038102ccb63b21014bb009c78c16cdf7b4b036ab34a09664", + "pair-00/postgres/handler.json": "b5e5f1a57bdab646401f26233438ed82620efc1e87f3c555aca9b75c07738c6e", + "pair-00/postgres/http-workload.json": "df60ae4fb88b12df15e852df64824aff791bc6b3405051895a57e900445ebbbf", + "pair-00/postgres/ingest-validation.json": "778e8698ce0a6a5e35115fe04be83e5f2d43feec741ce00dfc65ad2163b186fe", + "pair-00/postgres/ingest.jsonl": "f810161f5a3dab854a6adbdc821ae6d0bd9a1e9fa6525545f6c3cae2cc9f8242", + "pair-00/postgres/migration.json": "459423383c88da65af431c7998783c3af797f8f9ada3a6b4249cee15ac483601", + "pair-00/postgres/plans/advert-preservation.json": "372e245c87d0135fe52f559b9d7b0439cd0ee1bd2f4a954de23097c42e521ac2", + "pair-00/postgres/plans/retention-empty.json": "7412075f130422d9403e842cf08ccb71a25bc5d1fb507f0aa392891870f042ee", + "pair-00/postgres/processes.csv": "215337dc50e3aa8416d591e91f45510e40786dd58878191507524f592e678316", + "pair-00/postgres/requests.jsonl": "5b402922e164e51120ae2ff81703082012e7b4f0b8c387a7b0807c6a6a87808f", + "pair-00/postgres/resource-final.json": "b44406984880fd0a46ea8356dc9576efdd47f0c5d918951380cf87a080c9a8ed", + "pair-00/postgres/resources.csv": "4ede9d032a0b7b6cd18958c906d618a2c197a701ae5d107c43daa4b93e4f5055", + "pair-00/postgres/retention-empty.json": "3e1bab4285abffffcec28af2236591281a29ba0e527081a4d99a71a24b111650", + "pair-00/postgres/retention.json": "131eee10d5bfba57e84a067a17f85f642f5460363bf96df54960dba251e39863", + "pair-00/postgres/sql-queries.jsonl": "5dbf437ab1a9183d4ef70bbed26d05887cb6081a5db96211dda0c2ce115703c3", + "pair-00/postgres/startup-hot1.json": "d8fbdaf8125c841bd158eeb862e523a30b24a2f7a76de934ed40ff3e293ea121", + "pair-00/postgres/startup.json": "caa1e68dbf7485dd52bd18fb18d00acc123f907a865be564536e63710f225d6c", + "pair-00/postgres/validation-after.json": "4e23bff7b3d04a44bed3f1989ff8be8152a0201931af0bfc062dd1e68677eba8", + "pair-00/postgres/validation-before.json": "e6f8053a0e6f0efd8a201a530501877cc6f5e6b4a4cf0ac8ed1653bf4bc6754c", + "pair-00/postgres/visibility.json": "ccb73e5a797f60e91de7d8e6ba0ca76f8a8920200a6e52f32e25b2d5773b8b75", + "pair-00/postgres/websocket.jsonl": "dea51113de1355e850718800c40d8b93a9ce5e610536504aeeaa9e3643226b3a", + "pair-00/sqlite/client-validation.json": "0a09c0cc54f2e89daab23a80bb6b9219049848b662cc02cb2fcd45fc6613f34a", + "pair-00/sqlite/database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-00/sqlite/handler.json": "83882eccf4253ae57841185073a7ac1a7f51d8767e4ba464c4125579e0b5de36", + "pair-00/sqlite/http-workload.json": "df60ae4fb88b12df15e852df64824aff791bc6b3405051895a57e900445ebbbf", + "pair-00/sqlite/ingest-validation.json": "8437e3e393ac27bdfad88c2a767dd5b5bf2a38df92312fe98d3f7fa12c425670", + "pair-00/sqlite/ingest.jsonl": "899909b0d81172bb3079f3dd02c607fe0b8701de815f336e08d98fd984147dbd", + "pair-00/sqlite/plans/advert-preservation.json": "6f6b6d6e558851be2940d740effcff5a35830ea2eb5d3bfb348cf4830c884f87", + "pair-00/sqlite/plans/retention-empty.json": "afabe6af84b555d982616cf86a98f913e588e38e8e95c99bb6debbc1d43aae5a", + "pair-00/sqlite/processes.csv": "bd3d492df397dcd7661d86f07050b955e5f8e882451cfe1d66ed7988f50a3bb6", + "pair-00/sqlite/requests.jsonl": "799aadc7bdd0dfbae4735d822dfa21778c2e731ae1824c1fc392c3b9b2cc6022", + "pair-00/sqlite/resource-final.json": "840d8c64ea6c23131a517183f13139ae1503778ec72fa6bff47cdfb5d0cb2937", + "pair-00/sqlite/resources.csv": "1ad895ce5baee1af7785dc9e504d4437403ade59e5aa6faa81da1a038a8bbc93", + "pair-00/sqlite/retention-empty.json": "3f3a0015e80d3cec0b4c70255d497478e8a8670ae92dec7c7a3a17f38f24d164", + "pair-00/sqlite/retention.json": "98d9f0d60303d688991c4f590355897774d1094835862b9ba8459943139bf472", + "pair-00/sqlite/sql-queries.jsonl": "f2c3ead8c8b9b15b57e6bc9912dd96fa5330cf00298d532ac45df0a3f42fc16f", + "pair-00/sqlite/startup-hot1.json": "a01e43a567b6692c886cb55629370c11bb2212d96afd7187c3f22054bf6ff525", + "pair-00/sqlite/startup.json": "2388b1304cfd3586f4142c98e68c6273b785e742105dabaf3988beb4823d6b12", + "pair-00/sqlite/validation-after.json": "4e23bff7b3d04a44bed3f1989ff8be8152a0201931af0bfc062dd1e68677eba8", + "pair-00/sqlite/validation-before.json": "e6f8053a0e6f0efd8a201a530501877cc6f5e6b4a4cf0ac8ed1653bf4bc6754c", + "pair-00/sqlite/visibility.json": "0bbf0a9aeafb8da9a176dca870aa26b3ccc139634c8fdb42ef1373253af73d68", + "pair-00/sqlite/websocket.jsonl": "5823611d6e3aba964176c15b805cbb9750a9d32b9b0d19933e3fb7e62b8b6eaa", + "pair-01/corpus-offset.json": "12158a10e58c978253c5e05d15ec088ebcda1809476f28f646e4144f61b519ba", + "pair-01/events.json": "5c028b88f59de3037503da4cea91f9cbbcb425760a8cbfc92f2a60e53cdeab8e", + "pair-01/postgres/client-validation.json": "f176febf0ca3833427622275f9e8a114b54c0d7cd6179d2a5b0c5c7c4cd82ff1", + "pair-01/postgres/database-settings.json": "999ed1c72e72ba5c038102ccb63b21014bb009c78c16cdf7b4b036ab34a09664", + "pair-01/postgres/handler.json": "ad656602af944ac117c161fcb2021a319c9cec8b4ec6c2114479e1b86255403d", + "pair-01/postgres/http-workload.json": "6931a0c5bb191d0c60a03e11f04b55fa3cb786e7a94aaef9602b80477aa6b596", + "pair-01/postgres/ingest-validation.json": "29eea1f9d36483386b55fd7518b15d4e1003b1fd9a40a28d0d3778224c5f7626", + "pair-01/postgres/ingest.jsonl": "61636b5a47df447e6911a6f525b4376a8f25db9ed2d8596145094eb723d5ff15", + "pair-01/postgres/migration.json": "ff68a60b277d7548bd760ae55e9a131b01f727d6203b912f817cc82130e793cb", + "pair-01/postgres/plans/advert-preservation.json": "d3d60e731a81de3e85af1ae6435f2fed560905da428de1202cb6e2f3af6d3899", + "pair-01/postgres/plans/retention-empty.json": "3419c3e403b740a639593b9b9d092217440080059e23f4d4b3ab850d847b4274", + "pair-01/postgres/processes.csv": "9c9e2710ae5031104f2df4c874eb42be5f09d8c5842bf23fcce75fba30061a7a", + "pair-01/postgres/requests.jsonl": "76943e80317e96b6b7ba9cc557cba284d5dd4a47ed820d9c927cddfecbc66f08", + "pair-01/postgres/resource-final.json": "8c32b2886a1cfea0ade89f7daa425a21a2a98058eb98d93c06ecbdcc05efe961", + "pair-01/postgres/resources.csv": "d5f7d7dde906894d56e1aa3688f9a4940d51ca116140164c0b71cab8a93bd1cb", + "pair-01/postgres/retention-empty.json": "39d373f08a7d3cafef6f49f8b614edd45d189b7da81425e3da99841460ae5469", + "pair-01/postgres/retention.json": "7e9272d8bf484782879267a74afe3ffec5e62d6c10d6916ef12e9cfa5fd299e7", + "pair-01/postgres/sql-queries.jsonl": "b4d6beed30b1ebb4b897060b7e033b66a01188a8c053382e9928e8b6abd19221", + "pair-01/postgres/startup-hot1.json": "f3844d4e6ba0bb544cd39a22c2504ec831ca5ca08d18e71fe73212a004430e7e", + "pair-01/postgres/startup.json": "c3f7a613ba9a450b0398606dc180818cdc3790a35cdd52e2d09f6644bad3982a", + "pair-01/postgres/validation-after.json": "2bb797ad4c707e171be54aa6c37764eb58f97b212599531a42717bedeb769a2a", + "pair-01/postgres/validation-before.json": "877d32930f005f80bf357e500af0f1b5eb573b08ae23565d62ac9810200268aa", + "pair-01/postgres/visibility.json": "74e8d57fe43b23d6be361395bd39ba3b6edee5033252af8df5ebe2b0bb78c9c6", + "pair-01/postgres/websocket.jsonl": "05323e934243f7637e3ca134891e7a15a78ce6d3b96add5c00cc85e398706cda", + "pair-01/sqlite/client-validation.json": "3793c728b53da9435f2aab8f811b8f847cbeed539620ab9a17c87d93bc0184af", + "pair-01/sqlite/database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-01/sqlite/handler.json": "d2f982026bc3db8eaa29fd1f347e9131a5eb02b99eaa2bc40c1a4195c0a41162", + "pair-01/sqlite/http-workload.json": "6931a0c5bb191d0c60a03e11f04b55fa3cb786e7a94aaef9602b80477aa6b596", + "pair-01/sqlite/ingest-validation.json": "f219194a1db9e760236d526314cbeef8f86dbff3060183bcfe347c53d3415b6a", + "pair-01/sqlite/ingest.jsonl": "3414e9a2dce5a71d67ce49dae3cba3207f1afe0adb3b29ce2dcc4481a381d14a", + "pair-01/sqlite/plans/advert-preservation.json": "6f6b6d6e558851be2940d740effcff5a35830ea2eb5d3bfb348cf4830c884f87", + "pair-01/sqlite/plans/retention-empty.json": "8dfe03de0f46fef9a80f6ed7910618d5e4d5e7d73a27485ab0553f94f47de0aa", + "pair-01/sqlite/processes.csv": "3b42ff66bc1896d96d3e767028096f32e045764971d67f91ad1cf4b3a93cc7c7", + "pair-01/sqlite/requests.jsonl": "c97c96124e4e43e671ff5225621b7486f508b5545803099f191d66cbbda97229", + "pair-01/sqlite/resource-final.json": "6430b202ff14f7c2cead013a312611c9b340d0739efbc921aa273cb0ef35b87a", + "pair-01/sqlite/resources.csv": "6b761e6e872ca2ee95b4a8c6c9d966d9e33365aa520bc2f4ad0904cbc0127d1a", + "pair-01/sqlite/retention-empty.json": "64aefd505f541fe6fa132188a1d756ea9021822e093114eeee4d0196dc054fa0", + "pair-01/sqlite/retention.json": "8612195a68ac99d2d0c305ea69a690e5e18708fde63de304d1419d7996570362", + "pair-01/sqlite/sql-queries.jsonl": "06b248cf4d09efdee62e29ad85812d6741979cdb376679bf5c9365631bcfdfbe", + "pair-01/sqlite/startup-hot1.json": "8ecafaf365c6f31930375164872e82ec6d2d76ffbed7e42815d98b655d91391c", + "pair-01/sqlite/startup.json": "e035409cf284a58aa429e40e2d89a429eb2b780d9d4dd2fe6f578eecd97f4c84", + "pair-01/sqlite/validation-after.json": "2bb797ad4c707e171be54aa6c37764eb58f97b212599531a42717bedeb769a2a", + "pair-01/sqlite/validation-before.json": "877d32930f005f80bf357e500af0f1b5eb573b08ae23565d62ac9810200268aa", + "pair-01/sqlite/visibility.json": "71cafccb8f0b67004e6ad4f21ae63be5b871e46321f78cefc9f7252388bd5fb6", + "pair-01/sqlite/websocket.jsonl": "04c2c4f32123081b5abc10ffdd11b00bf4baebc2355714472e682251f967ae97", + "pair-02/corpus-offset.json": "d141dbf92c207e20123a02272e73071b18ef55ea92bda861d5594f52715a2f7e", + "pair-02/events.json": "ffb9ca481c7c31c21e32b3fc7f47de94596e20c7bf54fee8f7e7550224c20f70", + "pair-02/postgres/client-validation.json": "42480f3c2ffc0e46ecc09110e38dcb93be1aa8d7ddb11de5174ef48b110402ec", + "pair-02/postgres/database-settings.json": "999ed1c72e72ba5c038102ccb63b21014bb009c78c16cdf7b4b036ab34a09664", + "pair-02/postgres/handler.json": "1a46b8ecb1a7adfff7e36150c7618f2d6d2693a2f23162cd444e0be4b6b4a2b5", + "pair-02/postgres/http-workload.json": "d0b535e42a39cc4afe8a89b249268a12e71f8d4471cadd3df61059dd6a8e7d62", + "pair-02/postgres/ingest-validation.json": "a115ec64cfc5aea32fceff86f066c33a437706fcd1ef5a863472732172111998", + "pair-02/postgres/ingest.jsonl": "15ce0d04b7ecba6a3b387767668454ad948600c20685928f4e8065922d3caca6", + "pair-02/postgres/migration.json": "d24bc643f34f314145588b6a41c79864d92a206eda3d3f5eff58fa88298bc6a4", + "pair-02/postgres/plans/advert-preservation.json": "977127d5730ac047ae45a2a1967958333382dcba7a1c2a09c95e8d84e0404634", + "pair-02/postgres/plans/retention-empty.json": "dea5c6cd04238090585508aaba10bf040d1efb68835db4814eaf59d4164fa654", + "pair-02/postgres/processes.csv": "a44bc300d454f673726dcd1dee57cfe16e33c29b49db5b6178d289ed00ce98a8", + "pair-02/postgres/requests.jsonl": "4e4ccd7ba4386761e1656fde9450e50434455f857353d220f0ccbcc112a34dc3", + "pair-02/postgres/resource-final.json": "e0dd9c15e7a5cf55d50e98625fbdda70bb566f3ad5e834697709bcd48f08c94b", + "pair-02/postgres/resources.csv": "1a51dbfcc7e61dfefaa6ec74eb66491288f3c42c51969032fe1c491a0006ba4c", + "pair-02/postgres/retention-empty.json": "986ce7bfd7fe8d877ad1c6e8507dccca622226d8b575fca14fbbb1fb2a7e6b21", + "pair-02/postgres/retention.json": "3a6de0893a496d806966aa5b9e8cbf28fd3fc70188f8feddbb3e200fbf51a0b3", + "pair-02/postgres/sql-queries.jsonl": "d94a0c6189f13448e8dde342e0b45f3be2ced96bc46f5dc077ce536d59965ef6", + "pair-02/postgres/startup-hot1.json": "d9c9df611dfac41ff8f100e3ca58efa422b2acac5478d87a3eea9b97d3a25c8d", + "pair-02/postgres/startup.json": "f8e1bcb3fcb0b296955964034796c48f19ea56e2f51c9b4ec2f640eeac5a4e05", + "pair-02/postgres/validation-after.json": "f61176cb63616b0a5bfd2a627919edd4b1c8168d9d4d008754fbc1cb1643387a", + "pair-02/postgres/validation-before.json": "108cfbd317d2f83bc57bac1d55b67a65629332fb57164e512ec852902873e729", + "pair-02/postgres/visibility.json": "b16a9a041bbd39221ba4aa7a26d01e7400e54120b45a7bff5c494cb2494ed96b", + "pair-02/postgres/websocket.jsonl": "03a47ec400325a2ad955b604cd1cb1d87c986352b723e4a3c171dd49490129c5", + "pair-02/sqlite/client-validation.json": "c970080dd992a0d0003148d28c6b58705ed517e06aff4d2c9b2ab02a56d82478", + "pair-02/sqlite/database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-02/sqlite/handler.json": "9c6bf50dae42087fb0cdd39cc006fcd119660568f25102a0776cdac35fad4601", + "pair-02/sqlite/http-workload.json": "d0b535e42a39cc4afe8a89b249268a12e71f8d4471cadd3df61059dd6a8e7d62", + "pair-02/sqlite/ingest-validation.json": "e2198143a3c3b5bf198440f62704f00082150c7285822755d422138854af93dd", + "pair-02/sqlite/ingest.jsonl": "7ec642a934c661df4960c15baec1564a12697a5aba4fc6bf35dd55b304176c8d", + "pair-02/sqlite/plans/advert-preservation.json": "6f6b6d6e558851be2940d740effcff5a35830ea2eb5d3bfb348cf4830c884f87", + "pair-02/sqlite/plans/retention-empty.json": "b1fec67bf689c373364b097b79c1429dfb0b449b82d85795ef3917cf704c7257", + "pair-02/sqlite/processes.csv": "176a1a1d295c940b6511398b181d66179f9586e394e5c984030d74a8ba8f45db", + "pair-02/sqlite/requests.jsonl": "a1fd7a21a264d57d69533dc37e6a1ce3fc8128e14851f6d2c2d81eb087285a90", + "pair-02/sqlite/resource-final.json": "04e524872062c83b6e90f942391546a6285b34cd3f029921ebdae12ec355c1f3", + "pair-02/sqlite/resources.csv": "d5e930f71b159fb845fca766080979da4835f732f7e6a55618b74522f1a9e3fe", + "pair-02/sqlite/retention-empty.json": "64ab659c3d7f4c00bc8c1595cc1f60397f72f145e7611ce932be0a42139ad7eb", + "pair-02/sqlite/retention.json": "fcce7d1d4523f09eb5675ebf67aca18b08bb6a36b3a513b41da7238793fdbfd6", + "pair-02/sqlite/sql-queries.jsonl": "d957eb46cf1b8303d47ef788307c863373637cd2ba471bd2b1548a6be415d309", + "pair-02/sqlite/startup-hot1.json": "482ecda906ea563bd43192166ddca81739223b9959cda4903f9c141c58bb3607", + "pair-02/sqlite/startup.json": "9b5eab62d9f595a81a73d21dc1d473f7254cc1e901878b51e06ffdfdd54caa71", + "pair-02/sqlite/validation-after.json": "f61176cb63616b0a5bfd2a627919edd4b1c8168d9d4d008754fbc1cb1643387a", + "pair-02/sqlite/validation-before.json": "108cfbd317d2f83bc57bac1d55b67a65629332fb57164e512ec852902873e729", + "pair-02/sqlite/visibility.json": "3ee0bc5124d9da5359b06cc6ad876b80cf3c09942e6929f02e226ebbfb03f1c8", + "pair-02/sqlite/websocket.jsonl": "e2663885d378b344289069443ee37164a21c41f06e53265279948a45acaefec1", + "pair-03/corpus-offset.json": "4999ed14bbc7e0bc0d8e5501be711dfd4174d98be1f7275822fc4ea762110fa7", + "pair-03/events.json": "2d397fafc2172c46c1fdcadb50b066c51560bde241eb4e269aa876b343b7f486", + "pair-03/postgres/client-validation.json": "60520ca0cd72688e817efb8fab95b556f488248146e079bd5843e341beb4d9f4", + "pair-03/postgres/database-settings.json": "999ed1c72e72ba5c038102ccb63b21014bb009c78c16cdf7b4b036ab34a09664", + "pair-03/postgres/handler.json": "09baad87b3ed38b33132a4ec01dc96754cae24dfbd608b97c0e30d0719b24627", + "pair-03/postgres/http-workload.json": "553dc2b6dba343151bfa00dd6cd96cc8cbb6165bac087fd302728b125317595e", + "pair-03/postgres/ingest-validation.json": "fe296b8e425d4da92b722b8d60d2c9f69433a823833ec8bb3570822bafeade4f", + "pair-03/postgres/ingest.jsonl": "5822924874004261ccdb32512a985e226e50374feaff4419b866dd23f36d5126", + "pair-03/postgres/migration.json": "65b0632398569c3177c6d4ad9f2bb5e51e45ec62bdea17499a3d6d17b5853f14", + "pair-03/postgres/plans/advert-preservation.json": "27015349cd72a7b9af8b06c6e29c2ce0ca23e82094cdfd18c4026c42ff01366a", + "pair-03/postgres/plans/retention-empty.json": "d51e90863787e0fcdd0600d928e251c6bb80c03beb5b4397a65514c0cdea20ed", + "pair-03/postgres/processes.csv": "d0a2708dd447ce06e1355f06b4c86786723f4d9b9f5645c412802afa0e46ff27", + "pair-03/postgres/requests.jsonl": "8dca668c3a359977dacc0d793c3bc18a9fdfca3fc8894b02841ca290c4d64eda", + "pair-03/postgres/resource-final.json": "6d1fb666ad190480b0b90d3a9bd13438d93b58d98073889bb38906a0e513d82d", + "pair-03/postgres/resources.csv": "0ce0fb686f532f1026d83e256e9555345b25163c363add3468514fd2f144148b", + "pair-03/postgres/retention-empty.json": "f799d68420852495f926d4611bdd588fbdda15e5bf0a153f8d7e2ad7b4fc79c2", + "pair-03/postgres/retention.json": "024b0288ee92fa19de5d66aee3662602c31d69bf1acc85dc4ed21f70299f7b0b", + "pair-03/postgres/sql-queries.jsonl": "c617cc81a6cf73198bfac580ba54e8c68887d7fae998b987c14078e715c0d2d2", + "pair-03/postgres/startup-hot1.json": "8925a58dab558383583bd23b9448efa87ca3695777cc67e21530413f06cf6dc6", + "pair-03/postgres/startup.json": "2738fec95fb6e1aaa4cda8b55b54c04bb1c13cd6ffd9fc4c0fd7346739cf09c2", + "pair-03/postgres/validation-after.json": "81783421a01a1cecafb1f014cc314bdef9ceb71b3df691f0068bb959d32f9be8", + "pair-03/postgres/validation-before.json": "a7299d0640e1bae78bda2ce4630ce2f48b7828a492d21e2b89a7f8ad2c948965", + "pair-03/postgres/visibility.json": "6c94d6123bbb284e81cf5eee4b0e7398e109e3765d1ee2b72a4be789c9060248", + "pair-03/postgres/websocket.jsonl": "09c052514aa289599dcdbf03388db743f7972fcced85c855ae97e4fbb2ed2b96", + "pair-03/sqlite/client-validation.json": "04aa6fc778587f20e453258bd7c1c741ab9d7574c7438d71f4ed4f41c1465eff", + "pair-03/sqlite/database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-03/sqlite/handler.json": "0d5eb6ba78e49024f9aee8d1868f81d678ec58d7a2c470c0ea00fa62f409143e", + "pair-03/sqlite/http-workload.json": "553dc2b6dba343151bfa00dd6cd96cc8cbb6165bac087fd302728b125317595e", + "pair-03/sqlite/ingest-validation.json": "378a915a810db739671af7f66921cc87273ddc148e360990c958a709d2a81450", + "pair-03/sqlite/ingest.jsonl": "5833afb18f3be42ef4b2e7ed84173ca71c8d66e9c2b75345e17571d575638f2c", + "pair-03/sqlite/plans/advert-preservation.json": "6f6b6d6e558851be2940d740effcff5a35830ea2eb5d3bfb348cf4830c884f87", + "pair-03/sqlite/plans/retention-empty.json": "6d1f2e0a95a624415d66dfcacb950cfa597bf4f820be38467a77e836afee75b3", + "pair-03/sqlite/processes.csv": "515f6dab539a160ab2428bb5b52e1798b80f01f41d64431d783dbbb46c7b3b2b", + "pair-03/sqlite/requests.jsonl": "66df8806baebc45c433b2933356a4c7430f525cbfa345d5d11325e69a1e2df75", + "pair-03/sqlite/resource-final.json": "8dbf793c2d9db0638a4a1c2c0da07cafe3fd16b2e47dffab1d8775a8a8d9011e", + "pair-03/sqlite/resources.csv": "efc7b09cdbeea6551da0911bf09803fed693ff047b7d011caacb86abe4338860", + "pair-03/sqlite/retention-empty.json": "478a3287ba0389b83aeff8875504d8e5acaeef449a4808013395f24bbc78b437", + "pair-03/sqlite/retention.json": "c7c4b178d866825a60a0c3d9567b3fef9a4d21502d60b8c4af5d96ce10d148b8", + "pair-03/sqlite/sql-queries.jsonl": "7cde9514ba474df767bd732f39f9242592f8cb044f8fe4bc55611b415ada7ec7", + "pair-03/sqlite/startup-hot1.json": "b255d8ae00fb11ba086df57e0bec0a02baa45ccea1f469e861a918c0afa9f9c4", + "pair-03/sqlite/startup.json": "4a738e9039d78e4d0c252e9c3b69c3f6bd00fd820635b9f212aa41457025406d", + "pair-03/sqlite/validation-after.json": "81783421a01a1cecafb1f014cc314bdef9ceb71b3df691f0068bb959d32f9be8", + "pair-03/sqlite/validation-before.json": "a7299d0640e1bae78bda2ce4630ce2f48b7828a492d21e2b89a7f8ad2c948965", + "pair-03/sqlite/visibility.json": "2ead7da59f9f620dd11271451e9998933ebd48a2eb78b5492c29689e332df800", + "pair-03/sqlite/websocket.jsonl": "76526b57a8cfb3e1824254270d2c2b231b87add4866e56855cd6f78b8b99b9ce", + "pair-04/corpus-offset.json": "a4f3d21b79417937889f1555454ea19f43c78dd2795c46b261a633a6985c52e0", + "pair-04/events.json": "6fe3e3ae3ec316890c799801978d28a827ba418ef8b468c9c0d231459b085a7c", + "pair-04/postgres/client-validation.json": "f5945b02ad871b45fd721e8e36cc336f7312ad138cb427c5ef0e060ed5f5f6da", + "pair-04/postgres/database-settings.json": "999ed1c72e72ba5c038102ccb63b21014bb009c78c16cdf7b4b036ab34a09664", + "pair-04/postgres/handler.json": "3855863d418b0cfb5a7e28cb71dde71d98990f8608ed89037960b695f445d724", + "pair-04/postgres/http-workload.json": "b2399463776181b50c4707ba213f579e375d11ec984622a39586c7f5f58c92c9", + "pair-04/postgres/ingest-validation.json": "067b18bf75fc4311be0b6e129d794a70eae904fbe43857ca7bb90ab8e2759c0f", + "pair-04/postgres/ingest.jsonl": "aa104b8121a9a25afb78048fef85e331ebed14a2c9e120b8df37aef495465eaf", + "pair-04/postgres/migration.json": "2b96bebe5e42fae298508cd025179fccc955ce46bc8fd3d42d2b4a5d1fbfb3c9", + "pair-04/postgres/plans/advert-preservation.json": "fcc58407164df85b52b646ae9c19afb3ee3c554d083c0fea31064c676b6cf6f6", + "pair-04/postgres/plans/retention-empty.json": "a4e88cb2e13e372d73307a0dce61838720902f4058c8acae1e1c4ac152429356", + "pair-04/postgres/processes.csv": "13a11d04eb86ba0097155761cff0d23b469e022bebae6b74d9f94be0028fd57d", + "pair-04/postgres/requests.jsonl": "7e841567d6158de9e15175035b676ec77328d41efdd81e855e4fc5a9c87bdcc3", + "pair-04/postgres/resource-final.json": "a7f4739c1017278d622812216f994b0e5cce6f783ac61344e7de3647053f9858", + "pair-04/postgres/resources.csv": "985b85e22df8621ee4b3606c1ed48d41291a4fe19331ecb650646be117576fdc", + "pair-04/postgres/retention-empty.json": "8a97bd06ac181a9e04eb536ebe6790a7026272042fdde161753389ccc6415295", + "pair-04/postgres/retention.json": "30227c4f50470347b479b116f3b2815e7d29e15b47d5c09e8325229a11a53be4", + "pair-04/postgres/sql-queries.jsonl": "37c2de1685284bf89dfb8c689c158b2c700a4a8a221af8d5749b81e2b0bb9b14", + "pair-04/postgres/startup-hot1.json": "e4a7898254c92b25b056c86b4aa3519a777507f4fcf1f2b10200e6e46cb90352", + "pair-04/postgres/startup.json": "74264c21b03b78ce292982ded70677e0fcebc758686ecd508aa4038c07ad29a1", + "pair-04/postgres/validation-after.json": "0259715944229d5e6a5638b4918795885eccd2dad185b3cf9e1329ce2e334c09", + "pair-04/postgres/validation-before.json": "f7a6aa8342280bd2f1c036462854ef165a97308735ae165e9dc87307c273bac4", + "pair-04/postgres/visibility.json": "d6327f09993bdb500554f2255d1bfd8f20094252bbf332a875d01c3bd3084238", + "pair-04/postgres/websocket.jsonl": "5c40b92584014c3ebab054277a502d24a5ff73a6c38e570143406ff05f888adf", + "pair-04/sqlite/client-validation.json": "5df7ed7ac7fbd16a67d626b7b37116bf2d6ad5c41f31362cf6de0d8f6c30d573", + "pair-04/sqlite/database-settings.json": "e450fbf0d42086b047619a209e01fcf25211754e77415834c5a8b421819547e9", + "pair-04/sqlite/handler.json": "73bc91d3e4e478badd36926b9213d6f3f09646b99ba1ce8b17a46d2ed1da1a29", + "pair-04/sqlite/http-workload.json": "b2399463776181b50c4707ba213f579e375d11ec984622a39586c7f5f58c92c9", + "pair-04/sqlite/ingest-validation.json": "463f519107a2128b262583b815e29e569a4c94b48542715f13fab1687f180d27", + "pair-04/sqlite/ingest.jsonl": "491977c8c175cb807d9011170783b68fc2f28b7e83d138febfd99ba11fd93fbd", + "pair-04/sqlite/plans/advert-preservation.json": "6f6b6d6e558851be2940d740effcff5a35830ea2eb5d3bfb348cf4830c884f87", + "pair-04/sqlite/plans/retention-empty.json": "0bddc0ff94bbc9e93454763fd65b54a1f72702f158f122e5dd21f56f40e8d84e", + "pair-04/sqlite/processes.csv": "834d7fd76b98e0eac5fabf960034e9f9df13c60a1aa68636781c05dcb1aef8c8", + "pair-04/sqlite/requests.jsonl": "de0ceebe2883916ba3566ea0b613a77a50da73c9effe8f5e6d275604bf17d4fc", + "pair-04/sqlite/resource-final.json": "972ea9717de8d0b38ca1d6f10819d8e9cc4a806f526259de8ee99ce4557602da", + "pair-04/sqlite/resources.csv": "5916f3bc51e730e4f620462695f39193785861a81246614665beb0c5dbcba632", + "pair-04/sqlite/retention-empty.json": "c05294d9257e603df407210914bb2005b2c6410d6858ebec95d43cc67f01e68c", + "pair-04/sqlite/retention.json": "c7ad6f8fd1dbc52d80d507db797b7d762e7711cae21ea678699ed7ac11b5846c", + "pair-04/sqlite/sql-queries.jsonl": "82f34e4466c7339e92590e7244312a607247420c3fd362e27338e009913f718e", + "pair-04/sqlite/startup-hot1.json": "a33d4f2cc070380bc70fe3829c6c8f99f150768fd26c906ea05bef803ba53207", + "pair-04/sqlite/startup.json": "c85a68ac6f560b8bf18194b97fc51dd485fa99b5bd436b889d0aacc63e5a0e68", + "pair-04/sqlite/validation-after.json": "0259715944229d5e6a5638b4918795885eccd2dad185b3cf9e1329ce2e334c09", + "pair-04/sqlite/validation-before.json": "f7a6aa8342280bd2f1c036462854ef165a97308735ae165e9dc87307c273bac4", + "pair-04/sqlite/visibility.json": "874cd2bf5b579477bd50432e56c7319bc83f649fb317488383e5c24599f07fd7", + "pair-04/sqlite/websocket.jsonl": "f9df1e4d8ab6c3551d884c20b1d9f1e18f63e1fb20a92c21f644d8bd28c0a0f6", + "paired-summary.json": "532285b86c7e37bfa3f0a0c4b4d5fcaa3a38eb056b7f095845dd652d058e3682", + "postgres-preflight.json": "e926795faf9c07ff13185a7914625c659ad3a825a513fb7dd1006af59ca700ed", + "report.md": "025b21b645a7ce7261b9de2c1606a357daa10109cb8ffb8d2c88c4e7b64a26a8", + "resource-preflight.json": "16e6006274973fdff80b18b2afcd4a3ebb02de9fad49588ce88e77bdc443218e", + "summary-resources.csv": "08e9b525c74d23a101dff8da2071784c446e1f9879654c1c8a30776962a8e511", + "summary.csv": "2188045c5c05205e0ea1a683a1b7e76ccb9f1f5e6209b6e196b2d04332c4d57f" + }, + "artifact_id": 11647406303 + } + }, + "observer_followup": { + "description": "A separate Windows amd64 / Go 1.27.2 native SQLite response-build control used 2,048,000 observations, 128,000 transmissions, 128 observers and eight days, WAL/FULL, a 2,000 KiB reader page cache, GOMAXPROCS=2 and GOMEMLIMIT=512MiB. Each stage timed 50 uncached buildObserversDefaultResponse calls with warm OS/SQLite page caches. Before correction the upgraded layout had p50/p95 169.0/195.7 ms versus its same-run legacy control 19.46/20.29 ms. After the writer-side observer-statistics correction, upgraded p50/p95 were 19.59/20.42 ms versus same-run legacy 19.68/21.21 ms. Exact recent counts and full typed observer responses matched, excluding server_time. This supports the local planner repair; it is not full HTTP service, concurrent replay, PostgreSQL, or a replacement Linux B5 result.", + "source_note": "base `c6fee4e8872b6c0caf7af0aadc3013d48b704882`; before tree `0041649915878be1782e7d620a9336207493b83d`; corrected tree `1d9a08258b1350e512528075ca70ad988d569af7`. Frozen logs `observers-native-b-before.log` (SHA-256 `859686c1b225c5044e7ffa3175a8e7efc35576584cecd43f5b4c0601ca58cc4b`) and `observers-native-b-after.log` (`bb362f9d4544bef9bf31c0a2682346ef9dd1808f51d2d950c0e99dc9a5d0e50f`) were hash-checked and read.", + "manifest": { + "frozen": true, + "base_commit": "c6fee4e8872b6c0caf7af0aadc3013d48b704882", + "before_tree": "0041649915878be1782e7d620a9336207493b83d", + "after_tree": "1d9a08258b1350e512528075ca70ad988d569af7", + "paths": { + "cmd/server/db.go": { + "sha256": "06c54f644d7b5eb3e63907daa497c7b6457aee8e34de37b273900c837c37f136", + "bytes": 120755 + }, + "cmd/server/observer_planner_test.go": { + "sha256": "70adf7ed26a5194db72593859df47bb9867bc34f10aa7920a997247edec9b3aa", + "bytes": 7241 + }, + "internal/dbschema/sqlite.go": { + "sha256": "2dd15d5504ae03317008b9bbf5faf3feca0d5b3b71ae8738447c97909975f15b", + "bytes": 13432 + }, + "internal/dbschema/sqlite_test.go": { + "sha256": "c907455290946ed6f0c26d3ede44c0e5fc266ab2264e9e89baa419df874dc486", + "bytes": 16109 + } + }, + "scope": "Writer-side table-only SQLite observer planner-statistics refresh; no query/cache/role changes. SQL helper extraction exposes the actual aggregate to its regression.", + "benchmarks": { + "command": "go test -count=1 -run ^$ -bench ^BenchmarkObserverListSQLiteUpgrade$ -benchtime=50x ./...", + "platform": "Windows amd64; Intel Core i5-4690K; Go 1.27.2; GOMAXPROCS=2; GOMEMLIMIT=512MiB", + "grain": "50 timed uncached buildObserversDefaultResponse calls per stage; warm native SQLite page/OS caches; not full HTTP service or concurrent replay.", + "shape": { + "observations": 2048000, + "transmissions": 128000, + "observers": 128, + "nodes": 128, + "days": 8, + "recent_observations": 256000, + "average_observations_per_transmission": 16, + "reader_page_cache_kib": 2000, + "journal": "WAL", + "synchronous": "FULL" + }, + "before": { + "legacy_p50_ms": 19.46, + "legacy_p95_ms": 20.29, + "upgraded_p50_ms": 169.0, + "upgraded_p95_ms": 195.7 + }, + "after": { + "legacy_p50_ms": 19.68, + "legacy_p95_ms": 21.21, + "upgraded_p50_ms": 19.59, + "upgraded_p95_ms": 20.42 + }, + "semantics": "Exact per-observer recent counts and full typed observer response equality across layouts, excluding server_time.", + "limitation": "Supplementary deterministic SQL-only fixture, not the full Linux B5 protocol corpus/concurrent workload or a PostgreSQL performance measurement." + }, + "validation": { + "red": "Both native upgrade/repair cases fail: temporary GROUP BY tree (observers-planner-red.log), 3.560s.", + "planner_green": "Both cases pass, 3.882s.", + "sqlite_focused": "PASS 4.049s", + "postgres_focused": "PASS 5.840s; SQLite planner cases explicitly inapplicable", + "schema_full": "PASS dbschema 5.619s / legacy 0.761s", + "readonly": "Server opener remains mode=ro/query_only; schema refresh stays in writer ApplySQLite. Existing read-only schema check included in full schema suite.", + "initial_analysis": "Fresh schema test proves sqlite_stat1 remains absent so ingestor initial full analysis is not suppressed.", + "diff_check": "PASS" + }, + "logs": { + "observers-planner-red.log": "78e16d13ae816e3de601fc32b90a9c9403360b0f811f1148ffc9c1c5246c0091", + "observers-planner-green.log": "2528420ea4a5782507afdb635600f8c80def9507b92dda990399891e086ceea7", + "observers-native-b-before.log": "859686c1b225c5044e7ffa3175a8e7efc35576584cecd43f5b4c0601ca58cc4b", + "observers-native-b-after.log": "bb362f9d4544bef9bf31c0a2682346ef9dd1808f51d2d950c0e99dc9a5d0e50f", + "observers-focused-sqlite.log": "2b3820db71aa03fbaa72ca8afd8e3b0431ed30881154fdddf1c32a4470781cac", + "observers-focused-postgres.log": "12fd7c95cfed491bc83218a3483ad2cc0649ef20233d69f1caf43fd7dcf39ceb", + "observers-schema-full.log": "3c1bce4a106287c8d251f8dd7b5261872b8b7d588866dddf96ab71eb177c2537" + } + }, + "evidence_handoff_sha256": "14536dec64f1b09c3f3a331305bce3de870f3bbf42f09a42aa8ef39fe20818ab" + } +} diff --git a/docs/performance/optional-backend-b5-2026-10-09.md b/docs/performance/optional-backend-b5-2026-10-09.md new file mode 100644 index 000000000..fc5a37ff1 --- /dev/null +++ b/docs/performance/optional-backend-b5-2026-10-09.md @@ -0,0 +1,91 @@ +# Optional backend: two five-pair B experiments, 9 October 2026 + +Both experiments passed the benchmark data and coverage gates for `c03a2585eef3819cf06640911b2f7308d99339db`. They do **not** support a general speed claim. The measured c03 SQLite candidate has a material `/api/observers` regression, with a later correction separately evidenced by the native control below. Its Linux numbers remain unchanged. Candidate PostgreSQL greatly shortens this retention workload and its associated ingest queue, but costs more CPU, memory and storage and slows many reads. + +**Exact scope.** Baseline: `3e21b179ecccae005aa5f6f45bc095353c94b343`; measured application candidate: `c03a2585eef3819cf06640911b2f7308d99339db`. These are separate same-host paired studies, with different runners and HTTP mixes. Never rank candidate PostgreSQL against candidate SQLite by combining their numbers. The SQLite mix adds paths, neighbors and health requests and changes endpoint weights. Later test fixes and the observer correction were not measured by these Linux runs. + +| Study | Successful benchmark job / sanitized artifact | Harness | +|---|---|---| +| Native SQLite before/after | [Run 37985636346, job 114006837291](https://github.com/Kpa-clawbot/CoreScope/actions/runs/37985636346/job/114006837291), artifact `11645874366` | `4de419413c0a6fd93ecc2361cd704cd2a59e673e`; SHA-256 `f06f836de930a16d428b7c644231d8460e1a7ab736eed855e5d3eae7f2e23e96` | +| Upstream SQLite versus candidate PostgreSQL | [Run 37985643046, job 114006882924](https://github.com/Kpa-clawbot/CoreScope/actions/runs/37985643046/job/114006882924), artifact `11647406303` | Harness from `c03a2585eef3819cf06640911b2f7308d99339db`; SHA-256 `7c234ba61c93351688d7068c9d8405f7d2906558d797854738eb0ff059f35d84` | + +The PostgreSQL benchmark job succeeded independently of the workflow’s unrelated test-fixture failures. Smoke/pilot runs and previous failed studies are excluded. + +**Method and acceptance.** Each study has five alternating pairs, Linux amd64 / Go 1.27.2, enforced 3 CPU / 6 GiB for the whole measured stack, a 2 GiB packet-store limit and 3 GiB server memory setting. PostgreSQL is 18.6. The B corpus contains 128,000 transmissions and 2,048,000 observations over eight days. Every full and hot startup loaded the complete retained 112,000 / 1,792,000 records, with zero startup evictions. Accounts are disabled. + +Each of the ten legs per study completed all 12,000 offered ingest events and 4,800 HTTP requests: 50 ingest/s and 20 HTTP/s, 60 s warmup plus 180 s measured. Independently reconstructed schedules, warmup flags, replay identities, full WebSocket receipt coverage, errors and drops all passed. Post-replay validation recorded 118,000 transmissions / 1,802,200 observations in every leg after deleting 16,000 / 256,000; the empty second cleanup deleted zero. All 18 table digests match within each pair: full initial data and logical post-replay data. Post-replay comparison excludes only the declared operational receipt/maintenance timestamps listed in the JSON. SQLite comparison normalizes only the exact, verified `observers_identity_autoincrement_v1` migration marker; observer rowids and all other logical application values remain checked. + +All 1,000 paired isolated SQL-control responses per study matched in bytes and SHA-256. Static mixed-request controls (metrics, rx-coverage, packet detail and unfiltered RF) also matched. Mixed live response bodies are not universally byte-identical: public artifacts retain hashes, sizes and structural/count checks, not bodies sufficient to explain every difference. This is not proof of universal API response equality. + +Tables show medians of per-leg quantiles and the median of five candidate/baseline ratios, with their observed range; these are not confidence intervals or ratios of pooled requests. Lower is better for times and resource use. Service time excludes dispatch/replay queue wait; scheduled-to-complete includes it. HTTP service includes response reading and validation. p99 is omitted below 1,000 samples per leg; the 900 late-ingest events, individual HTTP classes and 100-per-class SQL controls therefore have no p99 claim. + +**SQLite before/after.** All five observer p95 comparisons regress (180 measured requests per leg). Its roughly 53× increase is the principal performance blocker for this measured candidate. Ingest scheduled latency improves, but duplicate service p99 increases and several reads regress. + +| Metric (ms) | Upstream SQLite | Candidate SQLite | Median paired ratio [range] | +|---|---:|---:|---:| +| Observers HTTP p95 | 3.653 | 194.9 | 53.36 [50.05, 54.08] | +| Additional ingest scheduled p95 | 2203 | 1583 | 0.7466 [0.6328, 0.9491] | +| Additional ingest scheduled p99 | 8066 | 6803 | 0.8422 [0.7889, 0.9025] | +| Additional ingest service p95 | 1.132 | 0.7674 | 0.6956 [0.6357, 0.7325] | +| New ingest service p95 | 1.572 | 1.174 | 0.7446 [0.7076, 0.763] | +| Duplicate ingest service p99 | 1.392 | 2.067 | 1.704 [0.5793, 2.745] | +| Health HTTP p95 | 73.54 | 1.288 | 0.01737 [0.003842, 0.939] | +| Paths HTTP p95 | 46.28 | 47.79 | 1.03 [0.9701, 1.038] | +| Reach HTTP p95 | 0.7175 | 0.8227 | 1.098 [0.947, 1.203] | +| Topology HTTP p95 | 37.84 | 37.72 | 1.102 [0.7627, 1.387] | +| Uncached channel SQL p95 | 90.02 | 96.89 | 1.075 [1.02, 1.082] | +| WebSocket signed return-to-receipt p99 | 1976 | 1317 | 0.5422 [0.4464, 0.9699] | + +Cleanup took 11.231 → 11.091 s (paired ratio 0.9949); it remains a substantial queue interruption. Full-start ready took 28.268 → 27.848 s (0.9891), but **candidate storage adoption separately took 42.03–42.59 s, median 42.50 s, before startup timing began**. The baseline needed no adoption. Neither leg drained ingest or HTTP work past the scheduled window’s end. + +| Metric (unit in row) | Upstream SQLite | Candidate SQLite | Median paired ratio [range] | +|---|---:|---:|---:| +| CPU consumed (s) | 54.68 | 53.52 | 0.9787 [0.9597, 1.023] | +| Cgroup sampled peak (GiB) | 3.988 | 3.944 | 0.9944 [0.9891, 1.002] | +| PSS sampled peak (GiB) | 2.897 | 2.87 | 0.9947 [0.9904, 1.005] | +| Selected DB/cluster + WAL peak (GiB) | 1.071 | 1.069 | 0.9987 [0.9896, 1] | +| WAL sampled peak (MiB) | 38.7 | 29.6 | 0.9496 [0.6983, 1.003] | +| Device 8:0 write delta (GiB) | 2.31 | 2.24 | 0.9694 [0.9652, 0.9812] | + +Memory and CPU differences are small: the normalized average-CPU paired ratio is 0.9837 and its range crosses 1. All SQLite measured memory-limit/OOM deltas were zero; candidate whole-leg kernel peaks were 4.045–4.120 GiB. This is not a large retained-memory win. + +**PostgreSQL comparison.** These ratios use the upstream SQLite leg of the PostgreSQL study, not the candidate SQLite experiment above. Cleanup fell from 11.268 to 0.974 s (paired ratio 0.08708). Scheduled ingest tails fell sharply while additional and duplicate service p95 increased, so the queue-inclusive result must not be presented as a universal per-write speedup. + +| Metric (ms) | Upstream SQLite | Candidate PostgreSQL | Median paired ratio [range] | +|---|---:|---:|---:| +| Additional ingest scheduled p95 | 2262 | 3.044 | 0.001346 [0.001186, 0.001457] | +| Additional ingest scheduled p99 | 8075 | 5.57 | 0.0006807 [0.0006152, 0.0007697] | +| Additional ingest service p95 | 1.627 | 2.239 | 1.373 [0.956, 1.513] | +| Duplicate ingest service p95 | 1.049 | 1.874 | 1.713 [1.626, 1.94] | +| New ingest service p95 | 2.689 | 2.238 | 0.8562 [0.6949, 0.9195] | +| New ingest service p99 | 5.492 | 4.041 | 0.7722 [0.6216, 0.8537] | +| Observers HTTP p95 | 3.762 | 4.06 | 1.079 [1.071, 1.154] | +| Nodes (50) HTTP p95 | 5.241 | 10.55 | 1.784 [1.656, 2.156] | +| Messages (offset 0) HTTP p95 | 69.11 | 114.9 | 1.654 [1.626, 2.519] | +| Packet detail, memory HTTP p95 | 1.644 | 4.013 | 2.407 [2.298, 2.553] | +| Uncached channel SQL p95 | 95.29 | 122.6 | 1.278 [1.219, 1.54] | +| WebSocket signed return-to-receipt p99 | 2035 | 998 | 0.5318 [0.4524, 0.5928] | + +Full-start ready was 28.965 → 27.712 s (0.9701), while hot-start first-ready was 9.768 → 11.271 s (1.1427). The separately measured import, grants and validation stage took a median **148.887 s**; neither startup number includes it. There was no work draining past the scheduled measured endpoint. + +| Metric (unit in row) | Upstream SQLite | Candidate PostgreSQL | Median paired ratio [range] | +|---|---:|---:|---:| +| CPU consumed (s) | 45.02 | 59.23 | 1.329 [1.31, 1.519] | +| Average CPU cores | 0.2506 | 0.3311 | 1.323 [1.308, 1.525] | +| Cgroup sampled peak (GiB) | 3.954 | 5.536 | 1.422 [1.371, 1.527] | +| PSS sampled peak (GiB) | 2.868 | 3.194 | 1.11 [1.088, 1.148] | +| Selected DB/cluster + WAL peak (GiB) | 1.072 | 2.273 | 2.119 [2.099, 2.146] | +| WAL sampled peak (MiB) | 30.71 | 1024 | 33.35 [25.54, 40.79] | +| Device 8:0 write delta (GiB) | 2.315 | 0.3907 | 0.1702 [0.1619, 0.2152] | + +**PostgreSQL memory pressure is a sizing constraint.** All five descriptor-local whole-leg kernel peaks reached the 6 GiB limit, with small kernel-accounting overshoots of 4–116 KiB. Measured-window `memory.events max` increments were `[338, 0, 522, 0, 0]`; OOM and OOM-kill increments were zero. The lowest sampled headroom was only 20 KiB. An absence of killed processes does not establish spare capacity. Cgroup memory includes file cache. PSS apportions shared pages; summed RSS (paired 1.853× here) double-counts shared PostgreSQL mappings and must not be interpreted as unique resident memory. + +PostgreSQL retained **1 GiB of WAL throughout every measured window after migration**. Its storage peak therefore includes that carryover and is not all newly generated runtime data. PostgreSQL storage counts the private cluster; SQLite counts its database and sidecars. Conversion source/recovery copies are additional. Lower windowed write accounting does not measure later checkpoints or steady-state write amplification. Minimum sampled free disk was 72.11 GiB in the PostgreSQL legs and 75.81 GiB in candidate SQLite’s separate study; these well-provisioned runners do not establish minimum operator disk sizing. + +**Limits.** Resource deltas use the first and last sample inside the exact 180 s measured window, not startup or warmup; sampled spans are about 178–180 s. Gauge peaks can miss subsecond spikes; whole-leg kernel peaks are separately reset/read through the same retained file descriptor and include setup/startup/validation. `memory.events` raw final counters are cumulative; only explicit differences are per-window events. I/O is device-level cgroup accounting, not a physical-device write-amplification measurement. Query plans were collected after retention. WebSocket timing is signed receipt minus ingest-return; no negative measured samples occurred here. OS cache is warm, tests are synthetic, accounts are disabled, and neither capacity, OS-cold behavior, a long soak, Windows performance, nor a generally superior backend follows. + +All per-run metrics, paired distributions, body-hash comparison counts, settings, source/binary identities and raw-file SHA-256 values are in the [accompanying JSON](optional-backend-b5-2026-10-09.json). Production hashes and both harness hashes were recomputed from immutable Git objects; public database digest attestations were checked without reopening private payload databases. + +**Separate observer follow-up control.** A separate Windows amd64 / Go 1.27.2 native SQLite response-build control used 2,048,000 observations, 128,000 transmissions, 128 observers and eight days, WAL/FULL, a 2,000 KiB reader page cache, GOMAXPROCS=2 and GOMEMLIMIT=512MiB. Each stage timed 50 uncached buildObserversDefaultResponse calls with warm OS/SQLite page caches. Before correction the upgraded layout had p50/p95 169.0/195.7 ms versus its same-run legacy control 19.46/20.29 ms. After the writer-side observer-statistics correction, upgraded p50/p95 were 19.59/20.42 ms versus same-run legacy 19.68/21.21 ms. Exact recent counts and full typed observer responses matched, excluding server_time. This supports the local planner repair; it is not full HTTP service, concurrent replay, PostgreSQL, or a replacement Linux B5 result. + +Source/evidence: base `c6fee4e8872b6c0caf7af0aadc3013d48b704882`; before tree `0041649915878be1782e7d620a9336207493b83d`; corrected tree `1d9a08258b1350e512528075ca70ad988d569af7`. Frozen logs `observers-native-b-before.log` (SHA-256 `859686c1b225c5044e7ffa3175a8e7efc35576584cecd43f5b4c0601ca58cc4b`) and `observers-native-b-after.log` (`bb362f9d4544bef9bf31c0a2682346ef9dd1808f51d2d950c0e99dc9a5d0e50f`) were hash-checked and read. diff --git a/docs/postgresql-performance.md b/docs/postgresql-performance.md index e601fb432..2e869616e 100644 --- a/docs/postgresql-performance.md +++ b/docs/postgresql-performance.md @@ -2,8 +2,8 @@ > Historical measurement of candidate `ce023e0`, before optional backend selection > and upstream's #2147 concurrency fix. It does not measure the current dual-backend -> implementation. Keep these results separate from the new comparison against -> upstream `3e21b17`. +> implementation. Keep these results separate from the +> [optional-backend comparison against upstream `3e21b17`](performance/optional-backend-b5-2026-10-09.md). The PostgreSQL conversion reduced retention-related ingestion delays in this test. Median writes and several reads were slower, and CPU, memory and storage diff --git a/docs/storage.md b/docs/storage.md index 185a906a3..7baeca2f7 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -1,6 +1,8 @@ # Choose, update and recover storage -CoreScope uses **SQLite by default**. PostgreSQL18.6 is optional. Both keep telemetry and accounts separate; the ingestor writes telemetry and the server reads it. Accounts remain at their recorded target while user management is disabled. If a legacy installation never initialized accounts, enabling them is an explicit setup operation: stop writers and run `./manage.sh setup` after editing the account configuration. Restart alone does not invent a missing account database. +CoreScope uses **SQLite by default**. PostgreSQL 18.6 is optional. Both keep telemetry and accounts separate; the ingestor writes telemetry and the server reads it. Accounts remain at their recorded target while user management is disabled. If a legacy installation never initialized accounts, enabling them is an explicit setup operation: stop writers and run `./manage.sh setup` after editing the account configuration. Restart alone does not invent a missing account database. + +Use the [measured backend comparison](performance/optional-backend-b5-2026-10-09.md) to compare latency, resource use and conversion overhead before choosing PostgreSQL. ## Install From c6c5c8f1d72f42d67c4d601d3e88ae12cdf6fe8e Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 19:19:26 -0400 Subject: [PATCH 24/67] fix(postgres): back up recorded public-schema targets --- cmd/server/backup_test.go | 20 ++++++- cmd/server/users_backup_test.go | 2 +- internal/pgutil/dump.go | 8 ++- internal/pgutil/dump_schema_test.go | 90 +++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+), 4 deletions(-) create mode 100644 internal/pgutil/dump_schema_test.go diff --git a/cmd/server/backup_test.go b/cmd/server/backup_test.go index 5a565aea1..d4c4f0c8f 100644 --- a/cmd/server/backup_test.go +++ b/cmd/server/backup_test.go @@ -3,6 +3,7 @@ package main import ( "net/http" "net/http/httptest" + "net/url" "os" "path/filepath" "strings" @@ -12,6 +13,23 @@ import ( // postgresMagic is the signature of a PostgreSQL custom-format archive. var postgresMagic = testNativeSQL("SQLite format 3\x00", "PGDMP") +// Installed PostgreSQL selections pin even the default schema explicitly. +func selectedBackupTestDSN(t *testing.T) string { + t.Helper() + dsn := testDatabaseDSN(t) + if testBackendValue() == "postgres" { + u, err := url.Parse(dsn) + if err != nil { + t.Fatal("invalid fixture URL") + } + q := u.Query() + q.Set("search_path", `"public"`) + u.RawQuery = q.Encode() + dsn = u.String() + } + return dsn +} + func TestBackupRequiresAPIKey(t *testing.T) { _, router := setupTestServerWithAPIKey(t, "test-secret-key-strong-enough") @@ -26,7 +44,7 @@ func TestBackupRequiresAPIKey(t *testing.T) { func TestBackupRestoresNativePostgresSnapshot(t *testing.T) { const apiKey = "test-secret-key-strong-enough" srv, router := setupTestServerWithAPIKey(t, apiKey) - srv.db = setupTestDBAtURL(t, testDatabaseDSN(t)) + srv.db = setupTestDBAtURL(t, selectedBackupTestDSN(t)) seedTestData(t, srv.db) writerURL := srv.db.path reader, err := openFixtureReader(t, writerURL) diff --git a/cmd/server/users_backup_test.go b/cmd/server/users_backup_test.go index 456d6011b..55b17a0f4 100644 --- a/cmd/server/users_backup_test.go +++ b/cmd/server/users_backup_test.go @@ -243,7 +243,7 @@ func TestInitUserManagementBacksUpAtStartup(t *testing.T) { var usersBackupFilenameRE = regexp.MustCompile(`^attachment; filename="corescope-users-\d{8}-\d{6}\.(?:db|dump)"$`) func TestAdminUsersBackupDownload(t *testing.T) { - f := newBackupAuthFixture(t, "boss@example.org") + f := newAuthFixtureWithURL(t, selectedBackupTestDSN(t), "boss@example.org") boss := f.registerAndActivate(t, "boss@example.org", "Boss", pw) uma := f.registerAndActivate(t, "uma@example.org", "Uma", pw) tmp := t.TempDir() diff --git a/internal/pgutil/dump.go b/internal/pgutil/dump.go index 64215f18f..135396dd5 100644 --- a/internal/pgutil/dump.go +++ b/internal/pgutil/dump.go @@ -50,8 +50,12 @@ func Dump(ctx context.Context, dsn, path string) (err error) { return err } u, _ := url.Parse(dsn) - if u.Query().Get("search_path") != "" { - return errors.New("postgres: native backup requires a database URL without search_path") + // Installed selections pin the default schema explicitly. pg_dump still + // exports the whole database; do not turn this into a schema-filtered dump. + switch strings.TrimSpace(u.Query().Get("search_path")) { + case "", "public", `"public"`: + default: + return errors.New("postgres: native backup requires the default public schema") } f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) if err != nil { diff --git a/internal/pgutil/dump_schema_test.go b/internal/pgutil/dump_schema_test.go new file mode 100644 index 000000000..c9f5bf1d8 --- /dev/null +++ b/internal/pgutil/dump_schema_test.go @@ -0,0 +1,90 @@ +package pgutil_test + +import ( + "context" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +func TestDumpPublicSchemaOwnerRestoresWholeDatabase(t *testing.T) { + for _, schema := range []string{"public", `"public"`} { + t.Run(schema, func(t *testing.T) { + dsn := pgtest.NewDatabase(t) + u, _ := url.Parse(dsn) + q := u.Query() + q.Set("search_path", schema) + u.RawQuery = q.Encode() + dsn = u.String() + db, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`CREATE TABLE sample(id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, value TEXT); + INSERT INTO sample(value) VALUES('kept'),('deleted'); DELETE FROM sample WHERE id=2; + CREATE SCHEMA auxiliary; CREATE TABLE auxiliary.retained(value TEXT); INSERT INTO auxiliary.retained VALUES('whole database')`); err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "owner.dump") + if err := pgutil.Dump(context.Background(), dsn, path); err != nil { + t.Fatal(err) + } + target := pgtest.NewDatabase(t) + env, err := pgutil.CommandEnv(target) + if err != nil { + t.Fatal(err) + } + cfg, err := pgutil.ParseConfig(target) + if err != nil { + t.Fatal(err) + } + cmd := exec.Command("pg_restore", "--no-password", "--exit-on-error", "--no-owner", "--no-privileges", "--dbname="+cfg.Database, path) + cmd.Env = env + if err := cmd.Run(); err != nil { + t.Fatal("native whole-database restore failed") + } + restored, err := pgutil.Open(target, false) + if err != nil { + t.Fatal(err) + } + defer restored.Close() + var kept, auxiliary string + if err := restored.QueryRow(`SELECT value FROM sample WHERE id=1`).Scan(&kept); err != nil || kept != "kept" { + t.Fatal("restored public data changed", err) + } + if err := restored.QueryRow(`SELECT value FROM auxiliary.retained`).Scan(&auxiliary); err != nil || auxiliary != "whole database" { + t.Fatal("public search_path incorrectly filtered the native archive", err) + } + var nextID int + if err := restored.QueryRow(`INSERT INTO sample(value) VALUES('next') RETURNING id`).Scan(&nextID); err != nil || nextID != 3 { + t.Fatal("restored identity high-water changed", err) + } + }) + } +} + +func TestDumpRejectsCustomOrMultipleSchemasBeforeOutput(t *testing.T) { + for _, schema := range []string{"custom", "public,custom", `"public","custom"`, `"Public"`, "$user,public", "pg_catalog"} { + t.Run(schema, func(t *testing.T) { + u, _ := url.Parse("postgresql://owner:private-sentinel@localhost/example?sslmode=disable") + q := u.Query() + q.Set("search_path", schema) + u.RawQuery = q.Encode() + path := filepath.Join(t.TempDir(), "refused.dump") + err := pgutil.Dump(context.Background(), u.String(), path) + if err == nil || !strings.Contains(err.Error(), "default public schema") || strings.Contains(err.Error(), "private-sentinel") { + t.Fatal("unsupported backup scope was accepted or exposed credentials") + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatal("rejected schema left an output file") + } + }) + } +} From 2d1e763a7f2f710066246dcba47ba8cb4f4652e1 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 9 Oct 2026 19:19:30 -0400 Subject: [PATCH 25/67] test(qa): exercise the selected native database backend --- .github/workflows/deploy.yml | 7 +- qa/scripts/blacklist-test.sh | 96 +++++++- qa/scripts/test-blacklist-postgres.sh | 247 +++++++++++++++++++++ qa/scripts/test-blacklist-sql.sh | 254 +-------------------- qa/scripts/test-blacklist-sqlite.sh | 303 ++++++++++++++++++++++++++ 5 files changed, 655 insertions(+), 252 deletions(-) create mode 100644 qa/scripts/test-blacklist-postgres.sh create mode 100644 qa/scripts/test-blacklist-sqlite.sh diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8f8df412b..c80b6688b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -354,7 +354,12 @@ jobs: run: bash scripts/staging/test-disk-monitor.sh - name: QA SQL parameter-binding unit tests (issue #1977) - run: bash qa/scripts/test-blacklist-sql.sh + run: | + if [ "$CORESCOPE_TEST_BACKEND" = sqlite ] && ! command -v sqlite3 >/dev/null; then + sudo apt-get update -qq + sudo apt-get install -y sqlite3 + fi + bash qa/scripts/test-blacklist-sql.sh - name: Lint CSS variables (issue #1128) run: | diff --git a/qa/scripts/blacklist-test.sh b/qa/scripts/blacklist-test.sh index 6904f2bdf..277f21ef8 100755 --- a/qa/scripts/blacklist-test.sh +++ b/qa/scripts/blacklist-test.sh @@ -15,10 +15,12 @@ # TARGET_CONFIG_PATH — absolute path to config.json on the target # TARGET_CONTAINER — docker container name on the target # Optional env: +# CORESCOPE_DB_BACKEND — target's selected backend: sqlite (default) or postgres +# TARGET_DB_PATH — selected SQLite path on the target for the native reader # CORESCOPE_READER_DATABASE_URL — private local PostgreSQL reader URL for §10.2 # PGSERVICE / PGDATABASE — alternatively configure private native libpq settings # CORESCOPE_QA_PYTHON — Python 3 executable for URL parsing (default python3) -# ADMIN_API_TOKEN — if /api/admin/transmissions exists, use it instead of the PostgreSQL probe +# ADMIN_API_TOKEN — if /api/admin/transmissions exists, use it instead of the DB probe # (read from env, not argv — never appears in ps) # CURL_TIMEOUT — per-request curl timeout, seconds (default 60) # RESTART_WAIT_S — max wait for /api/stats after restart (default 120) @@ -29,9 +31,9 @@ # hide-failed → blacklisted pubkey still surfaced via API (§10.1 fail) # retain-failed → no transmissions.from_pubkey rows (ADVERTs) for the # blacklisted pubkey in the DB (§10.2 fail), or the -# §10.2 probe cannot verify a ready, restricted PostgreSQL -# reader. Native parameter binding is mandatory; there is no -# interpolated SQL or SQLite runtime fallback. +# §10.2 probe cannot verify its selected native reader. +# Native parameter binding is mandatory; neither an +# interpolated query nor another backend is a fallback. # teardown-failed→ post-test removal did not restore listing # # Exit code = number of failures (0 = pass). @@ -164,6 +166,7 @@ node_visible() { # Queries and hex-only parameter values travel on stdin; connection credentials # stay in private libpq environment/service settings, never process arguments. POSTGRES_RUNNER="" # local | container | host +SQL_BACKEND="${CORESCOPE_DB_BACKEND:-sqlite}" POSTGRES_PROBE_TOKEN="corescope-probe-ok" RETAIN_COUNT="" POSTGRES_CONNECT_PY="$(cat "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/postgres-connect.py")" @@ -171,6 +174,7 @@ POSTGRES_CONNECT_PY="$(cat "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/postgr # A psql meta-command argument whose only variable bytes are [0-9a-f]. od -v # prevents repeated long values collapsing to '*'. Empty input binds as ''. sql_hex_literal() { + [[ "$SQL_BACKEND" != sqlite ]] || printf x printf "'%s'" "$(printf '%s' "$1" | od -An -v -tx1 | tr -d ' \n')" } @@ -198,7 +202,7 @@ BEGIN END $qa$; SQL } -transmission_count_sql() { +postgres_transmission_count_sql() { postgres_read_guard printf '%s\n' "SELECT COUNT(*) FROM transmissions WHERE from_pubkey = lower(convert_from(decode(\$1,'hex'),'UTF8'))" printf '\\bind %s\n\\g\nCOMMIT;\n' "$(sql_hex_literal "$1")" @@ -257,6 +261,73 @@ postgres_error_summary() { echo 'Check PostgreSQL 18 client availability and private reader connection settings.' >&2 fi } + +# Native SQLite CLI bindings from the original SQLite QA path (#1977). The +# blob-to-text cast keeps spaces/newlines out of dot-command argument parsing. +# -readonly forbids persistent writes; .parameter uses only a temporary table. +SQLITE_ARGS=(-readonly -batch -bail -init /dev/null -noheader -list) +SQLITE_PROBE_TOKEN="corescope-probe-ok" +SQLITE_RUNNER="" +sqlite_transmission_count_sql() { + printf '.parameter init\n' + printf '.parameter set :pubkey "cast(%s as text)"\n' "$(sql_hex_literal "$1")" + printf 'SELECT COUNT(*) FROM transmissions WHERE from_pubkey = lower(:pubkey);\n' +} +transmission_count_sql() { + case "$SQL_BACKEND" in + sqlite) sqlite_transmission_count_sql "$1" ;; + postgres) postgres_transmission_count_sql "$1" ;; + *) echo 'Unsupported QA database backend' >&2; return 2 ;; + esac +} +sqlite_probe_sql() { + printf '.parameter init\n' + printf '.parameter set :probe "cast(%s as text)"\n' "$(sql_hex_literal "$SQLITE_PROBE_TOKEN")" + printf 'SELECT :probe;\n' +} +resolve_sqlite_runner() { + local probe out + probe=$(sqlite_probe_sql) + SQLITE_RUNNER="" + if out=$(ssh_t "docker exec -i $(printf %q "$TARGET_CONTAINER") sqlite3 ${SQLITE_ARGS[*]} :memory:" \ + <<<"$probe" 2>>"$TMP/sqlite-probe.err" | tr -d '\r') && [[ "$out" == "$SQLITE_PROBE_TOKEN" ]]; then + SQLITE_RUNNER=container; return 0 + fi + if out=$(ssh_t "sqlite3 ${SQLITE_ARGS[*]} :memory:" <<<"$probe" 2>>"$TMP/sqlite-probe.err" | tr -d '\r') && [[ "$out" == "$SQLITE_PROBE_TOKEN" ]]; then + SQLITE_RUNNER=host; return 0 + fi + return 1 +} +run_sqlite() { + case "$SQLITE_RUNNER" in + container) ssh_t "docker exec -i $(printf %q "$TARGET_CONTAINER") sqlite3 ${SQLITE_ARGS[*]} $(printf %q "$TARGET_DB_PATH")" ;; + host) ssh_t "sqlite3 ${SQLITE_ARGS[*]} $(printf %q "$TARGET_DB_PATH")" ;; + *) echo 'run_sqlite: no runner resolved' >&2; return 127 ;; + esac | tr -d '\r' +} +read_sqlite_retain_count() { + if [[ -z "${TARGET_DB_PATH:-}" ]]; then + echo ' ❌ retain-failed: TARGET_DB_PATH is required for the selected SQLite reader' + return 1 + fi + if ! resolve_sqlite_runner; then + echo ' ❌ retain-failed: no SQLite client able to bind a parameter on the target' + cat "$TMP/sqlite-probe.err" >&2 + return 1 + fi + echo " SQLite reader runner: $SQLITE_RUNNER" + if ! RETAIN_COUNT=$(transmission_count_sql "$TEST_PUBKEY" | run_sqlite 2>"$TMP/sqlite.err"); then + echo " ❌ retain-failed: SQLite query failed via $SQLITE_RUNNER" + cat "$TMP/sqlite.err" >&2 + RETAIN_COUNT="" + return 1 + fi + if ! [[ "$RETAIN_COUNT" =~ ^[0-9]+$ ]]; then + echo ' ❌ retain-failed: SQLite query did not return one numeric count' + RETAIN_COUNT="" + return 1 + fi +} read_retain_count() { RETAIN_COUNT="" local code @@ -270,6 +341,11 @@ read_retain_count() { if [[ "$RETAIN_COUNT" =~ ^[0-9]+$ ]]; then return 0; fi RETAIN_COUNT="" fi + case "$SQL_BACKEND" in + sqlite) read_sqlite_retain_count; return $? ;; + postgres) ;; + *) echo ' ❌ retain-failed: unsupported QA database backend'; return 1 ;; + esac if ! resolve_postgres_runner; then echo ' ❌ retain-failed: no ready, restricted PostgreSQL reader with native parameter binding' postgres_error_summary "$TMP/postgres-probe.err" @@ -306,7 +382,9 @@ main() { TARGET_SSH_KEY="${TARGET_SSH_KEY:-/root/.ssh/id_ed25519}" TARGET_CONFIG_PATH="${TARGET_CONFIG_PATH:-}" TARGET_CONTAINER="${TARGET_CONTAINER:-}" + TARGET_DB_PATH="${TARGET_DB_PATH:-}" ADMIN_API_TOKEN="${ADMIN_API_TOKEN:-}" + case "$SQL_BACKEND" in sqlite|postgres) ;; *) echo 'error: CORESCOPE_DB_BACKEND must be sqlite or postgres' >&2; exit 2 ;; esac if [[ -z "$TEST_PUBKEY" || -z "$TARGET_SSH_HOST" || -z "$TARGET_CONFIG_PATH" || -z "$TARGET_CONTAINER" ]]; then echo "error: TEST_NODE_PUBKEY, TARGET_SSH_HOST, TARGET_CONFIG_PATH, TARGET_CONTAINER are required" >&2 @@ -332,8 +410,12 @@ main() { echo "error: TARGET_CONFIG_PATH must be a sane absolute path" >&2 exit 2 fi - if [[ -n "${TARGET_DB_PATH:-}" ]]; then - echo "error: TARGET_DB_PATH is obsolete; configure a private PostgreSQL reader connection" >&2 + if [[ "$SQL_BACKEND" == postgres && -n "$TARGET_DB_PATH" ]]; then + echo 'error: TARGET_DB_PATH is a SQLite path; configure the selected PostgreSQL reader connection' >&2 + exit 2 + fi + if [[ -n "$TARGET_DB_PATH" && ! "$TARGET_DB_PATH" =~ ^/[A-Za-z0-9_./-]+$ ]]; then + echo 'error: TARGET_DB_PATH must be a sane absolute SQLite path' >&2 exit 2 fi diff --git a/qa/scripts/test-blacklist-postgres.sh b/qa/scripts/test-blacklist-postgres.sh new file mode 100644 index 000000000..fec09b285 --- /dev/null +++ b/qa/scripts/test-blacklist-postgres.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash +# Native PostgreSQL regressions for the blacklist retention probe. No external +# endpoints are contacted: SSH is stubbed and all schemas/roles are disposable. +set -uo pipefail +set +x +export CORESCOPE_DB_BACKEND=postgres +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +# shellcheck source=blacklist-test.sh +. "$SCRIPT_DIR/blacklist-test.sh" +PYTHON="${CORESCOPE_QA_PYTHON:-python3}" +CONNECT="$SCRIPT_DIR/postgres-connect.py" +export QA_CONNECT="$CONNECT" +PASS=0; FAIL=0 +assert_eq() { + if [[ "$2" == "$3" ]]; then PASS=$((PASS+1)); else + FAIL=$((FAIL+1)); printf 'FAIL: %s — expected %q got %q\n' "$1" "$2" "$3" >&2 + fi +} +assert_true() { local label="$1"; shift; if "$@"; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); echo "FAIL: $label" >&2; fi; } +contains() { [[ "$1" == *"$2"* ]]; } +lacks() { [[ "$1" != *"$2"* ]]; } +hex_only() { [[ "$1" =~ ^\'[0-9a-f]*\'$ ]]; } +assert_eq 'hex of deadbeef' "'6465616462656566'" "$(sql_hex_literal deadbeef)" +assert_eq 'hex of empty value' "''" "$(sql_hex_literal '')" +for payload in "' OR 1=1 --" '"; DROP TABLE transmissions; --' 'a\b' '$(id) `id`' "$(printf 'a\nb')" 'héllo'; do + assert_true 'bound parameter alphabet is only hexadecimal' hex_only "$(sql_hex_literal "$payload")" +done +LONG=$(printf 'x%.0s' $(seq 1 4096)); LONG_HEX=$(sql_hex_literal "$LONG") +assert_true 'long bound parameter alphabet is hexadecimal' hex_only "$LONG_HEX" +assert_eq 'repeated lines are not collapsed by od' 8192 "$(( ${#LONG_HEX} - 2 ))" + +command -v "$PYTHON" >/dev/null || { echo 'FAIL: Python 3 is required' >&2; exit 1; } +"$PYTHON" "$SCRIPT_DIR/test-postgres-connect.py" || exit 1 +command -v psql >/dev/null || { echo 'FAIL: PostgreSQL 18 psql is required' >&2; exit 1; } +[[ -n "${CORESCOPE_TEST_POSTGRES_URL:-}" ]] || { echo 'FAIL: set CORESCOPE_TEST_POSTGRES_URL to a disposable PostgreSQL administrator URL' >&2; exit 1; } +FIXTURE_ROOT="$(cd "${TMPDIR:-/tmp}" && pwd -P)" +FIXTURE_DIR=$(mktemp -d "$FIXTURE_ROOT/corescope-qa-XXXXXX") +FIXTURE_DIR="$(cd "$FIXTURE_DIR" && pwd -P)" +SCHEMA="corescope_qa_${BASHPID}_${RANDOM}" +OWNER="${SCHEMA}_owner"; READER="${SCHEMA}_reader" +FIXTURE_PASSWORD=$(od -An -N24 -v -tx1 /dev/urandom | tr -d ' \n') +CREATED=0 +admin_sql() { CORESCOPE_READER_DATABASE_URL="$CORESCOPE_TEST_POSTGRES_URL" "$PYTHON" "$CONNECT" | tr -d '\r'; } +cleanup() { + local rc=$? + if [[ "$CREATED" == 1 && "$SCHEMA" =~ ^corescope_qa_[0-9]+_[0-9]+$ ]]; then + if ! admin_sql >/dev/null <&2; rc=1; fi + fi + case "$FIXTURE_DIR" in "$FIXTURE_ROOT"/corescope-qa-*) rm -rf -- "$FIXTURE_DIR" ;; *) echo 'FAIL: refusing unsafe temporary cleanup' >&2; rc=1 ;; esac + exit "$rc" +} +trap cleanup EXIT +# Build private role URLs without putting the original URL/password in argv or +# printing either. The connection launcher validates the URL before psql runs. +set_role_url() { + local base="$CORESCOPE_TEST_POSTGRES_URL" authority tail query part rest="" + authority=${base#*://}; tail=${authority#*/}; authority=${authority%%/*} + QA_ROLE_URL="${base%%://*}://$1:$FIXTURE_PASSWORD@${authority##*@}/${tail%%\?*}" + if [[ "$tail" == *\?* ]]; then + query=${tail#*\?} + while [[ -n "$query" ]]; do + part=${query%%&*}; if [[ "$query" == *'&'* ]]; then query=${query#*&}; else query=""; fi + [[ "${part%%=*}" == search_path ]] || rest+="${part}&" + done + fi + QA_ROLE_URL+="?${rest}search_path=$2" +} +if ! admin_sql >/dev/null <&2; exit 1; fi +CREATED=1 +set_role_url "$OWNER" "$SCHEMA"; OWNER_URL="$QA_ROLE_URL" +set_role_url "$READER" "$SCHEMA"; READER_URL="$QA_ROLE_URL" +set_role_url "$READER" "${SCHEMA}_bad"; BAD_URL="$QA_ROLE_URL" +set_role_url "$READER" "${SCHEMA}_empty"; EMPTY_URL="$QA_ROLE_URL" +set_role_url "$OWNER" "${SCHEMA}_import"; IMPORT_OWNER_URL="$QA_ROLE_URL" +set_role_url "$READER" "${SCHEMA}_import"; IMPORT_READER_URL="$QA_ROLE_URL" +MIGRATOR="${CORESCOPE_QA_MIGRATOR:-$FIXTURE_DIR/migrate}" +PREPARER="${CORESCOPE_QA_PREPARER:-$FIXTURE_DIR/prepare-fixture}" +if [[ -z "${CORESCOPE_QA_MIGRATOR:-}" ]]; then + (cd "$REPO_ROOT/cmd/migrate" && CGO_ENABLED=1 go build -o "$MIGRATOR" .) || exit 1 +fi +if [[ -z "${CORESCOPE_QA_PREPARER:-}" ]]; then + (cd "$REPO_ROOT/cmd/migrate" && CGO_ENABLED=1 go build -o "$PREPARER" ../../scripts/prepare-postgres-fixture.go) || exit 1 +fi +if ! CORESCOPE_DATABASE_URL="$OWNER_URL" CORESCOPE_USERS_DATABASE_URL="" "$MIGRATOR" >"$FIXTURE_DIR/bootstrap.out" 2>"$FIXTURE_DIR/bootstrap.err"; then + echo 'FAIL: shipping migration tool could not bootstrap the owner schema' >&2; exit 1 +fi +PK_A="0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" +PK_B="fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210" +PK_ABSENT="00000000000000000000000000000000000000000000000000000000000000ff" +PK_A_UPPER=$(printf '%s' "$PK_A" | tr 'a-f' 'A-F') +LONG_PK=$(printf 'ab%.0s' $(seq 1 5000)) +if ! admin_sql >/dev/null <&2; exit 1; fi +run_local() { CORESCOPE_READER_DATABASE_URL="$1" "$PYTHON" "$CONNECT" | tr -d '\r'; } +count() { transmission_count_sql "$1" | run_local "$2"; } + +# Positive controls must succeed, otherwise zero-valued injection results prove +# nothing. These exercise the actual canonical schema and restricted LOGIN role. +assert_eq 'fixture holds six rows' 6 "$(run_local "$READER_URL" <<<'SELECT count(*) FROM transmissions;')" +out=$(count "$PK_A" "$READER_URL"); rc=$? +assert_eq 'legitimate pubkey has three rows' 3 "$out" +assert_eq 'legitimate pubkey returns success' 0 "$rc" +assert_eq 'native parameter capability probe round-trips' "$POSTGRES_PROBE_TOKEN" "$(postgres_probe_sql | run_local "$READER_URL")" +COUNT_SQL=$(transmission_count_sql "$PK_A") +assert_true 'query uses the real attribution column' contains "$COUNT_SQL" 'WHERE from_pubkey = lower(convert_from(decode($1' +assert_true 'query has no invented from_node column' lacks "$COUNT_SQL" from_node +assert_true 'query contains no raw pubkey' lacks "$COUNT_SQL" "$PK_A" +assert_eq 'other legitimate pubkey has one row' 1 "$(count "$PK_B" "$READER_URL")" +assert_eq 'upper-case key matches' 3 "$(count "$PK_A_UPPER" "$READER_URL")" +assert_eq 'absent key has no rows' 0 "$(count "$PK_ABSENT" "$READER_URL")" +assert_eq 'prefix is not an exact key' 0 "$(count "${PK_A:0:16}" "$READER_URL")" +for payload in "' OR 1=1 --" "') OR 1=1 --" '" OR 1=1 --' "x' OR from_pubkey IS NOT NULL --" "$PK_A' OR '1'='1" \ + '"); .shell id; --' "$(printf 'a\n.shell id\nSELECT 99;')" "$(printf 'a\n\\! id\nSELECT 99;')" '$(id) `id`' 'a\b'; do + out=$(count "$payload" "$READER_URL" 2>"$FIXTURE_DIR/payload.err"); rc=$? + assert_eq 'hostile pubkey is a literal, not SQL/meta-command/shell code' 0 "$out" + assert_eq 'hostile literal query succeeds' 0 "$rc" +done +assert_eq 'unsafe interpolated control would leak all rows' 6 "$(run_local "$READER_URL" <<<"SELECT count(*) FROM transmissions WHERE from_pubkey = '' OR 1=1 --';")" +admin_sql >/dev/null <"$FIXTURE_DIR/schema.err"); rc=$? + assert_true 'bad schema returns nonzero' test "$rc" -ne 0 + assert_true 'bad schema exposes its PostgreSQL error class' contains "$(cat "$FIXTURE_DIR/schema.err")" "${spec##*|}" + assert_eq 'bad schema produces no count' '' "$out" +done +# Prove actual grants, independently of the read-only transaction setting. +out=$(run_local "$READER_URL" <<<'BEGIN READ WRITE; DELETE FROM transmissions;' 2>"$FIXTURE_DIR/write.err"); rc=$? +assert_true 'restricted reader cannot delete even in read-write transaction' test "$rc" -ne 0 +assert_true 'delete fails for privilege, not transaction mode' contains "$(cat "$FIXTURE_DIR/write.err")" '42501' +out=$(count "$PK_A" "$OWNER_URL" 2>"$FIXTURE_DIR/owner.err"); rc=$? +assert_true 'owner credential refused by runtime probe' test "$rc" -ne 0 +assert_eq 'owner refusal has no count' '' "$out" +admin_sql >/dev/null <<<"UPDATE $SCHEMA.corescope_schema SET ready=false;" +out=$(count "$PK_A" "$READER_URL" 2>"$FIXTURE_DIR/ready.err"); rc=$? +assert_true 'unfinished import refused' test "$rc" -ne 0 +assert_eq 'unfinished import produces no count' '' "$out" +admin_sql >/dev/null <<<"UPDATE $SCHEMA.corescope_schema SET ready=true;" + +# Preserve the committed fixture comparison through its explicit offline repair +# and shipping importer. The original SQLite file is never modified or queried +# as a runtime backend. +REAL_FIXTURE="$REPO_ROOT/test-fixtures/e2e-fixture.db" +if ! "$PREPARER" -source "$REAL_FIXTURE" -destination "$FIXTURE_DIR/prepared.sqlite" >"$FIXTURE_DIR/prepare.out" 2>"$FIXTURE_DIR/prepare.err"; then + echo 'FAIL: offline historical fixture preparation failed' >&2; exit 1 +fi +if ! CORESCOPE_DATABASE_URL="$IMPORT_OWNER_URL" CORESCOPE_USERS_DATABASE_URL="" "$MIGRATOR" -offline -from-sqlite "$FIXTURE_DIR/prepared.sqlite" -state-dir "$FIXTURE_DIR/import-state" >"$FIXTURE_DIR/import.out" 2>"$FIXTURE_DIR/import.err"; then + echo 'FAIL: shipping importer failed for the historical fixture' >&2; exit 1 +fi +admin_sql >/dev/null <<<"GRANT SELECT ON ALL TABLES IN SCHEMA ${SCHEMA}_import TO $READER;" +real_row=$(run_local "$IMPORT_READER_URL" <<<'SELECT from_pubkey,count(*) FROM transmissions WHERE from_pubkey IS NOT NULL GROUP BY from_pubkey ORDER BY count(*) DESC,from_pubkey LIMIT 1;') +real_pk=${real_row%|*}; real_n=${real_row##*|} +assert_true 'imported historical fixture has attributed pubkey' bash -c '[[ "$1" =~ ^[0-9a-f]{64}$ ]]' _ "$real_pk" +assert_true 'imported fixture has positive retained count' bash -c '[[ "$1" =~ ^[1-9][0-9]*$ ]]' _ "$real_n" +assert_eq 'bound query matches direct count on imported fixture' "$real_n" "$(count "$real_pk" "$IMPORT_READER_URL")" + +# Transport uses the real query/client but stubs all SSH/container activity. +STUB_ARGV="$FIXTURE_DIR/ssh.argv"; STUB_STDIN="$FIXTURE_DIR/ssh.stdin" +STUB_CONTAINER=0 +ssh_t() { + local cmd="$1" input + printf '%s\n' "$cmd" >>"$STUB_ARGV" + input=$(cat); printf '%s\n' "$input" >>"$STUB_STDIN" + case "$cmd" in + 'docker exec -i '*) + if [[ "$STUB_CONTAINER" != 1 ]]; then echo 'client unavailable in stub container' >&2; return 127; fi + cmd=${cmd#docker exec -i }; cmd=${cmd#* } ;; + esac + printf '%s\n' "$input" | CORESCOPE_READER_DATABASE_URL="$READER_URL" bash -c "$cmd" +} +reset_stub() { : >"$STUB_ARGV"; : >"$STUB_STDIN"; } +TMP="$FIXTURE_DIR"; ADMIN_API_TOKEN=''; TEST_PUBKEY="$PK_A" +TARGET_CONTAINER='corescope-stub'; TARGET_SSH_HOST='stub-host' +unset CORESCOPE_READER_DATABASE_URL PGSERVICE PGDATABASE +for runner in host container; do + reset_stub; [[ "$runner" == container ]] && STUB_CONTAINER=1 + read_retain_count >/dev/null 2>"$FIXTURE_DIR/transport.err"; rc=$? + assert_eq 'remote reader count succeeds' 0 "$rc" + assert_eq 'resolved runner is expected target' "$runner" "${POSTGRES_RUNNER:-}" + assert_eq 'transport preserves retained count' 3 "$RETAIN_COUNT" + argv=$(cat "$STUB_ARGV"); input=$(cat "$STUB_STDIN") + assert_true 'pubkey is absent from remote argv' lacks "$argv" "$PK_A" + assert_true 'credentials are absent from remote argv' lacks "$argv" "$FIXTURE_PASSWORD" + assert_true 'SQL is absent from remote argv' lacks "$argv" 'SELECT' + assert_true 'native parameter query arrives on stdin' contains "$input" 'decode($1' + assert_true 'raw pubkey is absent from stdin' lacks "$input" "$PK_A" +done +reset_stub +CORESCOPE_READER_DATABASE_URL="$READER_URL" read_retain_count >/dev/null 2>"$FIXTURE_DIR/local.err"; rc=$? +assert_eq 'explicit local reader URL succeeds' 0 "$rc" +assert_eq 'explicit URL selects local client' local "${POSTGRES_RUNNER:-}" +assert_eq 'explicit URL does not contact SSH' '' "$(cat "$STUB_ARGV")" +# Missing/unusable binding capability and query errors must fail closed. +ssh_t() { cat >/dev/null; printf '\\bind unsupported\n0\n'; } +read_retain_count >/dev/null 2>&1; rc=$? +assert_eq 'client that does not bind is rejected' 1 "$rc" +assert_eq 'failed capability leaves no count' '' "$RETAIN_COUNT" +POSTGRES_RUNNER='' +if run_postgres /dev/null 2>&1; then FAIL=$((FAIL+1)); echo 'FAIL: unresolved runner accepted' >&2; else PASS=$((PASS+1)); fi +for malformed in 'postgresql://test:secret-sentinel@/db' 'postgresql://test:secret-sentinel@localhost/db?unsupported=yes' 'postgresql://test:secret-sentinel%xx@localhost/db' "${READER_URL}&target_session_attrs=secret-sentinel"; do + out=$(CORESCOPE_READER_DATABASE_URL="$malformed" "$PYTHON" "$CONNECT" &1); rc=$? + assert_true 'malformed reader URL fails' test "$rc" -ne 0 + assert_true 'malformed URL never reveals credentials' lacks "$out" 'secret-sentinel' +done +# Native libpq configuration is also accepted without putting values in argv. +out=$(CORESCOPE_READER_DATABASE_URL="$READER_URL" "$PYTHON" -c 'import os,runpy,subprocess,sys; m=runpy.run_path(os.environ["QA_CONNECT"]); e=m["connection_env"](os.environ); e.pop("CORESCOPE_READER_DATABASE_URL",None); sys.exit(subprocess.run(["psql","-X","-qAt","-w","-v","ON_ERROR_STOP=1"],env=e).returncode)' <<<'SELECT 1977;' 2>"$FIXTURE_DIR/libpq.err" | tr -d '\r') +assert_eq 'private libpq fields reach the real database' 1977 "$out" +space_url=${READER_URL/search_path=$SCHEMA/search_path=$SCHEMA%2C%20pg_catalog} +assert_eq 'search_path option whitespace is escaped on the real client' "$SCHEMA, pg_catalog" "$(run_local "$space_url" <<<"SELECT current_setting('search_path');")" +echo "test-blacklist-sql.sh: $PASS passed, $FAIL failed" +[[ "$FAIL" == 0 ]] diff --git a/qa/scripts/test-blacklist-sql.sh b/qa/scripts/test-blacklist-sql.sh index bec4f04df..7e59014c1 100755 --- a/qa/scripts/test-blacklist-sql.sh +++ b/qa/scripts/test-blacklist-sql.sh @@ -1,246 +1,12 @@ #!/usr/bin/env bash -# Native PostgreSQL regressions for the blacklist retention probe. No external -# endpoints are contacted: SSH is stubbed and all schemas/roles are disposable. -set -uo pipefail -set +x +# Run the selected native engine's full SQL binding/security regressions. +# The default SQLite job deliberately requires no PostgreSQL service. +set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" -# shellcheck source=blacklist-test.sh -. "$SCRIPT_DIR/blacklist-test.sh" -PYTHON="${CORESCOPE_QA_PYTHON:-python3}" -CONNECT="$SCRIPT_DIR/postgres-connect.py" -export QA_CONNECT="$CONNECT" -PASS=0; FAIL=0 -assert_eq() { - if [[ "$2" == "$3" ]]; then PASS=$((PASS+1)); else - FAIL=$((FAIL+1)); printf 'FAIL: %s — expected %q got %q\n' "$1" "$2" "$3" >&2 - fi -} -assert_true() { local label="$1"; shift; if "$@"; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); echo "FAIL: $label" >&2; fi; } -contains() { [[ "$1" == *"$2"* ]]; } -lacks() { [[ "$1" != *"$2"* ]]; } -hex_only() { [[ "$1" =~ ^\'[0-9a-f]*\'$ ]]; } -assert_eq 'hex of deadbeef' "'6465616462656566'" "$(sql_hex_literal deadbeef)" -assert_eq 'hex of empty value' "''" "$(sql_hex_literal '')" -for payload in "' OR 1=1 --" '"; DROP TABLE transmissions; --' 'a\b' '$(id) `id`' "$(printf 'a\nb')" 'héllo'; do - assert_true 'bound parameter alphabet is only hexadecimal' hex_only "$(sql_hex_literal "$payload")" -done -LONG=$(printf 'x%.0s' $(seq 1 4096)); LONG_HEX=$(sql_hex_literal "$LONG") -assert_true 'long bound parameter alphabet is hexadecimal' hex_only "$LONG_HEX" -assert_eq 'repeated lines are not collapsed by od' 8192 "$(( ${#LONG_HEX} - 2 ))" - -command -v "$PYTHON" >/dev/null || { echo 'FAIL: Python 3 is required' >&2; exit 1; } -"$PYTHON" "$SCRIPT_DIR/test-postgres-connect.py" || exit 1 -command -v psql >/dev/null || { echo 'FAIL: PostgreSQL 18 psql is required' >&2; exit 1; } -[[ -n "${CORESCOPE_TEST_POSTGRES_URL:-}" ]] || { echo 'FAIL: set CORESCOPE_TEST_POSTGRES_URL to a disposable PostgreSQL administrator URL' >&2; exit 1; } -FIXTURE_ROOT="$(cd "${TMPDIR:-/tmp}" && pwd -P)" -FIXTURE_DIR=$(mktemp -d "$FIXTURE_ROOT/corescope-qa-XXXXXX") -FIXTURE_DIR="$(cd "$FIXTURE_DIR" && pwd -P)" -SCHEMA="corescope_qa_${BASHPID}_${RANDOM}" -OWNER="${SCHEMA}_owner"; READER="${SCHEMA}_reader" -FIXTURE_PASSWORD=$(od -An -N24 -v -tx1 /dev/urandom | tr -d ' \n') -CREATED=0 -admin_sql() { CORESCOPE_READER_DATABASE_URL="$CORESCOPE_TEST_POSTGRES_URL" "$PYTHON" "$CONNECT" | tr -d '\r'; } -cleanup() { - local rc=$? - if [[ "$CREATED" == 1 && "$SCHEMA" =~ ^corescope_qa_[0-9]+_[0-9]+$ ]]; then - if ! admin_sql >/dev/null <&2; rc=1; fi - fi - case "$FIXTURE_DIR" in "$FIXTURE_ROOT"/corescope-qa-*) rm -rf -- "$FIXTURE_DIR" ;; *) echo 'FAIL: refusing unsafe temporary cleanup' >&2; rc=1 ;; esac - exit "$rc" -} -trap cleanup EXIT -# Build private role URLs without putting the original URL/password in argv or -# printing either. The connection launcher validates the URL before psql runs. -set_role_url() { - local base="$CORESCOPE_TEST_POSTGRES_URL" authority tail query part rest="" - authority=${base#*://}; tail=${authority#*/}; authority=${authority%%/*} - QA_ROLE_URL="${base%%://*}://$1:$FIXTURE_PASSWORD@${authority##*@}/${tail%%\?*}" - if [[ "$tail" == *\?* ]]; then - query=${tail#*\?} - while [[ -n "$query" ]]; do - part=${query%%&*}; if [[ "$query" == *'&'* ]]; then query=${query#*&}; else query=""; fi - [[ "${part%%=*}" == search_path ]] || rest+="${part}&" - done - fi - QA_ROLE_URL+="?${rest}search_path=$2" -} -if ! admin_sql >/dev/null <&2; exit 1; fi -CREATED=1 -set_role_url "$OWNER" "$SCHEMA"; OWNER_URL="$QA_ROLE_URL" -set_role_url "$READER" "$SCHEMA"; READER_URL="$QA_ROLE_URL" -set_role_url "$READER" "${SCHEMA}_bad"; BAD_URL="$QA_ROLE_URL" -set_role_url "$READER" "${SCHEMA}_empty"; EMPTY_URL="$QA_ROLE_URL" -set_role_url "$OWNER" "${SCHEMA}_import"; IMPORT_OWNER_URL="$QA_ROLE_URL" -set_role_url "$READER" "${SCHEMA}_import"; IMPORT_READER_URL="$QA_ROLE_URL" -MIGRATOR="${CORESCOPE_QA_MIGRATOR:-$FIXTURE_DIR/migrate}" -PREPARER="${CORESCOPE_QA_PREPARER:-$FIXTURE_DIR/prepare-fixture}" -if [[ -z "${CORESCOPE_QA_MIGRATOR:-}" ]]; then - (cd "$REPO_ROOT/cmd/migrate" && CGO_ENABLED=1 go build -o "$MIGRATOR" .) || exit 1 -fi -if [[ -z "${CORESCOPE_QA_PREPARER:-}" ]]; then - (cd "$REPO_ROOT/cmd/migrate" && CGO_ENABLED=1 go build -o "$PREPARER" ../../scripts/prepare-postgres-fixture.go) || exit 1 -fi -if ! CORESCOPE_DATABASE_URL="$OWNER_URL" CORESCOPE_USERS_DATABASE_URL="" "$MIGRATOR" >"$FIXTURE_DIR/bootstrap.out" 2>"$FIXTURE_DIR/bootstrap.err"; then - echo 'FAIL: shipping migration tool could not bootstrap the owner schema' >&2; exit 1 -fi -PK_A="0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" -PK_B="fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210" -PK_ABSENT="00000000000000000000000000000000000000000000000000000000000000ff" -PK_A_UPPER=$(printf '%s' "$PK_A" | tr 'a-f' 'A-F') -LONG_PK=$(printf 'ab%.0s' $(seq 1 5000)) -if ! admin_sql >/dev/null <&2; exit 1; fi -run_local() { CORESCOPE_READER_DATABASE_URL="$1" "$PYTHON" "$CONNECT" | tr -d '\r'; } -count() { transmission_count_sql "$1" | run_local "$2"; } - -# Positive controls must succeed, otherwise zero-valued injection results prove -# nothing. These exercise the actual canonical schema and restricted LOGIN role. -assert_eq 'fixture holds six rows' 6 "$(run_local "$READER_URL" <<<'SELECT count(*) FROM transmissions;')" -out=$(count "$PK_A" "$READER_URL"); rc=$? -assert_eq 'legitimate pubkey has three rows' 3 "$out" -assert_eq 'legitimate pubkey returns success' 0 "$rc" -assert_eq 'native parameter capability probe round-trips' "$POSTGRES_PROBE_TOKEN" "$(postgres_probe_sql | run_local "$READER_URL")" -COUNT_SQL=$(transmission_count_sql "$PK_A") -assert_true 'query uses the real attribution column' contains "$COUNT_SQL" 'WHERE from_pubkey = lower(convert_from(decode($1' -assert_true 'query has no invented from_node column' lacks "$COUNT_SQL" from_node -assert_true 'query contains no raw pubkey' lacks "$COUNT_SQL" "$PK_A" -assert_eq 'other legitimate pubkey has one row' 1 "$(count "$PK_B" "$READER_URL")" -assert_eq 'upper-case key matches' 3 "$(count "$PK_A_UPPER" "$READER_URL")" -assert_eq 'absent key has no rows' 0 "$(count "$PK_ABSENT" "$READER_URL")" -assert_eq 'prefix is not an exact key' 0 "$(count "${PK_A:0:16}" "$READER_URL")" -for payload in "' OR 1=1 --" "') OR 1=1 --" '" OR 1=1 --' "x' OR from_pubkey IS NOT NULL --" "$PK_A' OR '1'='1" \ - '"); .shell id; --' "$(printf 'a\n.shell id\nSELECT 99;')" "$(printf 'a\n\\! id\nSELECT 99;')" '$(id) `id`' 'a\b'; do - out=$(count "$payload" "$READER_URL" 2>"$FIXTURE_DIR/payload.err"); rc=$? - assert_eq 'hostile pubkey is a literal, not SQL/meta-command/shell code' 0 "$out" - assert_eq 'hostile literal query succeeds' 0 "$rc" -done -assert_eq 'unsafe interpolated control would leak all rows' 6 "$(run_local "$READER_URL" <<<"SELECT count(*) FROM transmissions WHERE from_pubkey = '' OR 1=1 --';")" -admin_sql >/dev/null <"$FIXTURE_DIR/schema.err"); rc=$? - assert_true 'bad schema returns nonzero' test "$rc" -ne 0 - assert_true 'bad schema exposes its PostgreSQL error class' contains "$(cat "$FIXTURE_DIR/schema.err")" "${spec##*|}" - assert_eq 'bad schema produces no count' '' "$out" -done -# Prove actual grants, independently of the read-only transaction setting. -out=$(run_local "$READER_URL" <<<'BEGIN READ WRITE; DELETE FROM transmissions;' 2>"$FIXTURE_DIR/write.err"); rc=$? -assert_true 'restricted reader cannot delete even in read-write transaction' test "$rc" -ne 0 -assert_true 'delete fails for privilege, not transaction mode' contains "$(cat "$FIXTURE_DIR/write.err")" '42501' -out=$(count "$PK_A" "$OWNER_URL" 2>"$FIXTURE_DIR/owner.err"); rc=$? -assert_true 'owner credential refused by runtime probe' test "$rc" -ne 0 -assert_eq 'owner refusal has no count' '' "$out" -admin_sql >/dev/null <<<"UPDATE $SCHEMA.corescope_schema SET ready=false;" -out=$(count "$PK_A" "$READER_URL" 2>"$FIXTURE_DIR/ready.err"); rc=$? -assert_true 'unfinished import refused' test "$rc" -ne 0 -assert_eq 'unfinished import produces no count' '' "$out" -admin_sql >/dev/null <<<"UPDATE $SCHEMA.corescope_schema SET ready=true;" - -# Preserve the committed fixture comparison through its explicit offline repair -# and shipping importer. The original SQLite file is never modified or queried -# as a runtime backend. -REAL_FIXTURE="$REPO_ROOT/test-fixtures/e2e-fixture.db" -if ! "$PREPARER" -source "$REAL_FIXTURE" -destination "$FIXTURE_DIR/prepared.sqlite" >"$FIXTURE_DIR/prepare.out" 2>"$FIXTURE_DIR/prepare.err"; then - echo 'FAIL: offline historical fixture preparation failed' >&2; exit 1 -fi -if ! CORESCOPE_DATABASE_URL="$IMPORT_OWNER_URL" CORESCOPE_USERS_DATABASE_URL="" "$MIGRATOR" -offline -from-sqlite "$FIXTURE_DIR/prepared.sqlite" -state-dir "$FIXTURE_DIR/import-state" >"$FIXTURE_DIR/import.out" 2>"$FIXTURE_DIR/import.err"; then - echo 'FAIL: shipping importer failed for the historical fixture' >&2; exit 1 -fi -admin_sql >/dev/null <<<"GRANT SELECT ON ALL TABLES IN SCHEMA ${SCHEMA}_import TO $READER;" -real_row=$(run_local "$IMPORT_READER_URL" <<<'SELECT from_pubkey,count(*) FROM transmissions WHERE from_pubkey IS NOT NULL GROUP BY from_pubkey ORDER BY count(*) DESC,from_pubkey LIMIT 1;') -real_pk=${real_row%|*}; real_n=${real_row##*|} -assert_true 'imported historical fixture has attributed pubkey' bash -c '[[ "$1" =~ ^[0-9a-f]{64}$ ]]' _ "$real_pk" -assert_true 'imported fixture has positive retained count' bash -c '[[ "$1" =~ ^[1-9][0-9]*$ ]]' _ "$real_n" -assert_eq 'bound query matches direct count on imported fixture' "$real_n" "$(count "$real_pk" "$IMPORT_READER_URL")" - -# Transport uses the real query/client but stubs all SSH/container activity. -STUB_ARGV="$FIXTURE_DIR/ssh.argv"; STUB_STDIN="$FIXTURE_DIR/ssh.stdin" -STUB_CONTAINER=0 -ssh_t() { - local cmd="$1" input - printf '%s\n' "$cmd" >>"$STUB_ARGV" - input=$(cat); printf '%s\n' "$input" >>"$STUB_STDIN" - case "$cmd" in - 'docker exec -i '*) - if [[ "$STUB_CONTAINER" != 1 ]]; then echo 'client unavailable in stub container' >&2; return 127; fi - cmd=${cmd#docker exec -i }; cmd=${cmd#* } ;; - esac - printf '%s\n' "$input" | CORESCOPE_READER_DATABASE_URL="$READER_URL" bash -c "$cmd" -} -reset_stub() { : >"$STUB_ARGV"; : >"$STUB_STDIN"; } -TMP="$FIXTURE_DIR"; ADMIN_API_TOKEN=''; TEST_PUBKEY="$PK_A" -TARGET_CONTAINER='corescope-stub'; TARGET_SSH_HOST='stub-host' -unset CORESCOPE_READER_DATABASE_URL PGSERVICE PGDATABASE -for runner in host container; do - reset_stub; [[ "$runner" == container ]] && STUB_CONTAINER=1 - read_retain_count >/dev/null 2>"$FIXTURE_DIR/transport.err"; rc=$? - assert_eq 'remote reader count succeeds' 0 "$rc" - assert_eq 'resolved runner is expected target' "$runner" "${POSTGRES_RUNNER:-}" - assert_eq 'transport preserves retained count' 3 "$RETAIN_COUNT" - argv=$(cat "$STUB_ARGV"); input=$(cat "$STUB_STDIN") - assert_true 'pubkey is absent from remote argv' lacks "$argv" "$PK_A" - assert_true 'credentials are absent from remote argv' lacks "$argv" "$FIXTURE_PASSWORD" - assert_true 'SQL is absent from remote argv' lacks "$argv" 'SELECT' - assert_true 'native parameter query arrives on stdin' contains "$input" 'decode($1' - assert_true 'raw pubkey is absent from stdin' lacks "$input" "$PK_A" -done -reset_stub -CORESCOPE_READER_DATABASE_URL="$READER_URL" read_retain_count >/dev/null 2>"$FIXTURE_DIR/local.err"; rc=$? -assert_eq 'explicit local reader URL succeeds' 0 "$rc" -assert_eq 'explicit URL selects local client' local "${POSTGRES_RUNNER:-}" -assert_eq 'explicit URL does not contact SSH' '' "$(cat "$STUB_ARGV")" -# Missing/unusable binding capability and query errors must fail closed. -ssh_t() { cat >/dev/null; printf '\\bind unsupported\n0\n'; } -read_retain_count >/dev/null 2>&1; rc=$? -assert_eq 'client that does not bind is rejected' 1 "$rc" -assert_eq 'failed capability leaves no count' '' "$RETAIN_COUNT" -POSTGRES_RUNNER='' -if run_postgres /dev/null 2>&1; then FAIL=$((FAIL+1)); echo 'FAIL: unresolved runner accepted' >&2; else PASS=$((PASS+1)); fi -for malformed in 'postgresql://test:secret-sentinel@/db' 'postgresql://test:secret-sentinel@localhost/db?unsupported=yes' 'postgresql://test:secret-sentinel%xx@localhost/db' "${READER_URL}&target_session_attrs=secret-sentinel"; do - out=$(CORESCOPE_READER_DATABASE_URL="$malformed" "$PYTHON" "$CONNECT" &1); rc=$? - assert_true 'malformed reader URL fails' test "$rc" -ne 0 - assert_true 'malformed URL never reveals credentials' lacks "$out" 'secret-sentinel' -done -# Native libpq configuration is also accepted without putting values in argv. -out=$(CORESCOPE_READER_DATABASE_URL="$READER_URL" "$PYTHON" -c 'import os,runpy,subprocess,sys; m=runpy.run_path(os.environ["QA_CONNECT"]); e=m["connection_env"](os.environ); e.pop("CORESCOPE_READER_DATABASE_URL",None); sys.exit(subprocess.run(["psql","-X","-qAt","-w","-v","ON_ERROR_STOP=1"],env=e).returncode)' <<<'SELECT 1977;' 2>"$FIXTURE_DIR/libpq.err" | tr -d '\r') -assert_eq 'private libpq fields reach the real database' 1977 "$out" -space_url=${READER_URL/search_path=$SCHEMA/search_path=$SCHEMA%2C%20pg_catalog} -assert_eq 'search_path option whitespace is escaped on the real client' "$SCHEMA, pg_catalog" "$(run_local "$space_url" <<<"SELECT current_setting('search_path');")" -echo "test-blacklist-sql.sh: $PASS passed, $FAIL failed" -[[ "$FAIL" == 0 ]] +case "${CORESCOPE_TEST_BACKEND:-sqlite}" in + sqlite|postgres) + export CORESCOPE_DB_BACKEND="${CORESCOPE_TEST_BACKEND:-sqlite}" + exec bash "$SCRIPT_DIR/test-blacklist-$CORESCOPE_DB_BACKEND.sh" + ;; + *) echo 'FAIL: unsupported CORESCOPE_TEST_BACKEND; choose sqlite or postgres' >&2; exit 2 ;; +esac diff --git a/qa/scripts/test-blacklist-sqlite.sh b/qa/scripts/test-blacklist-sqlite.sh new file mode 100644 index 000000000..e38139c5d --- /dev/null +++ b/qa/scripts/test-blacklist-sqlite.sh @@ -0,0 +1,303 @@ +#!/usr/bin/env bash +# Native SQLite binding/transport regressions for blacklist retention. No SSH +# endpoint is contacted. Canonical fixtures come from the shipping bootstrap. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +REAL_FIXTURE="$REPO_ROOT/test-fixtures/e2e-fixture.db" +REAL_FIXTURE_SHA=$(sha256sum "$REAL_FIXTURE") || exit 1 +export CORESCOPE_DB_BACKEND=sqlite +# shellcheck source=blacklist-test.sh +. "$SCRIPT_DIR/blacklist-test.sh" + +PASS=0 +FAIL=0 + +assert_eq() { + local label="$1" expected="$2" actual="$3" + if [ "$expected" = "$actual" ]; then + PASS=$((PASS + 1)) + else + FAIL=$((FAIL + 1)) + echo "FAIL: $label — expected '$expected' got '$actual'" >&2 + fi +} + +assert_match() { + local label="$1" pattern="$2" actual="$3" + if [[ "$actual" =~ $pattern ]]; then + PASS=$((PASS + 1)) + else + FAIL=$((FAIL + 1)) + echo "FAIL: $label — '$actual' does not match /$pattern/" >&2 + fi +} + +assert_true() { + local label="$1"; shift + if "$@"; then PASS=$((PASS + 1)); else + FAIL=$((FAIL + 1)); echo "FAIL: $label" >&2 + fi +} + +contains() { [[ "$1" == *"$2"* ]]; } +lacks() { [[ "$1" != *"$2"* ]]; } + +# ----- sql_hex_literal ------------------------------------------------------ +# The security property: whatever goes in, the SQL text it produces is drawn +# from [0-9a-f] only. No caller-supplied byte can close a string literal or add +# a dot-command argument. Needs no sqlite3, so this group always runs. +assert_eq "hex of deadbeef" "x'6465616462656566'" "$(sql_hex_literal deadbeef)" +assert_eq "hex of empty" "x''" "$(sql_hex_literal "")" + +HEX_ONLY="^x'[0-9a-f]*'\$" +assert_match "alphabet: sql quote payload" "$HEX_ONLY" "$(sql_hex_literal "' OR 1=1 --")" +assert_match "alphabet: drop table" "$HEX_ONLY" "$(sql_hex_literal '"; DROP TABLE transmissions; --')" +assert_match "alphabet: backslash" "$HEX_ONLY" "$(sql_hex_literal 'a\b')" +assert_match "alphabet: dollar and backtick" "$HEX_ONLY" "$(sql_hex_literal '$(id) `id`')" +assert_match "alphabet: embedded newline" "$HEX_ONLY" "$(sql_hex_literal "$(printf 'a\nb')")" +assert_match "alphabet: multibyte" "$HEX_ONLY" "$(sql_hex_literal 'héllo')" + +# `od` without -v collapses runs of identical lines to '*'. A long repetitive +# value is the case that catches losing the flag. +LONG=$(printf 'x%.0s' $(seq 1 4096)) +LONG_HEX=$(sql_hex_literal "$LONG") +assert_match "alphabet: 4096 repeated bytes" "$HEX_ONLY" "$LONG_HEX" +# 4096 bytes → 8192 hex digits, plus the 3 chars of x''. A collapsed run would +# be far shorter and would also fail the alphabet check on '*'. +assert_eq "no od line-collapse in 4096-byte value" "8192" "$(( ${#LONG_HEX} - 3 ))" + +# ----- query text ------------------------------------------------------------- +# Synthetic 32-byte pubkeys in the ingestor's form (hex.EncodeToString → lowercase). +PK_A="0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" +PK_B="fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210" +PK_ABSENT="00000000000000000000000000000000000000000000000000000000000000ff" +PK_A_UPPER=$(printf '%s' "$PK_A" | tr 'a-f' 'A-F') + +COUNT_SQL=$(transmission_count_sql "$PK_A") +assert_true "query targets transmissions.from_pubkey" contains "$COUNT_SQL" "WHERE from_pubkey = lower(:pubkey)" +assert_true "query does not reference from_node" lacks "$COUNT_SQL" "from_node" +assert_true "query text does not carry the raw pubkey" lacks "$COUNT_SQL" "$PK_A" + +command -v sqlite3 >/dev/null 2>&1 || { echo 'FAIL: sqlite3 CLI is required for the selected SQLite backend' >&2; exit 1; } +FIXTURE_ROOT="$(cd "${TMPDIR:-/tmp}" && pwd -P)" +FIXTURE_DIR=$(mktemp -d "$FIXTURE_ROOT/corescope-qa-sqlite-XXXXXX") +cleanup() { + local rc=$? + case "$FIXTURE_DIR" in "$FIXTURE_ROOT"/corescope-qa-sqlite-*) rm -rf -- "$FIXTURE_DIR" ;; *) echo 'FAIL: refusing unsafe temporary cleanup' >&2; rc=1 ;; esac + exit "$rc" +} +trap cleanup EXIT +DB="$FIXTURE_DIR/fixture.db" +LEGACY_DB="$FIXTURE_DIR/legacy-from-node.db" +EMPTY_DB="$FIXTURE_DIR/no-table.db" +mkdir -p "$FIXTURE_DIR/state" +SELECTION_FILE="$FIXTURE_DIR/state/storage-selection.json" +case "$(uname -s)" in MINGW*|MSYS*|CYGWIN*) SELECTION_FILE=$(cygpath -w "$SELECTION_FILE") ;; esac +MIGRATOR="${CORESCOPE_QA_MIGRATOR:-$FIXTURE_DIR/migrate}" +if [[ -z "${CORESCOPE_QA_MIGRATOR:-}" ]]; then + (cd "$REPO_ROOT/cmd/migrate" && CGO_ENABLED=1 go build -o "$MIGRATOR" .) || exit 1 +fi +if ! CORESCOPE_DATABASE_URL='' CORESCOPE_USERS_DATABASE_URL='' CORESCOPE_USERS_OWNER_DATABASE_URL='' "$MIGRATOR" \ + -storage-action=init -backend=sqlite -offline -config-dir "$FIXTURE_DIR" -selection-file "$SELECTION_FILE" \ + -sqlite-path "$DB" -users-sqlite-path "$FIXTURE_DIR/users.db" >"$FIXTURE_DIR/bootstrap.out" 2>"$FIXTURE_DIR/bootstrap.err"; then + echo 'FAIL: shipping migration tool could not bootstrap native SQLite' >&2 + cat "$FIXTURE_DIR/bootstrap.err" >&2; exit 1 +fi +DDL=$(sqlite3 -readonly "$DB" "SELECT sql FROM sqlite_master WHERE type='table' AND name='transmissions';" | tr -d '\r') +assert_match 'shipping schema is native transmissions table' 'CREATE TABLE.*transmissions' "$DDL" +assert_match 'shipping schema has from_pubkey TEXT' 'from_pubkey[[:space:]]+TEXT' "$DDL" +assert_true 'shipping schema has no invented from_node' lacks "$DDL" from_node + +# Schema-realistic fixture: shipping SQLite bootstrap. Three ADVERTs from A, one +# from B, and two non-ADVERT rows whose from_pubkey is NULL (the ingestor only +# attributes ADVERTs), so a structural injection has 6 rows to leak. +sqlite3 "$DB" <&1); rc=$? + assert_eq "payload binds literally: $(printf %q "$payload")" "0" "$out" + assert_eq "payload exit 0: $(printf %q "$payload")" "0" "$rc" +done + +# Interpolating the same payload the old way returns the whole table. This is +# the behaviour the change removes; asserting it keeps the test honest about +# what "0" above is worth. +legacy="SELECT COUNT(*) FROM transmissions WHERE from_pubkey = '' OR 1=1 --';" +assert_eq "interpolated form leaks the table" "6" "$(run_local "$DB" <<<"$legacy")" + +# Multibyte, whitespace, empty and long values bind as themselves. +LONG_PK=$(printf 'ab%.0s' $(seq 1 5000)) +sqlite3 "$DB" "INSERT INTO transmissions(raw_hex,hash,first_seen,from_pubkey) VALUES + ('00','m1','t','héllo wörld'),('00','m2','t',''),('00','m3','t',' '),('00','m4','t','$LONG_PK');" +assert_eq "multibyte value with a space binds" "1" "$(count 'héllo wörld' "$DB")" +assert_eq "empty value binds as ''" "1" "$(count '' "$DB")" +assert_eq "whitespace value binds as itself" "1" "$(count ' ' "$DB")" +assert_eq "10000-byte value binds" "1" "$(count "$LONG_PK" "$DB")" + +out=$(run_local "$DB" <<<'DELETE FROM transmissions;' 2>"$FIXTURE_DIR/write.err"); rc=$? +assert_true 'native reader refuses writes' test "$rc" -ne 0 +assert_true 'write refusal is a read-only database error' contains "$(cat "$FIXTURE_DIR/write.err")" 'readonly' +assert_eq 'failed write preserves all ten rows' 10 "$(run_local "$DB" <<<'SELECT count(*) FROM transmissions;')" + +# REAL DATA: the committed staging-captured fixture. Its most-attributed +# pubkey, counted by the script's query, must equal a direct count. +if [ -f "$REAL_FIXTURE" ]; then + cp "$REAL_FIXTURE" "$FIXTURE_DIR/real.db" + real_row=$(run_local "$FIXTURE_DIR/real.db" <<<"SELECT from_pubkey,count(*) FROM transmissions WHERE from_pubkey IS NOT NULL GROUP BY from_pubkey ORDER BY count(*) DESC,from_pubkey LIMIT 1;") + real_pk=${real_row%|*}; real_n=${real_row##*|} + assert_match "real fixture has an attributed pubkey" '^[0-9a-f]{64}$' "$real_pk" + assert_match "real fixture count > 0" '^[1-9][0-9]*$' "$real_n" + assert_eq "script query on real fixture" "$real_n" "$(count "$real_pk" "$FIXTURE_DIR/real.db")" +else + FAIL=$((FAIL + 1)); echo "FAIL: $REAL_FIXTURE missing" >&2 +fi + +# ERROR SURFACING: a broken query must be distinguishable from an empty +# result — non-zero exit and something on stderr, not a silent "" or "0". +for bad in "$LEGACY_DB:no such column: from_pubkey" "$EMPTY_DB:no such table: transmissions"; do + bad_db=${bad%%:*}; want=${bad#*:} + err_file="$FIXTURE_DIR/err" + out=$(count "$PK_A" "$bad_db" 2>"$err_file"); rc=$? + assert_true "$want → non-zero exit (got $rc)" test "$rc" -ne 0 + assert_true "$want → named on stderr" contains "$(cat "$err_file")" "$want" + assert_eq "$want → no count on stdout" "" "$out" +done + +# ----- runner selection and transport, with ssh stubbed --------------------- +# ssh_t is replaced by a stub that logs its argv, captures stdin, and runs the +# remote command string through a real `bash -c`, so the script's own quoting +# is what gets parsed. "docker exec -i NAME" is peeled off when the stub +# container is said to have sqlite3; otherwise it fails like the app image. +STUB_ARGV="$FIXTURE_DIR/ssh.argv"; STUB_STDIN="$FIXTURE_DIR/ssh.stdin" +STUB_CONTAINER_SQLITE=0 +STUB_PATH="$PATH" +ssh_t() { + local cmd="$1" in + printf '%s\n' "$*" >>"$STUB_ARGV" + in=$(cat); printf '%s\n' "$in" >>"$STUB_STDIN" + case "$cmd" in + "docker exec -i "*) + if [ "$STUB_CONTAINER_SQLITE" != 1 ]; then + echo 'exec: "sqlite3": executable file not found in $PATH' >&2; return 126 + fi + cmd=${cmd#docker exec -i }; cmd=${cmd#* } ;; + esac + printf '%s\n' "$in" | PATH="$STUB_PATH" bash -c "$cmd" +} +reset_stub() { : >"$STUB_ARGV"; : >"$STUB_STDIN"; } +TMP="$FIXTURE_DIR"; ADMIN_API_TOKEN=""; TARGET_CONTAINER="corescope-stub" +TARGET_SSH_HOST="stub-host"; TARGET_DB_PATH="$DB"; TEST_PUBKEY="$PK_A" + +# Container has no sqlite3 → host runner, correct count. +reset_stub +read_retain_count >/dev/null 2>&1; rc=$? +assert_eq "fallback: rc" "0" "$rc" +assert_eq "fallback: runner" "host" "$SQLITE_RUNNER" +assert_eq "fallback: count" "3" "$RETAIN_COUNT" +argv=$(cat "$STUB_ARGV"); stdin=$(cat "$STUB_STDIN") +assert_true "transport: container tried first" contains "$(head -1 "$STUB_ARGV")" "docker exec -i corescope-stub sqlite3" +assert_true "transport: pubkey not in remote argv" lacks "$argv" "$PK_A" +assert_true "transport: SQL not in remote argv" lacks "$argv" "SELECT" +assert_true "transport: column not in remote argv" lacks "$argv" "from_pubkey" +assert_true "transport: SQL arrives on stdin" contains "$stdin" "SELECT COUNT(*) FROM transmissions WHERE from_pubkey = lower(:pubkey);" +assert_true "transport: pubkey arrives hex-encoded on stdin" contains "$stdin" "$(sql_hex_literal "$PK_A")" +assert_true "transport: raw pubkey never on stdin" lacks "$stdin" "$PK_A" + +# Container has sqlite3 → container runner. +reset_stub; STUB_CONTAINER_SQLITE=1 +read_retain_count >/dev/null 2>&1; rc=$? +assert_eq "container: rc" "0" "$rc" +assert_eq "container: runner" "container" "$SQLITE_RUNNER" +assert_eq "container: count" "3" "$RETAIN_COUNT" +STUB_CONTAINER_SQLITE=0 + +# A sqlite3 that cannot bind (.parameter prints help to stdout, exit 0, +# parameter left NULL) must be rejected by the probe, not trusted. +FAKE_BIN="$FIXTURE_DIR/fakebin"; mkdir -p "$FAKE_BIN" +cat >"$FAKE_BIN/sqlite3" <<'FAKE' +#!/usr/bin/env bash +cat >/dev/null +echo ".parameter CMD ... Manage SQL parameter bindings" +echo "0" +FAKE +chmod +x "$FAKE_BIN/sqlite3" +reset_stub; STUB_PATH="$FAKE_BIN:$PATH" +read_retain_count >/dev/null 2>&1; rc=$? +assert_eq "unbindable sqlite3: rejected" "1" "$rc" +assert_eq "unbindable sqlite3: no runner" "" "$SQLITE_RUNNER" +assert_eq "unbindable sqlite3: no count" "" "$RETAIN_COUNT" +STUB_PATH="$PATH" + +# A matching token from a failed client is not a successful capability probe. +cat >"$FAKE_BIN/sqlite3" <<'FAKE' +#!/usr/bin/env bash +cat >/dev/null +echo 'corescope-probe-ok' +exit 1 +FAKE +reset_stub; STUB_PATH="$FAKE_BIN:$PATH" +read_retain_count >/dev/null 2>&1; rc=$? +assert_eq 'failed client with matching probe token is rejected' 1 "$rc" +assert_eq 'failed client leaves no runner' '' "$SQLITE_RUNNER" +assert_eq 'failed client leaves no count' '' "$RETAIN_COUNT" +STUB_PATH="$PATH" + +# A query error on the target is a failure with no count, never "0". +reset_stub; TARGET_DB_PATH="$LEGACY_DB" +read_retain_count >/dev/null 2>&1; rc=$? +assert_eq "remote schema error: rc" "1" "$rc" +assert_eq "remote schema error: count" "" "$RETAIN_COUNT" +assert_true "remote schema error: stderr kept" contains "$(cat "$TMP/sqlite.err")" "no such column: from_pubkey" +TARGET_DB_PATH="$DB" + +# run_sqlite with no resolved runner must refuse rather than guess. +SQLITE_RUNNER="" +if run_sqlite /dev/null 2>&1; then + FAIL=$((FAIL + 1)); echo "FAIL: run_sqlite with no runner — expected non-zero exit" >&2 +else + PASS=$((PASS + 1)) +fi + +out=$(CORESCOPE_TEST_BACKEND=unsupported bash "$SCRIPT_DIR/test-blacklist-sql.sh" 2>&1); rc=$? +assert_eq 'unknown test backend is refused' 2 "$rc" +assert_true 'unknown backend fails before a fixture or connection' contains "$out" 'unsupported CORESCOPE_TEST_BACKEND' +assert_eq 'committed historical fixture is unchanged' "$REAL_FIXTURE_SHA" "$(sha256sum "$REAL_FIXTURE")" + +echo "test-blacklist-sqlite.sh: $PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ] From 3485ade60a3f19af1819845a0ee310a10540feba Mon Sep 17 00:00:00 2001 From: n30nex Date: Mon, 12 Oct 2026 10:31:31 -0400 Subject: [PATCH 26/67] fix(ingestor): normalize packet names before database writes --- cmd/ingestor/db.go | 8 +- cmd/ingestor/sig_validate_ingest_test.go | 111 +++++++++++++++++++++++ 2 files changed, 118 insertions(+), 1 deletion(-) diff --git a/cmd/ingestor/db.go b/cmd/ingestor/db.go index 7f4fd5931..5f67b63b3 100644 --- a/cmd/ingestor/db.go +++ b/cmd/ingestor/db.go @@ -580,6 +580,8 @@ func optionalWrite(tx *sql.Tx, fn func() error) error { // UpsertNode inserts or updates a node. func (s *Store) UpsertNode(pubKey, name, role string, lat, lon *float64, lastSeen string) error { + // Preserve the decoder's byte cap while making the name safe for SQL text. + name = strings.ToValidUTF8(name, "") ingestNow := time.Now().UTC().Format(time.RFC3339) now := lastSeen if now == "" { @@ -1324,9 +1326,13 @@ type DroppedPacket struct { // InsertDroppedPacket records a rejected packet in the dropped_packets table. func (s *Store) InsertDroppedPacket(dp *DroppedPacket) error { + // The decoder's byte cap can split a UTF-8 rune. Drop incomplete bytes + // without expanding the name past that cap; raw_hex retains the evidence. + nodeName := strings.ToValidUTF8(dp.NodeName, "") + observerName := clampObserverField(dp.ObserverName, maxObserverTextLen) _, err := s.db.Exec( `INSERT INTO dropped_packets (hash, raw_hex, reason, observer_id, observer_name, node_pubkey, node_name) VALUES (`+s.parameter(1)+`, `+s.parameter(2)+`, `+s.parameter(3)+`, `+s.parameter(4)+`, `+s.parameter(5)+`, `+s.parameter(6)+`, `+s.parameter(7)+`)`, - dp.Hash, dp.RawHex, dp.Reason, dp.ObserverID, dp.ObserverName, dp.NodePubKey, dp.NodeName, + dp.Hash, dp.RawHex, dp.Reason, dp.ObserverID, observerName, dp.NodePubKey, nodeName, ) if err != nil { s.Stats.WriteErrors.Add(1) diff --git a/cmd/ingestor/sig_validate_ingest_test.go b/cmd/ingestor/sig_validate_ingest_test.go index 6e7a29f19..2443606e4 100644 --- a/cmd/ingestor/sig_validate_ingest_test.go +++ b/cmd/ingestor/sig_validate_ingest_test.go @@ -4,8 +4,10 @@ import ( "crypto/ed25519" "encoding/binary" "encoding/hex" + "encoding/json" "strings" "testing" + "unicode/utf8" ) // buildAdvertHex constructs a full ADVERT packet hex string. @@ -166,6 +168,115 @@ func TestSigValidation_TruncatedAppdataDropped(t *testing.T) { } } +func TestSigValidation_DroppedPacketText(t *testing.T) { + cases := []struct { + name, nodeName, observerName, wantNode, wantObserver string + }{ + {"valid multibyte", strings.Repeat("é", 16), "Observer", strings.Repeat("é", 16), "Observer"}, + {"split two-byte rune", strings.Repeat("x", 31) + "é", "Observer", strings.Repeat("x", 31), "Observer"}, + {"split replacement rune", strings.Repeat("x", 30) + "\xff", "Observer", strings.Repeat("x", 30), "Observer"}, + {"split four-byte rune", strings.Repeat("x", 31) + "🌍", "Observer", strings.Repeat("x", 31), "Observer"}, + {"malformed name", "No\xffde", "Observer", "No\ufffdde", "Observer"}, + {"observer NUL", "Node", "Obs\x00erver", "Node", "Observer"}, + {"long observer", "Node", strings.Repeat("é", 129), "Node", strings.Repeat("é", maxObserverTextLen)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + store := newTestStore(t) + pub, priv, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + rawHex := buildAdvertHex(pub, priv, 1700000000, makeAppdata(tc.nodeName)) + raw, err := hex.DecodeString(rawHex) + if err != nil { + t.Fatal(err) + } + raw[38] ^= 1 // Invalidate the signature without changing the name bytes. + rawHex = hex.EncodeToString(raw) + payload, err := json.Marshal(struct { + Raw string `json:"raw"` + Origin string `json:"origin"` + }{rawHex, tc.observerName}) + if err != nil { + t.Fatal(err) + } + handleMessage(store, "test", MQTTSource{Name: "test"}, newMockMsg("meshcore/US/obs1/packet", string(payload)), nil, nil, &Config{}) + + var hash, storedRaw, nodeName, observerName string + if err := store.db.QueryRow("SELECT hash, raw_hex, node_name, observer_name FROM dropped_packets").Scan(&hash, &storedRaw, &nodeName, &observerName); err != nil { + t.Fatalf("rejected packet audit missing: %v (write errors=%d)", err, store.Stats.WriteErrors.Load()) + } + if storedRaw != rawHex || hash != ComputeContentHash(rawHex) { + t.Error("audit changed raw packet evidence or its hash") + } + if nodeName != tc.wantNode || !utf8.ValidString(nodeName) || len(nodeName) > 32 { + t.Errorf("node name=%q (%d bytes), want %q and valid UTF-8 within 32 bytes", nodeName, len(nodeName), tc.wantNode) + } + if observerName != tc.wantObserver || !utf8.ValidString(observerName) || utf8.RuneCountInString(observerName) > maxObserverTextLen { + t.Errorf("observer name=%q, want %q within observer text limit", observerName, tc.wantObserver) + } + if drops, errors := store.Stats.SignatureDrops.Load(), store.Stats.WriteErrors.Load(); drops != 1 || errors != 0 { + t.Errorf("signature drops=%d write errors=%d, want 1 and 0", drops, errors) + } + var transmissions int + if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&transmissions); err != nil || transmissions != 0 { + t.Fatalf("invalid signature stored as transmission: count=%d err=%v", transmissions, err) + } + }) + } +} + +func TestSigValidation_AcceptedNodeText(t *testing.T) { + pub, priv, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + cases := []struct { + name, nodeName, want string + update bool + }{ + {"insert split two-byte rune", strings.Repeat("x", 31) + "é", strings.Repeat("x", 31), false}, + {"update split replacement rune", strings.Repeat("x", 30) + "\xff", strings.Repeat("x", 30), true}, + {"update split four-byte rune", strings.Repeat("x", 31) + "🌍", strings.Repeat("x", 31), true}, + {"insert valid multibyte", strings.Repeat("é", 16), strings.Repeat("é", 16), false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + store := newTestStore(t) + wantUpserts := int64(1) + if tc.update { + seedHex := buildAdvertHex(pub, priv, 1700000000, makeAppdata("Original")) + handleMessage(store, "test", MQTTSource{Name: "test"}, newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+seedHex+`","origin":"Observer"}`), nil, nil, &Config{}) + if store.Stats.NodeUpserts.Load() != 1 { + t.Fatal("seed node not inserted") + } + wantUpserts++ + } + rawHex := buildAdvertHex(pub, priv, 1700000001, makeAppdata(tc.nodeName)) + msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+rawHex+`","origin":"Observer"}`) + handleMessage(store, "test", MQTTSource{Name: "test"}, msg, nil, nil, &Config{}) + + var nodeName, storedRaw string + if err := store.db.QueryRow("SELECT name FROM nodes WHERE public_key = "+store.parameter(1), hex.EncodeToString(pub)).Scan(&nodeName); err != nil { + t.Fatalf("accepted node missing: %v (write errors=%d)", err, store.Stats.WriteErrors.Load()) + } + if nodeName != tc.want || !utf8.ValidString(nodeName) || len(nodeName) > 32 { + t.Errorf("node name=%q (%d bytes), want %q and valid UTF-8 within 32 bytes", nodeName, len(nodeName), tc.want) + } + if err := store.db.QueryRow("SELECT raw_hex FROM transmissions WHERE hash = "+store.parameter(1), ComputeContentHash(rawHex)).Scan(&storedRaw); err != nil || storedRaw != rawHex { + t.Fatalf("accepted packet evidence changed: err=%v", err) + } + if upserts, errors := store.Stats.NodeUpserts.Load(), store.Stats.WriteErrors.Load(); upserts != wantUpserts || errors != 0 { + t.Errorf("node upserts=%d write errors=%d, want %d and 0", upserts, errors, wantUpserts) + } + if drops := store.Stats.SignatureDrops.Load(); drops != 0 { + t.Errorf("valid signature dropped: %d", drops) + } + }) + } +} + func TestSigValidation_DisabledByConfig(t *testing.T) { dbPath := t.TempDir() + "/test.db" store, err := openPostgresTestStoreInterval(t, dbPath, 300) From 5633088cfaeb5bb22878cd80bb5ac416d0ad9ad1 Mon Sep 17 00:00:00 2001 From: n30nex Date: Mon, 12 Oct 2026 10:58:20 -0400 Subject: [PATCH 27/67] fix(store): enforce cache budget throughout startup and history loading --- cmd/server/bounded_load_test.go | 12 +- cmd/server/chunked_load.go | 344 ++---------- cmd/server/issue1690_cold_load_test.go | 25 +- cmd/server/packet_load.go | 365 +++++++++++++ cmd/server/startup_memory_budget_test.go | 270 ++++++++++ cmd/server/startup_query_plan_test.go | 58 ++ cmd/server/store.go | 494 ++---------------- .../startup-cache-budget-2026-10-12.md | 65 +++ 8 files changed, 839 insertions(+), 794 deletions(-) create mode 100644 cmd/server/packet_load.go create mode 100644 cmd/server/startup_memory_budget_test.go create mode 100644 cmd/server/startup_query_plan_test.go create mode 100644 docs/performance/startup-cache-budget-2026-10-12.md diff --git a/cmd/server/bounded_load_test.go b/cmd/server/bounded_load_test.go index 639405547..1c5379429 100644 --- a/cmd/server/bounded_load_test.go +++ b/cmd/server/bounded_load_test.go @@ -40,8 +40,8 @@ func TestBoundedLoad_LimitedMemory(t *testing.T) { if loaded >= 5000 { t.Errorf("expected bounded load to limit packets, got %d/5000", loaded) } - if loaded < 1000 { - t.Errorf("expected at least 1000 packets (minimum), got %d", loaded) + if loaded == 0 || store.trackedBytes > 1048576 { + t.Errorf("expected a nonempty cache within 1 MiB, got %d packets / %d bytes", loaded, store.trackedBytes) } t.Logf("Loaded %d/5000 packets with 1MB budget", loaded) } @@ -84,8 +84,12 @@ func TestBoundedLoad_OldestLoadedSet(t *testing.T) { if store.oldestLoaded == "" { t.Fatal("oldestLoaded should be set after bounded load") } - if len(store.packets) > 0 && store.oldestLoaded != store.packets[0].FirstSeen { - t.Errorf("oldestLoaded (%s) should match first packet (%s)", store.oldestLoaded, store.packets[0].FirstSeen) + if len(store.packets) > 0 && store.oldestLoaded > store.packets[0].FirstSeen { + t.Errorf("coverage floor (%s) should include first packet (%s)", store.oldestLoaded, store.packets[0].FirstSeen) + } + oldestOmitted := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).Add(time.Duration(5000-len(store.packets)) * time.Minute).Format(time.RFC3339) + if store.oldestLoaded <= oldestOmitted { + t.Errorf("coverage floor %s claims omitted packet %s", store.oldestLoaded, oldestOmitted) } t.Logf("oldestLoaded = %s", store.oldestLoaded) } diff --git a/cmd/server/chunked_load.go b/cmd/server/chunked_load.go index 079e0a305..485e579a1 100644 --- a/cmd/server/chunked_load.go +++ b/cmd/server/chunked_load.go @@ -3,9 +3,10 @@ package main // Chunked startup load + early HTTP readiness for issue #1009. // // Design: -// * LoadChunked paginates transmissions in id-ordered chunks of +// * LoadChunked paginates transmissions in bounded chunks of // `chunkSize` (default 10000 via Config.DBLoadChunkSize). After the // first chunk is merged into the store, FirstChunkReady is closed. +// Memory-capped stores prioritize newest rows; unlimited stores use IDs. // main.go binds the HTTP listener on that signal and serves // partial data while remaining chunks stream in the background. // * RunStartupLoad is the orchestrator: it runs LoadChunked @@ -26,11 +27,9 @@ package main // HTTP handlers (which take s.mu.RLock) stay responsive. import ( - "database/sql" "fmt" "log" "net/http" - "sort" "strings" "sync" "sync/atomic" @@ -187,8 +186,8 @@ func (s *PacketStore) fireChunkCallbacks(rowsThisChunk, totalRows int) { // (no background work was needed). Coverage is computed against // row count so /api/perf does not report ratio=0.0 (dij #2). // - hotStartupHours > 0 success: terminal state is whatever -// loadBackgroundChunks set (done=true on full coverage, -// failed=true on partial / chunk errors — see #1690). +// loadBackgroundChunks set (done=true on full coverage or the configured +// memory limit; failed=true on unexpected partial coverage/chunk errors). // // Issue #1809 root cause: previously main.go spawned loadBackgroundChunks // at FirstChunkReady while LoadChunked was still merging the remainder @@ -266,7 +265,7 @@ func (s *PacketStore) RunStartupLoad(chunkSize int) error { } // LoadChunked streams transmissions + observations from PostgreSQL into -// the in-memory store in id-ordered chunks of `chunkSize` rows. Pass +// the in-memory store in bounded chunks of `chunkSize` rows. Pass // 0 to use the default (10000). // // After the first chunk is merged, FirstChunkReady is closed and the @@ -353,152 +352,47 @@ func (s *PacketStore) LoadChunked(chunkSize int) error { totalInDB = -1 } - // Memory cap honoured by clamping the maximum cursor walk. - var maxPackets int64 + // Polling starts after the first chunk. Capture the complete database's + // high-water marks before loading a subset, so older omitted rows are not + // mistaken for new ingest and replayed into the cache immediately. if s.maxMemoryMB > 0 { - avgBytes := int64(1000) - if sample := estimateStoreTxBytesTypical(10); sample > avgBytes { - avgBytes = sample - } - maxPackets = (int64(s.maxMemoryMB) * 1048576) / avgBytes - if maxPackets < 1000 { - maxPackets = 1000 + var maxTxID, maxObsID int + if err := s.db.conn.QueryRow(`SELECT + (SELECT COALESCE(MAX(id), 0) FROM transmissions), + (SELECT COALESCE(MAX(id), 0) FROM observations)`).Scan(&maxTxID, &maxObsID); err != nil { + return fmt.Errorf("load watermarks: %w", err) } + s.mu.Lock() + s.maxTxID, s.maxObsID = maxTxID, maxObsID + s.mu.Unlock() + } + var from time.Time + if hotCutoffStr != "" { + from, _ = time.Parse(time.RFC3339, hotCutoffStr) } - chunkIdx := 0 - totalLoaded := 0 - // Start the id cursor below the legacy fixture's zero ID so the - // first chunk's `t2.id > cursorID` predicate includes id=0. The - // e2e fixture seed for issue #1486 inserts the grouped-packet row - // with id=0 (so it sorts LAST in the default packets view via - // `ORDER BY id DESC` / oldest first_seen). Seeding the cursor at - // 0 silently excluded that row, leaving the page with no - // tr[data-hash] and timing out the playwright wait. Legacy Load() - // had no id cursor and loaded id=0 unconditionally — we restore - // that semantic by starting at -1. - var cursorID int64 = -1 - - // Relay-hop fallback inputs, fetched ONCE before the chunk-query loop. - // getCachedNodesAndPM issues its own DB query, so calling it while a - // chunk cursor is open would deadlock on a single-connection database - // pool. resolved_path is never persisted post-#1287, so scanAndMergeChunk - // re-resolves relay hops from path_json using these snapshots. - // PR #1643 R1 munger #1: cold load uses unique_prefix-only gate, so - // the neighbor graph is no longer consulted here (affinity-tier - // resolution against ≤168h-old observations would silently mis-attribute). - s.mu.RLock() - _, relayPM := s.getCachedNodesAndPM() - s.mu.RUnlock() - var coldLoadAmbiguousHopsSkipped int - - for { - conds := append([]string{}, loadConditions...) - conds = append(conds, fmt.Sprintf("t2.id > %d", cursorID)) - whereClause := "WHERE " + strings.Join(conds, " AND ") - - rpCol := "" - if s.db.hasResolvedPath { - rpCol = ", o.resolved_path" - } - obsRawHexCol := "" - if s.db.hasObsRawHex { - obsRawHexCol = ", o.raw_hex" - } - // #1751: scope_name is on the transmission row, so appending it as the - // last selected column is safe regardless of the observation fan-out. - scopeNameCol := "" - if s.db.hasScopeName { - scopeNameCol = ", t.scope_name" - } - - var chunkSQL string - if s.db.isV3 { - chunkSQL = s.db.nativeSQL(`SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, - t.payload_type, t.payload_version, t.decoded_json, - o.id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, - o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')`, `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, - t.payload_type, t.payload_version, t.decoded_json, - o.id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, - o.snr, o.rssi, o.score, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`) + obsRawHexCol + rpCol + scopeNameCol + ` - FROM (SELECT * FROM transmissions t2 ` + whereClause + ` ORDER BY t2.id ASC LIMIT ` + fmt.Sprintf("%d", chunkSize) + `) AS t - LEFT JOIN observations o ON o.transmission_id = t.id - LEFT JOIN observers obs ON obs.rowid = o.observer_idx - ORDER BY t.id ASC, o.timestamp DESC` - } else { - chunkSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, - t.payload_type, t.payload_version, t.decoded_json, - o.id, o.observer_id, o.observer_name, COALESCE(obs.iata, ''), o.direction, - o.snr, o.rssi, o.score, o.path_json, o.timestamp` + obsRawHexCol + rpCol + scopeNameCol + ` - FROM (SELECT * FROM transmissions t2 ` + whereClause + ` ORDER BY t2.id ASC LIMIT ` + fmt.Sprintf("%d", chunkSize) + `) AS t - LEFT JOIN observations o ON o.transmission_id = t.id - LEFT JOIN observers obs ON obs.id = o.observer_id - ORDER BY t.id ASC, o.timestamp DESC` - } - - // Acquire before opening the cursor: a feed waiting for the same - // connection must never hold this gate while we hold its cursor. - s.advertEvidenceMu.Lock() - rows, err := s.db.conn.Query(chunkSQL) - if err != nil { - s.advertEvidenceMu.Unlock() - return fmt.Errorf("chunk %d: query: %w", chunkIdx, err) - } - - chunkTxCount, lastID, err := s.scanAndMergeChunk(rows, relayPM, &coldLoadAmbiguousHopsSkipped) - rows.Close() - s.advertEvidenceMu.Unlock() - if err != nil { - return fmt.Errorf("chunk %d: scan: %w", chunkIdx, err) - } - - if chunkTxCount == 0 { - break - } - - cursorID = lastID - totalLoaded += chunkTxCount + totalLoaded, err := s.loadPacketRange(from, time.Time{}, chunkSize, func(count, total int) { chunkIdx++ - s.loadProgressRows.Store(int64(totalLoaded)) + s.loadProgressRows.Store(int64(total)) s.signalFirstChunk() - s.fireChunkCallbacks(chunkTxCount, totalLoaded) - - if maxPackets > 0 && int64(totalLoaded) >= maxPackets { - break - } - if chunkTxCount < chunkSize { - break - } + s.fireChunkCallbacks(count, total) + }) + if err != nil { + return fmt.Errorf("chunk %d: %w", chunkIdx, err) } - // Post-load: pick best observation, build indexes — same shape as - // legacy Load(). + // Each chunk publishes complete packets and indexes atomically in + // first_seen order. Build the derived analytics indexes once at the end. s.mu.Lock() - for _, tx := range s.packets { - pickBestObservation(tx) - s.trackedBytes += rechargeTx(tx) - s.indexByNode(tx) - } - // Restore the "s.packets sorted oldest-first by FirstSeen" invariant - // that legacy Load() got for free from "ORDER BY t.first_seen ASC". - // LoadChunked walks chunks in id-ASC order so the slice ends up - // id-ordered, which only equals first_seen-ordered when ids and - // timestamps are correlated. After tools/freshen-fixture.sh (or any - // real-world out-of-order ingest) they're not, leaving - // s.packets[0].FirstSeen pointing at the newest row — which then - // poisons oldestLoaded below and routes legitimate in-memory queries - // to the SQL fallback. GetTimestamps (store.go) and QueryPackets - // both rely on this invariant. See PR #1596 / mobile e2e regression. - sort.SliceStable(s.packets, func(i, j int) bool { - return s.packets[i].FirstSeen < s.packets[j].FirstSeen - }) s.buildSubpathIndex() s.buildPathHopIndex() s.buildDistanceIndex() - if s.hotStartupHours > 0 { - s.oldestLoaded = hotCutoffStr - } else if len(s.packets) > 0 { - s.oldestLoaded = s.packets[0].FirstSeen + if !s.startupMemoryLimited { + if s.hotStartupHours > 0 { + s.oldestLoaded = hotCutoffStr + } else if len(s.packets) > 0 { + s.oldestLoaded = s.packets[0].FirstSeen + } } s.loaded = true s.mu.Unlock() @@ -514,10 +408,6 @@ func (s *PacketStore) LoadChunked(chunkSize int) error { elapsed := time.Since(t0) log.Printf("[store] LoadChunked: %d transmissions (%d observations) across %d chunk(s) in %v (chunkSize=%d, DB total=%d)", totalLoaded, s.totalObs, chunkIdx, elapsed, chunkSize, totalInDB) - if coldLoadAmbiguousHopsSkipped > 0 { - log.Printf("[store] LoadChunked: skipped %d ambiguous-prefix relay hops (unique_prefix gate, PR #1643 R1)", - coldLoadAmbiguousHopsSkipped) - } s.loadMultibyteCapFromDB() // Mark complete on the success path only — see the function-level // defer above for why this is NOT in a deferred call. Probes that @@ -527,172 +417,6 @@ func (s *PacketStore) LoadChunked(chunkSize int) error { return nil } -// scanAndMergeChunk consumes one chunk's rows under s.mu.Lock and -// returns the number of distinct transmissions seen + the max -// transmission id (cursor for the next chunk). -func (s *PacketStore) scanAndMergeChunk(rows *sql.Rows, relayPM *prefixMap, coldLoadAmbiguousHopsSkipped *int) (int, int64, error) { - s.mu.Lock() - defer s.mu.Unlock() - - hopsSeen := make(map[string]bool) - seenTxIDs := make(map[int]bool) - var maxID int64 - - for rows.Next() { - var txID int - var rawHex, hash, firstSeen, decodedJSON sql.NullString - var routeType, payloadType, payloadVersion sql.NullInt64 - var obsID sql.NullInt64 - var observerID, observerName, observerIATA, direction, pathJSON, obsTimestamp sql.NullString - var snr, rssi sql.NullFloat64 - var score sql.NullFloat64 - var obsRawHex sql.NullString - var resolvedPathStr sql.NullString - var scopeName sql.NullString - - scanArgs := []interface{}{&txID, &rawHex, &hash, &firstSeen, &routeType, &payloadType, - &payloadVersion, &decodedJSON, - &obsID, &observerID, &observerName, &observerIATA, &direction, - &snr, &rssi, &score, &pathJSON, &obsTimestamp} - if s.db.hasObsRawHex { - scanArgs = append(scanArgs, &obsRawHex) - } - if s.db.hasResolvedPath { - scanArgs = append(scanArgs, &resolvedPathStr) - } - if s.db.hasScopeName { - scanArgs = append(scanArgs, &scopeName) - } - if err := rows.Scan(scanArgs...); err != nil { - log.Printf("[store] LoadChunked scan error: %v", err) - continue - } - - if int64(txID) > maxID { - maxID = int64(txID) - } - seenTxIDs[txID] = true - - hashStr := nullStrVal(hash) - tx := s.byHash[hashStr] - if tx == nil { - tx = &StoreTx{ - ID: txID, - RawHex: nullStrVal(rawHex), - Hash: hashStr, - FirstSeen: nullStrVal(firstSeen), - LatestSeen: nullStrVal(firstSeen), - RouteType: nullIntPtr(routeType), - PayloadType: nullIntPtr(payloadType), - DecodedJSON: nullStrVal(decodedJSON), - ScopeName: nullStrPtr(scopeName), - obsKeys: make(map[string]bool), - observerSet: make(map[string]bool), - } - s.byHash[hashStr] = tx - s.packets = append(s.packets, tx) - s.byTxID[txID] = tx - if txID > s.maxTxID { - s.maxTxID = txID - } - s.indexByNode(tx) - if tx.PayloadType != nil { - pt := *tx.PayloadType - s.byPayloadType[pt] = append(s.byPayloadType[pt], tx) - } - s.trackAdvertPubkey(tx) - s.trackedBytes += rechargeTx(tx) - } - - if obsID.Valid { - oid := int(obsID.Int64) - obsIDStr := nullStrVal(observerID) - obsPJ := nullStrVal(pathJSON) - - dk := obsIDStr + "|" + obsPJ - if tx.obsKeys[dk] { - continue - } - - obs := &StoreObs{ - ID: oid, - TransmissionID: txID, - ObserverID: obsIDStr, - ObserverName: nullStrVal(observerName), - ObserverIATA: nullStrVal(observerIATA), - Direction: nullStrVal(direction), - SNR: nullFloatPtr(snr), - RSSI: nullFloatPtr(rssi), - Score: nullFloatPtr(score), - PathJSON: obsPJ, - // Raw frames stay in the database; hash equality does not imply route - // equality. Only compact advert evidence is retained per tx. - // Packet-detail queries can read the original observation raw. - Timestamp: normalizeTimestamp(nullStrVal(obsTimestamp)), - } - - rpStr := nullStrVal(resolvedPathStr) - if rpStr != "" { - rp := unmarshalResolvedPath(rpStr) - pks := extractResolvedPubkeys(rp) - s.indexResolvedPathHops(tx, pks, hopsSeen) - } else if relayPM != nil && obsPJ != "" && obsPJ != "[]" { - // resolved_path is NULL on live (since #1287 relay data is - // persisted as neighbor_edges, not per-observation). Re-resolve - // relay-hop attribution from path_json so relay nodes keep their - // analytics history across a restart instead of rebuilding only - // from post-restart live traffic. relayPM is passed in from - // LoadChunked (fetched before any chunk cursor opened). - // byNode ONLY — see the Load() counterpart for why the - // resolved_path/path-hop indexes must NOT be populated here. - // PR #1643 R1 munger #1: unique_prefix-only gate. - rp := resolvePathForObsColdLoad(obsPJ, obsIDStr, tx, relayPM, coldLoadAmbiguousHopsSkipped) - for _, pk := range extractResolvedPubkeys(rp) { - s.addToByNode(tx, pk) - } - } - - tx.Observations = append(tx.Observations, obs) - tx.obsKeys[dk] = true - if obs.ObserverID != "" && !tx.observerSet[obs.ObserverID] { - tx.observerSet[obs.ObserverID] = true - tx.UniqueObserverCount++ - } - tx.ObservationCount++ - if obs.Timestamp > tx.LatestSeen { - tx.LatestSeen = obs.Timestamp - } - - s.byObsID[oid] = obs - if oid > s.maxObsID { - s.maxObsID = oid - } - if obsIDStr != "" { - s.byObserver[obsIDStr] = append(s.byObserver[obsIDStr], obs) - } - s.totalObs++ - s.trackedBytes += estimateStoreObsBytes(obs) - } - } - if err := rows.Err(); err != nil { - return len(seenTxIDs), maxID, err - } - rows.Close() - txs := make([]*StoreTx, 0, len(seenTxIDs)) - for id := range seenTxIDs { - txs = append(txs, s.byTxID[id]) - } - ids := advertTxIDs(txs) - s.mu.Unlock() - masks, err := s.db.advertEvidenceForIDs(ids) - s.mu.Lock() - if err != nil { - return len(seenTxIDs), maxID, err - } - s.mergeAdvertEvidence(masks) - return len(seenTxIDs), maxID, nil -} - // loadStatusMiddleware sets X-CoreScope-Load-Status on every response. // While LoadChunked is in flight the header reports // "loading; progress="; after completion it reports "ready". diff --git a/cmd/server/issue1690_cold_load_test.go b/cmd/server/issue1690_cold_load_test.go index 23f6cba39..9fc02bacd 100644 --- a/cmd/server/issue1690_cold_load_test.go +++ b/cmd/server/issue1690_cold_load_test.go @@ -14,7 +14,6 @@ package main import ( "encoding/json" - "strings" "testing" "time" ) @@ -77,10 +76,8 @@ func Test1690_ColdLoad_TimeAxis(t *testing.T) { } } -// Test1690_BackgroundLoadHonesty seeds 1000 transmissions but caps the -// store's memory budget so it can only fit a fraction. After -// loadBackgroundChunks runs, backgroundLoadDone must be FALSE and -// backgroundLoadFailed must be TRUE because actual coverage is < 90%. +// A configured memory cap is successful bounded loading, not a failed startup. +// Keep reporting actual database coverage rather than pretending it is 100%. func Test1690_BackgroundLoadHonesty(t *testing.T) { dbPath := postgresTestDSN(t) @@ -107,20 +104,14 @@ func Test1690_BackgroundLoadHonesty(t *testing.T) { } store.loadBackgroundChunks() - if store.backgroundLoadDone.Load() { - t.Errorf("backgroundLoadDone=true with only %d/5000 packets loaded; "+ - "must be false until coverage ≥ 90%%", len(store.packets)) + if !store.backgroundLoadDone.Load() { + t.Error("memory-bounded background load must reach a terminal state") } - if !store.backgroundLoadFailed.Load() { - t.Errorf("backgroundLoadFailed=false despite under-coverage "+ - "(%d/5000 packets loaded); must be true with a reason", len(store.packets)) + if store.backgroundLoadFailed.Load() || store.BackgroundLoadError() != "" { + t.Errorf("configured memory budget treated as failure: %s", store.BackgroundLoadError()) } - // The error message must mention a percentage so operators can see - // the actual ratio surface in the perf endpoint. - errMsg := store.BackgroundLoadError() - if !strings.Contains(errMsg, "%") { - t.Errorf("backgroundLoadError=%q; expected human-readable ratio "+ - "(e.g. 'loaded X%% of Y rows')", errMsg) + if got, want := store.LoadCoverageRatio(), float64(len(store.packets))/5000; got != want || got >= 0.9 { + t.Errorf("coverage = %g, want actual bounded fraction %g", got, want) } } diff --git a/cmd/server/packet_load.go b/cmd/server/packet_load.go new file mode 100644 index 000000000..7c2d03978 --- /dev/null +++ b/cmd/server/packet_load.go @@ -0,0 +1,365 @@ +package main + +import ( + "database/sql" + "fmt" + "strconv" + "strings" + "time" +) + +type packetLoadCursor struct { + id int + recency string +} + +type loadedPacket struct { + tx *StoreTx + cursor packetLoadCursor + bytes int64 + hops int + resolved map[string]bool + nodes map[string]bool +} + +// loadPacketRange is shared by the hot and background loaders. The SQL cursor +// walks newest-first for a bounded cache, including timestamp ties and ID zero. +// Unlimited loads keep the indexed ID walk; they do not need recency sorting. +// Memory is charged +// per observation, not from an assumed average fanout. An over-budget packet is +// discarded before publishing any of its indexes; SQL still owns its full data. +func (s *PacketStore) loadPacketRange(from, to time.Time, chunkSize int, onChunk func(int, int)) (int, error) { + _, relayPM := s.getCachedNodesAndPM() + var cursor packetLoadCursor + total := 0 + for { + s.mu.RLock() + limited := s.startupMemoryLimited + s.mu.RUnlock() + if limited { + return total, nil + } + count, next, err := s.loadPacketChunk(from, to, cursor, chunkSize, relayPM) + if err != nil { + return total, err + } + if count == 0 { + return total, nil + } + total += count + if onChunk != nil { + onChunk(count, total) + } + if count < chunkSize { + return total, nil + } + cursor = next + } +} + +func (s *PacketStore) packetLoadQuery(from, to time.Time, cursor packetLoadCursor, limit int) (string, []any) { + axis := "first_seen" + if s.db.hasLastSeen { + axis = "last_seen" + } + var conditions []string + var args []any + parameter := func(value any) string { + args = append(args, value) + return s.db.parameter(len(args)) + } + timestamp := func(t time.Time) any { + if s.db.hasLastSeen { + return t.Unix() + } + return t.UTC().Format(time.RFC3339) + } + if !from.IsZero() { + conditions = append(conditions, "t2."+axis+" >= "+parameter(timestamp(from))) + } + if !to.IsZero() { + conditions = append(conditions, "t2."+axis+" < "+parameter(timestamp(to))) + } + if cursor.recency != "" { + if s.maxMemoryMB > 0 { + // Separate bind slots also work with SQLite's positional '?' syntax. + conditions = append(conditions, "(t2."+axis+" < "+parameter(cursor.recency)+" OR (t2."+axis+" = "+parameter(cursor.recency)+" AND t2.id < "+parameter(cursor.id)+"))") + } else { + conditions = append(conditions, "t2.id > "+parameter(cursor.id)) + } + } + order := "id ASC" + if s.maxMemoryMB > 0 { + order = axis + " DESC, %s.id DESC" + } + innerOrder := strings.ReplaceAll("t2."+order, "%s", "t2") + outerOrder := strings.ReplaceAll("t."+order, "%s", "t") + where := "" + if len(conditions) > 0 { + where = " WHERE " + strings.Join(conditions, " AND ") + } + query := `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, + t.payload_type, t.decoded_json, o.id, ` + if s.db.isV3 { + query += s.db.nativeSQL(`obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, + o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')`, + `obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, + o.snr, o.rssi, o.score, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`) + } else { + query += `o.observer_id, o.observer_name, COALESCE(obs.iata, ''), o.direction, + o.snr, o.rssi, o.score, o.path_json, o.timestamp` + } + if s.db.hasResolvedPath { + query += ", o.resolved_path" + } + if s.db.hasScopeName { + query += ", t.scope_name" + } + query += ", t." + axis + ` FROM (SELECT * FROM transmissions t2` + where + + " ORDER BY " + innerOrder + " LIMIT " + strconv.Itoa(limit) + `) AS t + LEFT JOIN observations o ON o.transmission_id = t.id` + if s.db.isV3 { + query += " LEFT JOIN observers obs ON obs.rowid = o.observer_idx" + } else { + query += " LEFT JOIN observers obs ON obs.id = o.observer_id" + } + query += " ORDER BY " + outerOrder + ", o.timestamp DESC, o.id DESC" + + return query, args +} + +func (s *PacketStore) loadPacketChunk(from, to time.Time, cursor packetLoadCursor, limit int, relayPM *prefixMap) (int, packetLoadCursor, error) { + query, args := s.packetLoadQuery(from, to, cursor, limit) + + // Serialize with live advert ingestion before opening a cursor, but do not + // hold the packet-store lock while doing SQL or decoding observations. + s.advertEvidenceMu.Lock() + defer s.advertEvidenceMu.Unlock() + rows, err := s.db.conn.Query(query, args...) + if err != nil { + return 0, cursor, err + } + defer rows.Close() + s.mu.RLock() + remaining := int64(s.maxMemoryMB)*1048576 - s.trackedBytes + s.mu.RUnlock() + var packets []loadedPacket + var current *loadedPacket + var currentCursor packetLoadCursor + var bytes int64 + limited := false + seen := 0 + skippedHops := 0 + finish := func() { + if current == nil { + return + } + pickBestObservation(current.tx) + current.bytes += rechargeTx(current.tx) + packets = append(packets, *current) + bytes += current.bytes + cursor = currentCursor + current = nil + } + for rows.Next() { + var id int + var raw, hash, first, decoded, recency sql.NullString + var route, payload, obsID sql.NullInt64 + var observer, name, iata, direction, path, stamp, resolved, scope sql.NullString + var snr, rssi, score sql.NullFloat64 + scan := []any{&id, &raw, &hash, &first, &route, &payload, &decoded, + &obsID, &observer, &name, &iata, &direction, &snr, &rssi, &score, &path, &stamp} + if s.db.hasResolvedPath { + scan = append(scan, &resolved) + } + if s.db.hasScopeName { + scan = append(scan, &scope) + } + scan = append(scan, &recency) + if err := rows.Scan(scan...); err != nil { + return 0, cursor, err + } + if current == nil || current.tx.ID != id { + finish() + seen++ + currentCursor = packetLoadCursor{id: id, recency: recency.String} + current = &loadedPacket{cursor: currentCursor, tx: &StoreTx{ + ID: id, RawHex: raw.String, Hash: hash.String, FirstSeen: first.String, LatestSeen: first.String, + RouteType: nullIntPtr(route), PayloadType: nullIntPtr(payload), DecodedJSON: decoded.String, + ScopeName: nullStrPtr(scope), obsKeys: make(map[string]bool), observerSet: make(map[string]bool), + }, resolved: make(map[string]bool), nodes: make(map[string]bool)} + current.bytes = rechargeTx(current.tx) + } + tx := current.tx + if obsID.Valid { + key := observer.String + "|" + path.String + if tx.obsKeys[key] { + continue + } + obs := &StoreObs{ID: int(obsID.Int64), TransmissionID: id, + ObserverID: observer.String, ObserverName: name.String, ObserverIATA: iata.String, + Direction: direction.String, SNR: nullFloatPtr(snr), RSSI: nullFloatPtr(rssi), Score: nullFloatPtr(score), + PathJSON: path.String, Timestamp: normalizeTimestamp(stamp.String)} + tx.Observations = append(tx.Observations, obs) + tx.obsKeys[key] = true + if observer.String != "" && !tx.observerSet[observer.String] { + tx.observerSet[observer.String] = true + tx.UniqueObserverCount++ + } + tx.ObservationCount++ + if obs.Timestamp > tx.LatestSeen { + tx.LatestSeen = obs.Timestamp + } + current.bytes += estimateStoreObsBytes(obs) + if hops := pathLen(path.String); len(tx.Observations) == 1 || hops > current.hops { + current.hops = hops + tx.PathJSON = path.String + tx.pathParsed = false + current.bytes += rechargeTx(tx) + } + if resolved.String != "" { + for _, pk := range extractResolvedPubkeys(unmarshalResolvedPath(resolved.String)) { + current.resolved[pk] = true + } + } else if relayPM != nil && path.String != "" && path.String != "[]" { + for _, pk := range extractResolvedPubkeys(resolvePathForObsColdLoad(path.String, observer.String, tx, relayPM, &skippedHops)) { + current.nodes[pk] = true + } + } + } + if s.maxMemoryMB > 0 && bytes+current.bytes > remaining { + limited = true + break + } + } + if err := rows.Err(); err != nil { + return 0, cursor, err + } + rows.Close() + if !limited { + finish() + } + ids := make([]int, 0, len(packets)) + for _, packet := range packets { + if packet.tx.PayloadType != nil && *packet.tx.PayloadType == PayloadADVERT { + ids = append(ids, packet.tx.ID) + } + } + masks, err := s.db.advertEvidenceForIDs(ids) + if err != nil { + return 0, cursor, err + } + + s.mu.Lock() + defer s.mu.Unlock() + accepted := make([]*StoreTx, 0, len(packets)) + hopsSeen := make(map[string]bool) + changed := false + for _, packet := range packets { + tx := packet.tx + existing := s.byHash[tx.Hash] + observations := tx.Observations + charge := packet.bytes + if existing != nil { + observations = make([]*StoreObs, 0, len(tx.Observations)) + charge = 0 + for _, obs := range tx.Observations { + if !existing.obsKeys[obs.ObserverID+"|"+obs.PathJSON] { + observations = append(observations, obs) + charge += estimateStoreObsBytes(obs) + } + } + // Estimate the winning path before mutating the live pointer. Copy + // only the fields the estimator needs, never the decode sync.Once. + candidate := StoreTx{RawHex: existing.RawHex, Hash: existing.Hash, + DecodedJSON: existing.DecodedJSON, PathJSON: existing.PathJSON} + if (len(existing.Observations) == 0 && len(observations) > 0) || packet.hops > pathLen(existing.PathJSON) { + candidate.PathJSON = tx.PathJSON + candidate.pathParsed = false + } + charge += estimateStoreTxBytes(&candidate) - existing.accountedBytes + } + // Live observations can consume budget while the SQL cursor is open. + if s.maxMemoryMB > 0 && s.trackedBytes+charge > int64(s.maxMemoryMB)*1048576 { + limited = true + currentCursor = packet.cursor + break + } + if existing == nil { + s.byHash[tx.Hash], s.byTxID[tx.ID] = tx, tx + accepted = append(accepted, tx) + if tx.PayloadType != nil { + s.byPayloadType[*tx.PayloadType] = append(s.byPayloadType[*tx.PayloadType], tx) + } + s.trackAdvertPubkey(tx) + } else { + tx = existing + for _, obs := range observations { + tx.Observations = append(tx.Observations, obs) + tx.obsKeys[obs.ObserverID+"|"+obs.PathJSON] = true + if obs.ObserverID != "" && !tx.observerSet[obs.ObserverID] { + tx.observerSet[obs.ObserverID] = true + tx.UniqueObserverCount++ + } + tx.ObservationCount++ + if obs.Timestamp > tx.LatestSeen { + tx.LatestSeen = obs.Timestamp + } + } + pickBestObservation(tx) + // The preflight charge above includes this same path delta. + rechargeTx(tx) + } + changed = unionAdvertEvidence(tx, masks[tx.ID]) || changed || len(observations) > 0 + for _, obs := range observations { + s.byObsID[obs.ID] = obs + if obs.ObserverID != "" { + s.byObserver[obs.ObserverID] = append(s.byObserver[obs.ObserverID], obs) + } + if s.maxMemoryMB <= 0 && obs.ID > s.maxObsID { + s.maxObsID = obs.ID + } + } + if s.maxMemoryMB <= 0 && tx.ID > s.maxTxID { + s.maxTxID = tx.ID + } + s.indexByNode(tx) + pks := make([]string, 0, len(packet.resolved)) + for pk := range packet.resolved { + pks = append(pks, pk) + } + s.indexResolvedPathHops(tx, pks, hopsSeen) + for pk := range packet.nodes { + s.addToByNode(tx, pk) + } + s.totalObs += len(observations) + s.trackedBytes += charge + } + if len(accepted) > 0 { + s.packets = mergeChunkIntoPackets(accepted, s.packets) + } + if changed { + s.invalidateAdvertEvidence() + } + if limited { + s.startupMemoryLimited = true + // A partial timestamp group is not complete coverage. Use the next + // second as a conservative boundary, including last_seen's second + // precision when first_seen itself contains fractional seconds. + var omitted time.Time + if s.db.hasLastSeen { + unix, err := strconv.ParseInt(currentCursor.recency, 10, 64) + if err != nil { + return 0, cursor, fmt.Errorf("load cursor timestamp: %w", err) + } + omitted = time.Unix(unix, 0) + } else { + omitted, err = time.Parse(time.RFC3339Nano, currentCursor.recency) + if err != nil { + return 0, cursor, fmt.Errorf("load cursor timestamp: %w", err) + } + } + s.oldestLoaded = omitted.Add(time.Second).UTC().Format(time.RFC3339) + } + return seen, cursor, nil +} diff --git a/cmd/server/startup_memory_budget_test.go b/cmd/server/startup_memory_budget_test.go new file mode 100644 index 000000000..ca26a15e0 --- /dev/null +++ b/cmd/server/startup_memory_budget_test.go @@ -0,0 +1,270 @@ +package main + +import ( + "fmt" + "runtime" + "testing" + "time" +) + +// Each transmission has many distinct observer/path pairs. A transmission-count +// estimate cannot bound this fixture: every row is retained by the real deduper. +func startupMemoryFixture(t testing.TB, transmissions, observations int, recent bool) *DB { + t.Helper() + dsn := postgresTestDSN(t) + createTestDBAt(t, dsn, 0) + w, err := openFixtureSQL(dsn) + if err != nil { + t.Fatal(err) + } + defer w.Close() + if _, err := w.Exec("ALTER TABLE transmissions ADD COLUMN last_seen BIGINT"); err != nil { + t.Fatal(err) + } + // Use the production v3 epoch timestamp/observer-index representation, so + // SQL fallback and PostgreSQL parameter types are exercised realistically. + if _, err := w.Exec("DROP TABLE observations"); err != nil { + t.Fatal(err) + } + if _, err := w.Exec(`CREATE TABLE observations ( + id BIGINT PRIMARY KEY, transmission_id BIGINT, observer_idx BIGINT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp BIGINT, raw_hex TEXT + )`); err != nil { + t.Fatal(err) + } + now := time.Now().UTC().Truncate(time.Second) + stamp := func(n int) time.Time { + if recent || n == 0 { + return now.Add(-30 * time.Minute) + } + return now.Add(-2 * time.Hour).Add(-time.Duration(n/3) * time.Second) + } + copyTestRows(t, w, "transmissions", []string{"id", "raw_hex", "hash", "first_seen", "route_type", "payload_type", "decoded_json", "last_seen"}, transmissions, func(n int) []any { + return []any{n, "aabb", fmt.Sprintf("budget-%04d", n), stamp(n).Format(time.RFC3339), 0, 4, "{}", stamp(n).Unix()} + }) + copyTestRows(t, w, "observers", []string{"rowid", "id", "name"}, observations, func(n int) []any { + return []any{n, fmt.Sprintf("observer-%04d", n), "Synthetic observer"} + }) + copyTestRows(t, w, "observations", []string{"id", "transmission_id", "observer_idx", "direction", "snr", "rssi", "score", "path_json", "timestamp"}, transmissions*observations, func(n int) []any { + return []any{n + 1, n / observations, n % observations, "rx", 5.0, -80.0, 5, `["aa","bb","cc","dd","ee","ff","ab","bc","cd","de","ef","fa"]`, stamp(n / observations).Unix()} + }) + for _, query := range []string{ + "CREATE INDEX budget_observations ON observations(transmission_id, timestamp)", + } { + if _, err := w.Exec(query); err != nil { + t.Fatal(err) + } + } + r, err := openFixtureReader(t, dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { r.conn.Close() }) + return r +} + +func assertStartupBudget(t *testing.T, s *PacketStore, observations int) { + t.Helper() + if s.trackedBytes > int64(s.maxMemoryMB)*1048576 { + t.Fatalf("cache exceeds %d MiB budget: %.2f MiB, %d transmissions/%d observations", s.maxMemoryMB, s.trackedMemoryMB(), len(s.packets), s.totalObs) + } + for i, tx := range s.packets { + if len(tx.Observations) != observations { + t.Fatalf("partial transmission %d cached: %d/%d observations", tx.ID, len(tx.Observations), observations) + } + if s.byHash[tx.Hash] != tx || s.byTxID[tx.ID] != tx { + t.Fatalf("transmission %d missing from indexes", tx.ID) + } + if i > 0 && s.packets[i-1].FirstSeen > tx.FirstSeen { + t.Fatal("packet slice is not sorted oldest-first") + } + } + if len(s.byHash) != len(s.packets) || len(s.byTxID) != len(s.packets) || len(s.byObsID) != s.totalObs { + t.Fatal("partial transmission leaked into an index") + } +} + +func TestStartupMemoryBudget_HighFanout(t *testing.T) { + for _, hotHours := range []float64{0, 1} { + t.Run(fmt.Sprintf("hot-%g", hotHours), func(t *testing.T) { + db := startupMemoryFixture(t, 120, 300, hotHours == 0) + s := NewPacketStore(db, &PacketStoreConfig{MaxMemoryMB: 1, RetentionHours: 24, HotStartupHours: hotHours}) + if err := s.RunStartupLoad(10000); err != nil { + t.Fatal(err) + } + assertStartupBudget(t, s, 300) + if len(s.packets) == 0 || len(s.packets) >= 120 { + t.Fatalf("expected a useful bounded cache, got %d/120", len(s.packets)) + } + if !s.LoadComplete() || !s.backgroundLoadDone.Load() || s.backgroundLoadFailed.Load() { + t.Fatalf("configured memory limit must finish successfully: %q", s.BackgroundLoadError()) + } + if s.LoadCoverageRatio() >= 1 { + t.Fatal("bounded cache must not claim complete database coverage") + } + if s.maxTxID != 119 || s.maxObsID != 120*300 { + t.Fatalf("poll cursors replay excluded history: tx=%d, obs=%d", s.maxTxID, s.maxObsID) + } + if replayed, _ := s.IngestNewFromDB(s.MaxTransmissionID(), 500); len(replayed) != 0 { + t.Fatalf("poll replayed %d omitted historical transmissions", len(replayed)) + } + if replayed := s.IngestNewObservations(s.MaxObservationID(), 500); len(replayed) != 0 { + t.Fatalf("poll replayed %d omitted historical observations", len(replayed)) + } + if hotHours > 0 && s.byTxID[0] == nil { + t.Fatal("old transmission ID with newest traffic was lost") + } + result := s.QueryPackets(PacketQuery{Since: time.Now().UTC().Add(-24 * time.Hour).Format(time.RFC3339), Limit: 200}) + if result.Total != 120 { + t.Fatalf("SQL history fallback omitted transmissions: %d/120", result.Total) + } + }) + } +} + +func TestStartupMemoryBudget_OversizedTransmission(t *testing.T) { + db := startupMemoryFixture(t, 1, 5000, true) + s := NewPacketStore(db, &PacketStoreConfig{MaxMemoryMB: 1, RetentionHours: 24, HotStartupHours: 1}) + if err := s.RunStartupLoad(10000); err != nil { + t.Fatal(err) + } + assertStartupBudget(t, s, 5000) + if len(s.packets) != 0 || s.totalObs != 0 { + t.Fatal("oversized transmission must stay in SQL, not partially in cache") + } + if s.QueryPackets(PacketQuery{Hash: "budget-0000", Limit: 1}).Total != 1 { + t.Fatal("omitted transmission is unavailable without a date filter") + } + if s.QueryGroupedPackets(PacketQuery{Hash: "budget-0000", Limit: 1}).Total != 1 { + t.Fatal("omitted transmission is unavailable in grouped packet queries") + } + if !s.backgroundLoadDone.Load() || s.backgroundLoadFailed.Load() { + t.Fatalf("oversized row must not prevent readiness: %q", s.BackgroundLoadError()) + } +} + +func TestStartupMemoryBudget_TimeAndIDCursor(t *testing.T) { + db := startupMemoryFixture(t, 100, 30, true) + s := NewPacketStore(db, &PacketStoreConfig{MaxMemoryMB: 1, RetentionHours: 24}) + if err := s.RunStartupLoad(2); err != nil { + t.Fatal(err) + } + assertStartupBudget(t, s, 30) + if s.byTxID[99] == nil || s.byTxID[0] != nil { + t.Fatal("equal-time rows must load highest IDs first") + } + for id := 100 - len(s.packets); id < 100; id++ { + if s.byTxID[id] == nil { + t.Fatalf("time/ID cursor skipped transmission %d", id) + } + } + // All timestamps tie; the floor must be conservative across the omitted + // part of that second, rather than claiming that every tied row was loaded. + if s.oldestLoaded <= s.packets[0].FirstSeen { + t.Fatalf("cache coverage floor includes omitted tied rows: %s", s.oldestLoaded) + } +} + +func TestStartupMemoryBudget_MergesLiveObservations(t *testing.T) { + db, writer := openPostgresValueFixture(t) + now := time.Now().UTC().Truncate(time.Second) + for i := 1; i <= 4; i++ { + if _, err := writer.Exec(`INSERT INTO observers(rowid,id,name) VALUES($1,$2,$2)`, i, fmt.Sprintf("merge-%d", i)); err != nil { + t.Fatal(err) + } + } + if _, err := writer.Exec(`INSERT INTO transmissions(id,raw_hex,hash,first_seen,last_seen,payload_type) VALUES(1,'AA','merge-history',$1,$2,4)`, now.Add(-3*time.Hour).Format(time.RFC3339), now.Unix()); err != nil { + t.Fatal(err) + } + addObservation := func(id int) { + t.Helper() + if _, err := writer.Exec(`INSERT INTO observations(id,transmission_id,observer_idx,path_json,timestamp) VALUES($1,1,$1,'[]',$2)`, id, now.Add(time.Duration(id)*time.Minute).Unix()); err != nil { + t.Fatal(err) + } + } + addObservation(3) + s := NewPacketStore(db, &PacketStoreConfig{MaxMemoryMB: 1, RetentionHours: 24}) + s.IngestNewFromDB(0, 10) + original := s.byTxID[1] + if original == nil || len(original.Observations) != 1 { + t.Fatal("live observation setup failed") + } + addObservation(1) + addObservation(2) + // The loader has finished its SQL scan but has not published it yet. + // A live poll is queued behind the advert-evidence gate with an observation + // not present in the loader's snapshot. It must extend the same pointer. + hookCalls := 0 + pollDone := make(chan struct{}) + db.advertEvidenceReadHook = func() { + hookCalls++ + if hookCalls == 1 { + addObservation(4) + go func() { + s.IngestNewObservations(3, 10) + close(pollDone) + }() + } + } + if err := s.loadChunk(now.Add(-24*time.Hour), now.Add(time.Hour)); err != nil { + t.Fatal(err) + } + if hookCalls == 0 { + t.Fatal("interleaving hook did not run") + } + select { + case <-pollDone: + case <-time.After(5 * time.Second): + t.Fatal("live poll remained blocked after loading") + } + assertStartupBudget(t, s, 4) + if s.byTxID[1] != original || len(s.packets) != 1 { + t.Fatal("load replaced the live pointer or duplicated its packet") + } + latest, err := time.Parse(time.RFC3339Nano, original.LatestSeen) + if err != nil || !latest.Equal(now.Add(4*time.Minute)) { + t.Fatalf("newer live observation lost: %q", original.LatestSeen) + } + bytes := s.trackedBytes + if err := s.loadChunk(now.Add(-24*time.Hour), now.Add(time.Hour)); err != nil { + t.Fatal(err) + } + assertStartupBudget(t, s, 4) + if s.trackedBytes != bytes || len(s.byPayloadType[4]) != 1 { + t.Fatal("overlapping load duplicated accounting or payload index") + } +} + +func BenchmarkStartupMemoryBudget(b *testing.B) { + benchmarkStartupMemoryBudget(b, 6000, 65) +} + +func BenchmarkStartupMemoryBudgetTypicalFanout(b *testing.B) { + benchmarkStartupMemoryBudget(b, 30000, 6) +} + +func benchmarkStartupMemoryBudget(b *testing.B, transmissions, observations int) { + b.StopTimer() + db := startupMemoryFixture(b, transmissions, observations, false) + var retained uint64 + for i := 0; i < b.N; i++ { + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + s := NewPacketStore(db, &PacketStoreConfig{MaxMemoryMB: 16, RetentionHours: 24, HotStartupHours: 1}) + b.StartTimer() + if err := s.RunStartupLoad(10000); err != nil { + b.Fatal(err) + } + b.StopTimer() + runtime.GC() + runtime.ReadMemStats(&after) + if after.HeapAlloc > before.HeapAlloc { + retained += after.HeapAlloc - before.HeapAlloc + } + b.ReportMetric(s.trackedMemoryMB(), "cache-MiB") + runtime.KeepAlive(s) + } + b.ReportMetric(float64(retained)/float64(b.N)/1048576, "heap-MiB") +} diff --git a/cmd/server/startup_query_plan_test.go b/cmd/server/startup_query_plan_test.go new file mode 100644 index 000000000..78dc0eb20 --- /dev/null +++ b/cmd/server/startup_query_plan_test.go @@ -0,0 +1,58 @@ +package main + +import ( + "encoding/json" + "fmt" + "strconv" + "testing" + "time" +) + +func TestPostgresStartupUnlimitedCursorUsesPrimaryKeySeek(t *testing.T) { + postgresOnly(t) + db, writer := openPostgresValueFixture(t) + stamp := time.Now().UTC().Add(-2 * time.Hour).Truncate(time.Second) + copyTestRows(t, writer, "transmissions", []string{"id", "raw_hex", "hash", "first_seen", "last_seen"}, 20000, func(n int) []any { + // A large tie group proves the ID half of the cursor also seeks. + return []any{n, "AA", fmt.Sprintf("seek-%d", n), stamp.Format(time.RFC3339), stamp.Unix()} + }) + if _, err := writer.Exec("ANALYZE transmissions"); err != nil { + t.Fatal(err) + } + s := NewPacketStore(db, &PacketStoreConfig{}) + query, args := s.packetLoadQuery(stamp.Add(-time.Hour), time.Time{}, packetLoadCursor{id: 10000, recency: strconv.FormatInt(stamp.Unix(), 10)}, 100) + var raw []byte + if err := db.conn.QueryRow("EXPLAIN (ANALYZE, FORMAT JSON) "+query, args...).Scan(&raw); err != nil { + t.Fatal(err) + } + type planNode struct { + Type string `json:"Node Type"` + Relation string `json:"Relation Name"` + Index string `json:"Index Name"` + Rows float64 `json:"Actual Rows"` + Filtered float64 `json:"Rows Removed by Filter"` + Plans []planNode `json:"Plans"` + } + var plans []struct{ Plan planNode } + if err := json.Unmarshal(raw, &plans); err != nil || len(plans) != 1 { + t.Fatalf("invalid plan: %v", err) + } + pending := []planNode{plans[0].Plan} + indexed := false + for len(pending) > 0 { + node := pending[0] + pending = append(pending[1:], node.Plans...) + if node.Relation != "transmissions" { + continue + } + if node.Index == "transmissions_pkey" { + indexed = true + } + if node.Rows+node.Filtered > 100 { + t.Fatalf("100-row page rescanned history: %s (%s), rows=%g filtered=%g", node.Type, node.Index, node.Rows, node.Filtered) + } + } + if !indexed { + t.Fatal("unlimited startup cursor did not use the primary-key index") + } +} diff --git a/cmd/server/store.go b/cmd/server/store.go index aef042c79..e56b08045 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -456,9 +456,11 @@ type PacketStore struct { // Clock skew detection engine. clockSkew *ClockSkewEngine - // Bounded cold load: oldest packet timestamp loaded into memory. - // Empty string means all data is in memory (no limit applied). - oldestLoaded string + // Conservative timestamp boundary for complete cache coverage. A partial + // timestamp group stays behind this boundary and is queried from SQL. + // Empty string means no fallback boundary has been established. + oldestLoaded string + startupMemoryLimited bool // bounded cache is complete; omitted history remains in SQL // Hot startup atomic gates — see contract below. // @@ -468,13 +470,14 @@ type PacketStore struct { // formula. Only the ingestor or offline migration can repair hashes. // * backgroundLoadDone is set true exactly once, after // loadBackgroundChunks finishes its loop AND its post-load index - // rebuild. It gates "hot startup has finished filling - // retentionHours of data into memory" (used by /api/perf and the + // rebuild. It gates "hot startup has filled the configured retention + // window or reached its memory budget" (used by /api/perf and the // UI's hot-load banner). It says nothing about success — see // backgroundLoadFailed for the success/failure signal. // * backgroundLoadFailed is set true ONLY if at least one chunk - // errored during background load. /api/perf surfaces it so - // operators can distinguish "done & full" from "done & partial". + // errored or coverage unexpectedly fell short during background load. + // Reaching the configured byte budget is not an error; the separate + // loadCoverageRatio continues to report the actual cached fraction. // Read order MUST be: load backgroundLoadDone first; only if true // is backgroundLoadFailed meaningful. // 0 = disabled (current behavior). Background loader fills the rest. @@ -790,6 +793,9 @@ func NewPacketStore(db *DB, cfg *PacketStoreConfig, cacheTTLs ...map[string]inte // When maxMemoryMB > 0, loads only the newest N transmissions that fit // within the memory budget, avoiding OOM on large databases. func (s *PacketStore) Load() error { + if s.maxMemoryMB > 0 { + return s.LoadChunked(0) + } s.advertEvidenceMu.Lock() defer s.advertEvidenceMu.Unlock() s.mu.Lock() @@ -809,27 +815,6 @@ func (s *PacketStore) Load() error { t0 := time.Now() - // Count total transmissions for logging. - var totalInDB int - if err := s.db.conn.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&totalInDB); err != nil { - totalInDB = -1 // non-fatal - } - - // Calculate max packets to load based on memory budget. - var maxPackets int64 - if s.maxMemoryMB > 0 { - // Use a typical packet with ~10 observations as the estimate. - avgBytes := int64(1000) // conservative floor - if sample := estimateStoreTxBytesTypical(10); sample > avgBytes { - avgBytes = sample - } - maxPackets = (int64(s.maxMemoryMB) * 1048576) / avgBytes - if maxPackets < 1000 { - maxPackets = 1000 // minimum 1000 packets - } - } - // maxPackets == 0 means unlimited - var loadSQL string rpCol := "" if s.db.hasResolvedPath { @@ -871,15 +856,6 @@ func (s *PacketStore) Load() error { loadConditions = append(loadConditions, fmt.Sprintf("t.first_seen >= '%s'", hotCutoffStr)) } } - if maxPackets > 0 { - if s.db.hasLastSeen { - loadConditions = append(loadConditions, fmt.Sprintf( - "t.id IN (SELECT id FROM transmissions ORDER BY last_seen DESC LIMIT %d)", maxPackets)) - } else { - loadConditions = append(loadConditions, fmt.Sprintf( - "t.id IN (SELECT id FROM transmissions ORDER BY first_seen DESC LIMIT %d)", maxPackets)) - } - } filterClause := "" if len(loadConditions) > 0 { filterClause = "\n\t\t\tWHERE " + strings.Join(loadConditions, "\n\t\t\t AND ") @@ -1120,13 +1096,8 @@ func (s *PacketStore) Load() error { s.loaded = true elapsed := time.Since(t0) - if maxPackets > 0 && totalInDB > len(s.packets) { - log.Printf("[store] Loaded %d/%d transmissions (%d observations) in %v — bounded by %dMB budget (tracked ~%.0fMB, heap ~%.0fMB)", - len(s.packets), totalInDB, s.totalObs, elapsed, s.maxMemoryMB, s.trackedMemoryMB(), s.estimatedMemoryMB()) - } else { - log.Printf("[store] Loaded %d transmissions (%d observations) in %v (tracked ~%.0fMB, heap ~%.0fMB)", - len(s.packets), s.totalObs, elapsed, s.trackedMemoryMB(), s.estimatedMemoryMB()) - } + log.Printf("[store] Loaded %d transmissions (%d observations) in %v (tracked ~%.0fMB, heap ~%.0fMB)", + len(s.packets), s.totalObs, elapsed, s.trackedMemoryMB(), s.estimatedMemoryMB()) s.loadMultibyteCapFromDB() if coldLoadAmbiguousHopsSkipped > 0 { log.Printf("[store] cold load: skipped %d ambiguous-prefix relay hops (unique_prefix gate, PR #1643 R1)", @@ -1152,420 +1123,12 @@ func (s *PacketStore) Load() error { // distHops are NOT updated here — the caller (loadBackgroundChunks) rebuilds // those once after all chunks are merged. -// accumulateDedup appends pks to byTx[txID], deduplicating per-tx via the -// parallel seenByTx set. Used by loadChunk to build the per-tx relay-hop -// pubkey unions outside the merge critical section. -func accumulateDedup(byTx map[int][]string, seenByTx map[int]map[string]bool, txID int, pks []string) { - if len(pks) == 0 { - return - } - seen := seenByTx[txID] - if seen == nil { - seen = make(map[string]bool, len(pks)) - seenByTx[txID] = seen - } - for _, pk := range pks { - if seen[pk] { - continue - } - seen[pk] = true - byTx[txID] = append(byTx[txID], pk) - } -} - +// loadChunk uses the same observation-aware budget and bounded SQL pages as +// the initial load. A daily background window never becomes an unbounded +// intermediate slice before the cache can enforce its limit. func (s *PacketStore) loadChunk(from, to time.Time) error { - fromStr := from.UTC().Format(time.RFC3339) - toStr := to.UTC().Format(time.RFC3339) - fromUnix := from.UTC().Unix() - toUnix := to.UTC().Unix() - _ = fromStr - _ = toStr - - // Build the same SQL as Load() but with a [from, to) window. - rpCol := "" - if s.db.hasResolvedPath { - rpCol = ",\n\t\t\t\to.resolved_path" - } - obsRawHexCol := "" - if s.db.hasObsRawHex { - obsRawHexCol = ", o.raw_hex" - } - // #1751: scope_name is on the transmission row; append as the last column. - scopeNameCol := "" - if s.db.hasScopeName { - scopeNameCol = ", t.scope_name" - } - - // #1690: window on the denormalized last_seen (effective recency) - // rather than first_seen. See chunked_load.go for the full rationale. - // Test/legacy DBs without the column fall back to first_seen. - var filterClause string - var qArgs []interface{} - if s.db.hasLastSeen { - filterClause = "\n\t\t\tWHERE t.last_seen >= " + s.db.parameter(1) + " AND t.last_seen < " + s.db.parameter(2) - qArgs = []interface{}{fromUnix, toUnix} - } else { - filterClause = "\n\t\t\tWHERE t.first_seen >= " + s.db.parameter(1) + " AND t.first_seen < " + s.db.parameter(2) - qArgs = []interface{}{fromStr, toStr} - } - - var chunkSQL string - if s.db.isV3 { - chunkSQL = s.db.nativeSQL(`SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, - t.payload_type, t.payload_version, t.decoded_json, - o.id, obs.id, obs.name, o.direction, - o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')`, `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, - t.payload_type, t.payload_version, t.decoded_json, - o.id, obs.id, obs.name, o.direction, - o.snr, o.rssi, o.score, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`) + obsRawHexCol + rpCol + scopeNameCol + ` - FROM transmissions t - LEFT JOIN observations o ON o.transmission_id = t.id - LEFT JOIN observers obs ON obs.rowid = o.observer_idx` + filterClause + ` - ORDER BY t.first_seen ASC, o.timestamp DESC` - } else { - chunkSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, - t.payload_type, t.payload_version, t.decoded_json, - o.id, o.observer_id, o.observer_name, o.direction, - o.snr, o.rssi, o.score, o.path_json, o.timestamp` + obsRawHexCol + rpCol + scopeNameCol + ` - FROM transmissions t - LEFT JOIN observations o ON o.transmission_id = t.id` + filterClause + ` - ORDER BY t.first_seen ASC, o.timestamp DESC` - } - - // Relay-hop fallback inputs. observations.resolved_path is NULL on - // every live deployment (since #1287 the ingestor persists relay data - // as aggregate neighbor_edges, not per-observation resolved_path), so - // for this background-loaded older window we re-resolve relay hops - // from the persisted path_json using the prefix map. PR #1643 R1 - // munger #1: cold load resolves ONLY when the prefix is unique - // (affinity-tier resolution against ≤168h-old observations would - // silently mis-attribute hops). Fetched BEFORE opening the rows - // cursor below: getCachedNodesAndPM issues its own DB query, which - // would deadlock against the open cursor on a single-connection database - // pool. - s.mu.RLock() - _, relayPM := s.getCachedNodesAndPM() - s.mu.RUnlock() - var coldLoadAmbiguousHopsSkipped int - - rows, err := s.db.conn.Query(chunkSQL, qArgs...) - if err != nil { - return err - } - defer rows.Close() - - // Local data structures — built without holding the lock. - localByHash := make(map[string]*StoreTx) - localPackets := make([]*StoreTx, 0) - localByTxID := make(map[int]*StoreTx) - localByObsID := make(map[int]*StoreObs) - localByObserver := make(map[string][]*StoreObs) - // Issue #1558: accumulate the union of resolved_path relay-hop - // pubkeys per tx outside the lock. We unmarshal + dedupe here so the - // merge critical section below only does map-append work, mirroring - // the rest of loadChunk's "build local, merge under lock" shape. - localResolvedPKsByTx := make(map[int][]string) - localResolvedSeenByTx := make(map[int]map[string]bool) - // Path_json fallback pubkeys (resolved_path NULL): indexed into byNode - // ONLY at merge, kept separate from localResolvedPKsByTx because they must - // NOT enter the resolved_path/path-hop indexes (see the per-row comment). - localByNodePKsByTx := make(map[int][]string) - localByNodeSeenByTx := make(map[int]map[string]bool) - var localTotalObs int - var localTrackedBytes int64 - var localMaxTxID int - var localMaxObsID int - - for rows.Next() { - var txID int - var rawHex, hash, firstSeen, decodedJSON sql.NullString - var routeType, payloadType, payloadVersion sql.NullInt64 - var obsID sql.NullInt64 - var observerID, observerName, direction, pathJSON, obsTimestamp sql.NullString - var snr, rssi sql.NullFloat64 - var score sql.NullFloat64 - var obsRawHex sql.NullString - var resolvedPathStr sql.NullString - var scopeName sql.NullString - - scanArgs := []interface{}{&txID, &rawHex, &hash, &firstSeen, &routeType, &payloadType, - &payloadVersion, &decodedJSON, - &obsID, &observerID, &observerName, &direction, - &snr, &rssi, &score, &pathJSON, &obsTimestamp} - if s.db.hasObsRawHex { - scanArgs = append(scanArgs, &obsRawHex) - } - if s.db.hasResolvedPath { - scanArgs = append(scanArgs, &resolvedPathStr) - } - if s.db.hasScopeName { - scanArgs = append(scanArgs, &scopeName) - } - if err := rows.Scan(scanArgs...); err != nil { - log.Printf("[store] loadChunk scan error: %v", err) - continue - } - - hashStr := nullStrVal(hash) - tx := localByHash[hashStr] - if tx == nil { - tx = &StoreTx{ - ID: txID, - RawHex: nullStrVal(rawHex), - Hash: hashStr, - FirstSeen: nullStrVal(firstSeen), - LatestSeen: nullStrVal(firstSeen), - RouteType: nullIntPtr(routeType), - PayloadType: nullIntPtr(payloadType), - DecodedJSON: nullStrVal(decodedJSON), - ScopeName: nullStrPtr(scopeName), - obsKeys: make(map[string]bool), - observerSet: make(map[string]bool), - } - localByHash[hashStr] = tx - localPackets = append(localPackets, tx) - localByTxID[txID] = tx - if txID > localMaxTxID { - localMaxTxID = txID - } - localTrackedBytes += rechargeTx(tx) - } - - if obsID.Valid { - oid := int(obsID.Int64) - obsIDStr := nullStrVal(observerID) - obsPJ := nullStrVal(pathJSON) - - dk := obsIDStr + "|" + obsPJ - if tx.obsKeys[dk] { - continue - } - - obs := &StoreObs{ - ID: oid, - TransmissionID: txID, - ObserverID: obsIDStr, - ObserverName: nullStrVal(observerName), - Direction: nullStrVal(direction), - SNR: nullFloatPtr(snr), - RSSI: nullFloatPtr(rssi), - Score: nullFloatPtr(score), - PathJSON: obsPJ, - Timestamp: normalizeTimestamp(nullStrVal(obsTimestamp)), - } - - tx.Observations = append(tx.Observations, obs) - tx.obsKeys[dk] = true - if obs.ObserverID != "" && !tx.observerSet[obs.ObserverID] { - tx.observerSet[obs.ObserverID] = true - tx.UniqueObserverCount++ - } - tx.ObservationCount++ - if obs.Timestamp > tx.LatestSeen { - tx.LatestSeen = obs.Timestamp - } - - localByObsID[oid] = obs - if oid > localMaxObsID { - localMaxObsID = oid - } - if obsIDStr != "" { - localByObserver[obsIDStr] = append(localByObserver[obsIDStr], obs) - } - localTotalObs++ - localTrackedBytes += estimateStoreObsBytes(obs) - - // Issue #1558: collect resolved_path relay-hop pubkeys for - // this observation. We unmarshal + dedupe OUTSIDE the merge - // critical section so the lock-held work in the per-batch - // merge below stays bounded. Without this, background-loaded - // transmissions silently miss byNode entries for every relay - // they were heard via — Load() does the equivalent inline. - rpStr := nullStrVal(resolvedPathStr) - if rpStr != "" { - rp := unmarshalResolvedPath(rpStr) - pks := extractResolvedPubkeys(rp) - accumulateDedup(localResolvedPKsByTx, localResolvedSeenByTx, txID, pks) - } else if relayPM != nil && obsPJ != "" && obsPJ != "[]" { - // resolved_path not persisted — reconstruct relay hops from - // path_json so the older retention window keeps relay-node - // history across a restart (mirrors the hot-window fix in - // scanAndMergeChunk and the live ingest path). These go into a - // SEPARATE accumulator indexed into byNode ONLY at merge — they - // must not enter the resolved_path/path-hop indexes, which - // handleNodePaths cross-checks against the NULL resolved_path - // column (#1352). - // PR #1643 R1 munger #1: unique_prefix-only gate. - rp := resolvePathForObsColdLoad(obsPJ, obsIDStr, tx, relayPM, &coldLoadAmbiguousHopsSkipped) - pks := extractResolvedPubkeys(rp) - accumulateDedup(localByNodePKsByTx, localByNodeSeenByTx, txID, pks) - } - } - } - if err := rows.Err(); err != nil { - return err - } - - // Pick best observation for each local packet before merging. - for _, tx := range localPackets { - pickBestObservation(tx) - localTrackedBytes += rechargeTx(tx) - } - - if len(localPackets) == 0 { - return nil - } - rows.Close() - s.advertEvidenceMu.Lock() - defer s.advertEvidenceMu.Unlock() - ids := advertTxIDs(localPackets) - masks, err := s.db.advertEvidenceForIDs(ids) - if err != nil { - return err - } - for _, tx := range localPackets { - tx.AdvertRouteEvidence = masks[tx.ID] - } - - // PR #1187 r3 MUST-FIX 1: index↔slice consistency. - // - // The previous (r2 #6, commit 2ec762aa) merge order was: - // 1) prepend s.packets in one critical section - // 2) populate s.byHash/byTxID/byObsID/byObserver/byNode/byPayloadType - // in separate per-batch critical sections - // 3) bump counters in a third section - // - // Any RLock-holding reader between steps observed packets present in - // s.packets but missing from s.byHash → silent partial data loss in - // GetPacketByHash and the QueryPackets hash/node fast-paths. - // - // We invert the order: build the indexes FIRST in bounded per-batch - // critical sections, then under a SINGLE final critical section - // prepend s.packets AND bump counters AND advance maxIDs. The - // invariant "every tx in s.packets is in s.byHash/s.byTxID" holds at - // every RLock instant — readers either see the old state, or an - // intermediate state where indexes contain a superset of s.packets - // (which is harmless: nothing in s.packets dangles), or the fully - // merged new state. - const mergeBatchSize = 500 - - for batchStart := 0; batchStart < len(localPackets); batchStart += mergeBatchSize { - batchEnd := batchStart + mergeBatchSize - if batchEnd > len(localPackets) { - batchEnd = len(localPackets) - } - batch := localPackets[batchStart:batchEnd] - - // Build the per-batch index views outside the lock so the - // critical section below is O(batch), not O(total local). - batchHashes := make(map[string]*StoreTx, len(batch)) - batchTxIDs := make(map[int]*StoreTx, len(batch)) - batchObsIDs := make(map[int]*StoreObs) - batchByObserver := make(map[string][]*StoreObs) - for _, tx := range batch { - batchHashes[tx.Hash] = tx - batchTxIDs[tx.ID] = tx - for _, o := range tx.Observations { - batchObsIDs[o.ID] = o - if o.ObserverID != "" { - batchByObserver[o.ObserverID] = append(batchByObserver[o.ObserverID], o) - } - } - } - - s.mu.Lock() - // Issue #1558: hopsSeen scratch map for indexResolvedPathHops → - // addResolvedPubkeysToPathHopIndex. Allocated once per batch and - // reused across txs (clear()d on each call inside the helper). - hopsSeen := make(map[string]bool) - newObsIDs := make(map[int]bool, len(batchObsIDs)) - for k := range batchObsIDs { - if s.byObsID[k] == nil { - newObsIDs[k] = true - } - } - evidenceChanged := false - for k, v := range batchHashes { - if s.byHash[k] == nil { - s.byHash[k] = v - } else { - // A live-loaded copy may already carry newer bits. Union in - // both directions before mergeChunkIntoPackets chooses it. - if unionAdvertEvidence(s.byHash[k], v.AdvertRouteEvidence) { - evidenceChanged = true - } - v.AdvertRouteEvidence |= s.byHash[k].AdvertRouteEvidence - } - } - if evidenceChanged { - s.invalidateAdvertEvidence() - } - for k, v := range batchTxIDs { - if s.byTxID[k] == nil { - s.byTxID[k] = v - } - } - for k, v := range batchObsIDs { - if newObsIDs[k] { - s.byObsID[k] = v - } - } - for observerID, obsList := range batchByObserver { - for _, o := range obsList { - if newObsIDs[o.ID] { - s.byObserver[observerID] = append(s.byObserver[observerID], o) - } - } - } - for _, tx := range batch { - s.indexByNode(tx) - if tx.PayloadType != nil { - pt := *tx.PayloadType - s.byPayloadType[pt] = append(s.byPayloadType[pt], tx) - } - s.trackAdvertPubkey(tx) - // Issue #1558: mirror Load()'s 783-799 resolved-path branch. - // Without this, background-loaded transmissions never enter - // byNode under their relay-hop pubkeys → Home-page per-node - // stats collapse after restart for relay-heavy nodes. The - // pubkey union was pre-built outside the lock above. - if pks := localResolvedPKsByTx[tx.ID]; len(pks) > 0 { - s.indexResolvedPathHops(tx, pks, hopsSeen) - } - // path_json fallback (resolved_path NULL): byNode ONLY, so relay - // nodes keep their analytics history across a restart without - // polluting the resolved_path/path-hop indexes (#1352). - for _, pk := range localByNodePKsByTx[tx.ID] { - s.addToByNode(tx, pk) - } - } - s.mu.Unlock() - runtime.Gosched() - } - - // Final atomic step: now that every tx in localPackets is fully - // indexed, publish it into s.packets and bump counters in one short - // critical section. After this point the new state is fully visible; - // before it readers see the old slice (which is still fully indexed). - s.mu.Lock() - s.packets = mergeChunkIntoPackets(localPackets, s.packets) - s.invalidateAdvertEvidence() - s.totalObs += localTotalObs - s.trackedBytes += localTrackedBytes - if localMaxTxID > s.maxTxID { - s.maxTxID = localMaxTxID - } - if localMaxObsID > s.maxObsID { - s.maxObsID = localMaxObsID - } - s.mu.Unlock() - - log.Printf("[store] background chunk [%s, %s) merged: %d tx, %d obs", fromStr, toStr, len(localPackets), localTotalObs) - if coldLoadAmbiguousHopsSkipped > 0 { - log.Printf("[store] background chunk: skipped %d ambiguous-prefix relay hops (unique_prefix gate, PR #1643 R1)", - coldLoadAmbiguousHopsSkipped) - } - return nil + _, err := s.loadPacketRange(from, to, 500, nil) + return err } // loadBackgroundChunks fills the remaining retentionHours window by loading @@ -1645,9 +1208,10 @@ func (s *PacketStore) loadBackgroundChunks() { for { s.mu.RLock() oldest := s.oldestLoaded + limited := s.startupMemoryLimited s.mu.RUnlock() - if oldest == "" { + if limited || oldest == "" { break } chunkEnd, err := time.Parse(time.RFC3339, oldest) @@ -1673,7 +1237,7 @@ func (s *PacketStore) loadBackgroundChunks() { // Always advance oldestLoaded to chunkStart so the loop terminates, // even when the chunk was empty (loadChunk skips the update when 0 packets). s.mu.Lock() - if s.oldestLoaded > chunkStartStr || s.oldestLoaded == "" { + if !s.startupMemoryLimited && (s.oldestLoaded > chunkStartStr || s.oldestLoaded == "") { s.oldestLoaded = chunkStartStr } s.mu.Unlock() @@ -1724,6 +1288,7 @@ func (s *PacketStore) loadBackgroundChunks() { // criterion. s.mu.RLock() loadedCount := int64(len(s.packets)) + limited := s.startupMemoryLimited s.mu.RUnlock() var ratio float64 @@ -1735,7 +1300,7 @@ func (s *PacketStore) loadBackgroundChunks() { } s.bgErrMu.Lock() s.loadCoverageRatio = ratio - if totalInDB > 0 && ratio < 0.90 { + if totalInDB > 0 && ratio < 0.90 && !limited { s.backgroundLoadFailed.Store(true) s.backgroundLoadErr = fmt.Sprintf("loaded %.1f%% of %d rows (%d in memory)", ratio*100, totalInDB, loadedCount) @@ -1748,7 +1313,8 @@ func (s *PacketStore) loadBackgroundChunks() { bgErr := s.backgroundLoadErr s.bgErrMu.Unlock() - // Only flip done=true when the store truly reflects the DB. Operators + // Reaching the configured memory limit is a successful bounded cache. + // Coverage still reports the actual ratio; older queries use SQL. Operators // (and the analytics warm-up gate, #1659) read this as "no more rows // to load" — making it lie was the cascading failure. if !s.backgroundLoadFailed.Load() { @@ -1919,9 +1485,10 @@ func (s *PacketStore) QueryPackets(q PacketQuery) *PacketResult { // to the DB layer which covers the full database retention period. s.mu.RLock() oldest := s.oldestLoaded + limited := s.startupMemoryLimited s.mu.RUnlock() if oldest != "" { - needsSQL := (q.Since != "" && q.Since < oldest) || + needsSQL := (limited && q.Since == "") || (q.Since != "" && q.Since < oldest) || (q.Until != "" && q.Until < oldest) if needsSQL { if result, err := s.db.QueryPackets(q); err == nil { @@ -2006,9 +1573,10 @@ func (s *PacketStore) queryPacketsLocked(q PacketQuery, nodePK string) (*PacketR func (s *PacketStore) QueryGroupedPackets(q PacketQuery) *PacketResult { s.mu.RLock() oldest := s.oldestLoaded + limited := s.startupMemoryLimited s.mu.RUnlock() if oldest != "" { - needsSQL := (q.Since != "" && q.Since < oldest) || + needsSQL := (limited && q.Since == "") || (q.Since != "" && q.Since < oldest) || (q.Until != "" && q.Until < oldest) if needsSQL { if result, err := s.db.QueryGroupedPackets(q); err == nil { diff --git a/docs/performance/startup-cache-budget-2026-10-12.md b/docs/performance/startup-cache-budget-2026-10-12.md new file mode 100644 index 000000000..7f93d5b6c --- /dev/null +++ b/docs/performance/startup-cache-budget-2026-10-12.md @@ -0,0 +1,65 @@ +# Startup cache budget repair + +An ARM64 development instance repeatedly reached its 1.5 GiB application +container limit during recovery. The packet cache was configured for 256 MiB, +a one-hour hot startup and 24 hours of cached history. The initial loader used +a transmission-count estimate assuming ten observations per transmission; the +background loader then materialized whole daily windows without applying the +byte budget. Reducing the configured budget alone did not bound that backfill. + +Startup and background loading now share a reader which accounts for actual +observations and publishes only complete transmissions that fit the existing +`packetStore.maxMemoryMB` budget. Bounded loading selects recent traffic first, +including old transmission IDs heard again recently. Reaching the configured +limit is successful cache initialization, with honest coverage reporting and +SQL fallback for omitted history. Live observations and indexes survive an +overlapping history load. Poll watermarks cover the complete database rather +than replaying excluded historical rows. + +There is no schema change or new configuration setting. An unlimited cache +retains primary-key cursor traversal. Memory-capped pages require recency +sorting with the existing indexes; this repair does not add a full index to +every `last_seen` update. A native PostgreSQL plan regression verifies that an +unlimited 100-transmission page seeks past a 20,000-row timestamp tie without +rescanning those earlier rows. + +## Reproduction and measurements + +The committed benchmarks use canonical v3 fixtures with distinct observer/path +pairs, short synthetic packet metadata and twelve-hop paths. The hot window +contains one transmission; the remainder exercises historical backfill. +Both versions use a 16 MiB cache, a one-hour hot window and 24-hour retention. +The fixture has the production index layout, without an extra recency index. + +Baseline server code: `2d1e763a7f2f710066246dcba47ba8cb4f4652e1`, using the +same benchmark fixture through a Go source overlay. Three one-iteration runs +per version/workload were measured on Windows amd64, Intel i5-4690K, Go 1.25.1, +native SQLite, cgo enabled and `GOMAXPROCS=2`. Fixture creation is outside the +timer. Retained Go heap is the change in `HeapAlloc` after explicit GC, keeping +the store alive. Values below are medians of those three runs. + +| Synthetic workload | Startup before | Startup after | Tracked cache before / after | Retained Go heap before / after | +|---|---:|---:|---:|---:| +| 6,000 transmissions, 65 observations each | 5.882 s | 0.416 s | 226.4 / 15.96 MiB | 247.0 / 26.30 MiB | +| 30,000 transmissions, 6 observations each | 2.983 s | 0.277 s | 212.3 / 16.00 MiB | 151.2 / 20.27 MiB | + +Startup times across the three runs were 5.882/5.671/7.192 s versus +0.416/0.435/0.395 s for high fanout, and 3.274/2.983/2.944 s versus +0.304/0.277/0.274 s for typical fanout. This is a cache-budget comparison: +the repaired version intentionally retains less history in RAM. Database +history remains available through SQL. These measurements are not a comparison +of equal in-memory coverage, whole-container memory, or steady-state API and +ingestion performance. The local host was not isolated from other work. + +To repeat the candidate measurements from `cmd/server`: + +```sh +CORESCOPE_TEST_BACKEND=sqlite CGO_ENABLED=1 GOMAXPROCS=2 \ + go test -run '^$' -bench '^BenchmarkStartupMemoryBudget' \ + -benchtime=1x -count=3 -benchmem . +``` + +Regression tests also cover an oversized single transmission, tied timestamps, +ID zero, a recently heard old ID, native PostgreSQL values, complete SQL history +fallback, accurate byte accounting and a live poll interleaved with backfill. +The same startup tests run against both native database backends. From 0f339875a9bc7d795f180f2c6f49282d1e731636 Mon Sep 17 00:00:00 2001 From: n30nex Date: Mon, 12 Oct 2026 12:05:27 -0400 Subject: [PATCH 28/67] fix: page grouped SQL history before loading observations --- cmd/server/db.go | 105 ++++++------ cmd/server/grouped_history_page_test.go | 157 ++++++++++++++++++ .../startup-cache-budget-2026-10-12.md | 44 +++++ 3 files changed, 250 insertions(+), 56 deletions(-) create mode 100644 cmd/server/grouped_history_page_test.go diff --git a/cmd/server/db.go b/cmd/server/db.go index 297490533..291f7f7e3 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -663,62 +663,7 @@ func (db *DB) QueryGroupedPackets(q PacketQuery) (*PacketResult, error) { db.conn.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM transmissions t %s", w), args...).Scan(&total) } - // Build grouped query using transmissions table with correlated subqueries. - // #1189 R2: distinct_iatas is a NEW column — comma-separated DISTINCT IATA - // codes across all observers of the transmission, with empty/NULL IATAs - // excluded. Frontend needs this on the DEFAULT COLLAPSED VIEW (where - // p._children is empty), so we compute it server-side. - // - // scope_name lives on the transmission row, so appending it as the last - // selected column is safe for both query shapes. - scopeNameCol := "" - if db.hasScopeName { - scopeNameCol = ", t.scope_name" - } - var querySQL string - if db.isV3 { - querySQL = fmt.Sprintf(db.nativeSQL(`SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type, - COALESCE((SELECT COUNT(*) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS count, - COALESCE((SELECT COUNT(DISTINCT oi.observer_idx) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS observer_count, - COALESCE((SELECT MAX(strftime('%%Y-%%m-%%dT%%H:%%M:%%fZ', oi.timestamp, 'unixepoch')) FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen) AS latest, - obs.id AS observer_id, obs.name AS observer_name, COALESCE(obs.iata, '') AS observer_iata, - o.snr, o.rssi, o.path_json, - COALESCE((SELECT group_concat(DISTINCT obi.iata) FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas`, `SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type, - COALESCE((SELECT COUNT(*) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS count, - COALESCE((SELECT COUNT(DISTINCT oi.observer_idx) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS observer_count, - COALESCE((SELECT MAX(to_char(to_timestamp(oi.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')) FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen) AS latest, - obs.id AS observer_id, obs.name AS observer_name, COALESCE(obs.iata, '') AS observer_iata, - o.snr, o.rssi, o.path_json, - COALESCE((SELECT string_agg(DISTINCT obi.iata, ',') FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas`)+scopeNameCol+` - FROM transmissions t - LEFT JOIN observations o ON o.id = ( - SELECT id FROM observations WHERE transmission_id = t.id - ORDER BY length(COALESCE(path_json,'')) DESC NULLS LAST LIMIT 1 - ) - LEFT JOIN observers obs ON obs.rowid = o.observer_idx - %s ORDER BY latest DESC NULLS LAST LIMIT %s OFFSET %s`, w, db.parameter(len(args)+1), db.parameter(len(args)+2)) - } else { - querySQL = fmt.Sprintf(db.nativeSQL(`SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type, - COALESCE((SELECT COUNT(*) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS count, - COALESCE((SELECT COUNT(DISTINCT oi.observer_id) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS observer_count, - COALESCE((SELECT CAST(MAX(oi.timestamp) AS TEXT) FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen) AS latest, - o.observer_id, o.observer_name, COALESCE(obs2.iata, '') AS observer_iata, - o.snr, o.rssi, o.path_json, - COALESCE((SELECT group_concat(DISTINCT obi.iata) FROM observations oi JOIN observers obi ON obi.id = oi.observer_id WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas`, `SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type, - COALESCE((SELECT COUNT(*) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS count, - COALESCE((SELECT COUNT(DISTINCT oi.observer_id) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS observer_count, - COALESCE((SELECT MAX(oi.timestamp)::text FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen) AS latest, - o.observer_id, o.observer_name, COALESCE(obs2.iata, '') AS observer_iata, - o.snr, o.rssi, o.path_json, - COALESCE((SELECT string_agg(DISTINCT obi.iata, ',') FROM observations oi JOIN observers obi ON obi.id = oi.observer_id WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas`)+scopeNameCol+` - FROM transmissions t - LEFT JOIN observations o ON o.id = ( - SELECT id FROM observations WHERE transmission_id = t.id - ORDER BY length(COALESCE(path_json,'')) DESC NULLS LAST LIMIT 1 - ) - LEFT JOIN observers obs2 ON obs2.id = o.observer_id - %s ORDER BY latest DESC NULLS LAST LIMIT %s OFFSET %s`, w, db.parameter(len(args)+1), db.parameter(len(args)+2)) - } + querySQL := db.groupedPacketSelectSQL(w, len(args)) qArgs := make([]interface{}, len(args)) copy(qArgs, args) @@ -773,6 +718,54 @@ func (db *DB) QueryGroupedPackets(q PacketQuery) (*PacketResult, error) { return &PacketResult{Packets: packets, Total: total}, nil } +// groupedPacketSelectSQL ranks and pages matching transmissions before loading +// their observation details. The timestamp index can answer numeric MAX with +// one backwards lookup; formatting every observation before MAX prevents that. +func (db *DB) groupedPacketSelectSQL(w string, argumentCount int) string { + observerKey := "observer_id" + observerID, observerName := "o.observer_id", "o.observer_name" + observerJoin := "obs.id = o.observer_id" + iataJoin := "obi.id = oi.observer_id" + latest := `COALESCE((SELECT CAST(MAX(oi.timestamp) AS TEXT) FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen)` + if db.isV3 { + observerKey = "observer_idx" + observerID, observerName = "obs.id", "obs.name" + observerJoin = "obs.rowid = o.observer_idx" + iataJoin = "obi.rowid = oi.observer_idx" + latest = db.nativeSQL( + `COALESCE((SELECT strftime('%Y-%m-%dT%H:%M:%fZ', MAX(oi.timestamp), 'unixepoch') FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen)`, + `COALESCE((SELECT to_char(to_timestamp(MAX(oi.timestamp)) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen)`) + } + // Keep latest independent of the observation filters: those decide which + // transmissions match, while the returned group describes all observations. + // LIMIT inside the derived table prevents both engines from joining/sorting + // best-path observations for transmissions outside the requested page. + page := fmt.Sprintf(`SELECT t.id, %s AS latest FROM transmissions t %s + ORDER BY latest DESC NULLS LAST, t.id DESC LIMIT %s OFFSET %s`, + latest, w, db.parameter(argumentCount+1), db.parameter(argumentCount+2)) + iatas := db.nativeSQL("group_concat(DISTINCT obi.iata)", "string_agg(DISTINCT obi.iata, ',')") + scopeNameCol := "" + if db.hasScopeName { + scopeNameCol = ", t.scope_name" + } + return fmt.Sprintf(`SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type, + COALESCE((SELECT COUNT(*) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS count, + COALESCE((SELECT COUNT(DISTINCT oi.%s) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS observer_count, + page.latest, %s AS observer_id, %s AS observer_name, COALESCE(obs.iata, '') AS observer_iata, + o.snr, o.rssi, o.path_json, + COALESCE((SELECT %s FROM observations oi JOIN observers obi ON %s + WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas%s + FROM (%s) AS page + JOIN transmissions t ON t.id = page.id + LEFT JOIN observations o ON o.id = ( + SELECT id FROM observations WHERE transmission_id = t.id + ORDER BY length(COALESCE(path_json,'')) DESC NULLS LAST LIMIT 1 + ) + LEFT JOIN observers obs ON %s + ORDER BY page.latest DESC NULLS LAST, t.id DESC`, + observerKey, observerID, observerName, iatas, iataJoin, scopeNameCol, page, observerJoin) +} + // parseDistinctIatasCSV turns PostgreSQL string_agg output ("SJC,SFO,OAK") into // a sorted, deduped []string. Returns an empty (non-nil) slice when the input // is empty/nil so JSON serialization stays consistent (`[]` not `null`). diff --git a/cmd/server/grouped_history_page_test.go b/cmd/server/grouped_history_page_test.go new file mode 100644 index 000000000..374a5fb7e --- /dev/null +++ b/cmd/server/grouped_history_page_test.go @@ -0,0 +1,157 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + "time" +) + +func TestGroupedHistoryPagePreservesFiltersAndDetails(t *testing.T) { + db, writer := openPostgresValueFixture(t) + now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC) + exec := func(query string, args ...any) { + t.Helper() + if _, err := writer.Exec(query, args...); err != nil { + t.Fatal(err) + } + } + exec(`INSERT INTO observers(rowid,id,name,iata) VALUES(1,'history-a','Observer A','AAA'),(2,'history-b','Observer B','BBB')`) + for id, age := range []time.Duration{48 * time.Hour, time.Minute, 2 * time.Minute, 12 * time.Hour} { + exec(`INSERT INTO transmissions(id,raw_hex,hash,first_seen,last_seen,payload_type,scope_name) VALUES($1,'AA',$2,$3,0,4,'history-scope')`, id+1, fmt.Sprintf("history-%d", id+1), now.Add(-age).Format(time.RFC3339)) + } + exec(`INSERT INTO observations(id,transmission_id,observer_idx,path_json,timestamp,snr) VALUES + (1,1,1,'["aa","bb","cc"]',$1,3), + (2,1,2,'["dd"]',$2,2), + (3,1,1,'["aa"]',$3,1), + (4,2,1,'[]',$4,4), + (5,4,1,'[]',$2,5)`, now.Add(-25*time.Hour).Unix(), now.Add(-5*time.Minute).Unix(), now.Add(-26*time.Hour).Unix(), now.Add(-72*time.Hour).Unix()) + q := PacketQuery{Since: now.Add(-24 * time.Hour).Format(time.RFC3339), Observer: "history-a", Limit: 1, Offset: 1} + result, err := db.QueryGroupedPackets(q) + if err != nil { + t.Fatal(err) + } + if result.Total != 2 || len(result.Packets) != 1 { + t.Fatalf("filtered page/total = %#v", result) + } + row := result.Packets[0] + // Equal latest timestamps are paged deterministically by descending ID. + // The since/observer predicates match independently, as before; details + // still include every observation, including ones outside that window. + if row["hash"] != "history-1" || row["count"] != 3 || row["observer_count"] != 2 || row["observer_id"] != "history-a" || row["path_json"] != `["aa","bb","cc"]` || row["scope_name"] != "history-scope" { + t.Fatalf("history details changed or tie page unstable: %#v", row) + } + if got := strings.Join(row["distinct_iatas"].([]string), ","); got != "AAA,BBB" { + t.Fatalf("IATAs restricted to filter window: %q", got) + } + if row["latest"] != now.Add(-5*time.Minute).Format("2006-01-02T15:04:05.000Z") { + t.Fatalf("latest changed: %#v", row["latest"]) + } + all, err := db.QueryGroupedPackets(PacketQuery{Limit: 10}) + if err != nil || all.Total != 4 || len(all.Packets) != 4 { + t.Fatalf("all history: %#v / %v", all, err) + } + if all.Packets[0]["hash"] != "history-3" || all.Packets[0]["latest"] != now.Add(-2*time.Minute).Format(time.RFC3339) { + t.Fatalf("no-observation first_seen fallback changed: %#v", all.Packets[0]) + } + if all.Packets[3]["hash"] != "history-2" { + t.Fatalf("ordered by first_seen/denormalized last_seen instead of observations: %#v", all.Packets) + } + end, err := db.QueryGroupedPackets(PacketQuery{Limit: 1, Offset: 10}) + if err != nil || end.Total != 4 || len(end.Packets) != 0 { + t.Fatalf("empty page lost total: %#v / %v", end, err) + } +} + +func TestPostgresGroupedHistoryHydratesOnlyPage(t *testing.T) { + postgresOnly(t) + db := startupMemoryFixture(t, 1000, 65, false) + sql := db.groupedPacketSelectSQL("", 0) + var raw []byte + if err := db.conn.QueryRow("EXPLAIN (ANALYZE, VERBOSE, FORMAT JSON) "+sql, 2, 10).Scan(&raw); err != nil { + t.Fatal(err) + } + type planNode struct { + Type string `json:"Node Type"` + Rows float64 `json:"Actual Rows"` + Loops float64 `json:"Actual Loops"` + Output []string `json:"Output"` + SortKey []string `json:"Sort Key"` + Relation string `json:"Relation Name"` + Direction string `json:"Scan Direction"` + Condition string `json:"Index Cond"` + Plans []planNode `json:"Plans"` + } + var plans []struct{ Plan planNode } + if err := json.Unmarshal(raw, &plans); err != nil || len(plans) != 1 { + t.Fatalf("invalid plan: %v", err) + } + pending := []planNode{plans[0].Plan} + foundBestPath := false + foundLatestSeek := false + for len(pending) > 0 { + n := pending[0] + pending = append(pending[1:], n.Plans...) + if strings.Contains(strings.Join(n.SortKey, " "), "length(") { + foundBestPath = true + if n.Loops > 2 { + t.Fatalf("two-row page hydrated %g best-path groups", n.Loops) + } + } + if n.Type == "Aggregate" && strings.Contains(strings.Join(n.Output, " "), "max(to_char(") { + t.Fatal("latest still formats every observation before aggregating") + } + if n.Relation == "observations" { + t.Logf("observation scan: %s direction=%s rows=%g loops=%g condition=%s", n.Type, n.Direction, n.Rows, n.Loops, n.Condition) + } + condition := strings.ReplaceAll(n.Condition, `"`, "") + if n.Relation == "observations" && strings.Contains(n.Type, "Index") && n.Direction == "Backward" && strings.Contains(condition, "timestamp IS NOT NULL") { + foundLatestSeek = true + if n.Rows != 1 || !strings.Contains(condition, "transmission_id") { + t.Fatalf("latest lookup scanned observations instead of seeking one: rows=%g, condition=%s", n.Rows, n.Condition) + } + } + } + if !foundBestPath { + t.Fatal("plan did not exercise best-path selection") + } + if !foundLatestSeek { + t.Fatal("latest lookup did not use the existing timestamp index") + } +} + +func TestGroupedHistoryLegacyV2(t *testing.T) { + dsn := postgresTestDSN(t) + createTestDBAt(t, dsn, 2) + db, err := openFixtureReader(t, dsn) + if err != nil { + t.Fatal(err) + } + defer db.Close() + result, err := db.QueryGroupedPackets(PacketQuery{Limit: 1}) + if err != nil || result.Total != 2 || len(result.Packets) != 1 { + t.Fatalf("legacy page: %#v / %v", result, err) + } + row := result.Packets[0] + if row["hash"] != "h0002" || row["latest"] != "2026-01-01T00:02:00Z" || row["count"] != 1 || row["observer_id"] != "obs1" { + t.Fatalf("legacy timestamp or observer changed: %#v", row) + } +} + +func BenchmarkGroupedPacketHistory(b *testing.B) { + db := startupMemoryFixture(b, 6000, 65, false) + for _, limit := range []int{1, 5000} { + b.Run(fmt.Sprintf("page-%d", limit), func(b *testing.B) { + q := PacketQuery{Since: time.Now().UTC().Add(-24 * time.Hour).Format(time.RFC3339), Limit: limit} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + result, err := db.QueryGroupedPackets(q) + if err != nil || result.Total != 6000 || len(result.Packets) != limit { + b.Fatalf("history query: %v / %v", result, err) + } + } + }) + } +} diff --git a/docs/performance/startup-cache-budget-2026-10-12.md b/docs/performance/startup-cache-budget-2026-10-12.md index 7f93d5b6c..fcb01ec8e 100644 --- a/docs/performance/startup-cache-budget-2026-10-12.md +++ b/docs/performance/startup-cache-budget-2026-10-12.md @@ -63,3 +63,47 @@ Regression tests also cover an oversized single transmission, tied timestamps, ID zero, a recently heard old ID, native PostgreSQL values, complete SQL history fallback, accurate byte accounting and a live poll interleaved with backfill. The same startup tests run against both native database backends. + +## Grouped SQL history after bounded startup + +The bounded cache exposes the grouped SQL history path more often. Its old +query formatted every observation timestamp before taking `MAX`, and selected +best-path observations before pagination. The follow-up takes numeric `MAX` +first, then pages only transmission IDs and latest timestamps before joining +payloads and computing observation details. No index or schema change is needed. +The PostgreSQL plan regression checks one backwards indexed timestamp row per +matching transmission and two best-path selections for a two-transmission page. +Filtering, full group counts/IATAs, no-observation timestamp fallback and legacy +v2 results have regression coverage; timestamp ties now use descending ID. + +`BenchmarkGroupedPacketHistory` compares baseline `5633088` with this follow-up +using the same 6,000 transmissions and 390,000 observations, a 24-hour filter, +and three one-iteration runs. Fixture creation is outside the timer. Both +versions return the requested page and report a total of 6,000. The client is +the same Windows amd64/i5-4690K/Go 1.25.1 host with `GOMAXPROCS=2`; SQLite runs +locally, while native PostgreSQL 18 runs in an isolated ARM64 test container on +a Raspberry Pi 5 through a local SSH forward, with normal durability enabled. + +| Backend | Returned page | Before median | After median | +|---|---:|---:|---:| +| SQLite | 1 | 1.654 s | 0.102 s | +| SQLite | 5,000 | 2.693 s | 0.727 s | +| PostgreSQL | 1 | 5.021 s | 0.553 s | +| PostgreSQL | 5,000 | 4.858 s | 4.516 s | + +The PostgreSQL 5,000-row samples overlap substantially: 4.842–5.619 s before, +3.894–6.685 s after. These shared-host measurements support the small-page +improvement, not a robust large-page speedup. A one-row result still ranks all +matching transmissions; the 5,000-row case returns only part of the fixture. +Short synthetic metadata and empty IATA fields are not a live 50,000-row browser +workload. These timings cover database calls and result construction, excluding +HTTP encoding, transfer and browser rendering; they are not a cross-backend +hardware comparison or a whole-process memory measurement. + +Run from `cmd/server`, using `CORESCOPE_TEST_BACKEND=sqlite` or `postgres` with +`CORESCOPE_TEST_POSTGRES_URL` pointing at an isolated test database: + +```sh +CGO_ENABLED=1 GOMAXPROCS=2 go test -run '^$' \ + -bench '^BenchmarkGroupedPacketHistory$' -benchtime=1x -count=3 -benchmem . +``` From 01b2a941feb9210b37a72eeab4d196b106bbadb4 Mon Sep 17 00:00:00 2001 From: n30nex Date: Mon, 12 Oct 2026 12:06:05 -0400 Subject: [PATCH 29/67] fix(server): include timestamp in fallback packet broadcasts --- cmd/server/db.go | 1 + cmd/server/poller_timestamp_test.go | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 cmd/server/poller_timestamp_test.go diff --git a/cmd/server/db.go b/cmd/server/db.go index 291f7f7e3..b0829d0c8 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -2371,6 +2371,7 @@ func (db *DB) GetNewTransmissionsSince(lastID int, limit int) ([]map[string]inte "raw_hex": nullStr(rawHex), "hash": nullStr(hash), "first_seen": nullStr(firstSeen), + "timestamp": nullStr(firstSeen), "route_type": nullInt(routeType), "payload_type": nullInt(payloadType), "payload_version": nullInt(payloadVersion), diff --git a/cmd/server/poller_timestamp_test.go b/cmd/server/poller_timestamp_test.go new file mode 100644 index 000000000..95bae7cc0 --- /dev/null +++ b/cmd/server/poller_timestamp_test.go @@ -0,0 +1,23 @@ +package main + +import "testing" + +func TestGetNewTransmissionsSinceIncludesTimestamp(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + const firstSeen = "2026-01-16T10:00:00Z" + if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen) VALUES ('AABB', 'poller-timestamp', $1)`, firstSeen); err != nil { + t.Fatal(err) + } + + packets, err := db.GetNewTransmissionsSince(0, 1) + if err != nil || len(packets) != 1 { + t.Fatalf("poller query returned %d packets: %v", len(packets), err) + } + if got := packets[0]["timestamp"]; got != firstSeen { + t.Errorf("fallback packet timestamp=%v, want first_seen %q", got, firstSeen) + } + if got := packets[0]["first_seen"]; got != firstSeen { + t.Errorf("first_seen=%v, want %q", got, firstSeen) + } +} From c4882423d31033530a66e6dcb930f0e3e4bc71a5 Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:53 -0400 Subject: [PATCH 30/67] fix: resolve cold packet filters outside store lock --- cmd/server/packet_filter_lock_test.go | 304 ++++++++++++++++++++ cmd/server/store.go | 114 ++++---- cmd/server/store_lock_sql_invariant_test.go | 4 +- 3 files changed, 368 insertions(+), 54 deletions(-) create mode 100644 cmd/server/packet_filter_lock_test.go diff --git a/cmd/server/packet_filter_lock_test.go b/cmd/server/packet_filter_lock_test.go new file mode 100644 index 000000000..80f58bfd5 --- /dev/null +++ b/cmd/server/packet_filter_lock_test.go @@ -0,0 +1,304 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "path/filepath" + "reflect" + "sync" + "sync/atomic" + "testing" + "time" +) + +func packetFilterLockStore(t *testing.T) *PacketStore { + t.Helper() + db := setupTestDB(t) + mustExecDB(t, db, `INSERT INTO observers (id, iata) VALUES ('obs-sjc', 'SJC'), ('obs-lax', 'LAX')`) + mustExecDB(t, db, `INSERT INTO nodes (public_key, lat, lon) VALUES ('inside-node', 50.85, 4.35), ('outside-node', 48.0, 4.35)`) + s := newTestStoreWithDB(t, db, &Config{Areas: map[string]AreaEntry{ + "BEL": {Polygon: [][2]float64{{50, 2.5}, {51.5, 2.5}, {51.5, 6.4}, {50, 6.4}}}, + "EMPTY": {Polygon: [][2]float64{{0, 0}, {1, 0}, {1, 1}, {0, 1}}}, + }}) + pt := PayloadADVERT + s.packets = []*StoreTx{ + {ID: 1, Hash: "inside", FirstSeen: "2026-01-01T00:00:01Z", LatestSeen: "2026-01-01T00:00:01Z", PayloadType: &pt, + DecodedJSON: `{"public_key":"inside-node"}`, Observations: []*StoreObs{{ObserverID: "obs-sjc"}}}, + {ID: 2, Hash: "other-region", FirstSeen: "2026-01-01T00:00:02Z", LatestSeen: "2026-01-01T00:00:02Z", PayloadType: &pt, + DecodedJSON: `{"public_key":"inside-node"}`, Observations: []*StoreObs{{ObserverID: "obs-lax"}}}, + {ID: 3, Hash: "outside-area", FirstSeen: "2026-01-01T00:00:03Z", LatestSeen: "2026-01-01T00:00:03Z", PayloadType: &pt, + DecodedJSON: `{"public_key":"outside-node"}`, Observations: []*StoreObs{{ObserverID: "obs-sjc"}}}, + } + return s +} + +func packetFilterHashes(result *PacketResult) []string { + hashes := make([]string, 0, len(result.Packets)) + for _, packet := range result.Packets { + hashes = append(hashes, packet["hash"].(string)) + } + return hashes +} + +func assertColdFilterAllowsStoreReaders(t *testing.T, s *PacketStore, kind string, query func()) { + t.Helper() + hook, _, entered, release := blockingLoadHook(kind) + s.cacheLoadHook = hook + var workers sync.WaitGroup + var releaseOnce sync.Once + releaseSQL := func() { releaseOnce.Do(func() { close(release) }) } + defer func() { releaseSQL(); workers.Wait() }() + oldReadiness := readiness.Swap(1) + t.Cleanup(func() { readiness.Store(oldReadiness) }) + t.Setenv("CORESCOPE_INGESTOR_STATS", filepath.Join(t.TempDir(), "missing.json")) + + queryDone := make(chan struct{}) + workers.Add(1) + go func() { defer workers.Done(); query(); close(queryDone) }() + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("query did not reach the cold filter lookup") + } + + // Queue the poller writer while the filter's database work is blocked. + // Both the writer and healthz must complete before SQL is released. + writerStarted, writerDone := make(chan struct{}), make(chan struct{}) + workers.Add(1) + go func() { + defer workers.Done() + close(writerStarted) + s.mu.Lock() + s.mu.Unlock() + close(writerDone) + }() + <-writerStarted + healthDone := make(chan int, 1) + workers.Add(1) + go func() { + defer workers.Done() + w := httptest.NewRecorder() + (&Server{store: s}).handleHealthz(w, httptest.NewRequest(http.MethodGet, "/api/healthz", nil)) + healthDone <- w.Code + }() + select { + case <-writerDone: + case <-time.After(time.Second): + t.Error("store writer blocked behind cold filter SQL") + } + select { + case status := <-healthDone: + if status != http.StatusOK { + t.Errorf("healthz status = %d, want 200", status) + } + case <-time.After(time.Second): + t.Error("healthz blocked behind cold filter SQL and the queued writer") + } + releaseSQL() + select { + case <-queryDone: + case <-time.After(time.Second): + t.Fatal("query did not finish after releasing filter SQL") + } +} + +func TestColdPacketFiltersDoNotHoldStoreLock(t *testing.T) { + for _, grouped := range []bool{false, true} { + for _, filter := range []struct { + name, kind string + q PacketQuery + want []string + }{ + {"region", "regionObs", PacketQuery{Region: "SJC"}, []string{"outside-area", "inside"}}, + {"area", "areaNodes", PacketQuery{Area: "BEL"}, []string{"other-region", "inside"}}, + } { + name := "raw/" + filter.name + if grouped { + name = "grouped/" + filter.name + } + t.Run(name, func(t *testing.T) { + s := packetFilterLockStore(t) + query := s.QueryPackets + if grouped { + query = s.QueryGroupedPackets + } + var got *PacketResult + assertColdFilterAllowsStoreReaders(t, s, filter.kind, func() { got = query(filter.q) }) + if hashes := packetFilterHashes(got); !reflect.DeepEqual(hashes, filter.want) || got.Total != len(filter.want) { + t.Errorf("filtered packets = %v, total %d; want %v", hashes, got.Total, filter.want) + } + }) + } + } +} + +func TestColdRegionConsumersDoNotHoldStoreLock(t *testing.T) { + for _, tc := range []struct { + name string + query func(*PacketStore) + }{ + {"channels", func(s *PacketStore) { s.GetChannels("SJC") }}, + {"encrypted-channels", func(s *PacketStore) { s.GetEncryptedChannels("SJC") }}, + {"channel-messages", func(s *PacketStore) { s.GetChannelMessages("public", 10, 0, "SJC") }}, + {"analytics-channels", func(s *PacketStore) { s.computeAnalyticsChannels("SJC", "", TimeWindow{}) }}, + {"analytics-rf", func(s *PacketStore) { s.computeAnalyticsRF("SJC", "", TimeWindow{}) }}, + {"windowed-subpaths", func(s *PacketStore) { s.computeAnalyticsSubpathsWindowed("SJC", 2, 3, 10, TimeWindow{}) }}, + {"subpaths", func(s *PacketStore) { s.computeAnalyticsSubpaths("SJC", 2, 3, 10) }}, + } { + t.Run(tc.name, func(t *testing.T) { + s := packetFilterLockStore(t) + s.nodePM, s.nodeCacheTime = buildPrefixMap(nil), time.Now() + assertColdFilterAllowsStoreReaders(t, s, "regionObs", func() { tc.query(s) }) + }) + } +} + +func TestPacketFilterEmptyAndUnknownSets(t *testing.T) { + all := []string{"outside-area", "other-region", "inside"} + for _, grouped := range []bool{false, true} { + for _, tc := range []struct { + name string + q PacketQuery + want []string + }{ + {"no-filters", PacketQuery{}, all}, + {"unknown-region", PacketQuery{Region: "UNKNOWN"}, []string{}}, + {"unknown-region-with-area", PacketQuery{Region: "UNKNOWN", Area: "BEL"}, []string{}}, + {"unknown-area", PacketQuery{Area: "UNKNOWN"}, all}, + {"empty-area", PacketQuery{Area: "EMPTY"}, all}, + {"combined", PacketQuery{Region: "SJC", Area: "BEL"}, []string{"inside"}}, + } { + name := "raw/" + tc.name + if grouped { + name = "grouped/" + tc.name + } + t.Run(name, func(t *testing.T) { + s := packetFilterLockStore(t) + if tc.q.Region == "UNKNOWN" { + s.cacheLoadHook = func(kind string) { + if kind == "areaNodes" { + t.Error("empty region should short-circuit the area lookup") + } + } + } + query := s.QueryPackets + if grouped { + query = s.QueryGroupedPackets + } + got := query(tc.q) + if hashes := packetFilterHashes(got); !reflect.DeepEqual(hashes, tc.want) || got.Total != len(tc.want) { + t.Errorf("filtered packets = %v, total %d; want %v", hashes, got.Total, tc.want) + } + }) + } + } +} + +func TestPacketFilterUsesResolvedSnapshotAfterCacheEviction(t *testing.T) { + for _, grouped := range []bool{false, true} { + name := "raw" + if grouped { + name = "grouped" + } + t.Run(name, func(t *testing.T) { + s := packetFilterLockStore(t) + query := s.QueryPackets + if grouped { + query = s.QueryGroupedPackets + } + hook, _, entered, release := blockingLoadHook("areaNodes") + var regionLoads atomic.Int32 + s.cacheLoadHook = func(kind string) { + if kind == "regionObs" { + regionLoads.Add(1) + } + hook(kind) + } + var releaseOnce sync.Once + releaseSQL := func() { releaseOnce.Do(func() { close(release) }) } + var worker sync.WaitGroup + defer func() { releaseSQL(); worker.Wait() }() + result := make(chan *PacketResult, 1) + worker.Add(1) + go func() { + defer worker.Done() + result <- query(PacketQuery{Region: "SJC", Area: "BEL"}) + }() + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("query did not resolve its region before the area lookup") + } + // The refresh may evict a region between its lookup and the + // locked packet scan. Retain the set; do not query it again. + s.regionObsMu.Lock() + delete(s.regionObsCache, "SJC") + s.regionObsMu.Unlock() + releaseSQL() + got := <-result + if hashes := packetFilterHashes(got); !reflect.DeepEqual(hashes, []string{"inside"}) { + t.Errorf("snapshot-filtered packets = %v, want [inside]", hashes) + } + if loads := regionLoads.Load(); loads != 1 { + t.Errorf("region loaded %d times; query must retain its first snapshot", loads) + } + }) + } +} + +func TestPacketFiltersPreserveWarmAndStaleCaches(t *testing.T) { + for _, grouped := range []bool{false, true} { + for _, stale := range []bool{false, true} { + for _, kind := range []string{"regionObs", "areaNodes"} { + name := "raw/warm/" + kind + if grouped { + name = "grouped/warm/" + kind + } + if stale { + name += "/stale" + } + t.Run(name, func(t *testing.T) { + s := packetFilterLockStore(t) + query := s.QueryPackets + if grouped { + query = s.QueryGroupedPackets + } + cachedAt := time.Now() + if stale { + cachedAt = cachedAt.Add(-time.Hour) + } + q, want, refreshKey := PacketQuery{Region: "SJC"}, "other-region", "regionObs" + // Deliberately differ from the database, proving the cached + // set is served while the refresh is blocked. + s.regionObsCache["SJC"] = map[string]bool{"obs-lax": true} + s.regionObsCacheTime = cachedAt + if kind == "areaNodes" { + q, want, refreshKey = PacketQuery{Area: "BEL"}, "outside-area", "areaNodes:BEL" + s.areaNodeCache["BEL"] = map[string]bool{"outside-node": true} + s.areaNodeCacheTimes["BEL"] = cachedAt + } + hook, calls, entered, release := blockingLoadHook(kind) + s.cacheLoadHook = hook + defer func() { + close(release) + waitUntil(t, "filter refresh to finish", func() bool { return !s.cacheRefresh.running(refreshKey) }) + }() + var got *PacketResult + returnsWithin(t, time.Second, "cached packet filter", func() { got = query(q) }) + if hashes := packetFilterHashes(got); !reflect.DeepEqual(hashes, []string{want}) { + t.Errorf("cached packets = %v, want [%s]", hashes, want) + } + if stale { + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("stale filter did not start a background refresh") + } + } else if atomic.LoadInt64(calls) != 0 { + t.Fatal("warm filter queried the database") + } + }) + } + } + } +} diff --git a/cmd/server/store.go b/cmd/server/store.go index b4debd4ac..60df8c9d6 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -1615,8 +1615,9 @@ func (s *PacketStore) QueryPackets(q PacketQuery) *PacketResult { } atomic.AddInt64(&s.queryCount, 1) nodePK := s.resolveQueryNode(q.Node) + filterSets := s.resolvePacketFilterSets(q) s.mu.RLock() - result, jobs := s.queryPacketsLocked(q, nodePK) + result, jobs := s.queryPacketsLocked(q, nodePK, filterSets) s.mu.RUnlock() s.applyResolvedPaths(jobs) return result @@ -1624,7 +1625,7 @@ func (s *PacketStore) QueryPackets(q PacketQuery) *PacketResult { // queryPacketsLocked is the in-memory part of QueryPackets. Caller holds // s.mu and runs the returned resolved_path jobs after releasing it. -func (s *PacketStore) queryPacketsLocked(q PacketQuery, nodePK string) (*PacketResult, []rpJob) { +func (s *PacketStore) queryPacketsLocked(q PacketQuery, nodePK string, filterSets packetFilterSets) (*PacketResult, []rpJob) { if q.Limit <= 0 { q.Limit = 50 } @@ -1632,7 +1633,7 @@ func (s *PacketStore) queryPacketsLocked(q PacketQuery, nodePK string) (*PacketR q.Order = "DESC" } - results := s.filterPackets(q, nodePK) + results := s.filterPackets(q, nodePK, filterSets) total := len(results) // #1345: order by ingest id, not insertion-into-s.packets order. After @@ -1728,8 +1729,9 @@ func (s *PacketStore) QueryGroupedPackets(q PacketQuery) *PacketResult { // Collect StoreTx pointers under read lock; sort outside it. nodePK := s.resolveQueryNode(q.Node) + filterSets := s.resolvePacketFilterSets(q) s.mu.RLock() - results := s.filterPackets(q, nodePK) + results := s.filterPackets(q, nodePK, filterSets) txs := make([]*StoreTx, len(results)) copy(txs, results) s.mu.RUnlock() @@ -3179,9 +3181,30 @@ func (s *PacketStore) resolveQueryNode(node string) string { return s.db.resolveNodePubkey(node) } +// packetFilterSets holds immutable cache snapshots for one packet query. +// Resolve them before taking s.mu: even a cold cache may wait on SQL. +type packetFilterSets struct { + regionObservers map[string]bool + areaNodes map[string]bool +} + +func (s *PacketStore) resolvePacketFilterSets(q PacketQuery) packetFilterSets { + var sets packetFilterSets + if q.Region != "" { + sets.regionObservers = s.resolveRegionObservers(q.Region) + if len(sets.regionObservers) == 0 { + return sets + } + } + if q.Area != "" { + sets.areaNodes = s.resolveAreaNodes(q.Area) + } + return sets +} + // filterPackets applies PacketQuery filters to the in-memory packet list. -// nodePK is q.Node resolved by resolveQueryNode. Caller holds s.mu. -func (s *PacketStore) filterPackets(q PacketQuery, nodePK string) []*StoreTx { +// nodePK and sets are resolved before taking s.mu. Caller holds s.mu. +func (s *PacketStore) filterPackets(q PacketQuery, nodePK string, sets packetFilterSets) []*StoreTx { // Fast path: single-key index lookups if q.Hash != "" && q.Type == nil && q.Route == nil && q.Observer == "" && q.Region == "" && q.Area == "" && q.Node == "" && q.Channel == "" && q.Since == "" && q.Until == "" { @@ -3230,19 +3253,9 @@ func (s *PacketStore) filterPackets(q PacketQuery, nodePK string) []*StoreTx { } } - // Pre-compute region observer set. - var regionObservers map[string]bool - if q.Region != "" { - regionObservers = s.resolveRegionObservers(q.Region) - if len(regionObservers) == 0 { - return nil - } - } - - // Pre-compute area node set. - var areaNodes map[string]bool - if q.Area != "" { - areaNodes = s.resolveAreaNodes(q.Area) + regionObservers, areaNodes := sets.regionObservers, sets.areaNodes + if q.Region != "" && len(regionObservers) == 0 { + return nil } // Pre-compute node filter parameters. @@ -3410,9 +3423,9 @@ func (s *PacketStore) transmissionsForObserver(observerIDs string, from []*Store const regionObsTTL = 30 * time.Second // resolveRegionObservers returns a set of observer IDs for a given IATA region. -// Callers often hold s.mu, so a cache past its TTL is served as-is while one -// background refresh re-queries it (#2146). A region seen for the first time -// is queried inline, without holding regionObsMu. +// Call without s.mu held: a region seen for the first time queries SQL inline. +// Past its TTL the immutable cached set is served while one background refresh +// replaces it (#2146), so callers may retain the returned set for their query. func (s *PacketStore) resolveRegionObservers(region string) map[string]bool { s.regionObsMu.Lock() m, cached := s.regionObsCache[region] @@ -3499,9 +3512,9 @@ const areaNodeTTL = 30 * time.Second // resolveAreaNodes returns a set of node pubkeys whose GPS coordinates fall // inside the named area polygon. Returns nil if the area key is not in config. -// Results are cached per key. Callers often hold s.mu, so an entry past its -// TTL is served as-is while one background refresh re-queries it (#2146). -// Uses its own RWMutex so callers holding s.mu won't deadlock. +// Call without s.mu held: a cold entry queries SQL inline. Results are immutable +// and cached per key. Past its TTL a set is served as-is while one background +// refresh replaces it (#2146). func (s *PacketStore) resolveAreaNodes(areaKey string) map[string]bool { if s.config == nil || s.config.Areas == nil { return nil @@ -5336,12 +5349,12 @@ func (s *PacketStore) GetChannels(region string) []map[string]interface{} { hasRegion bool } - // Copy only the fields needed — release the lock before JSON unmarshal. - s.mu.RLock() var regionObs map[string]bool if region != "" { regionObs = s.resolveRegionObservers(region) } + // Copy only the fields needed — release the lock before JSON unmarshal. + s.mu.RLock() grpTxts := s.byPayloadType[5] snapshots := make([]txSnapshot, 0, len(grpTxts)) for _, tx := range grpTxts { @@ -5490,11 +5503,11 @@ func (s *PacketStore) GetChannels(region string) []map[string]interface{} { // GetEncryptedChannels returns undecryptable GRP_TXT channels from in-memory packets. func (s *PacketStore) GetEncryptedChannels(region string) []map[string]interface{} { - s.mu.RLock() var regionObs map[string]bool if region != "" { regionObs = s.resolveRegionObservers(region) } + s.mu.RLock() grpTxts := s.byPayloadType[5] type encInfo struct { @@ -5568,6 +5581,12 @@ func (s *PacketStore) GetEncryptedChannels(region string) []map[string]interface // GetChannelMessages returns deduplicated messages for a channel from in-memory packets. func (s *PacketStore) GetChannelMessages(channelHash string, limit, offset int, region ...string) ([]map[string]interface{}, int) { + regionParam := "" + if len(region) > 0 { + regionParam = region[0] + } + regionObs := s.resolveRegionObservers(regionParam) + s.mu.RLock() defer s.mu.RUnlock() @@ -5582,12 +5601,6 @@ func (s *PacketStore) GetChannelMessages(channelHash string, limit, offset int, } msgMap := map[string]*msgEntry{} var msgOrder []string - regionParam := "" - if len(region) > 0 { - regionParam = region[0] - } - regionObs := s.resolveRegionObservers(regionParam) - // Iterate type-5 packets oldest-first (byPayloadType is ASC = oldest first) type decodedMsg struct { Type string `json:"type"` @@ -5841,13 +5854,12 @@ func (s *PacketStore) computeAnalyticsChannels(region, area string, window TimeW areaNodes = s.resolveAreaNodes(area) } - s.mu.RLock() - defer s.mu.RUnlock() - var regionObs map[string]bool if region != "" { regionObs = s.resolveRegionObservers(region) } + s.mu.RLock() + defer s.mu.RUnlock() type decodedGrp struct { Type string `json:"type"` @@ -6100,15 +6112,14 @@ func (s *PacketStore) computeAnalyticsRF(region, area string, window TimeWindow) areaNodes = s.resolveAreaNodes(area) } - s.mu.RLock() - defer s.mu.RUnlock() - - ptNames := payloadTypeNames - var regionObs map[string]bool if region != "" { regionObs = s.resolveRegionObservers(region) } + s.mu.RLock() + defer s.mu.RUnlock() + + ptNames := payloadTypeNames // Collect all observations matching the region estCap := s.totalObs @@ -10088,6 +10099,10 @@ func (s *PacketStore) GetAnalyticsSubpathsWithWindow(region string, minLen, maxL // region-filters. Mirrors computeSubpathsSlow but with the window predicate // inlined and without requiring a non-empty region. func (s *PacketStore) computeAnalyticsSubpathsWindowed(region string, minLen, maxLen, limit int, window TimeWindow) map[string]interface{} { + var regionObs map[string]bool + if region != "" { + regionObs = s.resolveRegionObservers(region) + } s.mu.RLock() defer s.mu.RUnlock() @@ -10110,11 +10125,6 @@ func (s *PacketStore) computeAnalyticsSubpathsWindowed(region string, minLen, ma return hop } - var regionObs map[string]bool - if region != "" { - regionObs = s.resolveRegionObservers(region) - } - subpathCounts := make(map[string]*subpathAccum) totalPaths := 0 @@ -10308,6 +10318,10 @@ type subpathAccum struct { } func (s *PacketStore) computeAnalyticsSubpaths(region string, minLen, maxLen, limit int) map[string]interface{} { + var regionObs map[string]bool + if region != "" { + regionObs = s.resolveRegionObservers(region) + } s.mu.RLock() defer s.mu.RUnlock() @@ -10335,7 +10349,7 @@ func (s *PacketStore) computeAnalyticsSubpaths(region string, minLen, maxLen, li // For region queries fall back to packet iteration (region filtering // requires per-transmission observer checks). if region != "" { - return s.computeSubpathsSlow(region, minLen, maxLen, limit, resolveHop) + return s.computeSubpathsSlow(regionObs, minLen, maxLen, limit, resolveHop) } top, totalPaths := s.topNamedSubpaths([]subpathGroup{{MinLen: minLen, MaxLen: maxLen, Limit: limit}}, resolveHop) @@ -10344,9 +10358,7 @@ func (s *PacketStore) computeAnalyticsSubpaths(region string, minLen, maxLen, li // computeSubpathsSlow is the original O(N) packet-iteration path, used only // for region-filtered queries where we must check per-transmission observers. -func (s *PacketStore) computeSubpathsSlow(region string, minLen, maxLen, limit int, resolveHop func(string) string) map[string]interface{} { - regionObs := s.resolveRegionObservers(region) - +func (s *PacketStore) computeSubpathsSlow(regionObs map[string]bool, minLen, maxLen, limit int, resolveHop func(string) string) map[string]interface{} { subpathCounts := make(map[string]*subpathAccum) totalPaths := 0 diff --git a/cmd/server/store_lock_sql_invariant_test.go b/cmd/server/store_lock_sql_invariant_test.go index 03f38e887..9be0c2d3f 100644 --- a/cmd/server/store_lock_sql_invariant_test.go +++ b/cmd/server/store_lock_sql_invariant_test.go @@ -24,9 +24,7 @@ import ( // cache; past their TTL they serve the cached value and refresh in the // background (cache_refresh.go). var sqlUnderLockAllowedCallees = map[string]bool{ - "PacketStore.getCachedNodesAndPM": true, - "PacketStore.resolveRegionObservers": true, - "PacketStore.resolveAreaNodes": true, + "PacketStore.getCachedNodesAndPM": true, } // sqlUnderLockAllowedHolders run at startup, before or while the store is From 08a230d224d1b178cd804ac406348a12f8708f59 Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:53 -0400 Subject: [PATCH 31/67] fix(accounts): keep pending registration updates atomic --- cmd/server/auth_handlers.go | 10 +- cmd/server/auth_reregister_test.go | 121 ++++++++++++++++++++ internal/users/pending_registration_test.go | 61 ++++++++++ internal/users/users.go | 12 ++ 4 files changed, 200 insertions(+), 4 deletions(-) create mode 100644 cmd/server/auth_reregister_test.go create mode 100644 internal/users/pending_registration_test.go diff --git a/cmd/server/auth_handlers.go b/cmd/server/auth_handlers.go index 1632df079..1ee47c2bd 100644 --- a/cmd/server/auth_handlers.go +++ b/cmd/server/auth_handlers.go @@ -51,11 +51,13 @@ func (s *Server) handleRegister(w http.ResponseWriter, r *http.Request) { // password, so a squatter cannot plant credentials and the owner // can recover a squatted address. Failures stay invisible to the // client (same response as every other branch). - if err := a.st.SetPassword(existing.ID, hash); err != nil { + switch err := a.st.UpdatePendingRegistration(existing.ID, hash, name); { + case errors.Is(err, users.ErrAccountChanged): + // Activation or another account change won the race. Keep the + // generic response without changing credentials or issuing mail. + case err != nil: log.Printf("[users] register: update pending user #%d: %v", existing.ID, err) - } else if err := a.st.SetDisplayName(existing.ID, name); err != nil { - log.Printf("[users] register: update pending user #%d: %v", existing.ID, err) - } else { + default: existing.DisplayName = name _ = a.mailToken(r.Context(), existing, users.PurposeActivate, 48*time.Hour, "", "activate", func(tok string) mailer.Message { return a.activationMail(existing, tok) }) diff --git a/cmd/server/auth_reregister_test.go b/cmd/server/auth_reregister_test.go new file mode 100644 index 000000000..44f033603 --- /dev/null +++ b/cmd/server/auth_reregister_test.go @@ -0,0 +1,121 @@ +package main + +import ( + "context" + "net/http/httptest" + "testing" + "time" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/users" +) + +// Re-registration changes the password and name together or neither. This +// failure injection runs against both supported account backends. +func TestAuthReregisterProfileFailureKeepsPassword(t *testing.T) { + f := newAuthFixture(t) + expectStatus(t, f.do("POST", "/api/auth/register", registerRequest{Email: "pending@example.org", DisplayName: "Original", Password: pw}), 200) + u, err := f.st.GetByEmail("pending@example.org") + if err != nil { + t.Fatal(err) + } + sent := len(f.fake.Sent()) + f.execDB(t, testNativeSQL( + `CREATE TRIGGER reject_name BEFORE UPDATE OF display_name ON users BEGIN SELECT RAISE(ABORT, 'profile update failed'); END`, + `CREATE FUNCTION reject_name() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'profile update failed'; END $$; CREATE TRIGGER reject_name BEFORE UPDATE OF display_name ON users FOR EACH ROW EXECUTE FUNCTION reject_name()`)) + w := f.do("POST", "/api/auth/register", registerRequest{Email: u.Email, DisplayName: "Replacement", Password: "replacement password"}) + expectStatus(t, w, 200) // enumeration-safe response even on a store failure + got, err := f.st.GetByID(u.ID) + if err != nil { + t.Fatal(err) + } + if got.PasswordHash != u.PasswordHash || got.DisplayName != u.DisplayName { + t.Fatal("failed re-registration changed part of the account") + } + if len(f.fake.Sent()) != sent || f.unusedTokens(t, u.ID, users.PurposeActivate) != 1 { + t.Fatal("failed re-registration replaced the original activation link") + } +} + +// Hold activation uncommitted so the HTTP registration sees the old pending +// row. Observe its UPDATE waiting on that exact transaction, then commit the +// activation. No scheduling sleeps or production test hooks are needed. +func TestAuthReregisterCannotOverwriteActivatedAccount(t *testing.T) { + if testBackend(t) != dbconfig.Postgres { + t.Skip("PostgreSQL lock-wait interleaving; pending-only store contract runs on both backends") + } + f := newAuthFixture(t) + expectStatus(t, f.do("POST", "/api/auth/register", registerRequest{Email: "activation@example.org", DisplayName: "Original", Password: pw}), 200) + u, err := f.st.GetByEmail("activation@example.org") + if err != nil { + t.Fatal(err) + } + sent := len(f.fake.Sent()) + owner, err := openAccountFixtureSQL(f.ownerURL) + if err != nil { + t.Fatal(err) + } + defer owner.Close() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + tx, err := owner.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + var blocker int + if err := tx.QueryRowContext(ctx, `SELECT pg_backend_pid()`).Scan(&blocker); err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, `UPDATE users SET status='active', activated_at=$1 WHERE id=$2`, time.Now().Unix(), u.ID); err != nil { + t.Fatal(err) + } + result := make(chan *httptest.ResponseRecorder, 1) + done := make(chan struct{}) + go func() { + defer close(done) + result <- f.serve("POST", "/api/auth/register", registerRequest{Email: u.Email, DisplayName: "Replacement", Password: "replacement password"}) + }() + t.Cleanup(func() { + select { + case <-done: + case <-time.After(10 * time.Second): + t.Error("registration did not finish after the activation lock was released") + } + }) + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for { + var blocked bool + if err := owner.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE wait_event_type='Lock' AND $1=ANY(pg_blocking_pids(pid)))`, blocker).Scan(&blocked); err != nil { + t.Fatal(err) + } + if blocked { + break + } + select { + case <-ctx.Done(): + t.Fatal("registration did not reach the activation row lock") + case <-ticker.C: + } + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + select { + case w := <-result: + expectStatus(t, w, 200) + case <-ctx.Done(): + t.Fatal("registration did not finish") + } + got, err := f.st.GetByID(u.ID) + if err != nil { + t.Fatal(err) + } + if got.Status != users.StatusActive || got.PasswordHash != u.PasswordHash || got.DisplayName != u.DisplayName { + t.Fatal("registration overwrote an account activated after its pending lookup") + } + if len(f.fake.Sent()) != sent { + t.Fatal("registration issued an activation mail after the account became active") + } +} diff --git a/internal/users/pending_registration_test.go b/internal/users/pending_registration_test.go new file mode 100644 index 000000000..5f14c3b47 --- /dev/null +++ b/internal/users/pending_registration_test.go @@ -0,0 +1,61 @@ +package users + +import ( + "errors" + "testing" +) + +func TestUpdatePendingRegistrationUpdatesBothFields(t *testing.T) { + st, _ := newTestStore(t) + u := mustCreate(t, st, "pending@example.org", "Original") + if err := st.UpdatePendingRegistration(u.ID, "replacement-hash", "Replacement"); err != nil { + t.Fatal(err) + } + got, err := st.GetByID(u.ID) + if err != nil { + t.Fatal(err) + } + if got.PasswordHash != "replacement-hash" || got.DisplayName != "Replacement" || got.Status != StatusPending { + t.Fatal("pending registration was not updated together") + } +} + +// The caller holds a stale pending snapshot after activation/disable/deletion. +// Both SQLite and PostgreSQL must evaluate status at the write, not rely on it. +func TestUpdatePendingRegistrationRejectsChangedAccount(t *testing.T) { + for _, state := range []string{"active", "disabled", "deleted"} { + t.Run(state, func(t *testing.T) { + st, _ := newTestStore(t) + u := mustCreate(t, st, "changed@example.org", "Original") + if err := st.Activate(u.ID, RoleUser, nil); err != nil { + t.Fatal(err) + } + if state == "disabled" { + if err := st.SetStatus(u.ID, StatusDisabled); err != nil { + t.Fatal(err) + } + } + if state == "deleted" { + if err := st.Delete(u.ID); err != nil { + t.Fatal(err) + } + } + if err := st.UpdatePendingRegistration(u.ID, "replacement-hash", "Replacement"); !errors.Is(err, ErrAccountChanged) { + t.Fatalf("stale registration error = %v, want ErrAccountChanged", err) + } + got, err := st.GetByID(u.ID) + if state == "deleted" { + if !errors.Is(err, ErrNotFound) { + t.Fatalf("deleted account lookup = %v", err) + } + return + } + if err != nil { + t.Fatal(err) + } + if got.PasswordHash != u.PasswordHash || got.DisplayName != u.DisplayName { + t.Fatal("stale registration changed an account that is no longer pending") + } + }) + } +} diff --git a/internal/users/users.go b/internal/users/users.go index 7a7f38979..edca2bc0f 100644 --- a/internal/users/users.go +++ b/internal/users/users.go @@ -327,3 +327,15 @@ func (s *Store) hashMailLogTx(tx *sql.Tx, userID int64) error { } return nil } + +// UpdatePendingRegistration replaces both registration fields only while the +// account is still pending. A caller's earlier lookup cannot authorize a write +// after activation, disablement or deletion has committed. +func (s *Store) UpdatePendingRegistration(id int64, passwordHash, displayName string) error { + err := expectOne(s.db.Exec(`UPDATE users SET password_hash = `+s.p(1)+`, display_name = `+s.p(2)+` + WHERE id = `+s.p(3)+` AND status = 'pending'`, passwordHash, displayName, id)) + if errors.Is(err, ErrNotFound) { + return ErrAccountChanged + } + return err +} From 05a2ed983eaf97180350fff41c9b82353061e896 Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:54 -0400 Subject: [PATCH 32/67] perf(graph): reuse indexed shortest-path workspaces --- cmd/server/bridge_score.go | 98 +++++++-------- cmd/server/coverage_score.go | 18 +-- cmd/server/graph_scores_bench_test.go | 67 ++++++++++ cmd/server/graph_scores_parity_test.go | 164 +++++++++++++++++++++++++ cmd/server/graph_weighted.go | 108 ++++++++++++---- 5 files changed, 376 insertions(+), 79 deletions(-) create mode 100644 cmd/server/graph_scores_bench_test.go create mode 100644 cmd/server/graph_scores_parity_test.go diff --git a/cmd/server/bridge_score.go b/cmd/server/bridge_score.go index 283c67cbc..4f1d09d91 100644 --- a/cmd/server/bridge_score.go +++ b/cmd/server/bridge_score.go @@ -69,57 +69,46 @@ const bridgeMinWeightEpsilon = 1e-9 func ComputeBridgeScores(edges []BridgeEdge) map[string]float64 { // 1. Build the distance adjacency (cost = 1/weight) — shared with the // Coverage axis (graph_weighted.go) so both see identical structure. - adj := weightedDistanceAdjacency(edges) - if len(adj) == 0 { + graph := weightedDistanceAdjacency(edges) + if len(graph.nodes) == 0 { return map[string]float64{} } - nodes := make([]string, 0, len(adj)) - for n := range adj { - nodes = append(nodes, n) - } - - bc := make(map[string]float64, len(nodes)) - for _, n := range nodes { - bc[n] = 0 - } + size := len(graph.nodes) + bc := make([]float64, size) + stack := make([]int, 0, size) + pred := make([][]int, size) + sigma, delta := make([]float64, size), make([]float64, size) + work := newDistanceWorkspace(size) // 2. Brandes outer loop: one Dijkstra-based single-source shortest // path computation per source vertex. - for _, s := range nodes { - stack := make([]string, 0, len(nodes)) - pred := make(map[string][]string, len(nodes)) - sigma := make(map[string]float64, len(nodes)) - dist := make(map[string]float64, len(nodes)) - for _, n := range nodes { - sigma[n] = 0 - dist[n] = math.Inf(1) + for s := range graph.nodes { + for _, node := range work.reached { + pred[node] = pred[node][:0] + sigma[node], delta[node] = 0, 0 } + stack = stack[:0] + work.reset(s) sigma[s] = 1 - dist[s] = 0 - pq := &bridgePQ{} - heap.Init(pq) - heap.Push(pq, bridgePQItem{node: s, dist: 0}) - - visited := make(map[string]bool, len(nodes)) - for pq.Len() > 0 { - top := heap.Pop(pq).(bridgePQItem) + for work.queue.Len() > 0 { + top := heap.Pop(&work.queue).(*bridgePQItem) v := top.node - if visited[v] { + if work.visited[v] { continue } - visited[v] = true + work.visited[v] = true stack = append(stack, v) - for w, edgeDist := range adj[v] { - alt := dist[v] + edgeDist - if alt < dist[w]-1e-12 { - dist[w] = alt + for _, edge := range graph.adj[v] { + w := edge.node + alt := work.dist[v] + edge.dist + if alt < work.dist[w]-1e-12 { + work.improve(w, alt) sigma[w] = sigma[v] pred[w] = append(pred[w][:0], v) - heap.Push(pq, bridgePQItem{node: w, dist: alt}) - } else if math.Abs(alt-dist[w]) <= 1e-12 { + } else if math.Abs(alt-work.dist[w]) <= 1e-12 { sigma[w] += sigma[v] pred[w] = append(pred[w], v) } @@ -127,7 +116,6 @@ func ComputeBridgeScores(edges []BridgeEdge) map[string]float64 { } // 3. Back-propagation: walk the stack in reverse order. - delta := make(map[string]float64, len(nodes)) for i := len(stack) - 1; i >= 0; i-- { w := stack[i] for _, v := range pred[w] { @@ -143,8 +131,8 @@ func ComputeBridgeScores(edges []BridgeEdge) map[string]float64 { } // 4. Undirected graphs double-count each (s,t) pair, so halve. - for k := range bc { - bc[k] /= 2.0 + for i := range bc { + bc[i] /= 2.0 } // 5. Normalize by max so scores live in [0, 1]. If max is 0 @@ -155,31 +143,43 @@ func ComputeBridgeScores(edges []BridgeEdge) map[string]float64 { maxBC = v } } - if maxBC > 0 { - for k, v := range bc { - bc[k] = v / maxBC + out := make(map[string]float64, size) + for i, key := range graph.nodes { + out[key] = bc[i] + if maxBC > 0 { + out[key] /= maxBC } } - return bc + return out } // ─── min-heap for Dijkstra ───────────────────────────────────────────────────── type bridgePQItem struct { - node string - dist float64 + node int + dist float64 + index int } -type bridgePQ []bridgePQItem +type bridgePQ []*bridgePQItem -func (h bridgePQ) Len() int { return len(h) } -func (h bridgePQ) Less(i, j int) bool { return h[i].dist < h[j].dist } -func (h bridgePQ) Swap(i, j int) { h[i], h[j] = h[j], h[i] } -func (h *bridgePQ) Push(x interface{}) { *h = append(*h, x.(bridgePQItem)) } +func (h bridgePQ) Len() int { return len(h) } +func (h bridgePQ) Less(i, j int) bool { return h[i].dist < h[j].dist } +func (h bridgePQ) Swap(i, j int) { + h[i], h[j] = h[j], h[i] + h[i].index, h[j].index = i, j +} +func (h *bridgePQ) Push(x interface{}) { + item := x.(*bridgePQItem) + item.index = len(*h) + *h = append(*h, item) +} func (h *bridgePQ) Pop() interface{} { old := *h n := len(old) it := old[n-1] + old[n-1] = nil *h = old[:n-1] + it.index = -1 return it } diff --git a/cmd/server/coverage_score.go b/cmd/server/coverage_score.go index bc9e6d68a..41d294d0f 100644 --- a/cmd/server/coverage_score.go +++ b/cmd/server/coverage_score.go @@ -45,25 +45,27 @@ package main // // Pure (no global state, no locks); safe to call concurrently. func ComputeCoverageScores(edges []BridgeEdge) map[string]float64 { - adj := weightedDistanceAdjacency(edges) - if len(adj) == 0 { + graph := weightedDistanceAdjacency(edges) + if len(graph.nodes) == 0 { return map[string]float64{} } - harmonic := make(map[string]float64, len(adj)) - for s := range adj { - // dijkstraFrom returns only reachable nodes, so every distance is + harmonic := make(map[string]float64, len(graph.nodes)) + work := newDistanceWorkspace(len(graph.nodes)) + for s, key := range graph.nodes { + // dijkstraFrom records only reachable nodes, so every distance is // finite — unreachable peers simply contribute nothing (harmonic // reach treats them as 1/∞ = 0). - dist := dijkstraFrom(adj, s) + dijkstraFrom(graph.adj, s, work) var reach float64 - for t, d := range dist { + for _, t := range work.reached { + d := work.dist[t] if t == s || d <= 0 { continue } reach += 1.0 / d } - harmonic[s] = reach + harmonic[key] = reach } // Normalize by the max so the best-reaching repeater is 1.0. If max is diff --git a/cmd/server/graph_scores_bench_test.go b/cmd/server/graph_scores_bench_test.go new file mode 100644 index 000000000..16e39f5c3 --- /dev/null +++ b/cmd/server/graph_scores_bench_test.go @@ -0,0 +1,67 @@ +package main + +import ( + "fmt" + "math/rand" + "testing" +) + +// graphScoreFixture makes connected mesh components with realistic 64-byte +// pubkeys, duplicate-free edges and reproducible affinity weights. +func graphScoreFixture(n, edgeCount, components int, seed int64) []BridgeEdge { + rng := rand.New(rand.NewSource(seed)) + keys := make([]string, n) + for i := range keys { + keys[i] = fmt.Sprintf("%064x", i+1) + } + edges := make([]BridgeEdge, 0, edgeCount) + seen := make(map[[2]int]bool, edgeCount) + add := func(a, b int) { + if a > b { + a, b = b, a + } + pair := [2]int{a, b} + if a == b || seen[pair] { + return + } + seen[pair] = true + edges = append(edges, BridgeEdge{A: keys[a], B: keys[b], Weight: 1 / float64(1+rng.Intn(9))}) + } + size := n / components + for i := 0; i < n; i++ { + start := i / size * size + add(i, start+(i-start+1)%size) + } + for len(edges) < edgeCount { + a := rng.Intn(n) + add(a, a/size*size+rng.Intn(size)) + } + return edges +} + +func BenchmarkGraphScores(b *testing.B) { + for _, shape := range []struct { + name string + n, edges, components int + }{ + {"mesh600", 600, 2000, 1}, + {"mesh2000", 2000, 10000, 1}, + {"mesh5000", 5000, 30000, 1}, + {"fragmented2000", 2000, 8000, 20}, + } { + edges := graphScoreFixture(shape.n, shape.edges, shape.components, 42) + for _, axis := range []struct { + name string + compute func([]BridgeEdge) map[string]float64 + }{{"bridge", ComputeBridgeScores}, {"coverage", ComputeCoverageScores}} { + b.Run(shape.name+"/"+axis.name, func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if got := axis.compute(edges); len(got) != shape.n { + b.Fatalf("got %d nodes, want %d", len(got), shape.n) + } + } + }) + } + } +} diff --git a/cmd/server/graph_scores_parity_test.go b/cmd/server/graph_scores_parity_test.go new file mode 100644 index 000000000..9631a38d7 --- /dev/null +++ b/cmd/server/graph_scores_parity_test.go @@ -0,0 +1,164 @@ +package main + +import ( + "math" + "math/rand" + "sort" + "testing" +) + +// Enumerating every simple path is an independent oracle for tiny graphs. +// It deliberately does not use Dijkstra, Brandes, or production graph helpers. +func simplePathScores(edges []BridgeEdge) (map[string]float64, map[string]float64) { + keySet := make(map[string]bool) + for _, e := range edges { + keySet[e.A], keySet[e.B] = true, true + } + keys := make([]string, 0, len(keySet)) + for k := range keySet { + keys = append(keys, k) + } + sort.Strings(keys) + index := make(map[string]int, len(keys)) + cost := make([][]float64, len(keys)) + for i, k := range keys { + index[k] = i + cost[i] = make([]float64, len(keys)) + } + for _, e := range edges { + a, b := index[e.A], index[e.B] + cost[a][b], cost[b][a] = 1/e.Weight, 1/e.Weight + } + bridge, coverage := make([]float64, len(keys)), make([]float64, len(keys)) + for source := range keys { + for target := source + 1; target < len(keys); target++ { + best, count := math.Inf(1), 0 + via := make([]int, len(keys)) + path := []int{source} + visited := make([]bool, len(keys)) + visited[source] = true + var walk func(int, float64) + walk = func(node int, distance float64) { + if distance > best+1e-12 { + return + } + if node == target { + if distance < best-1e-12 { + best, count = distance, 0 + clear(via) + } + count++ + for _, mid := range path[1 : len(path)-1] { + via[mid]++ + } + return + } + for next, length := range cost[node] { + if length == 0 || visited[next] { + continue + } + visited[next] = true + path = append(path, next) + walk(next, distance+length) + path = path[:len(path)-1] + visited[next] = false + } + } + walk(source, 0) + if count == 0 { + continue + } + coverage[source] += 1 / best + coverage[target] += 1 / best + for i, paths := range via { + bridge[i] += float64(paths) / float64(count) + } + } + } + normalize := func(values []float64) map[string]float64 { + maxValue := 0.0 + for _, v := range values { + maxValue = math.Max(maxValue, v) + } + out := make(map[string]float64, len(keys)) + for i, k := range keys { + out[k] = 0 + if maxValue > 0 { + out[k] = values[i] / maxValue + } + } + return out + } + return normalize(bridge), normalize(coverage) +} + +func assertGraphScores(t *testing.T, got, want map[string]float64) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("got %d nodes, want %d", len(got), len(want)) + } + for key, expected := range want { + value, ok := got[key] + if !ok || math.IsNaN(value) || math.Abs(value-expected) > 1e-9 { + t.Errorf("score[%s] = %v (present %v), want %v", key, value, ok, expected) + } + } +} + +func TestGraphScoresMatchSimplePaths(t *testing.T) { + for seed := int64(0); seed < 20; seed++ { + for _, edges := range [][]BridgeEdge{ + graphScoreFixture(7, 7+int(seed)%10, 1, seed), + graphScoreFixture(8, 10, 2, seed), + } { + bridge, coverage := simplePathScores(edges) + assertGraphScores(t, ComputeBridgeScores(edges), bridge) + assertGraphScores(t, ComputeCoverageScores(edges), coverage) + } + } +} + +func TestGraphScoresNormalizeAndFilterEdges(t *testing.T) { + clean := []BridgeEdge{{"a", "b", 1}, {"b", "c", .5}, {"c", "d", 1}, {"b", "d", .25}} + dirty := []BridgeEdge{ + {" A ", "B", 1}, {"b", "c ", .5}, {"C", "d", 1}, {"B", " D", .25}, + {"b", "a", .1}, {"A", "B", .5}, {"a", "a", 1}, {"", "z", 1}, + {"x", "y", math.NaN()}, {"x", "y", math.Inf(1)}, {"x", "y", math.Inf(-1)}, + {"x", "y", 0}, {"x", "y", -1}, {"x", "y", bridgeMinWeightEpsilon / 2}, + } + bridge, coverage := simplePathScores(clean) + for seed := int64(0); seed < 10; seed++ { + rand.New(rand.NewSource(seed)).Shuffle(len(dirty), func(i, j int) { dirty[i], dirty[j] = dirty[j], dirty[i] }) + assertGraphScores(t, ComputeBridgeScores(dirty), bridge) + assertGraphScores(t, ComputeCoverageScores(dirty), coverage) + } +} + +func TestBridgeScoresPreserveDistanceTieTolerance(t *testing.T) { + for _, delta := range []float64{0, 5e-13, 2e-12} { + edges := []BridgeEdge{{"a", "b", 1}, {"b", "d", 1}, {"a", "c", 1}, {"c", "d", 1}, {"a", "d", 1 / (2 + delta)}} + middle := 2.0 / 3 + if delta > 1e-12 { + middle = 1 + } + want := map[string]float64{"a": 1, "b": middle, "c": middle, "d": 1} + assertGraphScores(t, ComputeBridgeScores(edges), want) + } +} + +func TestGraphScoresAllocationBudget(t *testing.T) { + edges := graphScoreFixture(600, 2000, 1, 42) + // String maps allocated ~1.5M objects; indexed state with boxed heap + // values still allocated ~1.05M. Bound queue storage by graph size instead. + for _, axis := range []struct { + name string + compute func([]BridgeEdge) map[string]float64 + }{{"bridge", ComputeBridgeScores}, {"coverage", ComputeCoverageScores}} { + t.Run(axis.name, func(t *testing.T) { + allocs := testing.AllocsPerRun(1, func() { axis.compute(edges) }) + if allocs > 25000 { + t.Fatalf("allocated %.0f objects, budget is 25000", allocs) + } + }) + } +} diff --git a/cmd/server/graph_weighted.go b/cmd/server/graph_weighted.go index 94bcffa4a..7e061f17a 100644 --- a/cmd/server/graph_weighted.go +++ b/cmd/server/graph_weighted.go @@ -17,6 +17,16 @@ import ( "strings" ) +type weightedNeighbor struct { + node int + dist float64 +} + +type weightedGraph struct { + nodes []string + adj [][]weightedNeighbor +} + // weightedDistanceAdjacency builds the symmetric distance adjacency from a // weighted edge list: cost = 1/weight (lower distance = stronger affinity), // keeping the cheapest edge per pair. Self-loops and edges with a non-finite @@ -25,7 +35,7 @@ import ( // for NaN, so NaN must be rejected explicitly; otherwise 1/NaN = NaN would // poison every reach computation downstream (#1762 review). Keys are the // lowercased pubkey form. -func weightedDistanceAdjacency(edges []BridgeEdge) map[string]map[string]float64 { +func weightedDistanceAdjacency(edges []BridgeEdge) weightedGraph { adj := make(map[string]map[string]float64) addOrMerge := func(a, b string, dist float64) { m, ok := adj[a] @@ -51,34 +61,88 @@ func weightedDistanceAdjacency(edges []BridgeEdge) map[string]map[string]float64 addOrMerge(a, b, dist) addOrMerge(b, a, dist) } - return adj + // Convert pubkeys once, outside the all-sources shortest-path loops. + // Keep the normalization and cheapest-duplicate rules above shared. + graph := weightedGraph{nodes: make([]string, 0, len(adj)), adj: make([][]weightedNeighbor, len(adj))} + index := make(map[string]int, len(adj)) + for node := range adj { + index[node] = len(graph.nodes) + graph.nodes = append(graph.nodes, node) + } + for i, node := range graph.nodes { + graph.adj[i] = make([]weightedNeighbor, 0, len(adj[node])) + for next, distance := range adj[node] { + graph.adj[i] = append(graph.adj[i], weightedNeighbor{node: index[next], dist: distance}) + } + } + return graph } -// dijkstraFrom returns the shortest-path distance from src to every reachable -// node over the distance adjacency (unreachable nodes are absent). Reuses the -// bridgePQ min-heap. Used by the Coverage axis; the Bridge axis runs its own -// Brandes-coupled SSSP that additionally tracks predecessors and path counts. -func dijkstraFrom(adj map[string]map[string]float64, src string) map[string]float64 { - dist := map[string]float64{src: 0} - pq := &bridgePQ{} - heap.Init(pq) - heap.Push(pq, bridgePQItem{node: src, dist: 0}) +// distanceWorkspace reuses per-source state. Reset only reached vertices so +// disconnected graphs do not pay an all-vertices scan for every small component. +type distanceWorkspace struct { + dist []float64 + visited []bool + reached []int + queue bridgePQ + items []bridgePQItem +} + +func newDistanceWorkspace(size int) *distanceWorkspace { + w := &distanceWorkspace{ + dist: make([]float64, size), visited: make([]bool, size), + reached: make([]int, 0, size), queue: make(bridgePQ, 0, size), + items: make([]bridgePQItem, size), + } + for i := range w.dist { + w.dist[i] = math.Inf(1) + w.items[i] = bridgePQItem{node: i, index: -1} + } + return w +} + +func (w *distanceWorkspace) reset(src int) { + for _, node := range w.reached { + w.dist[node] = math.Inf(1) + w.visited[node] = false + } + w.reached = w.reached[:0] + w.queue = w.queue[:0] + w.improve(src, 0) +} + +// One item per vertex avoids boxing a new heap value on every relaxation. +// Decrease-key keeps the standard-library heap bounded to at most V entries. +func (w *distanceWorkspace) improve(node int, distance float64) { + if math.IsInf(w.dist[node], 1) { + w.reached = append(w.reached, node) + } + w.dist[node] = distance + item := &w.items[node] + item.dist = distance + if item.index < 0 { + heap.Push(&w.queue, item) + } else { + heap.Fix(&w.queue, item.index) + } +} - visited := make(map[string]bool) - for pq.Len() > 0 { - top := heap.Pop(pq).(bridgePQItem) +// dijkstraFrom fills distances and reached vertices in the reusable workspace. +// Bridge uses the same graph/workspace with Brandes predecessor/path-count logic. +func dijkstraFrom(adj [][]weightedNeighbor, src int, work *distanceWorkspace) { + work.reset(src) + for work.queue.Len() > 0 { + top := heap.Pop(&work.queue).(*bridgePQItem) v := top.node - if visited[v] { + if work.visited[v] { continue } - visited[v] = true - for w, edgeDist := range adj[v] { - alt := top.dist + edgeDist - if cur, ok := dist[w]; !ok || alt < cur { - dist[w] = alt - heap.Push(pq, bridgePQItem{node: w, dist: alt}) + work.visited[v] = true + for _, edge := range adj[v] { + alt := top.dist + edge.dist + if alt < work.dist[edge.node] { + work.improve(edge.node, alt) } } } - return dist } From aa7875ad6f53a492f7bf851d625a6fcfc4c6747f Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:54 -0400 Subject: [PATCH 33/67] perf(startup): leave chunked distance indexing lazy --- cmd/server/chunked_distance_lazy_test.go | 83 ++++++++++++++++++++++++ cmd/server/chunked_load.go | 3 +- 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 cmd/server/chunked_distance_lazy_test.go diff --git a/cmd/server/chunked_distance_lazy_test.go b/cmd/server/chunked_distance_lazy_test.go new file mode 100644 index 000000000..4db55c604 --- /dev/null +++ b/cmd/server/chunked_distance_lazy_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestLoadChunkedDistanceIndexRemainsLazy(t *testing.T) { + db := setupTestDB(t) + seedTestData(t, db) + // Two known, located relay nodes make an eager distance build observable. + mustExecDB(t, db, `UPDATE observations SET path_json = '["aa","11"]' WHERE id = 1`) + store := NewPacketStore(db, nil) + if err := store.RunStartupLoad(2); err != nil { + t.Fatal(err) + } + if len(store.distHops) != 0 || len(store.distPaths) != 0 || store.DistanceIndexBuilt() { + t.Fatalf("chunked startup built the lazy distance index: %d hops, %d paths", len(store.distHops), len(store.distPaths)) + } + if !store.LoadComplete() || !store.SubpathIndexReady() || !store.PathHopIndexReady() { + t.Fatal("chunked load must still publish its required indexes and completion") + } + stopRecomputers := store.StartAnalyticsRecomputers(time.Hour) + defer stopRecomputers() + // Drain the initial and post-load passes before requesting distance data, + // so a coincidental post-load refresh cannot hide a missing lazy refresh. + waitUntil(t, "post-load distance snapshot", func() bool { return store.recompDistance.ComputeRuns() >= 2 }) + runsBeforeBuild := store.recompDistance.ComputeRuns() + + entered, release := make(chan struct{}, 2), make(chan struct{}) + var releaseOnce sync.Once + var builds atomic.Int32 + store.distanceBuildHook = func() { builds.Add(1); entered <- struct{}{}; <-release } + defer func() { + releaseOnce.Do(func() { close(release) }) + waitUntil(t, "lazy distance build", store.DistanceIndexBuilt) + }() + srv := NewServer(db, &Config{}, nil) + srv.store = store + request := func() *httptest.ResponseRecorder { + w := httptest.NewRecorder() + srv.handleAnalyticsDistance(w, httptest.NewRequest(http.MethodGet, "/api/analytics/distance", nil)) + return w + } + if w := request(); w.Code != http.StatusAccepted || w.Header().Get("Retry-After") == "" { + t.Fatalf("first distance response = %d, want 202 with Retry-After", w.Code) + } + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("first distance request did not start the lazy build") + } + if w := request(); w.Code != http.StatusAccepted { + t.Fatalf("distance response during build = %d, want 202", w.Code) + } + releaseOnce.Do(func() { close(release) }) + waitUntil(t, "lazy distance build", store.DistanceIndexBuilt) + if builds.Load() != 1 || len(store.distHops) == 0 { + t.Fatalf("want one nonempty lazy build, got %d builds and %d hops", builds.Load(), len(store.distHops)) + } + w := request() + if w.Code != http.StatusOK { + t.Fatalf("distance response after build = %d, want 200", w.Code) + } + var body struct { + TopHops []json.RawMessage `json:"topHops"` + TopPaths []json.RawMessage `json:"topPaths"` + } + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if len(body.TopHops) == 0 || len(body.TopPaths) == 0 { + t.Fatalf("ready response served an empty distance snapshot: %s", w.Body.String()) + } + if store.recompDistance.ComputeRuns() <= runsBeforeBuild { + t.Fatal("lazy build did not refresh the real distance recomputer before returning 200") + } +} diff --git a/cmd/server/chunked_load.go b/cmd/server/chunked_load.go index 176450b33..c2a55a0b2 100644 --- a/cmd/server/chunked_load.go +++ b/cmd/server/chunked_load.go @@ -385,7 +385,8 @@ func (s *PacketStore) LoadChunked(chunkSize int) error { // first_seen order. Build the derived analytics indexes once at the end. s.mu.Lock() s.buildPathHopIndex() - s.buildDistanceIndex() + // Distance is built on its first request, matching Load() and the + // background loader. An eager build here is repeated by the lazy gate. if !s.startupMemoryLimited { if s.hotStartupHours > 0 { s.oldestLoaded = hotCutoffStr From d80ed8c6ce6f45c5ff9300604be17d020cdaf0cc Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:54 -0400 Subject: [PATCH 34/67] perf(neighbors): refresh snapshots outside request paths --- cmd/server/neighbor_api.go | 56 +++++++--- cmd/server/neighbor_graph_cache.go | 1 + cmd/server/neighbor_refresh_test.go | 165 ++++++++++++++++++++++++++++ cmd/server/routes.go | 8 +- 4 files changed, 211 insertions(+), 19 deletions(-) create mode 100644 cmd/server/neighbor_refresh_test.go diff --git a/cmd/server/neighbor_api.go b/cmd/server/neighbor_api.go index ebdcc5724..a7f252d8e 100644 --- a/cmd/server/neighbor_api.go +++ b/cmd/server/neighbor_api.go @@ -80,25 +80,49 @@ type GraphStats struct { // ─── Graph accessor on Server ────────────────────────────────────────────────── -// getNeighborGraph returns the current neighbor graph, rebuilding if stale. +// getNeighborGraph serves the last complete snapshot regardless of age. Only +// the first request with no snapshot builds synchronously; the recomputer owns +// later refreshes (#2221). func (s *Server) getNeighborGraph() *NeighborGraph { s.neighborMu.Lock() - defer s.neighborMu.Unlock() - - if s.neighborGraph == nil || s.neighborGraph.IsStale() { - if s.store != nil { - opts := BuildOptions{MaxEdgeKm: DefaultMaxEdgeKm} - if s.cfg != nil { - opts.EnableLog = s.cfg.DebugAffinity - opts.MaxEdgeKm = s.cfg.NeighborMaxEdgeKm() - opts.PathTrust = s.cfg.PathTrust - } - s.neighborGraph = BuildFromStoreWithOptions(s.store, opts) - } else { - s.neighborGraph = NewNeighborGraph() - } + graph := s.neighborGraph + s.neighborMu.Unlock() + if graph != nil { + return graph + } + return s.rebuildNeighborGraph(false) +} + +// rebuildNeighborGraph serializes builders without blocking snapshot readers. +// Cold callers recheck after waiting; background refreshes always force a build +// so a tick just before the old TTL expires cannot skip the refresh. +func (s *Server) rebuildNeighborGraph(force bool) *NeighborGraph { + s.neighborBuildMu.Lock() + defer s.neighborBuildMu.Unlock() + if !force { + s.neighborMu.Lock() + graph := s.neighborGraph + s.neighborMu.Unlock() + if graph != nil { + return graph + } + } + var graph *NeighborGraph + if s.store != nil { + opts := BuildOptions{MaxEdgeKm: DefaultMaxEdgeKm} + if s.cfg != nil { + opts.EnableLog = s.cfg.DebugAffinity + opts.MaxEdgeKm = s.cfg.NeighborMaxEdgeKm() + opts.PathTrust = s.cfg.PathTrust + } + graph = BuildFromStoreWithOptions(s.store, opts) + } else { + graph = NewNeighborGraph() } - return s.neighborGraph + s.neighborMu.Lock() + s.neighborGraph = graph + s.neighborMu.Unlock() + return graph } // ─── Handlers ────────────────────────────────────────────────────────────────── diff --git a/cmd/server/neighbor_graph_cache.go b/cmd/server/neighbor_graph_cache.go index 319af56b1..ed63d0628 100644 --- a/cmd/server/neighbor_graph_cache.go +++ b/cmd/server/neighbor_graph_cache.go @@ -73,6 +73,7 @@ func (s *Server) recomputeNeighborGraphCache() { } }() start := time.Now() + s.rebuildNeighborGraph(true) resp := s.buildDefaultNeighborGraphResponse() var buf bytes.Buffer if err := json.NewEncoder(&buf).Encode(resp); err != nil { diff --git a/cmd/server/neighbor_refresh_test.go b/cmd/server/neighbor_refresh_test.go new file mode 100644 index 000000000..5b591840f --- /dev/null +++ b/cmd/server/neighbor_refresh_test.go @@ -0,0 +1,165 @@ +package main + +import ( + "sync" + "sync/atomic" + "testing" + "time" +) + +// A warm request must not visit the packet store, even after the old TTL. The +// locked store makes a request-path rebuild fail deterministically (#2221). +func TestNeighborGraphStaleSnapshotDoesNotRebuildOnRequest(t *testing.T) { + store := ngTestStore(nil, nil) + graph := makeTestGraph() + graph.builtAt = time.Now().Add(-2 * neighborGraphTTL) + s := &Server{store: store, neighborGraph: graph} + store.mu.Lock() + result := make(chan *NeighborGraph, 1) + go func() { result <- s.getNeighborGraph() }() + select { + case got := <-result: + store.mu.Unlock() + if got != graph { + t.Fatal("request replaced an existing stale snapshot") + } + case <-time.After(500 * time.Millisecond): + store.mu.Unlock() + <-result + t.Fatal("stale snapshot request waited for a packet-store rebuild") + } +} + +func TestNeighborGraphBackgroundRefreshRebuildsFreshSnapshot(t *testing.T) { + store := ngTestStore(nil, nil) + s := &Server{store: store} + first := s.getNeighborGraph() + if first.IsStale() { + t.Fatal("fixture must start with a fresh snapshot") + } + observations := make([]*StoreObs, 20) // above the default min-count and score filters + for i := range observations { + observations[i] = ngMakeObs("bbbb2222", `[]`, time.Now().UTC().Format(time.RFC3339), nil) + } + store.mu.Lock() + store.packets = []*StoreTx{ngMakeTx(1, PayloadADVERT, ngFromNodeJSON("aaaa1111"), observations)} + store.mu.Unlock() + + s.recomputeNeighborGraphCache() + second := s.getNeighborGraph() + if second == first { + t.Fatal("background refresh reused the almost-stale graph instead of rebuilding") + } + edges := second.Neighbors("aaaa1111") + if len(edges) != 1 || edges[0].Count != len(observations) { + t.Fatal("background refresh did not include newly received observations") + } + for _, entry := range []*neighborGraphCacheEntry{s.neighborGraphCache.ptr.Load(), s.neighborGraphCache.unfilteredPtr.Load()} { + if entry == nil || len(entry.resp.Edges) != 1 || entry.resp.Edges[0].Weight != len(observations) { + t.Fatal("background response cache did not use the rebuilt graph") + } + } + s.recomputeNeighborGraphCache() + if s.getNeighborGraph() == second { + t.Fatal("a second background tick skipped a still-fresh graph") + } +} + +func TestNeighborGraphConcurrentColdRequestsShareOneSnapshot(t *testing.T) { + store := ngTestStore(nil, nil) + s := &Server{store: store} + const callers = 32 + results := make(chan *NeighborGraph, callers) + start := make(chan struct{}) + var ready sync.WaitGroup + ready.Add(callers) + for i := 0; i < callers; i++ { + go func() { + ready.Done() + <-start + results <- s.getNeighborGraph() + }() + } + ready.Wait() + close(start) + first := <-results + if first == nil || first.IsStale() { + t.Fatal("cold initialization did not build a fresh snapshot") + } + for i := 1; i < callers; i++ { + if got := <-results; got != first { + t.Fatal("concurrent cold requests built different snapshots") + } + } +} + +func TestNeighborGraphRefreshPanicPreservesSnapshot(t *testing.T) { + graph := makeTestGraph() + store := ngTestStore(nil, []*StoreTx{nil}) // builder failure, before publication + s := &Server{store: store, neighborGraph: graph} + s.recomputeNeighborGraphCache() + if s.getNeighborGraph() != graph || atomic.LoadUint64(&s.neighborGraphCacheRebuildFailures) != 1 { + t.Fatal("failed rebuild did not retain the last complete snapshot and count its failure") + } + store.mu.Lock() + store.packets = nil + store.mu.Unlock() + done := make(chan struct{}) + go func() { + s.recomputeNeighborGraphCache() + close(done) + }() + select { + case <-done: + if s.getNeighborGraph() == graph { + t.Fatal("a later refresh could not recover after a build panic") + } + case <-time.After(time.Second): + t.Fatal("build panic stranded the refresh lock") + } +} + +func TestNeighborGraphBackgroundRefreshKeepsReadersAvailable(t *testing.T) { + graph := makeTestGraph() + store := ngTestStore(nil, nil) + s := &Server{store: store, neighborGraph: graph} + store.mu.Lock() // keep the actual graph builder in flight + done := make(chan struct{}) + go func() { + s.recomputeNeighborGraphCache() + close(done) + }() + defer func() { + store.mu.Unlock() + select { + case <-done: + case <-time.After(time.Second): + t.Error("background refresh did not finish after releasing the store") + } + }() + deadline := time.After(time.Second) + ticker := time.NewTicker(time.Millisecond) + defer ticker.Stop() + for s.neighborBuildMu.TryLock() { + s.neighborBuildMu.Unlock() + select { + case <-deadline: + t.Fatal("background refresh did not start its builder") + case <-ticker.C: + } + } + if !s.neighborMu.TryLock() { + t.Fatal("graph builder held the snapshot pointer mutex") + } + s.neighborMu.Unlock() + result := make(chan *NeighborGraph, 1) + go func() { result <- s.getNeighborGraph() }() + select { + case got := <-result: + if got != graph { + t.Fatal("request lost the last completed snapshot during refresh") + } + case <-time.After(500 * time.Millisecond): + t.Fatal("request waited for the background builder") + } +} diff --git a/cmd/server/routes.go b/cmd/server/routes.go index 374f2acff..e6b3cec82 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -88,9 +88,11 @@ type Server struct { // can't race on the .tmp file or interleave disk/memory updates. saveMu sync.Mutex - // Neighbor affinity graph (lazy-built, cached with TTL) - neighborMu sync.Mutex - neighborGraph *NeighborGraph + // Neighbor affinity snapshot. Slow builders serialize separately so reads + // of the last complete graph only take the pointer mutex. + neighborMu sync.Mutex + neighborBuildMu sync.Mutex + neighborGraph *NeighborGraph // Cached /api/scope-stats response — per-window, recomputed at most once every 30s scopeStatsMu sync.Mutex From 867a6f8239da6f4169bca4c0190e24160d85a14d Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:54 -0400 Subject: [PATCH 35/67] fix(readiness): require successful startup loading --- cmd/server/healthz.go | 15 ++++- cmd/server/healthz_test.go | 96 ++++++++++++++++++++++++++- cmd/server/packet_filter_lock_test.go | 2 + 3 files changed, 108 insertions(+), 5 deletions(-) diff --git a/cmd/server/healthz.go b/cmd/server/healthz.go index 3f554952f..3039a5fa0 100644 --- a/cmd/server/healthz.go +++ b/cmd/server/healthz.go @@ -7,15 +7,24 @@ import ( ) // readiness tracks whether background init goroutines have completed. -// Set to 1 once store.Load, pickBestObservation, and neighbor graph build are done. +// Startup loading is gated separately by StartupLoadDone in handleHealthz. var readiness atomic.Int32 // handleHealthz returns 200 when the server is ready to serve queries, -// or 503 while background initialization is still running. +// or 503 while startup is incomplete or the startup load has failed. func (s *Server) handleHealthz(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - if readiness.Load() == 0 { + ready := readiness.Load() != 0 + if ready && s.store != nil { + select { + case <-s.store.StartupLoadDone(): + ready = !s.store.backgroundLoadFailed.Load() + default: + ready = false + } + } + if !ready { w.WriteHeader(http.StatusServiceUnavailable) json.NewEncoder(w).Encode(map[string]interface{}{ "ready": false, diff --git a/cmd/server/healthz_test.go b/cmd/server/healthz_test.go index f343b0c7f..3bbae1494 100644 --- a/cmd/server/healthz_test.go +++ b/cmd/server/healthz_test.go @@ -12,7 +12,9 @@ func TestHealthzNotReady(t *testing.T) { readiness.Store(0) defer readiness.Store(0) - srv := &Server{store: &PacketStore{}} + store := &PacketStore{clockSkew: &ClockSkewEngine{}} + store.signalStartupLoadDone() + srv := &Server{store: store} req := httptest.NewRequest("GET", "/api/healthz", nil) w := httptest.NewRecorder() @@ -38,7 +40,9 @@ func TestHealthzReady(t *testing.T) { readiness.Store(1) defer readiness.Store(0) - srv := &Server{store: &PacketStore{}} + store := &PacketStore{clockSkew: &ClockSkewEngine{}} + store.signalStartupLoadDone() + srv := &Server{store: store} req := httptest.NewRequest("GET", "/api/healthz", nil) w := httptest.NewRecorder() @@ -78,3 +82,91 @@ func TestHealthzAntiTautology(t *testing.T) { t.Fatal("anti-tautology: handler returned 200 when readiness=0; gating is broken") } } + +// FirstChunkReady permits HTTP serving while either startup phase is still +// loading. Deployment readiness must wait for both phases and reject failures. +func TestHealthzWaitsForSuccessfulStartupLoad(t *testing.T) { + oldReadiness := readiness.Swap(1) + t.Cleanup(func() { readiness.Store(oldReadiness) }) + for _, phase := range []string{"hot_chunks", "background_fill"} { + for _, fail := range []bool{false, true} { + name := phase + "/success" + if fail { + name = phase + "/failure" + } + t.Run(name, func(t *testing.T) { + dbPath := createTestDBWithAgedPackets(t, 6, 6) + db, err := openFixtureReader(t, dbPath) + if err != nil { + t.Fatal(err) + } + defer db.conn.Close() + store := NewPacketStore(db, &PacketStoreConfig{RetentionHours: 72, HotStartupHours: 1}) + srv := &Server{store: store} + checkHealth := func(want int) { + t.Helper() + w := httptest.NewRecorder() + srv.handleHealthz(w, httptest.NewRequest(http.MethodGet, "/api/healthz", nil)) + if w.Code != want { + t.Errorf("health status = %d, want %d; body=%s", w.Code, want, w.Body.String()) + } + var response struct { + Ready bool `json:"ready"` + } + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if response.Ready != (want == http.StatusOK) { + t.Errorf("ready=%t, want %t", response.Ready, want == http.StatusOK) + } + } + hookCalled := false + checkLoading := func() { + if hookCalled { + return + } + hookCalled = true + if !isClosedForTest(store.FirstChunkReady()) { + t.Error("HTTP startup must be unblocked before all chunks finish") + } + if isClosedForTest(store.StartupLoadDone()) { + t.Error("startup unexpectedly complete inside loader") + } + if got, want := store.LoadComplete(), phase == "background_fill"; got != want { + t.Errorf("LoadComplete=%t, want %t during %s", got, want, phase) + } + checkHealth(http.StatusServiceUnavailable) + if fail { + if err := db.conn.Close(); err != nil { + t.Fatal(err) + } + } + } + if phase == "hot_chunks" { + store.OnChunkLoaded(func(_, _ int) { checkLoading() }) + } else { + store.bgLoaderEntryHook = checkLoading + } + err = store.RunStartupLoad(2) + if fail && phase == "hot_chunks" { + if err == nil { + t.Error("expected a late chunk read error") + } + } else if err != nil { + t.Fatal(err) + } + if !hookCalled || !isClosedForTest(store.StartupLoadDone()) { + t.Fatal("startup did not exercise the hook and reach a terminal state") + } + if store.backgroundLoadFailed.Load() != fail { + t.Fatalf("backgroundLoadFailed=%t, want %t", store.backgroundLoadFailed.Load(), fail) + } + want := http.StatusOK + if fail { + want = http.StatusServiceUnavailable + } + checkHealth(want) + }) + } + } +} diff --git a/cmd/server/packet_filter_lock_test.go b/cmd/server/packet_filter_lock_test.go index 80f58bfd5..d0ef623d8 100644 --- a/cmd/server/packet_filter_lock_test.go +++ b/cmd/server/packet_filter_lock_test.go @@ -42,6 +42,8 @@ func packetFilterHashes(result *PacketResult) []string { func assertColdFilterAllowsStoreReaders(t *testing.T, s *PacketStore, kind string, query func()) { t.Helper() + s.clockSkew = &ClockSkewEngine{} + s.signalStartupLoadDone() hook, _, entered, release := blockingLoadHook(kind) s.cacheLoadHook = hook var workers sync.WaitGroup From d2745c6191bc5bbf3d16d922369d9c2249ca96c7 Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:54 -0400 Subject: [PATCH 36/67] fix(analytics): defer distance build until initial chunks finish --- cmd/server/chunked_distance_lazy_test.go | 101 +++++++++++++++++++++++ cmd/server/store.go | 6 ++ 2 files changed, 107 insertions(+) diff --git a/cmd/server/chunked_distance_lazy_test.go b/cmd/server/chunked_distance_lazy_test.go index 4db55c604..1e82f5943 100644 --- a/cmd/server/chunked_distance_lazy_test.go +++ b/cmd/server/chunked_distance_lazy_test.go @@ -81,3 +81,104 @@ func TestLoadChunkedDistanceIndexRemainsLazy(t *testing.T) { t.Fatal("lazy build did not refresh the real distance recomputer before returning 200") } } + +func TestDistanceRequestDuringChunkedLoadWaitsForLaterPackets(t *testing.T) { + db := setupTestDB(t) + seedTestData(t, db) + mustExecDB(t, db, `UPDATE observations SET path_json = '["aa","11"]' WHERE transmission_id = 1`) + mustExecDB(t, db, `UPDATE observations SET path_json = '["11","aa"]' WHERE transmission_id = 3`) + store := NewPacketStore(db, &PacketStoreConfig{HotStartupHours: 0}) + firstChunk, releaseLoad, loadDone := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + store.OnChunkLoaded(func(_, total int) { + if total == 1 { + close(firstChunk) + <-releaseLoad + } + }) + var loadErr error + go func() { + defer close(loadDone) + loadErr = store.RunStartupLoad(1) + }() + t.Cleanup(func() { + releaseOnce.Do(func() { close(releaseLoad) }) + select { + case <-loadDone: + case <-time.After(5 * time.Second): + t.Error("chunked load did not finish") + } + }) + select { + case <-firstChunk: + case <-time.After(5 * time.Second): + t.Fatal("first chunk did not load") + } + if store.LoadComplete() || store.LoadProgress() != 1 { + t.Fatal("fixture must hold the loader after its first packet") + } + stopRecomputers := store.StartAnalyticsRecomputers(time.Hour) + defer stopRecomputers() + buildEntered := make(chan struct{}, 1) + var builds atomic.Int32 + store.distanceBuildHook = func() { builds.Add(1); buildEntered <- struct{}{} } + srv := NewServer(db, &Config{}, nil) + srv.store = store + request := func() *httptest.ResponseRecorder { + w := httptest.NewRecorder() + srv.handleAnalyticsDistance(w, httptest.NewRequest(http.MethodGet, "/api/analytics/distance", nil)) + return w + } + if w := request(); w.Code != http.StatusAccepted || w.Header().Get("Retry-After") == "" { + t.Fatalf("distance response while loading = %d, want 202 with Retry-After", w.Code) + } + select { + case <-buildEntered: + // Keep the later chunks held until the erroneous early build finishes, + // so the regression cannot pass by racing the build against the load. + waitUntil(t, "premature distance build", store.DistanceIndexBuilt) + t.Error("distance build started before the initial chunks finished") + case <-time.After(100 * time.Millisecond): + } + if w := request(); w.Code != http.StatusAccepted || w.Header().Get("Retry-After") == "" { + t.Errorf("distance poll while loading = %d, want 202 with Retry-After", w.Code) + } + releaseOnce.Do(func() { close(releaseLoad) }) + select { + case <-loadDone: + case <-time.After(5 * time.Second): + t.Fatal("chunked load did not finish") + } + if loadErr != nil { + t.Fatal(loadErr) + } + // Let the real post-load recompute settle before the lazy build, so it + // cannot hide a missing refresh of the newly built distance index. + waitUntil(t, "post-load distance snapshot", func() bool { return store.recompDistance.ComputeRuns() >= 2 }) + runsBeforeBuild := store.recompDistance.ComputeRuns() + if w := request(); w.Code != http.StatusAccepted { + t.Errorf("first distance request after loading = %d, want 202", w.Code) + } + waitUntil(t, "full distance build", store.DistanceIndexBuilt) + w := request() + if w.Code != http.StatusOK { + t.Fatalf("distance response after build = %d, want 200", w.Code) + } + var body DistanceAnalyticsResponse + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + paths := make(map[string]bool, len(body.TopPaths)) + for _, path := range body.TopPaths { + if path.TotalDist > 0 { + paths[path.Hash] = true + } + } + if !paths["abc123def4567890"] || !paths["def456abc1230099"] { + t.Errorf("distance paths must include both first and later chunks: %v", paths) + } + if builds.Load() != 1 || store.recompDistance.ComputeRuns() <= runsBeforeBuild { + t.Errorf("want one full build followed by a fresh recompute; builds=%d, runs=%d -> %d", + builds.Load(), runsBeforeBuild, store.recompDistance.ComputeRuns()) + } +} diff --git a/cmd/server/store.go b/cmd/server/store.go index 60df8c9d6..8bcb33679 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -4415,6 +4415,12 @@ func (s *PacketStore) DistanceIndexBuilding() bool { // the last build OR at most once per 5 minutes — whichever is more // restrictive. The first-ever build always runs. func (s *PacketStore) TriggerDistanceIndexBuild() { + // HTTP may start after the first chunk, but later initial chunks do not + // update this index. Keep requests on the 202 retry path until those + // chunks finish, without changing legacy Load() or manual stores. + if s.firstChunkSignaled.Load() && !s.LoadComplete() { + return + } s.distLazyMu.Lock() if s.distLazyBuilding { s.distLazyMu.Unlock() From ffd35fc0283e51d9ec078f89f884647ff2a3f31e Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:54 -0400 Subject: [PATCH 37/67] fix(neighbors): bound topology refresh independently of response cache --- cmd/server/config.go | 1 + cmd/server/neighbor_graph_cache.go | 35 ++++- cmd/server/neighbor_refresh_cadence_test.go | 139 ++++++++++++++++++++ cmd/server/neighbor_refresh_test.go | 89 +++++++------ 4 files changed, 220 insertions(+), 44 deletions(-) create mode 100644 cmd/server/neighbor_refresh_cadence_test.go diff --git a/cmd/server/config.go b/cmd/server/config.go index de1376f1e..aad31ea07 100644 --- a/cmd/server/config.go +++ b/cmd/server/config.go @@ -350,6 +350,7 @@ type NeighborGraphConfig struct { // recomputer that rebuilds the default-shape neighbor-graph // response (#1481 P0-1). 0/missing = default 300 (5 min). // Lower = fresher data, more CPU per minute. #1483. + // The underlying topology refresh cadence remains capped at 5 minutes. CacheRecomputeIntervalSeconds int `json:"cacheRecomputeIntervalSeconds,omitempty"` } diff --git a/cmd/server/neighbor_graph_cache.go b/cmd/server/neighbor_graph_cache.go index ed63d0628..1a67d1660 100644 --- a/cmd/server/neighbor_graph_cache.go +++ b/cmd/server/neighbor_graph_cache.go @@ -40,8 +40,9 @@ type neighborGraphCacheField struct { } // startNeighborGraphRecomputer launches a background goroutine that -// rebuilds the default-shape response every interval. Returns when -// the stop channel is closed. +// rebuilds the default-shape response every interval. Longer response +// intervals must not let the shared topology exceed its own refresh cadence. +// Returns when the stop channel is closed. func (s *Server) startNeighborGraphRecomputer(interval time.Duration, stop <-chan struct{}) { if interval <= 0 { interval = neighborGraphCacheInterval @@ -50,13 +51,32 @@ func (s *Server) startNeighborGraphRecomputer(interval time.Duration, stop <-cha s.recomputeNeighborGraphCache() t := time.NewTicker(interval) defer t.Stop() + var graphTimer *time.Timer + var graphTick <-chan time.Time + if interval > neighborGraphTTL { + graphTimer = time.NewTimer(neighborGraphTTL) + graphTick = graphTimer.C + defer graphTimer.Stop() + } for { + refreshResponses := false select { case <-t.C: - s.recomputeNeighborGraphCache() + refreshResponses = true + case <-graphTick: + // Coalesce coincident graph/response ticks into one build. + select { + case <-t.C: + refreshResponses = true + default: + } case <-stop: return } + if graphTimer != nil { + graphTimer.Reset(neighborGraphTTL) + } + s.recomputeNeighborGraph(refreshResponses) } }() } @@ -66,6 +86,12 @@ func (s *Server) startNeighborGraphRecomputer(interval time.Duration, stop <-cha // rebuild doesn't kill the background goroutine — but logs the panic // and increments a counter so operators see the failure (#1483 follow-up). func (s *Server) recomputeNeighborGraphCache() { + s.recomputeNeighborGraph(true) +} + +// Graph-only ticks keep node-neighbor/debug/filter reads current without +// paying for scoring and marshaling both analytics responses each time. +func (s *Server) recomputeNeighborGraph(refreshResponses bool) { defer func() { if r := recover(); r != nil { log.Printf("[neighbor-graph-cache] rebuild panic: %v\n%s", r, debug.Stack()) @@ -74,6 +100,9 @@ func (s *Server) recomputeNeighborGraphCache() { }() start := time.Now() s.rebuildNeighborGraph(true) + if !refreshResponses { + return + } resp := s.buildDefaultNeighborGraphResponse() var buf bytes.Buffer if err := json.NewEncoder(&buf).Encode(resp); err != nil { diff --git a/cmd/server/neighbor_refresh_cadence_test.go b/cmd/server/neighbor_refresh_cadence_test.go new file mode 100644 index 000000000..240785a47 --- /dev/null +++ b/cmd/server/neighbor_refresh_cadence_test.go @@ -0,0 +1,139 @@ +package main + +import ( + "sync/atomic" + "testing" + "testing/synctest" + "time" +) + +func TestNeighborGraphLongResponseIntervalStillRefreshesTopology(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + store := ngTestStore(nil, nil) + s := &Server{store: store} + var responseBuilds atomic.Int32 + s.computeNeighborGraphResponseFn = func(minCount int, minScore float64, region, role string) NeighborGraphResponse { + responseBuilds.Add(1) + return s.computeNeighborGraphResponse(minCount, minScore, region, role) + } + stop := make(chan struct{}) + defer close(stop) + // Deliberately not a multiple of the graph TTL: refreshing graph data + // must neither delay the configured JSON cadence nor repeat JSON work. + interval := 3*neighborGraphTTL + 2*time.Minute + s.startNeighborGraphRecomputer(interval, stop) + synctest.Wait() + cached := s.neighborGraphCache.ptr.Load() + unfiltered := s.neighborGraphCache.unfilteredPtr.Load() + graph := s.getNeighborGraph() + if cached == nil || unfiltered == nil || responseBuilds.Load() != 2 { + t.Fatal("initial graph and both response caches were not built") + } + + observations := make([]*StoreObs, 20) + for i := range observations { + observations[i] = ngMakeObs("bbbb2222", `[]`, time.Now().UTC().Format(time.RFC3339), nil) + } + store.mu.Lock() + store.packets = []*StoreTx{ngMakeTx(1, PayloadADVERT, ngFromNodeJSON("aaaa1111"), observations)} + store.mu.Unlock() + for i := 0; i < 3; i++ { + time.Sleep(neighborGraphTTL) + synctest.Wait() + fresh := s.getNeighborGraph() + if fresh == graph { + t.Fatalf("topology did not refresh after TTL at tick %d with a long response interval", i+1) + } + if edges := fresh.Neighbors("aaaa1111"); len(edges) != 1 || edges[0].Count != len(observations) { + t.Fatal("refreshed topology omitted new observations") + } + if s.neighborGraphCache.ptr.Load() != cached || s.neighborGraphCache.unfilteredPtr.Load() != unfiltered || responseBuilds.Load() != 2 { + t.Fatal("graph-only refresh rebuilt the response caches before their configured interval") + } + graph = fresh + } + time.Sleep(2 * time.Minute) + synctest.Wait() + if s.getNeighborGraph() == graph || responseBuilds.Load() != 4 { + t.Fatal("configured response tick did not force a fresh graph and rebuild both response caches") + } + for _, entry := range []*neighborGraphCacheEntry{s.neighborGraphCache.ptr.Load(), s.neighborGraphCache.unfilteredPtr.Load()} { + if len(entry.resp.Edges) != 1 || entry.resp.Edges[0].Weight != len(observations) { + t.Fatal("response cache did not include the refreshed topology") + } + } + }) +} + +func TestNeighborGraphOnlyRefreshRecoversAfterPanic(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + store := ngTestStore(nil, nil) + s := &Server{store: store} + stop := make(chan struct{}) + defer func() { + select { + case <-stop: + default: + close(stop) + } + }() + s.startNeighborGraphRecomputer(time.Hour, stop) + synctest.Wait() + graph := s.getNeighborGraph() + cached := s.neighborGraphCache.ptr.Load() + store.mu.Lock() + store.packets = []*StoreTx{nil} + store.mu.Unlock() + time.Sleep(neighborGraphTTL) + synctest.Wait() + if s.getNeighborGraph() != graph || atomic.LoadUint64(&s.neighborGraphCacheRebuildFailures) != 1 { + t.Fatal("failed graph-only refresh lost the old snapshot or failed to count the error") + } + store.mu.Lock() + store.packets = nil + store.mu.Unlock() + time.Sleep(neighborGraphTTL) + synctest.Wait() + if s.getNeighborGraph() == graph || s.neighborGraphCache.ptr.Load() != cached { + t.Fatal("graph-only refresh did not recover independently of the response cache") + } + close(stop) + synctest.Wait() + graph = s.getNeighborGraph() + time.Sleep(time.Hour) + synctest.Wait() + if s.getNeighborGraph() != graph || s.neighborGraphCache.ptr.Load() != cached { + t.Fatal("graph or response refreshed after the recomputer stopped") + } + }) +} + +func TestNeighborGraphDefaultAndShortResponseIntervals(t *testing.T) { + for _, interval := range []time.Duration{0, -time.Second, neighborGraphCacheInterval, time.Minute} { + t.Run(interval.String(), func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + s := &Server{store: ngTestStore(nil, nil)} + stop := make(chan struct{}) + defer close(stop) + s.startNeighborGraphRecomputer(interval, stop) + synctest.Wait() + graph := s.getNeighborGraph() + cached := s.neighborGraphCache.ptr.Load() + delay := interval + if delay <= 0 { + delay = neighborGraphCacheInterval + } + time.Sleep(delay - time.Nanosecond) + synctest.Wait() + if s.getNeighborGraph() != graph || s.neighborGraphCache.ptr.Load() != cached { + t.Fatal("graph or response refreshed before the configured interval") + } + time.Sleep(time.Nanosecond) + synctest.Wait() + if s.getNeighborGraph() == graph || s.neighborGraphCache.ptr.Load() == cached { + t.Fatal("configured interval did not refresh both the graph and response") + } + }) + }) + } +} diff --git a/cmd/server/neighbor_refresh_test.go b/cmd/server/neighbor_refresh_test.go index 5b591840f..ef9a1559c 100644 --- a/cmd/server/neighbor_refresh_test.go +++ b/cmd/server/neighbor_refresh_test.go @@ -120,46 +120,53 @@ func TestNeighborGraphRefreshPanicPreservesSnapshot(t *testing.T) { } func TestNeighborGraphBackgroundRefreshKeepsReadersAvailable(t *testing.T) { - graph := makeTestGraph() - store := ngTestStore(nil, nil) - s := &Server{store: store, neighborGraph: graph} - store.mu.Lock() // keep the actual graph builder in flight - done := make(chan struct{}) - go func() { - s.recomputeNeighborGraphCache() - close(done) - }() - defer func() { - store.mu.Unlock() - select { - case <-done: - case <-time.After(time.Second): - t.Error("background refresh did not finish after releasing the store") - } - }() - deadline := time.After(time.Second) - ticker := time.NewTicker(time.Millisecond) - defer ticker.Stop() - for s.neighborBuildMu.TryLock() { - s.neighborBuildMu.Unlock() - select { - case <-deadline: - t.Fatal("background refresh did not start its builder") - case <-ticker.C: - } - } - if !s.neighborMu.TryLock() { - t.Fatal("graph builder held the snapshot pointer mutex") - } - s.neighborMu.Unlock() - result := make(chan *NeighborGraph, 1) - go func() { result <- s.getNeighborGraph() }() - select { - case got := <-result: - if got != graph { - t.Fatal("request lost the last completed snapshot during refresh") - } - case <-time.After(500 * time.Millisecond): - t.Fatal("request waited for the background builder") + for _, tc := range []struct { + name string + responses bool + }{{"graph-only", false}, {"response-cache", true}} { + t.Run(tc.name, func(t *testing.T) { + graph := makeTestGraph() + store := ngTestStore(nil, nil) + s := &Server{store: store, neighborGraph: graph} + store.mu.Lock() // keep the actual graph builder in flight + done := make(chan struct{}) + go func() { + s.recomputeNeighborGraph(tc.responses) + close(done) + }() + defer func() { + store.mu.Unlock() + select { + case <-done: + case <-time.After(time.Second): + t.Error("background refresh did not finish after releasing the store") + } + }() + deadline := time.After(time.Second) + ticker := time.NewTicker(time.Millisecond) + defer ticker.Stop() + for s.neighborBuildMu.TryLock() { + s.neighborBuildMu.Unlock() + select { + case <-deadline: + t.Fatal("background refresh did not start its builder") + case <-ticker.C: + } + } + if !s.neighborMu.TryLock() { + t.Fatal("graph builder held the snapshot pointer mutex") + } + s.neighborMu.Unlock() + result := make(chan *NeighborGraph, 1) + go func() { result <- s.getNeighborGraph() }() + select { + case got := <-result: + if got != graph { + t.Fatal("request lost the last completed snapshot during refresh") + } + case <-time.After(500 * time.Millisecond): + t.Fatal("request waited for the background builder") + } + }) } } From 4cf594aa9637ec8d791eeba9ffcd23fba7626205 Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:54 -0400 Subject: [PATCH 38/67] fix(analytics): guard pending views and discard stale loads --- public/analytics.js | 37 +++- tests/e2e/test-analytics-loading-e2e.js | 127 +++++++++++++ tests/unit/test-analytics-loading.js | 229 ++++++++++++++++++++++++ 3 files changed, 387 insertions(+), 6 deletions(-) create mode 100644 tests/e2e/test-analytics-loading-e2e.js create mode 100644 tests/unit/test-analytics-loading.js diff --git a/public/analytics.js b/public/analytics.js index 1bfb12b89..f680b4205 100644 --- a/public/analytics.js +++ b/public/analytics.js @@ -2,7 +2,10 @@ 'use strict'; (function () { - let _analyticsData = {}; + let _analyticsData = null; + let _analyticsLoadId = 0; + let _analyticsLoadError = ''; + const _coreAnalyticsTabs = new Set(['overview', 'rf', 'topology', 'channels', 'hashsizes', 'collisions']); const sf = (v, d) => (v != null ? v.toFixed(d) : '–'); // safe toFixed function esc(s) { return s ? String(s).replace(/&/g,'&').replace(//g,'>').replace(/"/g,'"').replace(/'/g,''') : ''; } @@ -270,9 +273,21 @@ var _themeRefreshHandler = null; let _currentTab = 'overview'; + function renderAnalyticsLoadState(el) { + el.innerHTML = _analyticsLoadError + ? `` + : '
Loading analytics…
'; + } + async function loadAnalytics() { + const el = document.getElementById('analyticsContent'); + if (!el) return; + const loadId = ++_analyticsLoadId; + _analyticsData = null; + _analyticsLoadError = ''; + if (_coreAnalyticsTabs.has(_currentTab)) renderAnalyticsLoadState(el); + else renderTab(_currentTab); try { - _analyticsData = {}; const rqs = RegionFilter.regionQueryString(); // "®ion=..." or "" const aqs = AreaFilter.areaQueryString(); // "&area=..." or "" // Time window picker (#842) — append &window=… when set. @@ -299,16 +314,26 @@ api('/analytics/hash-collisions' + sepBase, { ttl: CLIENT_TTL.analyticsRF }), api('/analytics/relay-airtime-share' + sepWin, { ttl: CLIENT_TTL.analyticsRF }).catch(() => ({ rows: [] })), ]); + // A filter change or page leave makes every earlier response obsolete. + if (loadId !== _analyticsLoadId || document.getElementById('analyticsContent') !== el) return; _analyticsData = { hashData, rfData, topoData, chanData, collisionData, airtimeData }; - renderTab(_currentTab); + if (_coreAnalyticsTabs.has(_currentTab)) renderTab(_currentTab); } catch (e) { - document.getElementById('analyticsContent').innerHTML = - ``; + if (loadId !== _analyticsLoadId || document.getElementById('analyticsContent') !== el) return; + _analyticsLoadError = String(e.message || e); + if (_coreAnalyticsTabs.has(_currentTab)) renderAnalyticsLoadState(el); } } async function renderTab(tab) { const el = document.getElementById('analyticsContent'); + if (!el) return; + // Theme refreshes and tab clicks can arrive before the bulk load settles. + // Lazy tabs use their own data and remain available during that load. + if (!_analyticsData && _coreAnalyticsTabs.has(tab)) { + renderAnalyticsLoadState(el); + return; + } const d = _analyticsData; switch (tab) { case 'overview': renderOverview(el, d); break; @@ -3164,7 +3189,7 @@ } } -function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _stopDistanceRetry(); _analyticsData = {}; _channelData = null; if (_ngState && _ngState.animId) { cancelAnimationFrame(_ngState.animId); } _ngState = null; if (_themeRefreshHandler) { window.removeEventListener('theme-refresh', _themeRefreshHandler); _themeRefreshHandler = null; } } +function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRefresh(); _stopScopesRefresh(); _stopDistanceRetry(); _analyticsData = null; _channelData = null; if (_ngState && _ngState.animId) { cancelAnimationFrame(_ngState.animId); } _ngState = null; if (_themeRefreshHandler) { window.removeEventListener('theme-refresh', _themeRefreshHandler); _themeRefreshHandler = null; } } // Expose for testing if (typeof window !== 'undefined') { diff --git a/tests/e2e/test-analytics-loading-e2e.js b/tests/e2e/test-analytics-loading-e2e.js new file mode 100644 index 000000000..a15c13600 --- /dev/null +++ b/tests/e2e/test-analytics-loading-e2e.js @@ -0,0 +1,127 @@ +#!/usr/bin/env node +/* Local browser regression: delayed analytics, theme refreshes and filter races. */ +'use strict'; +const assert = require('assert'); +const { chromium } = require('playwright'); +const BASE = process.env.BASE_URL || 'http://localhost:13581'; +assert.ok(['localhost', '127.0.0.1', '[::1]'].includes(new URL(BASE).hostname), 'Use a local fixture server, never production'); + +function deferred() { + let resolve; + const promise = new Promise(r => { resolve = r; }); + return { promise, resolve }; +} + +const FIXTURES = { + 'hash-sizes': { total: 0, distribution: {}, hourly: [], topHops: [], multiByteNodes: [] }, + topology: { uniqueNodes: 4, avgHops: 1, maxHops: 1, hopDistribution: [] }, + channels: { activeChannels: 0, decryptable: 0 }, + 'hash-collisions': {}, + 'relay-airtime-share': { rows: [] }, + 'neighbor-graph': { + nodes: [ + { pubkey: 'aa'.repeat(32), role: 'companion', neighbor_count: 1 }, + { pubkey: 'bb'.repeat(32), role: 'companion', neighbor_count: 1 }, + { pubkey: 'cc'.repeat(32), role: 'repeater', neighbor_count: 1 }, + { pubkey: 'dd'.repeat(32), role: 'repeater', neighbor_count: 1 }, + ], + edges: [ + { source: 'aa'.repeat(32), target: 'bb'.repeat(32), score: 0.9, ambiguous: false }, + { source: 'cc'.repeat(32), target: 'dd'.repeat(32), score: 0.9, ambiguous: false }, + ], + stats: {}, + }, +}; + +(async () => { + const browser = await chromium.launch({ headless: true, executablePath: process.env.CHROMIUM_PATH || undefined }); + try { + const page = await browser.newPage({ viewport: { width: 1280, height: 900 } }); + page.setDefaultTimeout(15000); + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + const gates = new Map(['', '1h', '24h', '7d'].map(window => [window, { response: deferred(), arrived: deferred() }])); + await page.route('**/api/analytics/*', async route => { + const url = new URL(route.request().url()); + const endpoint = url.pathname.split('/').pop(); + let body = FIXTURES[endpoint]; + if (endpoint === 'rf') { + const gate = gates.get(url.searchParams.get('window') || ''); + gate.arrived.resolve(url.pathname.slice('/api'.length) + url.search); + const total = await gate.response.promise; + body = { + totalTransmissions: total, totalPackets: total, packetsPerHour: [], payloadTypes: [], + snr: { avg: 0, min: 0, max: 0 }, rssi: { avg: -100, min: -100, max: -100 }, + avgPacketSize: 20, minPacketSize: 20, maxPacketSize: 20, timeSpanHours: 1, + }; + } + if (body === undefined) return route.continue(); + await route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(body) }); + }); + + async function release(window, total) { + const gate = gates.get(window); + const requestPath = await gate.arrived.promise; + gate.response.resolve(total); + // Wait until the real api() has consumed the response, not just until + // the server has sent it; then its Promise.all continuation has run. + await page.waitForFunction(path => _apiPerf.log.some(entry => entry.path === path), requestPath); + } + async function theme() { + await page.evaluate(() => window.dispatchEvent(new CustomEvent('theme-refresh'))); + } + async function selectWindow(window) { + const request = page.waitForRequest(req => { + const url = new URL(req.url()); + return url.pathname === '/api/analytics/rf' && url.searchParams.get('window') === window; + }); + await page.locator('#analyticsTimeWindow').selectOption(window); + await request; + } + + await page.goto(BASE + '/#/analytics', { waitUntil: 'domcontentloaded' }); + await page.locator('#analyticsTimeWindow').waitFor(); + await theme(); + await page.locator('#analyticsTabs [data-tab="hashsizes"]').click(); + await theme(); + assert.match(await page.locator('#analyticsContent').innerText(), /Loading analytics/); + assert.deepStrictEqual(errors, []); + console.log(' ✓ pending core tabs and theme refresh retain the loading state'); + + await page.locator('#analyticsTabs [data-tab="neighbor-graph"]').click(); + const companion = page.locator('#ngRoleChecks input[data-role="companion"]'); + await page.waitForFunction(() => document.querySelector('#ngStats .stat-value')?.textContent === '4'); + await companion.uncheck(); + const canvas = await page.locator('#ngCanvas').elementHandle(); + await release('', 111); + await theme(); + assert.strictEqual(await companion.isChecked(), false, 'bulk completion and theme refresh must preserve graph filters'); + assert.strictEqual(await canvas.evaluate(el => el.isConnected), true, 'the graph DOM must not be rebuilt'); + console.log(' ✓ a lazy graph retains its filters and canvas after core data and theme refresh'); + + await page.locator('#analyticsTabs [data-tab="overview"]').click(); + assert.strictEqual(await page.locator('#analyticsContent .stat-value').first().innerText(), '111'); + await selectWindow('1h'); + await theme(); + assert.match(await page.locator('#analyticsContent').innerText(), /Loading analytics/); + await selectWindow('24h'); + await release('24h', 222); + await page.waitForFunction(() => document.querySelector('#analyticsContent .stat-value')?.textContent === '222'); + await release('1h', 999); + await theme(); + assert.strictEqual(await page.locator('#analyticsContent .stat-value').first().innerText(), '222'); + assert.strictEqual(await page.locator('#analyticsTimeWindow').inputValue(), '24h'); + console.log(' ✓ the newest time window wins when responses complete out of order'); + + await selectWindow('7d'); + await page.evaluate(() => { location.hash = '#/home'; }); + await page.locator('#analyticsContent').waitFor({ state: 'detached' }); + await release('7d', 333); + assert.strictEqual(await page.locator('#analyticsContent').count(), 0); + assert.deepStrictEqual(errors, [], 'analytics must not throw during loading, refresh or page leave'); + console.log(' ✓ late completion after page leave is harmless'); + console.log('Analytics loading browser regression: 4 passed'); + } finally { + await browser.close(); + } +})().catch(error => { console.error(error); process.exitCode = 1; }); diff --git a/tests/unit/test-analytics-loading.js b/tests/unit/test-analytics-loading.js new file mode 100644 index 000000000..46382b9dc --- /dev/null +++ b/tests/unit/test-analytics-loading.js @@ -0,0 +1,229 @@ +/* Exercise the real analytics page lifecycle with controllable API promises. */ +'use strict'; +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const vm = require('vm'); + +const source = fs.readFileSync(path.join(__dirname, '../../public/analytics.js'), 'utf8'); +const settle = () => new Promise(resolve => setImmediate(resolve)); +const unhandled = []; +process.on('unhandledRejection', error => unhandled.push(error.message)); + +function fixture(url, total) { + switch (url.split('?')[0]) { + case '/analytics/rf': return { totalTransmissions: total, totalPackets: total, packetsPerHour: [], payloadTypes: [], snr: {}, rssi: {} }; + case '/analytics/topology': return { uniqueNodes: 0, hopDistribution: [] }; + case '/analytics/hash-sizes': return { total: 0, distribution: {}, hourly: [], topHops: [], multiByteNodes: [] }; + case '/analytics/channels': return { activeChannels: 0, decryptable: 0 }; + default: return { rows: [] }; + } +} + +async function harness(initialTab = '') { + let page; + const requests = [], events = new Map(); + function element() { + let html = ''; + return { + get innerHTML() { return html; }, + set innerHTML(value) { html = value; this.writes++; }, + writes: 0, value: '', style: {}, listeners: {}, + classList: { add() {}, remove() {} }, + addEventListener(name, fn) { this.listeners[name] = fn; }, + querySelectorAll() { return []; }, + }; + } + const elements = Object.fromEntries(['analyticsContent', 'analyticsTabs', 'analyticsTimeWindow', 'analyticsRegionFilter', 'analyticsAreaFilter'].map(id => [id, element()])); + elements.analyticsTabs.querySelector = () => ({ classList: { add() {} } }); + function filter() { + return { init() {}, onChange(fn) { this.change = fn; }, regionQueryString: () => '', areaQueryString: () => '' }; + } + const sandbox = { + window: null, console, URLSearchParams, + document: { getElementById: id => elements[id] || null, querySelectorAll: () => [], documentElement: {} }, + location: { hash: '#/analytics' + (initialTab ? '?tab=' + initialTab : '') }, + localStorage: { getItem: () => null }, + RegionFilter: filter(), AreaFilter: filter(), + CLIENT_TTL: { analyticsRF: 1000 }, + initTabBar() {}, + registerPage(name, handlers) { assert.strictEqual(name, 'analytics'); page = handlers; }, + addEventListener(name, fn) { events.set(name, fn); }, + removeEventListener(name) { events.delete(name); }, + requestAnimationFrame: () => 0, cancelAnimationFrame() {}, + clearInterval() {}, clearTimeout() {}, + api(url) { + return new Promise((resolve, reject) => requests.push({ url, resolve, reject })); + }, + }; + sandbox.window = sandbox; + vm.runInNewContext(source, sandbox, { filename: 'analytics.js' }); + async function init() { + elements.analyticsContent = element(); + elements.analyticsContent.innerHTML = 'Loading analytics…'; + await page.init(element()); + } + await init(); + return { + page, elements, requests, init, + html: () => elements.analyticsContent.innerHTML, + theme: () => events.get('theme-refresh')?.(), + tab(tab) { + const button = { dataset: { tab }, classList: { add() {} } }; + elements.analyticsTabs.listeners.click({ target: { closest: () => button } }); + }, + window(value) { + elements.analyticsTimeWindow.value = value; + elements.analyticsTimeWindow.listeners.change(); + }, + refresh: () => sandbox.RegionFilter.change(), + async resolve(batch, total) { + requests.slice(batch * 6, batch * 6 + 6).forEach(r => r.resolve(fixture(r.url, total))); + await settle(); + }, + async reject(batch, message) { + requests[batch * 6].reject(new Error(message)); + await settle(); + }, + }; +} + +let passed = 0, failed = 0; +async function test(name, fn) { + const before = unhandled.length; + try { + try { await fn(); } finally { await settle(); } + assert.deepStrictEqual(unhandled.slice(before), [], 'page lifecycle must not reject in the background'); + passed++; + console.log(' ✓ ' + name); + } catch (error) { + failed++; + console.error(' ✗ ' + name + ': ' + error.message.split('\n')[0]); + } +} + +(async () => { + await test('theme refresh and every core tab wait for complete analytics', async () => { + const h = await harness(); + h.theme(); + for (const tab of ['overview', 'rf', 'topology', 'channels', 'hashsizes', 'collisions']) { + h.tab(tab); + h.theme(); + assert.match(h.html(), /Loading analytics/); + } + assert.strictEqual(h.requests.length, 6, 'refreshes must not issue extra requests'); + }); + + await test('ready data still renders and tabs and theme refresh keep working', async () => { + const h = await harness(); + await h.resolve(0, 111); + assert.match(h.html(), /stat-value">111111 { + const h = await harness(); + h.window('1h'); + assert.match(h.requests[7].url, /window=1h/); + await h.resolve(1, 222); + assert.match(h.html(), /stat-value">222222 { + const h = await harness(); + h.refresh(); + await h.resolve(1, 222); + await h.reject(0, 'stale failure'); + assert.match(h.html(), /stat-value">222222 { + const h = await harness(); + await h.reject(0, ''); + for (const tab of ['overview', 'hashsizes']) { + h.tab(tab); + h.theme(); + assert.match(h.html(), /Failed to load: <img/); + assert.doesNotMatch(h.html(), /333 { + const h = await harness(); + await h.resolve(0, 111); + h.window('7d'); + h.theme(); + assert.match(h.html(), /Loading analytics/); + assert.doesNotMatch(h.html(), /stat-value">111 { + const h = await harness(); + h.tab('my-repeaters'); + assert.doesNotMatch(h.html(), /Loading analytics/); + assert.match(h.html(), /repeater/i); + h.tab('overview'); + assert.match(h.html(), /Loading analytics/); + }); + + await test('deep-linked lazy tabs do not wait for the core requests', async () => { + const h = await harness('my-repeaters'); + assert.match(h.html(), /repeater/i); + assert.doesNotMatch(h.html(), /Loading analytics/); + assert.strictEqual(h.requests.length, 6); + }); + + await test('core success or failure cannot rebuild an interactive lazy tab', async () => { + for (const reject of [false, true]) { + const h = await harness(); + h.tab('my-repeaters'); + const writes = h.elements.analyticsContent.writes; + if (reject) await h.reject(0, 'core failure'); + else await h.resolve(0, 111); + assert.strictEqual(h.elements.analyticsContent.writes, writes); + h.tab('overview'); + assert.match(h.html(), reject ? /Failed to load: core failure/ : /stat-value">111 { + for (const reject of [false, true]) { + const h = await harness(); + h.page.destroy(); + delete h.elements.analyticsContent; + if (reject) await h.reject(0, 'left page'); + else await h.resolve(0, 111); + } + }); + + await test('old results cannot overwrite a new analytics visit', async () => { + const h = await harness(); + h.page.destroy(); + await h.init(); + await h.resolve(0, 111); + assert.match(h.html(), /Loading analytics/); + await h.resolve(1, 222); + assert.match(h.html(), /stat-value">222 Date: Thu, 15 Oct 2026 17:40:54 -0400 Subject: [PATCH 39/67] fix(frontend): avoid extra rejections during API cleanup --- public/app.js | 4 +- tests/unit/test-api-inflight-cleanup.js | 145 ++++++++++++++++++++++++ 2 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test-api-inflight-cleanup.js diff --git a/public/app.js b/public/app.js index 3e0478797..822cc24ae 100644 --- a/public/app.js +++ b/public/app.js @@ -203,7 +203,9 @@ async function api(path, { ttl = 0, bust = false } = {}) { } })(); _inflight.set(path, promise); - promise.finally(() => _inflight.delete(path)); + // Handle both outcomes so cleanup does not create an unobserved rejection. + const cleanup = () => _inflight.delete(path); + promise.then(cleanup, cleanup); return promise; } diff --git a/tests/unit/test-api-inflight-cleanup.js b/tests/unit/test-api-inflight-cleanup.js new file mode 100644 index 000000000..ade3592ad --- /dev/null +++ b/tests/unit/test-api-inflight-cleanup.js @@ -0,0 +1,145 @@ +'use strict'; + +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const vm = require('vm'); + +const source = fs.readFileSync(path.join(__dirname, '../../public/app.js'), 'utf8'); +const start = source.indexOf('// --- Utilities ---'); +const end = source.indexOf('// Fetch the COMPLETE /api/nodes set', start); +assert.ok(start >= 0 && end > start, 'locate the real API/cache/warmup helpers'); + +function response(status, data = {}) { + return { status, ok: status >= 200 && status < 300, headers: { get: () => '1' }, json: async () => data }; +} + +function makeContext(fetchImpl) { + const events = []; + const clock = { now: 1000 }; + const context = vm.createContext({ + console, + Date: { now: () => clock.now }, + performance: { now: () => 0 }, + window: { onWarmup_1659: visible => events.push(visible) }, + fetch: url => url === '/api/config/cache' ? Promise.resolve(response(200)) : fetchImpl(url), + setTimeout: fn => { fn(); return 0; }, + }); + // Execute the actual helpers, including their maps and warmup counter, without SPA startup. + vm.runInContext(source.slice(start, end), context, { filename: 'public/app.js' }); + return { context, events, clock }; +} + +function assertSettled(context) { + assert.strictEqual(vm.runInContext('_inflight.size', context), 0, 'settled requests leave the inflight map'); + assert.strictEqual(vm.runInContext('_warmupInflight_1659', context), 0, 'warmup counter stays balanced'); +} + +let passed = 0; +let failed = 0; +async function test(name, fn) { + const unhandled = []; + const capture = reason => unhandled.push(reason); + process.on('unhandledRejection', capture); + let failure; + try { await fn(); } catch (error) { failure = error; } + // Node reports unhandled rejections after the promise microtasks have drained. + await new Promise(setImmediate); + process.removeListener('unhandledRejection', capture); + try { + if (failure) throw failure; + assert.strictEqual(unhandled.length, 0, + 'handled API requests must not create unhandled rejections: ' + unhandled.map(error => error.message).join(', ')); + passed++; + console.log(' PASS ' + name); + } catch (error) { + failed++; + console.error(' FAIL ' + name + ': ' + error.message); + } +} + +(async () => { + console.log('\n=== API inflight cleanup ==='); + + await test('success shares one request, clears inflight, and preserves cache TTL and bust', async () => { + let release; + const pending = new Promise(resolve => { release = resolve; }); + let calls = 0; + const { context, clock } = makeContext(async () => { + calls++; + return calls === 1 ? pending : response(200, { call: calls }); + }); + const first = context.api('/cleanup', { ttl: 100 }); + const second = context.api('/cleanup', { ttl: 100 }); + assert.strictEqual(calls, 1, 'concurrent callers share one fetch'); + assert.strictEqual(vm.runInContext('_inflight.size', context), 1); + const body = { call: 1 }; + release(response(200, body)); + const results = await Promise.all([first, second]); + results.forEach(result => assert.strictEqual(result, body)); + assertSettled(context); + clock.now += 99; + assert.strictEqual(await context.api('/cleanup', { ttl: 100 }), body); + assert.strictEqual(calls, 1, 'unexpired cached data avoids another fetch'); + clock.now++; + assert.deepStrictEqual(await context.api('/cleanup', { ttl: 100 }), { call: 2 }); + assert.deepStrictEqual(await context.api('/cleanup', { ttl: 100, bust: true }), { call: 3 }); + assert.strictEqual(calls, 3, 'expiry and bust each start a fresh request'); + assertSettled(context); + }); + + const networkError = new TypeError('Network unavailable'); + const jsonError = new SyntaxError('Invalid JSON'); + const failures = [ + { name: 'HTTP failure', fetch: () => response(500), message: /^API 500: \/cleanup$/, calls: 1 }, + { name: 'network failure', fetch: () => { throw networkError; }, error: networkError, calls: 1 }, + { name: 'JSON parse failure', fetch: () => ({ ...response(200), json: async () => { throw jsonError; } }), error: jsonError, calls: 1 }, + { name: 'exhausted warmup retries', fetch: () => response(503), message: /^API 503: \/cleanup$/, calls: 7, warmup: true }, + { name: 'network failure after warmup', fetch: call => { + if (call === 1) return response(503); + throw networkError; + }, error: networkError, calls: 2, warmup: true }, + ]; + + for (const fixture of failures) { + await test(fixture.name + ' reaches both callers without an extra rejection and permits retry', async () => { + let calls = 0; + let failing = true; + const body = { recovered: true }; + const { context, events } = makeContext(async () => { + calls++; + return failing ? fixture.fetch(calls) : response(200, body); + }); + const results = await Promise.allSettled([ + context.api('/cleanup', { ttl: 100 }), + context.api('/cleanup', { ttl: 100 }), + ]); + assert.strictEqual(calls, fixture.calls, 'concurrent callers share one request/retry sequence'); + results.forEach(result => assert.strictEqual(result.status, 'rejected', 'the caller still receives the failure')); + assert.strictEqual(results[0].reason, results[1].reason, 'both callers receive the original error'); + if (fixture.error) assert.strictEqual(results[0].reason, fixture.error); + else assert.match(results[0].reason.message, fixture.message); + assertSettled(context); + assert.strictEqual(vm.runInContext('_apiCache.size', context), 0, 'failures are not cached'); + assert.deepStrictEqual(events, fixture.warmup ? [true, false] : []); + failing = false; + assert.strictEqual(await context.api('/cleanup', { ttl: 100 }), body); + assert.strictEqual(calls, fixture.calls + 1, 'a failed request does not block a fresh fetch'); + assertSettled(context); + }); + } + + await test('successful warmup retries share one sequence and balance the banner', async () => { + let calls = 0; + const body = { ready: true }; + const { context, events } = makeContext(async () => ++calls < 3 ? response(503) : response(200, body)); + const results = await Promise.all([context.api('/cleanup'), context.api('/cleanup')]); + results.forEach(result => assert.strictEqual(result, body)); + assert.strictEqual(calls, 3); + assert.deepStrictEqual(events, [true, false]); + assertSettled(context); + }); + + console.log('\n' + passed + ' passed, ' + failed + ' failed'); + if (failed) process.exitCode = 1; +})().catch(error => { console.error(error); process.exitCode = 1; }); From 91e26e7e4fd328d61f0e5c40617df26930ed0b69 Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:55 -0400 Subject: [PATCH 40/67] fix(analytics): discard obsolete lazy renders and callbacks --- public/analytics.js | 159 +++++++++++--- tests/e2e/test-analytics-loading-e2e.js | 79 ++++++- tests/unit/test-analytics-loading.js | 254 +++++++++++++++++++++- tests/unit/test-my-repeaters-dashboard.js | 4 +- 4 files changed, 453 insertions(+), 43 deletions(-) diff --git a/public/analytics.js b/public/analytics.js index f680b4205..06c76799b 100644 --- a/public/analytics.js +++ b/public/analytics.js @@ -4,11 +4,26 @@ (function () { let _analyticsData = null; let _analyticsLoadId = 0; + let _analyticsRenderId = 0; + const _analyticsRequests = new WeakMap(); let _analyticsLoadError = ''; const _coreAnalyticsTabs = new Set(['overview', 'rf', 'topology', 'channels', 'hashsizes', 'collisions']); const sf = (v, d) => (v != null ? v.toFixed(d) : '–'); // safe toFixed function esc(s) { return s ? String(s).replace(/&/g,'&').replace(//g,'>').replace(/"/g,'"').replace(/'/g,''') : ''; } + // Capture the current page/filter/tab render. A target also makes requests + // within that view latest-wins (detail panels, range picks, auto-refresh). + // Weak keys let replaced DOM nodes and their request tokens be collected. + function analyticsGuard(target) { + const el = document.getElementById('analyticsContent'); + const loadId = _analyticsLoadId, renderId = _analyticsRenderId; + const request = {}; + if (target) _analyticsRequests.set(target, request); + return () => loadId === _analyticsLoadId && renderId === _analyticsRenderId && + document.getElementById('analyticsContent') === el && + (!target || _analyticsRequests.get(target) === request); + } + // #1085 — Roles tab helpers (hoisted from renderRolesTab so they're not // re-allocated per render). function _rolesEmoji(role) { @@ -262,7 +277,10 @@ // still picks up the new theme: node colors are read live per frame // from window.ROLE_COLORS, role swatches use CSS tokens, and the one // cached value (_labelColor) is re-read on restart. - if (_currentTab === 'neighbor-graph' && _ngState) { startGraphRenderer(); return; } + if (_currentTab === 'neighbor-graph') { + if (_ngState) startGraphRenderer(); + return; + } renderTab(_currentTab); }; window.addEventListener('theme-refresh', _themeRefreshHandler); @@ -328,6 +346,14 @@ async function renderTab(tab) { const el = document.getElementById('analyticsContent'); if (!el) return; + _analyticsRenderId++; + const isCurrent = analyticsGuard(); + _stopRolesRefresh(); + _stopScopesRefresh(); + _stopDistanceRetry(); + el.onclick = null; + if (_ngState && _ngState.animId) cancelAnimationFrame(_ngState.animId); + _ngState = null; // Theme refreshes and tab clicks can arrive before the bulk load settles. // Lazy tabs use their own data and remain available during that load. if (!_analyticsData && _coreAnalyticsTabs.has(tab)) { @@ -354,8 +380,10 @@ case 'prefix-tool': await renderPrefixTool(el); break; case 'scopes': await renderScopesTab(el); break; } + if (!isCurrent()) return; // Auto-apply column resizing to all analytics tables requestAnimationFrame(() => { + if (!isCurrent()) return; el.querySelectorAll('.analytics-table').forEach((tbl, i) => { tbl.id = tbl.id || `analytics-tbl-${tab}-${i}`; if (typeof makeColumnsResizable === 'function') makeColumnsResizable('#' + tbl.id, `meshcore-analytics-${tab}-${i}-col-widths`); @@ -374,6 +402,7 @@ const sectionId = new URLSearchParams((location.hash.split('?')[1] || '')).get('section'); if (sectionId) { setTimeout(() => { + if (!isCurrent()) return; const target = document.getElementById(sectionId); if (target) target.scrollIntoView({ behavior: 'smooth', block: 'start' }); }, 400); @@ -382,6 +411,7 @@ // ===================== OVERVIEW ===================== function renderOverview(el, d) { + const isCurrent = analyticsGuard(); const rf = d.rfData, topo = d.topoData, ch = d.chanData, hs = d.hashData; el.innerHTML = `
@@ -481,6 +511,7 @@ fetch('/api/debug/affinity', { headers: { 'X-API-Key': apiKey } }) .then(function (r) { return r.ok ? r.json() : null; }) .then(function (data) { + if (!isCurrent()) return; if (!data || !data.stats) return; var s = data.stats; var total = s.resolvedCount + s.ambiguousCount + s.unresolvedCount; @@ -882,6 +913,7 @@ var _rtxRequestSeq = 0; async function loadRetransmissionPressure() { if (!document.getElementById('rtxPressureChart')) return; + const isCurrent = analyticsGuard(); var seq = ++_rtxRequestSeq; var twEl = document.getElementById('analyticsTimeWindow'); var tw = twEl ? twEl.value : ''; @@ -891,13 +923,14 @@ var html; try { var data = await api('/analytics/retransmissions?' + qs.slice(1), { ttl: CLIENT_TTL.analyticsRF }); + if (!isCurrent()) return; html = renderRetransmissionChart(data); } catch (e) { html = ''; } // A newer request (window/region change) or a tab switch wins. var host = document.getElementById('rtxPressureChart'); - if (seq !== _rtxRequestSeq || !host) return; + if (!isCurrent() || seq !== _rtxRequestSeq || !host) return; host.innerHTML = html; } @@ -1340,6 +1373,7 @@ } function renderChannels(el, ch) { + const isCurrent = analyticsGuard(); // Decorate first so grouping/display name reflect locally-stored PSK keys. // buildHashKeyMap is async; render once with a sync best-effort empty map, // then upgrade once keys resolve. That keeps first paint fast and avoids @@ -1359,7 +1393,7 @@ // newer rawChannels / decoration with stale-key data. var myGen = ++_channelRenderGen; function applyDecorate(map) { - if (myGen !== _channelRenderGen) return; // superseded + if (!isCurrent() || myGen !== _channelRenderGen) return; // superseded var labels = (typeof ChannelDecrypt !== 'undefined' && ChannelDecrypt.getLabels) ? ChannelDecrypt.getLabels() : {}; _channelData = decorateAnalyticsChannels(rawChannels, map, labels); @@ -1569,6 +1603,7 @@ } function renderMultiByteAdopters(nodes, caps) { + const isCurrent = analyticsGuard(); // Merge capability status into adopter nodes var capByPubkey = {}; (caps || []).forEach(function(c) { capByPubkey[c.pubkey] = c; }); @@ -1644,6 +1679,7 @@ // Use setTimeout for event delegation on the stable section container setTimeout(function() { + if (!isCurrent()) return; var section = document.getElementById('mbAdoptersSection'); if (!section) return; var currentFilter = 'all'; @@ -2216,7 +2252,8 @@ Distant >${t200}: beyond 915MHz range — internet bridge, MQTT gateway, or separate networks
`; } - async function renderSubpaths(el) { + async function renderSubpaths(el) { + const isCurrent = analyticsGuard(el); el.innerHTML = '
Analyzing route patterns…
'; try { const rq = RegionFilter.regionQueryString(); @@ -2226,6 +2263,7 @@ const twVal = twEl ? twEl.value : ''; const tws = twVal ? '&window=' + encodeURIComponent(twVal) : ''; const bulk = await api('/analytics/subpaths-bulk?groups=2-2:50,3-3:30,4-4:20,5-8:15' + rq + tws, { ttl: CLIENT_TTL.analyticsRF }); + if (!isCurrent()) return; const [d2, d3, d4, d5] = bulk.results; function renderTable(data, title) { @@ -2314,13 +2352,14 @@
`; // Click handler for rows - el.addEventListener('click', e => { + el.onclick = e => { + if (!isCurrent()) return; const tr = e.target.closest('tr[data-hops]'); if (!tr) return; el.querySelectorAll('tr.subpath-selected').forEach(r => r.classList.remove('subpath-selected')); tr.classList.add('subpath-selected'); loadSubpathDetail(tr.dataset.hops); - }); + }; // Jump nav — scroll within list panel el.querySelectorAll('.subpath-jump-nav a').forEach(a => { @@ -2341,18 +2380,23 @@ toggle.addEventListener('change', applyCollisionFilter); applyCollisionFilter(); } catch (e) { + if (!isCurrent()) return; el.innerHTML = `
Error loading subpath data: ${e.message}
`; } } async function loadSubpathDetail(hopsStr) { const panel = document.getElementById('subpathDetail'); + if (!panel) return; + const isCurrent = analyticsGuard(panel); panel.classList.remove('collapsed'); panel.innerHTML = '
Loading…
'; try { const data = await api('/analytics/subpath-detail?hops=' + encodeURIComponent(hopsStr), { ttl: CLIENT_TTL.analyticsRF }); + if (!isCurrent()) return; renderSubpathDetail(panel, data); } catch (e) { + if (!isCurrent()) return; panel.innerHTML = `
Error: ${e.message}
`; } } @@ -2456,6 +2500,7 @@ } async function renderNodesTab(el) { + const isCurrent = analyticsGuard(el); el.innerHTML = '
Loading node analytics…
'; try { const rq = RegionFilter.regionQueryString() + AreaFilter.areaQueryString(); @@ -2463,6 +2508,7 @@ fetchAllNodes('&sortBy=lastSeen' + rq, { ttl: CLIENT_TTL.nodeList }), api('/nodes/bulk-health?limit=50' + rq, { ttl: CLIENT_TTL.analyticsRF }) ]); + if (!isCurrent()) return; const nodes = nodesResp.nodes || nodesResp; const myNodes = JSON.parse(localStorage.getItem('meshcore-my-nodes') || '[]'); const myKeys = new Set(myNodes.map(n => n.pubkey)); @@ -2612,6 +2658,7 @@ `; } catch (e) { + if (!isCurrent()) return; el.innerHTML = `
Failed to load node analytics: ${esc(e.message)}
`; } } @@ -2655,6 +2702,7 @@ } async function renderMyRepeatersTab(el) { + const isCurrent = analyticsGuard(el); const haveFavs = () => (typeof getFavorites === 'function' ? getFavorites() : []); if (!haveFavs().length) { el.innerHTML = _mrEmptyState(); return; } el.innerHTML = '
Loading your repeaters…
'; @@ -2675,6 +2723,7 @@ api('/nodes/clock-skew', { ttl: CLIENT_TTL.analyticsRF }).catch(() => []), api('/analytics/neighbor-graph?min_count=1&min_score=0', { ttl: CLIENT_TTL.analyticsRF }).catch(() => ({ nodes: [], edges: [] })), ]); + if (!isCurrent()) return; const allNodes = nodesResp.nodes || nodesResp; const clockByPk = {}; (Array.isArray(clockData) ? clockData : []).forEach(c => { if (c && c.pubkey) clockByPk[c.pubkey] = c; }); @@ -2712,6 +2761,7 @@ // un-star drops the row with no refetch. Re-binds its stars after each // paint (the innerHTML reset clears listeners). function paint() { + if (!isCurrent()) return; const favs = new Set(haveFavs()); if (!favs.size) { el.innerHTML = _mrEmptyState(); return; } const myNodes = allNodes.filter(n => favs.has(n.public_key) && (n.role === 'repeater' || n.role === 'room')); @@ -2832,6 +2882,7 @@ } paint(); } catch (e) { + if (!isCurrent()) return; el.innerHTML = `
Failed to load your repeaters: ${esc(e.message)}
`; } } @@ -2991,10 +3042,12 @@ } async function renderRepeaterMetricsTab(el) { + const isCurrent = analyticsGuard(el); el.innerHTML = '
Loading repeater metrics…
'; try { const rq = RegionFilter.regionQueryString() + AreaFilter.areaQueryString(); const resp = await fetchAllNodes('&sortBy=lastSeen' + rq, { ttl: CLIENT_TTL.nodeList }); + if (!isCurrent()) return; const nodes = resp.nodes || resp; const favs = new Set(typeof getFavorites === 'function' ? getFavorites() : []); const points = _toScatterPoints(nodes, favs); @@ -3072,11 +3125,13 @@ wireAxis(ySel, 'meshcore-repeater-scatter-y'); draw(); } catch (e) { + if (!isCurrent()) return; el.innerHTML = `
Failed to load repeater metrics: ${esc(e.message)}
`; } } async function renderDistanceTab(el) { + const isCurrent = analyticsGuard(el); // A new render supersedes any retry still pending from an earlier one, // so the two can never both write into the tab. _stopDistanceRetry(); @@ -3084,6 +3139,7 @@ const rqs = RegionFilter.regionQueryString(); const sep = rqs ? '?' + rqs.slice(1) : ''; const data = await api('/analytics/distance' + sep, { ttl: CLIENT_TTL.analyticsRF }); + if (!isCurrent()) return; // #1997: the lazy index (#1011) answers 202 {status:"building"} with no // summary until it has been built. Rendering that as data is what threw @@ -3094,12 +3150,9 @@ 'Building the distance index…' + '
This runs once per instance. Retrying in ' + secs + 's.
'; _distanceRetryTimer = setTimeout(function () { + if (!isCurrent()) return; _distanceRetryTimer = null; - // The user may have switched tabs or left while this was pending. - if (_currentTab !== 'distance') return; - const cur = document.getElementById('analyticsContent'); - if (!cur) return; - renderDistanceTab(cur); + renderDistanceTab(el); }, _distanceRetryDelayMs(data)); return; } @@ -3185,6 +3238,7 @@ }); }); } catch (e) { + if (!isCurrent()) return; el.innerHTML = `
Failed to load distance analytics: ${esc(e.message)}
`; } } @@ -3216,6 +3270,7 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef let _ngState = null; // neighbor graph state async function renderNeighborGraphTab(el) { + const isCurrent = analyticsGuard(el); el.innerHTML = `

Neighbor Graph

@@ -3268,10 +3323,12 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef try { graphData = await api('/analytics/neighbor-graph' + sep + (sep ? '&' : '?') + 'min_count=1&min_score=0', { ttl: CLIENT_TTL.analyticsRF }); } catch (e) { + if (!isCurrent()) return; el.innerHTML = `

Failed to load neighbor graph: ${esc(e.message)}

`; return; } + if (!isCurrent()) return; _ngState = createGraphState(graphData); // Render through the filter path so the first paint already respects the // default filters (observers unchecked, saved min-score) and the @@ -3295,7 +3352,7 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef // Debounce the O(N+E) refilter so dragging the slider doesn't rebuild the // node set on every pixel (review of #1758). clearTimeout(ngScoreDebounce); - ngScoreDebounce = setTimeout(applyNGFilters, 50); + ngScoreDebounce = setTimeout(() => { if (isCurrent()) applyNGFilters(); }, 50); }); document.getElementById('ngConfidence').addEventListener('change', applyNGFilters); rcEl.addEventListener('change', applyNGFilters); @@ -3547,6 +3604,7 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef function startGraphRenderer() { if (!_ngState) return; + const state = _ngState; // Re-entrant: the filter path calls this to restart against the new node // set, so cancel any loop already running before deciding what to do. @@ -3616,8 +3674,8 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef // Performance: 500 nodes brute-force repulsion: avg ~4ms/frame = 250fps headroom (measured Chrome 120, M1) var _perfFrameTimes = [], _perfLastTime = 0; function tick() { - if (_ngState._runEpoch !== myEpoch) { return; } // a newer startGraphRenderer run superseded this loop - if (!document.getElementById('ngCanvas')) { _ngState.animId = null; return; } + if (_ngState !== state || state._runEpoch !== myEpoch) return; + if (document.getElementById('ngCanvas') !== canvas) { state.animId = null; return; } var now = performance.now(); if (_perfLastTime) _perfFrameTimes.push(now - _perfLastTime); _perfLastTime = now; @@ -3733,6 +3791,7 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef // --- Prefix Tool --- async function renderPrefixTool(el) { + const isCurrent = analyticsGuard(el); el.innerHTML = '
Loading prefix data…
'; const rq = RegionFilter.regionQueryString() + AreaFilter.areaQueryString(); @@ -3748,10 +3807,12 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef api('/analytics/hash-sizes' + rq, { ttl: CLIENT_TTL.analyticsRF }).catch(() => null), ]); } catch (e) { + if (!isCurrent()) return; el.innerHTML = ``; return; } + if (!isCurrent()) return; // Deduplicate by public_key, require at least 6 hex chars to build all 3 tiers const nodeMap = new Map(); (nodesResp.nodes || nodesResp).forEach(n => { @@ -4204,10 +4265,10 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef // Auto-run from URL params if (initPrefix) { doCheck(initPrefix); - setTimeout(() => { document.getElementById('ptChecker')?.scrollIntoView({ behavior: 'smooth', block: 'start' }); }, 150); + setTimeout(() => { if (isCurrent()) document.getElementById('ptChecker')?.scrollIntoView({ behavior: 'smooth', block: 'start' }); }, 150); } else if (initGenerate) { doGenerate(); - setTimeout(() => { document.getElementById('ptGenerator')?.scrollIntoView({ behavior: 'smooth', block: 'start' }); }, 150); + setTimeout(() => { if (isCurrent()) document.getElementById('ptGenerator')?.scrollIntoView({ behavior: 'smooth', block: 'start' }); }, 150); } } @@ -4243,6 +4304,7 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef } async function renderRFHealthTab(el) { + const isCurrent = analyticsGuard(); // Restore state from URL const hashParams = new URLSearchParams((location.hash.split('?')[1] || '')); if (hashParams.get('range')) _rfHealthState.range = hashParams.get('range'); @@ -4280,8 +4342,11 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef // Range button handlers el.querySelectorAll('.rf-range-btn[data-range]').forEach(btn => { btn.addEventListener('click', () => { + if (!isCurrent()) return; const range = btn.dataset.range; _rfHealthState.range = range; + _analyticsRequests.delete(document.getElementById('rfHealthGrid')); + _analyticsRequests.delete(document.getElementById('rfHealthDetail')); el.querySelectorAll('.rf-range-btn').forEach(b => b.classList.remove('active')); btn.classList.add('active'); const customInputs = el.querySelector('.rf-custom-inputs'); @@ -4296,6 +4361,7 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef const applyBtn = document.getElementById('rfCustomApply'); if (applyBtn) { applyBtn.addEventListener('click', () => { + if (!isCurrent()) return; _rfHealthState.customFrom = document.getElementById('rfFrom').value; _rfHealthState.customTo = document.getElementById('rfTo').value; rfHealthUpdateHash(); @@ -4309,12 +4375,17 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef async function loadRFHealthData(el) { const grid = document.getElementById('rfHealthGrid'); const detail = document.getElementById('rfHealthDetail'); + const isCurrent = analyticsGuard(grid); + // A range change invalidates any detail already in flight, even while + // its new summary is still loading or no longer contains that observer. + if (detail) _analyticsRequests.delete(detail); try { // Compute window string for summary endpoint const windowMap = { '1h':'1h', '3h':'3h', '6h':'6h', '12h':'12h', '24h':'24h', '3d':'3d', '7d':'7d', '30d':'30d' }; const window = windowMap[_rfHealthState.range] || '24h'; const summaryData = await api('/observers/metrics/summary?window=' + window + (RegionFilter.regionQueryString() || '')); + if (!isCurrent()) return; const observers = summaryData.observers || []; // Filter to observers with sufficient sparkline data (≥2 non-null noise_floor values) @@ -4363,6 +4434,7 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef // Click handler for cells grid.querySelectorAll('.rf-cell').forEach(cell => { cell.addEventListener('click', () => { + if (!isCurrent()) return; const obsId = cell.dataset.observer; grid.querySelectorAll('.rf-cell').forEach(c => c.classList.remove('rf-cell-selected')); cell.classList.add('rf-cell-selected'); @@ -4393,17 +4465,21 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef } } } catch (e) { + if (!isCurrent()) return; grid.innerHTML = `
Failed to load RF health data: ${esc(e.message)}
`; } } async function loadRFSparkline(observerId) { + const container = document.getElementById(`rf-spark-${observerId}`); + if (!container) return; + const isCurrent = analyticsGuard(container); const { since, until } = rfHealthTimeRangeToParams(_rfHealthState.range, _rfHealthState.customFrom, _rfHealthState.customTo); try { const data = await api(`/observers/${observerId}/metrics?since=${encodeURIComponent(since)}&until=${encodeURIComponent(until)}`); + if (!isCurrent()) return; const metrics = data.metrics || []; const nfValues = metrics.map(m => m.noise_floor).filter(v => v != null); - const container = document.getElementById(`rf-spark-${observerId}`); if (container && nfValues.length > 1) { container.innerHTML = rfNFSparkline(nfValues, 140, 24); } else if (container) { @@ -4438,6 +4514,7 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef } async function loadRFHealthDetail(observerId, container) { + const isCurrent = analyticsGuard(container); container.classList.remove('rf-panel-empty'); container.innerHTML = '
Loading detail…
'; @@ -4449,6 +4526,7 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef try { const data = await api(`/observers/${observerId}/metrics?since=${encodeURIComponent(since)}&until=${encodeURIComponent(until)}&resolution=${resolution}`); + if (!isCurrent()) return; const metrics = data.metrics || []; const reboots = (data.reboots || []).map(r => new Date(r).getTime()); const name = data.observer_name || observerId.substring(0, 8); @@ -4498,6 +4576,7 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef // Close button container.querySelector('.rf-detail-close').addEventListener('click', () => { + _analyticsRequests.delete(container); container.classList.add('rf-panel-empty'); container.innerHTML = 'Select an observer to view details'; _rfHealthState.selectedObserver = null; @@ -4542,6 +4621,7 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef } } } catch (e) { + if (!isCurrent()) return; container.innerHTML = `
Failed to load detail: ${esc(e.message)}
`; } } @@ -4871,10 +4951,12 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef // #690 — Clock Health fleet view (M3) async function renderClockHealthTab(el) { + const isCurrent = analyticsGuard(el); el.innerHTML = '
Loading clock health data…
'; try { const aqs = AreaFilter.areaQueryString(); var data = await (await fetch('/api/nodes/clock-skew' + (aqs ? '?' + aqs.slice(1) : ''))).json(); + if (!isCurrent()) return; if (!Array.isArray(data) || !data.length) { el.innerHTML = '
No clock skew data available. Nodes need recent adverts for clock analysis.
'; return; @@ -4973,6 +5055,7 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef render(); } catch (err) { + if (!isCurrent()) return; el.innerHTML = '
Failed to load clock health data: ' + esc(String(err)) + '
'; } } @@ -5006,6 +5089,7 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef } async function renderScopesTab(el) { + const isCurrent = analyticsGuard(); var winKey = 'scopes_window'; var selectedWindow = (typeof sessionStorage !== 'undefined' && sessionStorage.getItem(winKey)) || '24h'; @@ -5030,28 +5114,34 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef '
' + '
'; - // Attach window-button click listeners (once) - el.querySelectorAll('[data-win]').forEach(function(btn) { - btn.addEventListener('click', function() { - selectedWindow = btn.dataset.win; - if (typeof sessionStorage !== 'undefined') sessionStorage.setItem(winKey, selectedWindow); - el.querySelectorAll('[data-win]').forEach(function(b) { b.classList.toggle('active', b.dataset.win === selectedWindow); }); - load(selectedWindow); - }); - }); } + // The frame survives theme/filter refreshes; replace its handlers so + // window picks and the refresh timer share the current render's state. + el.querySelectorAll('[data-win]').forEach(function(btn) { + btn.onclick = function() { + if (!isCurrent()) return; + selectedWindow = btn.dataset.win; + if (typeof sessionStorage !== 'undefined') sessionStorage.setItem(winKey, selectedWindow); + el.querySelectorAll('[data-win]').forEach(function(b) { b.classList.toggle('active', b.dataset.win === selectedWindow); }); + load(selectedWindow); + }; + }); + function pct(n, total) { if (!total) return '—'; return (n / total * 100).toFixed(1) + '%'; } async function load(w) { + if (!isCurrent()) return; + const requestCurrent = analyticsGuard(document.getElementById('scopes-cards')); var loadingEl = document.getElementById('scopes-loading'); if (loadingEl) loadingEl.style.display = ''; try { // Fix 4: use api() instead of raw fetch() var data = await api('/scope-stats?window=' + encodeURIComponent(w), { ttl: 30000 }); + if (!requestCurrent()) return; if (loadingEl) loadingEl.style.display = 'none'; if (data.error) { var cardsEl2 = document.getElementById('scopes-cards'); @@ -5060,6 +5150,7 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef } updateData(data, w); } catch (err) { + if (!requestCurrent()) return; if (loadingEl) loadingEl.style.display = 'none'; var cardsEl3 = document.getElementById('scopes-cards'); if (cardsEl3) cardsEl3.innerHTML = '
Failed to load scope stats: ' + esc(String(err)) + '
'; @@ -5167,9 +5258,7 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef // Fix 6: auto-refresh every 60s _stopScopesRefresh(); _scopesRefreshTimer = setInterval(function() { - if (_currentTab !== 'scopes') { _stopScopesRefresh(); return; } - var cur = document.getElementById('analyticsContent'); - if (!cur) { _stopScopesRefresh(); return; } + if (!isCurrent()) return; load(selectedWindow); }, 60000); } @@ -5179,22 +5268,23 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef // Auto-refreshes every 60s while the Roles tab is active (matches the // behavior of the former standalone roles-page.js). async function renderRolesTab(el) { + const isCurrent = analyticsGuard(); el.innerHTML = '
Loading roles…
'; await _renderRolesTabBody(el); + if (!isCurrent()) return; // (Re)start the 60s auto-refresh. _stopRolesRefresh(); _rolesRefreshTimer = setInterval(function () { - // Bail if the user navigated away from the Roles tab. - if (_currentTab !== 'roles') { _stopRolesRefresh(); return; } - var cur = document.getElementById('analyticsContent'); - if (!cur) { _stopRolesRefresh(); return; } - _renderRolesTabBody(cur); + if (!isCurrent()) return; + _renderRolesTabBody(el); }, 60000); } async function _renderRolesTabBody(el) { + const isCurrent = analyticsGuard(el); try { var data = await api('/analytics/roles', { ttl: CLIENT_TTL.analyticsRF }); + if (!isCurrent()) return; var roles = (data && data.roles) || []; var total = (data && data.totalNodes) || 0; if (!roles.length) { @@ -5246,6 +5336,7 @@ function destroy() { _analyticsLoadId++; _analyticsLoadError = ''; _stopRolesRef '' + rows + '' + ''; } catch (err) { + if (!isCurrent()) return; el.innerHTML = '
Failed to load roles: ' + esc(String(err.message || err)) + '
'; } } diff --git a/tests/e2e/test-analytics-loading-e2e.js b/tests/e2e/test-analytics-loading-e2e.js index a15c13600..57689648f 100644 --- a/tests/e2e/test-analytics-loading-e2e.js +++ b/tests/e2e/test-analytics-loading-e2e.js @@ -18,6 +18,7 @@ const FIXTURES = { channels: { activeChannels: 0, decryptable: 0 }, 'hash-collisions': {}, 'relay-airtime-share': { rows: [] }, + 'subpaths-bulk': { results: Array.from({ length: 4 }, () => ({ subpaths: [] })) }, 'neighbor-graph': { nodes: [ { pubkey: 'aa'.repeat(32), role: 'companion', neighbor_count: 1 }, @@ -41,6 +42,7 @@ const FIXTURES = { const errors = []; page.on('pageerror', error => errors.push(error.message)); const gates = new Map(['', '1h', '24h', '7d'].map(window => [window, { response: deferred(), arrived: deferred() }])); + const lazyGates = new Map(['E2E-OLD', 'E2E-NEW', 'E2E-LATE', 'E2E-ERROR', 'E2E-FAIL', 'E2E-LEFT'].map(region => [region, { response: deferred(), arrived: deferred() }])); await page.route('**/api/analytics/*', async route => { const url = new URL(route.request().url()); const endpoint = url.pathname.split('/').pop(); @@ -58,6 +60,23 @@ const FIXTURES = { if (body === undefined) return route.continue(); await route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(body) }); }); + // Use synthetic nodes and bulk health; hold the lazy response independently + // of the six core requests so these exercise the actual renderer race. + await page.route('**/api/nodes**', async route => { + const url = new URL(route.request().url()); + const region = url.searchParams.get('region'); + const gate = lazyGates.get(region); + if (!gate || !['/api/nodes', '/api/nodes/bulk-health'].includes(url.pathname)) return route.continue(); + const publicKey = 'ab'.repeat(32); + let status = 200; + let body = { nodes: [{ public_key: publicKey, name: region + ' synthetic node', role: 'repeater', last_seen: new Date().toISOString() }], has_more: false, counts: { repeater: 1 }, total: 1 }; + if (url.pathname.endsWith('/bulk-health')) { + gate.arrived.resolve(url.pathname.slice('/api'.length) + url.search); + status = await gate.response.promise; + body = [{ public_key: publicKey, stats: { totalPackets: 22, packetsToday: 2, avgSnr: 3 }, observers: [] }]; + } + await route.fulfill({ status, contentType: 'application/json', body: JSON.stringify(body) }); + }); async function release(window, total) { const gate = gates.get(window); @@ -78,6 +97,18 @@ const FIXTURES = { await page.locator('#analyticsTimeWindow').selectOption(window); await request; } + async function selectRegion(region, window) { + await page.evaluate(region => RegionFilter.setSelected([region]), region); + await selectWindow(window); + } + async function releaseLazy(region, status = 200) { + const gate = lazyGates.get(region); + const requestPath = await gate.arrived.promise; + gate.response.resolve(status); + // Errors do not enter _apiPerf.log. Wait for the real api() cleanup for + // both success and rejection, then inspect the renderer's settled view. + await page.waitForFunction(path => !_inflight.has(path), requestPath); + } await page.goto(BASE + '/#/analytics', { waitUntil: 'domcontentloaded' }); await page.locator('#analyticsTimeWindow').waitFor(); @@ -120,7 +151,53 @@ const FIXTURES = { assert.strictEqual(await page.locator('#analyticsContent').count(), 0); assert.deepStrictEqual(errors, [], 'analytics must not throw during loading, refresh or page leave'); console.log(' ✓ late completion after page leave is harmless'); - console.log('Analytics loading browser regression: 4 passed'); + + await page.evaluate(() => { + RegionFilter.setSelected(['E2E-OLD']); + location.hash = '#/analytics?tab=nodes'; + }); + await lazyGates.get('E2E-OLD').arrived.promise; + assert.match(await page.locator('#analyticsContent').innerText(), /Loading node analytics/); + await selectRegion('E2E-NEW', '24h'); + await releaseLazy('E2E-NEW'); + await page.getByRole('link', { name: 'E2E-NEW synthetic node', exact: true }).first().waitFor(); + await releaseLazy('E2E-OLD'); + assert.match(await page.locator('#analyticsContent').innerText(), /E2E-NEW synthetic node/); + assert.doesNotMatch(await page.locator('#analyticsContent').innerText(), /E2E-OLD synthetic node/); + console.log(' ✓ lazy Nodes keeps the newest filter when the old response arrives last'); + + for (const [region, status] of [['E2E-LATE', 200], ['E2E-ERROR', 500]]) { + await page.evaluate(region => RegionFilter.setSelected([region]), region); + await page.locator('#analyticsTabs [data-tab="nodes"]').click(); + await lazyGates.get(region).arrived.promise; + await page.locator('#analyticsTabs [data-tab="subpaths"]').click(); + await page.getByRole('heading', { name: 'Route Pattern Analysis' }).waitFor(); + await releaseLazy(region, status); + assert.match(await page.locator('#analyticsContent').innerText(), /Route Pattern Analysis/); + assert.doesNotMatch(await page.locator('#analyticsContent').innerText(), /Failed to load node|synthetic node/); + } + console.log(' ✓ late lazy success and HTTP 500 failure preserve the newly selected tab'); + + await page.evaluate(() => RegionFilter.setSelected(['E2E-FAIL'])); + await page.locator('#analyticsTabs [data-tab="nodes"]').click(); + await releaseLazy('E2E-FAIL', 500); + await page.getByText(/Failed to load node analytics: API 500/).waitFor(); + assert.deepStrictEqual(errors, [], 'handled lazy HTTP failures must not become unhandled rejections'); + lazyGates.set('E2E-FAIL', { response: deferred(), arrived: deferred() }); + await theme(); + await releaseLazy('E2E-FAIL'); + await page.getByRole('link', { name: 'E2E-FAIL synthetic node', exact: true }).first().waitFor(); + console.log(' ✓ a current lazy failure is visible and a subsequent refresh recovers'); + + await selectRegion('E2E-LEFT', '1h'); + await lazyGates.get('E2E-LEFT').arrived.promise; + await page.evaluate(() => { location.hash = '#/home'; }); + await page.locator('#analyticsContent').waitFor({ state: 'detached' }); + await releaseLazy('E2E-LEFT'); + assert.strictEqual(await page.locator('#analyticsContent').count(), 0); + assert.deepStrictEqual(errors, [], 'lazy completion after destroy must not throw or restart rendering'); + console.log(' ✓ a delayed lazy response after page leave is harmless'); + console.log('Analytics loading browser regression: 8 passed'); } finally { await browser.close(); } diff --git a/tests/unit/test-analytics-loading.js b/tests/unit/test-analytics-loading.js index 46382b9dc..b671b782a 100644 --- a/tests/unit/test-analytics-loading.js +++ b/tests/unit/test-analytics-loading.js @@ -16,22 +16,36 @@ function fixture(url, total) { case '/analytics/topology': return { uniqueNodes: 0, hopDistribution: [] }; case '/analytics/hash-sizes': return { total: 0, distribution: {}, hourly: [], topHops: [], multiByteNodes: [] }; case '/analytics/channels': return { activeChannels: 0, decryptable: 0 }; + case '/nodes': return { nodes: [], total }; + case '/nodes/bulk-health': + case '/api/nodes/clock-skew': + case '/nodes/clock-skew': return []; + case '/analytics/subpaths-bulk': return { results: Array.from({ length: 4 }, () => ({ subpaths: [] })) }; + case '/analytics/neighbor-graph': return { nodes: [], edges: [], stats: {} }; + case '/analytics/distance': return { status: 'building', retry_after_seconds: 5 }; + case '/observers/metrics/summary': return { observers: [] }; + case '/analytics/roles': return { roles: [], totalNodes: 0 }; + case '/scope-stats': return { summary: { transportTotal: total, scoped: 0, unscoped: total, unknownScope: 0 } }; default: return { rows: [] }; } } async function harness(initialTab = '') { let page; - const requests = [], events = new Map(); + const requests = [], events = new Map(), timers = new Map(), frames = []; + let timerId = 0; function element() { let html = ''; return { get innerHTML() { return html; }, set innerHTML(value) { html = value; this.writes++; }, writes: 0, value: '', style: {}, listeners: {}, - classList: { add() {}, remove() {} }, + classList: { add() {}, remove() {}, toggle() {}, contains: () => false }, addEventListener(name, fn) { this.listeners[name] = fn; }, + setAttribute() {}, querySelectorAll() { return []; }, + querySelector(selector) { return elements[selector.slice(1)] || null; }, + getContext() { return { scale() {} }; }, }; } const elements = Object.fromEntries(['analyticsContent', 'analyticsTabs', 'analyticsTimeWindow', 'analyticsRegionFilter', 'analyticsAreaFilter'].map(id => [id, element()])); @@ -43,19 +57,27 @@ async function harness(initialTab = '') { window: null, console, URLSearchParams, document: { getElementById: id => elements[id] || null, querySelectorAll: () => [], documentElement: {} }, location: { hash: '#/analytics' + (initialTab ? '?tab=' + initialTab : '') }, - localStorage: { getItem: () => null }, + history: { replaceState() {} }, + localStorage: { getItem: () => null, setItem() {} }, + sessionStorage: { getItem: () => null, setItem() {} }, RegionFilter: filter(), AreaFilter: filter(), CLIENT_TTL: { analyticsRF: 1000 }, initTabBar() {}, registerPage(name, handlers) { assert.strictEqual(name, 'analytics'); page = handlers; }, addEventListener(name, fn) { events.set(name, fn); }, removeEventListener(name) { events.delete(name); }, - requestAnimationFrame: () => 0, cancelAnimationFrame() {}, - clearInterval() {}, clearTimeout() {}, + requestAnimationFrame(fn) { frames.push(fn); return frames.length; }, cancelAnimationFrame() {}, + setInterval(fn) { timers.set(++timerId, fn); return timerId; }, + setTimeout(fn) { const id = ++timerId; timers.set(id, () => { timers.delete(id); fn(); }); return id; }, + clearInterval(id) { timers.delete(id); }, clearTimeout(id) { timers.delete(id); }, + getComputedStyle: () => ({ getPropertyValue: () => '' }), + getFavorites: () => [], api(url) { return new Promise((resolve, reject) => requests.push({ url, resolve, reject })); }, }; + sandbox.fetchAllNodes = query => sandbox.api('/nodes?' + query); + sandbox.fetch = url => sandbox.api(url).then(data => ({ ok: true, json: async () => data })); sandbox.window = sandbox; vm.runInNewContext(source, sandbox, { filename: 'analytics.js' }); async function init() { @@ -65,7 +87,10 @@ async function harness(initialTab = '') { } await init(); return { - page, elements, requests, init, + page, elements, requests, init, timers, frames, + element, + add(...ids) { ids.forEach(id => { elements[id] = element(); }); }, + favorites() { sandbox.getFavorites = () => ['aa'.repeat(32)]; }, html: () => elements.analyticsContent.innerHTML, theme: () => events.get('theme-refresh')?.(), tab(tab) { @@ -224,6 +249,223 @@ async function test(name, fn) { assert.match(h.html(), /stat-value">222 reject ? r.reject(new Error('obsolete response')) : r.resolve(fixture(r.url, total))); + await settle(); + } + + for (const tab of lazyTabs) { + for (const reject of [false, true]) { + await test(`${tab}: stale ${reject ? 'failure' : 'success'} cannot write or restart work after a tab switch`, async () => { + const h = await harness(); + addLazyElements(h); + const start = h.requests.length; + h.tab(tab); + const pending = h.requests.slice(start); + assert.ok(pending.length, 'exercise the real lazy request'); + h.tab('overview'); + const writes = Object.values(h.elements).reduce((n, el) => n + el.writes, 0); + const frames = h.frames.length; + await complete(pending, reject); + assert.match(h.html(), /Loading analytics/); + assert.strictEqual(Object.values(h.elements).reduce((n, el) => n + el.writes, 0), writes, 'stale result must not mutate any view'); + assert.strictEqual(h.frames.length, frames, 'stale completion must not schedule layout or graph work'); + assert.strictEqual(h.timers.size, 0, 'stale completion must not restart retries or refresh timers'); + }); + } + } + + await test('Nodes uses the newest filter response and ignores an older failure', async () => { + for (const reject of [false, true]) { + const h = await harness(); + h.tab('nodes'); + const old = h.requests.slice(6); + const next = h.requests.length; + h.refresh(); + await complete(h.requests.slice(next, next + 2), false, 222); + assert.match(h.html(), />222222111 { + for (const tab of lazyTabs) { + const h = await harness(); + addLazyElements(h); + h.tab(tab); + const pending = h.requests.slice(6); + const previous = h.elements.analyticsContent; + h.page.destroy(); + const writes = previous.writes; + await h.init(); + const currentWrites = h.elements.analyticsContent.writes; + await complete(pending); + assert.strictEqual(previous.writes, writes, tab + ' must not write into the detached page'); + assert.strictEqual(h.elements.analyticsContent.writes, currentWrites, tab + ' must not write into the new page'); + } + }); + + await test('current lazy responses render normally and distance retries only while current', async () => { + for (const tab of lazyTabs) { + const h = await harness(); + addLazyElements(h); + h.tab(tab); + await complete(h.requests.slice(6)); + assert.doesNotMatch(h.html(), /Failed to load|Error loading|Loading node|Loading roles/); + if (tab === 'distance') { + assert.match(h.html(), /Building the distance index/); + const retry = [...h.timers.values()][0]; + const before = h.requests.length; + retry(); + assert.strictEqual(h.requests.length, before + 1); + h.tab('overview'); + retry(); + assert.strictEqual(h.requests.length, before + 1, 'a queued stale retry cannot fetch again'); + await complete(h.requests.slice(before)); + assert.strictEqual(h.timers.size, 0); + } + } + }); + + await test('Roles refreshes are latest-wins and obsolete timer callbacks cannot restart work', async () => { + const h = await harness('roles'); + await complete(h.requests.slice(0, 1)); + const refresh = [...h.timers.values()][0]; + assert.ok(refresh, 'current response starts the refresh timer'); + refresh(); + const old = h.requests.at(-1); + refresh(); + const current = h.requests.at(-1); + current.resolve({ totalNodes: 222, roles: [{ role: 'repeater', nodeCount: 222 }] }); + await settle(); + old.resolve({ totalNodes: 111, roles: [{ role: 'repeater', nodeCount: 111 }] }); + await settle(); + assert.match(h.html(), /222<\/strong>/); + h.tab('overview'); + const before = h.requests.length; + refresh(); + assert.strictEqual(h.requests.length, before); + assert.strictEqual(h.timers.size, 0); + }); + + await test('subpath detail selections are latest-wins and stop after a filter change', async () => { + const h = await harness(); + addLazyElements(h); + h.tab('subpaths'); + await complete(h.requests.slice(6)); + function select(hops) { + const row = h.element(); + row.dataset = { hops }; + h.elements.analyticsContent.onclick({ target: { closest: () => row } }); + return h.requests.at(-1); + } + function detail(count) { + return { nodes: [], hops: ['aa', 'bb'], totalMatches: count, hourDistribution: [], signal: {}, observers: [], parentPaths: [] }; + } + const old = select('aa,bb'), current = select('cc,dd'); + current.resolve(detail(222)); + await settle(); + old.reject(new Error('obsolete detail')); + await settle(); + assert.match(h.elements.subpathDetail.innerHTML, /222 occurrences/); + const pending = select('ee,ff'); + h.refresh(); + const writes = h.elements.subpathDetail.writes; + pending.resolve(detail(111)); + await settle(); + assert.strictEqual(h.elements.subpathDetail.writes, writes); + }); + + await test('RF range and observer selections reject stale successes and failures', async () => { + const h = await harness(); + addLazyElements(h); + const range = h.element(); + range.dataset = { range: '1h' }; + h.elements.analyticsContent.querySelectorAll = selector => selector.includes('data-range') ? [range] : []; + const cells = ['observer-a', 'observer-b'].map(id => Object.assign(h.element(), { dataset: { observer: id } })); + h.elements.rfHealthGrid.querySelectorAll = () => cells; + const summary = { observers: cells.map(c => ({ observer_id: c.dataset.observer, sparkline: [-110, -111], sample_count: 2 })) }; + h.tab('rf-health'); + const old = h.requests.at(-1); + range.listeners.click(); + const current = h.requests.at(-1); + current.resolve(summary); + await settle(); + const gridWrites = h.elements.rfHealthGrid.writes; + old.reject(new Error('obsolete summary')); + await settle(); + assert.strictEqual(h.elements.rfHealthGrid.writes, gridWrites); + cells[0].listeners.click(); + const firstDetail = h.requests.at(-1); + cells[1].listeners.click(); + const secondDetail = h.requests.at(-1); + secondDetail.resolve({ observer_name: 'Newest observer', metrics: [] }); + await settle(); + firstDetail.resolve({ observer_name: 'Old observer', metrics: [] }); + await settle(); + assert.match(h.elements.rfHealthDetail.innerHTML, /Newest observer/); + cells[0].listeners.click(); + const pending = h.requests.at(-1); + range.dataset.range = 'custom'; + range.listeners.click(); + const detailWrites = h.elements.rfHealthDetail.writes; + pending.reject(new Error('obsolete range detail')); + await settle(); + assert.strictEqual(h.elements.rfHealthDetail.writes, detailWrites); + }); + + await test('Scopes keeps current window handlers after refresh and ignores overlapping requests', async () => { + const h = await harness(); + addLazyElements(h); + const button = Object.assign(h.element(), { dataset: { win: '1h' } }); + h.elements.analyticsContent.querySelectorAll = selector => selector === '[data-win]' ? [button] : []; + h.tab('scopes'); + const old = h.requests.at(-1); + h.theme(); + button.onclick(); + const current = h.requests.at(-1); + assert.match(current.url, /window=1h/); + current.resolve(fixture(current.url, 222)); + await settle(); + old.reject(new Error('obsolete scopes')); + await settle(); + assert.match(h.elements['scopes-cards'].innerHTML, />222 { + const h = await harness(); + addLazyElements(h); + h.tab('neighbor-graph'); + await complete(h.requests.slice(6)); + const slider = h.elements.ngMinScore; + slider.value = '80'; + slider.listeners.input.call(slider); + const debounce = [...h.timers.values()][0]; + const frames = h.frames.slice(); + h.page.destroy(); + delete h.elements.analyticsContent; + delete h.elements.ngCanvas; + debounce(); + frames.forEach(fn => fn()); + assert.strictEqual(h.frames.length, frames.length, 'old callbacks cannot schedule more animation or layout'); + }); + console.log(`Analytics loading: ${passed} passed, ${failed} failed`); process.exitCode = failed ? 1 : 0; })(); diff --git a/tests/unit/test-my-repeaters-dashboard.js b/tests/unit/test-my-repeaters-dashboard.js index 30664a579..98d35ef86 100644 --- a/tests/unit/test-my-repeaters-dashboard.js +++ b/tests/unit/test-my-repeaters-dashboard.js @@ -67,10 +67,10 @@ const window = { ROLE_COLORS: { repeater: '#3b82f6', room: '#a855f7' }, aaBadgeS const M = new Function( 'esc', 'getFavorites', 'RegionFilter', 'AreaFilter', 'fetchAllNodes', 'api', 'CLIENT_TTL', - 'getHealthThresholds', 'renderSkewBadge', 'currentSkewValue', 'favStar', 'timeAgo', 'bindFavStars', 'window', + 'getHealthThresholds', 'renderSkewBadge', 'currentSkewValue', 'favStar', 'timeAgo', 'bindFavStars', 'window', 'analyticsGuard', block + '\nreturn { renderMyRepeatersTab, _mrStatusCell, _mrSummaryCard };')( esc, getFavorites, RegionFilter, AreaFilter, fetchAllNodes, api, CLIENT_TTL, - getHealthThresholds, renderSkewBadge, currentSkewValue, favStar, timeAgo, bindFavStars, window); + getHealthThresholds, renderSkewBadge, currentSkewValue, favStar, timeAgo, bindFavStars, window, () => () => true); function makeEl() { return { _h: '', set innerHTML(v) { this._h = v; }, get innerHTML() { return this._h; } }; } const tick = () => new Promise(r => setTimeout(r, 0)); From c5e4656eccaa3110aeca23148c409f275120ec6c Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:55 -0400 Subject: [PATCH 41/67] fix(analytics): count node roles within the selected filters --- public/analytics.js | 12 ++--- tests/e2e/test-analytics-loading-e2e.js | 14 +++++- tests/unit/test-analytics-loading.js | 58 ++++++++++++++++++++++--- 3 files changed, 72 insertions(+), 12 deletions(-) diff --git a/public/analytics.js b/public/analytics.js index 06c76799b..98b0139a8 100644 --- a/public/analytics.js +++ b/public/analytics.js @@ -2524,11 +2524,14 @@ const byObservers = [...enriched].sort((a, b) => (b.health.observers?.length || 0) - (a.health.observers?.length || 0)); const byRecent = [...enriched].filter(n => n.health.stats.lastHeard).sort((a, b) => new Date(b.health.stats.lastHeard) - new Date(a.health.stats.lastHeard)); - // Compute network status client-side from loaded nodes using shared getHealthThresholds() + // Status and roles must describe the same filtered nodes; API counts are global. + const roleColors = ROLE_COLORS; + const roleCounts = Object.fromEntries(Object.keys(roleColors).map(role => [role, 0])); const now = Date.now(); let active = 0, degraded = 0, silent = 0; nodes.forEach(function(n) { - const role = n.role || 'unknown'; + const role = Object.prototype.hasOwnProperty.call(roleCounts, n.role) ? n.role : 'unknown'; + roleCounts[role]++; const th = getHealthThresholds(role); const lastMs = n.last_heard ? new Date(n.last_heard).getTime() : n.last_seen ? new Date(n.last_seen).getTime() @@ -2539,7 +2542,6 @@ else silent++; }); const totalNodes = nodesResp.total || nodes.length; - const roleCounts = nodesResp.counts || {}; function nodeLink(n) { return `${esc(n.name || n.public_key.slice(0, 12))}`; @@ -2575,8 +2577,8 @@

Role Breakdown

${Object.entries(roleCounts).sort((a,b) => b[1]-a[1]).map(([role, count]) => { - const c = ROLE_COLORS[role] || '#6b7280'; - return `${role}: ${count}`; + const c = roleColors[role]; + return `${esc(role)}: ${count}`; }).join('')}
diff --git a/tests/e2e/test-analytics-loading-e2e.js b/tests/e2e/test-analytics-loading-e2e.js index 57689648f..bec67cc84 100644 --- a/tests/e2e/test-analytics-loading-e2e.js +++ b/tests/e2e/test-analytics-loading-e2e.js @@ -69,7 +69,7 @@ const FIXTURES = { if (!gate || !['/api/nodes', '/api/nodes/bulk-health'].includes(url.pathname)) return route.continue(); const publicKey = 'ab'.repeat(32); let status = 200; - let body = { nodes: [{ public_key: publicKey, name: region + ' synthetic node', role: 'repeater', last_seen: new Date().toISOString() }], has_more: false, counts: { repeater: 1 }, total: 1 }; + let body = { nodes: [{ public_key: publicKey, name: region + ' synthetic node', role: 'repeater', last_seen: new Date().toISOString() }], has_more: false, counts: { repeater: 9000, companion: 8000, room: 7000 }, total: 1 }; if (url.pathname.endsWith('/bulk-health')) { gate.arrived.resolve(url.pathname.slice('/api'.length) + url.search); status = await gate.response.promise; @@ -164,7 +164,17 @@ const FIXTURES = { await releaseLazy('E2E-OLD'); assert.match(await page.locator('#analyticsContent').innerText(), /E2E-NEW synthetic node/); assert.doesNotMatch(await page.locator('#analyticsContent').innerText(), /E2E-OLD synthetic node/); - console.log(' ✓ lazy Nodes keeps the newest filter when the old response arrives last'); + const subset = await page.evaluate(() => { + const content = document.getElementById('analyticsContent'); + const heading = Array.from(content.querySelectorAll('h3')).find(el => el.textContent.trim() === 'Role Breakdown'); + return { + roles: Array.from(heading.nextElementSibling.querySelectorAll('.badge'), el => el.textContent.trim()), + total: content.querySelectorAll('.analytics-stat-card')[3].firstElementChild.textContent.trim(), + }; + }); + assert.deepStrictEqual(subset.roles, ['repeater: 1', 'companion: 0', 'room: 0', 'sensor: 0', 'observer: 0', 'unknown: 0']); + assert.strictEqual(subset.total, '1', 'role counts must agree with the selected subset total'); + console.log(' ✓ lazy Nodes keeps the newest filter and its role totals when the old response arrives last'); for (const [region, status] of [['E2E-LATE', 200], ['E2E-ERROR', 500]]) { await page.evaluate(region => RegionFilter.setSelected([region]), region); diff --git a/tests/unit/test-analytics-loading.js b/tests/unit/test-analytics-loading.js index b671b782a..e6ac55a08 100644 --- a/tests/unit/test-analytics-loading.js +++ b/tests/unit/test-analytics-loading.js @@ -6,6 +6,8 @@ const path = require('path'); const vm = require('vm'); const source = fs.readFileSync(path.join(__dirname, '../../public/analytics.js'), 'utf8'); +const rolesSource = fs.readFileSync(path.join(__dirname, '../../public/roles.js'), 'utf8'); +const airtimeSource = fs.readFileSync(path.join(__dirname, '../../public/observer-airtime.js'), 'utf8'); const settle = () => new Promise(resolve => setImmediate(resolve)); const unhandled = []; process.on('unhandledRejection', error => unhandled.push(error.message)); @@ -50,17 +52,22 @@ async function harness(initialTab = '') { } const elements = Object.fromEntries(['analyticsContent', 'analyticsTabs', 'analyticsTimeWindow', 'analyticsRegionFilter', 'analyticsAreaFilter'].map(id => [id, element()])); elements.analyticsTabs.querySelector = () => ({ classList: { add() {} } }); - function filter() { - return { init() {}, onChange(fn) { this.change = fn; }, regionQueryString: () => '', areaQueryString: () => '' }; + function filter(key) { + let query = ''; + return { + init() {}, onChange(fn) { this.change = fn; }, + setSelected(value) { query = value ? '&' + key + '=' + encodeURIComponent(value) : ''; this.change(); }, + regionQueryString: () => query, areaQueryString: () => query, + }; } const sandbox = { window: null, console, URLSearchParams, - document: { getElementById: id => elements[id] || null, querySelectorAll: () => [], documentElement: {} }, + document: { getElementById: id => elements[id] || null, querySelectorAll: () => [], documentElement: {}, readyState: 'loading', addEventListener() {} }, location: { hash: '#/analytics' + (initialTab ? '?tab=' + initialTab : '') }, history: { replaceState() {} }, localStorage: { getItem: () => null, setItem() {} }, sessionStorage: { getItem: () => null, setItem() {} }, - RegionFilter: filter(), AreaFilter: filter(), + RegionFilter: filter('region'), AreaFilter: filter('area'), CLIENT_TTL: { analyticsRF: 1000 }, initTabBar() {}, registerPage(name, handlers) { assert.strictEqual(name, 'analytics'); page = handlers; }, @@ -77,8 +84,11 @@ async function harness(initialTab = '') { }, }; sandbox.fetchAllNodes = query => sandbox.api('/nodes?' + query); - sandbox.fetch = url => sandbox.api(url).then(data => ({ ok: true, json: async () => data })); + sandbox.fetch = url => (url === '/api/config/client' ? Promise.resolve({}) : sandbox.api(url)).then(data => ({ ok: true, json: async () => data })); sandbox.window = sandbox; + vm.runInNewContext(rolesSource, sandbox, { filename: 'roles.js' }); + vm.runInNewContext(airtimeSource, sandbox, { filename: 'observer-airtime.js' }); + await sandbox.MeshConfigReady; vm.runInNewContext(source, sandbox, { filename: 'analytics.js' }); async function init() { elements.analyticsContent = element(); @@ -102,6 +112,8 @@ async function harness(initialTab = '') { elements.analyticsTimeWindow.listeners.change(); }, refresh: () => sandbox.RegionFilter.change(), + region: value => sandbox.RegionFilter.setSelected(value), + area: value => sandbox.AreaFilter.setSelected(value), async resolve(batch, total) { requests.slice(batch * 6, batch * 6 + 6).forEach(r => r.resolve(fixture(r.url, total))); await settle(); @@ -296,6 +308,42 @@ async function test(name, fn) { } }); + await test('Nodes role counts follow region and area changes without extra requests', async () => { + const h = await harness(); + const cases = [ + { select: () => h.tab('nodes'), query: {}, roles: ['repeater', 'repeater', 'companion'], counts: { repeater: 2, companion: 1 }, requests: 2 }, + { select: () => h.region('E2E-REGION'), query: { region: 'E2E-REGION' }, roles: ['room', 'sensor', 'observer'], counts: { room: 1, sensor: 1, observer: 1 }, requests: 8 }, + { select: () => h.area('E2E-AREA'), query: { region: 'E2E-REGION', area: 'E2E-AREA' }, roles: ['sensor'], counts: { sensor: 1 }, requests: 8 }, + { select: () => h.area('E2E-EMPTY'), query: { region: 'E2E-REGION', area: 'E2E-EMPTY' }, roles: [], counts: {}, requests: 8 }, + { select: () => h.area(''), query: { region: 'E2E-REGION' }, roles: ['unknown', null, undefined, '', 'new-role', '__proto__', 'constructor', 'toString', ''], counts: { unknown: 9 }, requests: 8 }, + { select: () => h.region(''), query: {}, roles: ['repeater', 'companion'], counts: { repeater: 1, companion: 1 }, requests: 8 }, + ]; + for (const entry of cases) { + const start = h.requests.length; + entry.select(); + const pending = h.requests.slice(start); + const nodes = entry.roles.map((role, index) => ({ public_key: String(index), role })); + const nodeRequests = pending.filter(r => r.url.startsWith('/nodes')); + assert.strictEqual(nodeRequests.length, 2, 'only the existing node-list and bulk-health requests'); + nodeRequests.forEach(r => { + const params = new URLSearchParams(r.url.split('?')[1]); + for (const key of ['region', 'area']) assert.strictEqual(params.get(key), entry.query[key] || null); + }); + pending.forEach(r => r.resolve(r.url.startsWith('/nodes?') + ? { nodes, total: nodes.length, counts: { repeater: 9000, companion: 8000, room: 7000 } } + : fixture(r.url, 1))); + await settle(); + assert.doesNotMatch(h.html(), /Failed to load|')[1].split('
')[0]; + const counts = Object.fromEntries([...breakdown.matchAll(/>([^<>]+): (\d+)<\/span>/g)].map(match => [match[1], Number(match[2])])); + assert.deepStrictEqual(counts, { repeater: 0, companion: 0, room: 0, sensor: 0, observer: 0, unknown: 0, ...entry.counts }); + assert.strictEqual(Object.values(counts).reduce((sum, count) => sum + count, 0), nodes.length); + const cards = [...h.html().matchAll(/font-size:28px[^>]*>(\d+)<\/div>/g)].map(match => Number(match[1])); + assert.strictEqual(cards[3], nodes.length, 'role badges and Total Nodes use the same subset'); + assert.strictEqual(h.requests.length, start + entry.requests, 'counting roles cannot trigger more fetches'); + } + }); + await test('destroyed lazy requests cannot write into detached content or a new visit', async () => { for (const tab of lazyTabs) { const h = await harness(); From 8ab5fd5e7e92bd91f50cb72eff725c492b5ca8d5 Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:55 -0400 Subject: [PATCH 42/67] fix: ignore stale node analytics responses --- public/node-analytics.js | 22 ++- tests/e2e/test-e2e-playwright.js | 134 ++++++++++++++- tests/unit/test-node-analytics-lifecycle.js | 179 ++++++++++++++++++++ 3 files changed, 323 insertions(+), 12 deletions(-) create mode 100644 tests/unit/test-node-analytics-lifecycle.js diff --git a/public/node-analytics.js b/public/node-analytics.js index 3344bab28..0c63ccc70 100644 --- a/public/node-analytics.js +++ b/public/node-analytics.js @@ -9,6 +9,7 @@ let charts = []; let currentDays = 7; let currentPubkey = null; + let loadGeneration = 0; function destroyCharts() { charts.forEach(c => { try { c.destroy(); } catch {} }); @@ -31,6 +32,8 @@ } async function loadAnalytics(container, pubkey, days) { + const generation = ++loadGeneration; + const isCurrent = () => generation === loadGeneration && container.isConnected && currentPubkey === pubkey; currentPubkey = pubkey; currentDays = days; destroyCharts(); @@ -42,9 +45,11 @@ try { data = await api('/nodes/' + encodeURIComponent(pubkey) + '/analytics?days=' + days, { ttl: CLIENT_TTL.nodeAnalytics }); } catch (e) { + if (!isCurrent()) return; container.innerHTML = '
Failed to load analytics: ' + escapeHtml(e.message) + '
'; return; } + if (!isCurrent()) return; const n = data.node; const s = data.computedStats; @@ -158,6 +163,7 @@ // Time range buttons container.querySelectorAll('#timeRangeBtns button').forEach(btn => { btn.addEventListener('click', () => { + if (!isCurrent()) return; const d = Number(btn.dataset.days); loadAnalytics(container, pubkey, d); }); @@ -170,7 +176,7 @@ buildObserverChart(data); buildHopChart(data); buildHeatmap(data); - loadBatteryChart(pubkey, currentDays); + loadBatteryChart(pubkey, currentDays, isCurrent); NodeHopAnalytics.load(document.getElementById('hopCountSection'), pubkey, currentDays); } @@ -302,18 +308,21 @@ } } - async function loadBatteryChart(pubkey, days) { + async function loadBatteryChart(pubkey, days, isCurrent) { + // Keep this render's elements; matching IDs may belong to another node/range + // by the time the request finishes. + const ctx = document.getElementById('batteryChart'); + const empty = document.getElementById('batteryEmpty'); + const badge = document.getElementById('batteryStatusBadge'); let data; try { data = await api('/nodes/' + encodeURIComponent(pubkey) + '/battery?days=' + days); } catch (e) { - const empty = document.getElementById('batteryEmpty'); + if (!isCurrent() || (empty && !empty.isConnected)) return; if (empty) { empty.style.display = 'block'; empty.textContent = 'Battery data unavailable: ' + e.message; } return; } - const ctx = document.getElementById('batteryChart'); - const empty = document.getElementById('batteryEmpty'); - const badge = document.getElementById('batteryStatusBadge'); + if (!isCurrent() || (ctx && !ctx.isConnected)) return; const samples = (data && data.samples) || []; const thr = (data && data.thresholds) || { low_mv: 3300, critical_mv: 3000 }; @@ -375,6 +384,7 @@ } function destroy() { + loadGeneration++; destroyCharts(); NodeHopAnalytics.destroy(); currentPubkey = null; diff --git a/tests/e2e/test-e2e-playwright.js b/tests/e2e/test-e2e-playwright.js index 43a5a1ddc..3a6b4bd08 100644 --- a/tests/e2e/test-e2e-playwright.js +++ b/tests/e2e/test-e2e-playwright.js @@ -66,6 +66,127 @@ async function run() { console.log(`\nRunning E2E tests against ${BASE}\n`); + // #2218 Node Analytics lifecycle: delayed responses retain their own page/range. + const analyticsKey = 'aa'.repeat(32); + const otherAnalyticsKey = 'bb'.repeat(32); + function nodeAnalyticsFixture(key) { + return { + node: { public_key: key, name: key === analyticsKey ? 'Delayed node' : 'Current node', role: 'repeater' }, + computedStats: { totalPackets: 1, availabilityPct: 100, signalGrade: 'A', avgPacketsPerDay: 1, uniqueObservers: 1, relayPct: 0 }, + activityTimeline: [{ bucket: '2026-01-01T00:00:00Z', count: 1 }], + snrTrend: [], packetTypeBreakdown: [], observerCoverage: [], hopDistribution: [], uptimeHeatmap: [], peerInteractions: [], + }; + } + function nodeBatteryFixture(mv) { + return { status: 'ok', latest_mv: mv, samples: [{ timestamp: '2026-01-01T00:00:00Z', battery_mv: mv }] }; + } + async function nodeAnalyticsHarness(heldPath, failure) { + const ctx = await browser.newContext(); + const p = await ctx.newPage(); + const errors = []; + p.on('pageerror', error => errors.push(error.message)); + let release; + const held = new Promise(resolve => { release = resolve; }); + const started = p.waitForRequest(request => { + const url = new URL(request.url()); + return url.pathname + url.search === heldPath; + }, { timeout: 10000 }); + started.catch(() => {}); // The awaited promise still reports a missing request. + await p.route('**/api/nodes/**', async route => { + const url = new URL(route.request().url()); + const path = url.pathname + url.search; + if (path === heldPath) { + await held; + if (failure) return route.fulfill({ status: 500, json: { error: 'Delayed request failed' } }); + } + if (url.pathname.endsWith('/analytics')) { + return route.fulfill({ json: nodeAnalyticsFixture(url.pathname.split('/')[3]) }); + } + if (url.pathname.endsWith('/battery')) { + return route.fulfill({ json: nodeBatteryFixture(path === heldPath ? 3200 : 4000) }); + } + if (url.pathname.endsWith('/hop_analytics')) return route.fulfill({ json: { packets: [], ambiguous: 0 } }); + return route.continue(); + }); + return { + page: p, started, errors, + async release() { + const response = p.waitForResponse(r => new URL(r.url()).pathname + new URL(r.url()).search === heldPath); + release(); + await (await response).finished(); + await p.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))); + }, + async close() { release(); await ctx.close(); }, + }; + } + for (const failure of [false, true]) { + await test('Node Analytics late ' + (failure ? 'failure' : 'success') + ' cannot replace Nodes after leaving', async () => { + const h = await nodeAnalyticsHarness('/api/nodes/' + analyticsKey + '/analytics?days=7', failure); + try { + await h.page.goto(BASE + '/#/nodes/' + analyticsKey + '/analytics', { waitUntil: 'domcontentloaded' }); + await h.started; + await h.page.goto(BASE + '/#/nodes', { waitUntil: 'domcontentloaded' }); + await h.page.waitForSelector('#nodesLeft[data-loaded="true"]'); + await h.release(); + assert(await h.page.locator('#nodesLeft[data-loaded="true"]').isVisible(), 'old analytics response replaced the Nodes list'); + assert(await h.page.locator('#nodesBody tr[data-action="select"]').count() > 0, 'Nodes list lost its rows'); + assert(h.errors.length === 0, h.errors.join('\n')); + } finally { await h.close(); } + }); + await test('Node Analytics late battery ' + (failure ? 'failure' : 'success') + ' cannot replace the new range', async () => { + const h = await nodeAnalyticsHarness('/api/nodes/' + analyticsKey + '/battery?days=7', failure); + try { + await h.page.goto(BASE + '/#/nodes/' + analyticsKey + '/analytics', { waitUntil: 'domcontentloaded' }); + await h.started; + await h.page.click('#timeRangeBtns button[data-days="1"]'); + await h.page.waitForFunction(() => { + const chart = Chart.getChart(document.getElementById('batteryChart')); + return chart && chart.data.datasets[0].data[0] === 4000; + }); + await h.release(); + const current = await h.page.evaluate(() => ({ + days: document.querySelector('#timeRangeBtns .active').dataset.days, + battery: Chart.getChart(document.getElementById('batteryChart')).data.datasets[0].data[0], + emptyHidden: getComputedStyle(document.getElementById('batteryEmpty')).display === 'none', + })); + assert(current.days === '1' && current.battery === 4000 && current.emptyHidden, + 'stale battery response changed the new range: ' + JSON.stringify(current)); + assert(h.errors.length === 0, h.errors.join('\n')); + } finally { await h.close(); } + }); + } + await test('Node Analytics late response cannot replace a different node', async () => { + const h = await nodeAnalyticsHarness('/api/nodes/' + analyticsKey + '/analytics?days=7', false); + try { + await h.page.goto(BASE + '/#/nodes/' + analyticsKey + '/analytics', { waitUntil: 'domcontentloaded' }); + await h.started; + await h.page.goto(BASE + '/#/nodes/' + otherAnalyticsKey + '/analytics', { waitUntil: 'domcontentloaded' }); + await h.page.waitForFunction(() => document.querySelector('#app h2')?.textContent.includes('Current node')); + await h.release(); + assert((await h.page.locator('#app h2').textContent()).includes('Current node'), 'old response replaced the current node'); + assert(h.errors.length === 0, h.errors.join('\n')); + } finally { await h.close(); } + }); + await test('Node Analytics reentry renders only the current charts from a shared pending request', async () => { + const h = await nodeAnalyticsHarness('/api/nodes/' + analyticsKey + '/analytics?days=7', false); + try { + const url = BASE + '/#/nodes/' + analyticsKey + '/analytics'; + await h.page.goto(url, { waitUntil: 'domcontentloaded' }); + await h.started; + await h.page.goto(BASE + '/#/nodes', { waitUntil: 'domcontentloaded' }); + await h.page.waitForSelector('#nodesLeft[data-loaded="true"]'); + await h.page.goto(url, { waitUntil: 'domcontentloaded' }); + await h.release(); + await h.page.waitForFunction(() => Chart.getChart(document.getElementById('batteryChart'))); + const charts = await h.page.evaluate(() => ({ + count: Object.values(Chart.instances).length, + detached: Object.values(Chart.instances).filter(chart => !chart.canvas.isConnected).length, + })); + assert(charts.count === 6 && charts.detached === 0, 'reentry left stale charts: ' + JSON.stringify(charts)); + assert(h.errors.length === 0, h.errors.join('\n')); + } finally { await h.close(); } + }); + for (const [view, width, full, interaction] of [ ['desktop side pane', 1280, false], ['desktop full view', 1280, true], @@ -565,12 +686,13 @@ async function run() { // Use page.click() to avoid detached-element race with WebSocket auto-refresh. await page.click('table tbody tr:not([id^=vscroll])'); await page.waitForSelector('.node-detail'); - // Find the Details link in the side panel - await page.waitForSelector('#nodesRight a.btn-primary[href^="#/nodes/"]'); - const href = await page.$eval('#nodesRight a.btn-primary[href^="#/nodes/"]', el => el.getAttribute('href')); - assert(href, 'Details link not found in side panel'); - // Click the Details link — this should navigate to the full detail page - await page.click('#nodesRight a.btn-primary[href^="#/nodes/"]'); + // Details is a button; the first anchor now opens Analytics. + await page.waitForSelector('#nodesRight .node-detail-btn'); + const key = await page.locator('#nodesRight .node-detail-btn').getAttribute('data-pubkey'); + assert(key, 'Details button has no node key'); + const href = '#/nodes/' + key; + await page.click('#nodesRight .node-detail-btn'); + await page.waitForSelector('#nodeFullBody'); // Wait for navigation — the full detail page has sections like neighbors/packets await page.waitForFunction((expectedHash) => { return location.hash === expectedHash; diff --git a/tests/unit/test-node-analytics-lifecycle.js b/tests/unit/test-node-analytics-lifecycle.js new file mode 100644 index 000000000..c5716d749 --- /dev/null +++ b/tests/unit/test-node-analytics-lifecycle.js @@ -0,0 +1,179 @@ +/* #2218: exercise the shipped Node Analytics lifecycle with controllable requests. */ +'use strict'; +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const vm = require('vm'); +const source = fs.readFileSync(path.join(__dirname, '../../public/node-analytics.js'), 'utf8'); +const settle = () => new Promise(resolve => setImmediate(resolve)); +const unhandled = []; +process.on('unhandledRejection', error => unhandled.push(error.message)); + +function analytics(name = 'Current node') { + return { + node: { name, public_key: 'aa'.repeat(32), role: 'repeater' }, + computedStats: {}, peerInteractions: [], activityTimeline: [], snrTrend: [], + packetTypeBreakdown: [], observerCoverage: [], hopDistribution: [], uptimeHeatmap: [], + }; +} +function battery(mv = 3800) { + return { status: 'ok', latest_mv: mv, samples: [{ timestamp: '2026-01-01T00:00:00Z', battery_mv: mv }] }; +} + +function harness() { + const requests = [], chartInstances = [], hopLoads = []; + let page, elements = {}, buttons = []; + const container = { isConnected: true }; + function element(id) { + return { + id, innerHTML: '', textContent: '', style: {}, dataset: {}, listeners: {}, + get isConnected() { return container.isConnected && elements[id] === this; }, + addEventListener(name, fn) { this.listeners[name] = fn; }, + }; + } + let html = ''; + Object.defineProperty(container, 'innerHTML', { + get() { return html; }, + set(value) { + html = value; + elements = Object.fromEntries(Array.from(value.matchAll(/id="([^"]+)"/g), m => [m[1], element(m[1])])); + buttons = value.includes('id="timeRangeBtns"') ? [1, 7, 30, 365].map(days => { + const btn = element('range-' + days); + elements[btn.id] = btn; + btn.dataset.days = String(days); + return btn; + }) : []; + }, + }); + container.querySelectorAll = selector => selector === '#timeRangeBtns button' ? buttons : []; + class Chart { + constructor(canvas, config) { this.canvas = canvas; this.config = config; chartInstances.push(this); } + destroy() { this.destroyed = true; } + } + Chart.defaults = {}; + const sandbox = { + console, Chart, Date, CLIENT_TTL: { nodeAnalytics: 1000 }, + document: { documentElement: {}, getElementById: id => elements[id] || null }, + getComputedStyle: () => ({ getPropertyValue: () => '' }), + escapeHtml: value => String(value), timeAgo: () => 'now', + NodeHopAnalytics: { load: (...args) => hopLoads.push(args), destroy() {} }, + registerPage(name, handlers) { assert.strictEqual(name, 'node-analytics'); page = handlers; }, + api(url) { return new Promise((resolve, reject) => requests.push({ url, resolve, reject })); }, + }; + vm.runInNewContext(source, sandbox, { filename: 'node-analytics.js' }); + return { + page, container, requests, chartInstances, hopLoads, + init: (key = 'aa') => page.init(container, key.repeat(32) + '/analytics'), + element: id => elements[id], + button: days => buttons.find(btn => btn.dataset.days === String(days)), + async resolve(request, data) { request.resolve(data); await settle(); }, + async reject(request, message) { request.reject(new Error(message)); await settle(); }, + }; +} + +let passed = 0, failed = 0; +async function test(name, run) { + const before = unhandled.length; + try { + await run(); + await settle(); + assert.deepStrictEqual(unhandled.slice(before), [], 'no background rejection'); + passed++; + console.log(' ✓ ' + name); + } catch (error) { + failed++; + console.error(' ✗ ' + name + ': ' + error.message.split('\n')[0]); + } +} + +(async () => { + for (const outcome of ['success', 'failure']) { + await test('late analytics ' + outcome + ' cannot replace the page after leaving', async () => { + const h = harness(); h.init(); h.page.destroy(); h.container.innerHTML = 'Nodes list'; + if (outcome === 'success') await h.resolve(h.requests[0], analytics('Old node')); + else await h.reject(h.requests[0], 'old error'); + assert.strictEqual(h.container.innerHTML, 'Nodes list'); + assert.strictEqual(h.chartInstances.length, 0); + assert.strictEqual(h.hopLoads.length, 0); + assert.strictEqual(h.requests.length, 1, 'stale main response must not start battery requests'); + }); + await test('late analytics ' + outcome + ' cannot replace a remounted same node', async () => { + const h = harness(); h.init(); h.page.destroy(); h.init(); + await h.resolve(h.requests[1], analytics('New render')); + const current = h.container.innerHTML; + if (outcome === 'success') await h.resolve(h.requests[0], analytics('Old render')); + else await h.reject(h.requests[0], 'old error'); + assert.strictEqual(h.container.innerHTML, current); + assert.strictEqual(h.chartInstances.length, 5); + }); + await test('late battery ' + outcome + ' cannot alter the newer range', async () => { + const h = harness(); h.init(); await h.resolve(h.requests[0], analytics()); + const oldBattery = h.requests[1]; + h.button(30).listeners.click(); + await h.resolve(h.requests[2], analytics('New range')); + await h.resolve(h.requests[3], battery(4000)); + const currentBadge = h.element('batteryStatusBadge').textContent; + const currentEmpty = h.element('batteryEmpty').textContent; + const count = h.chartInstances.length; + if (outcome === 'success') await h.resolve(oldBattery, battery(3200)); + else await h.reject(oldBattery, 'old battery error'); + assert.strictEqual(h.element('batteryStatusBadge').textContent, currentBadge); + assert.strictEqual(h.element('batteryEmpty').textContent, currentEmpty); + assert.strictEqual(h.chartInstances.length, count); + }); + await test('late battery ' + outcome + ' cannot alter another page with matching IDs', async () => { + const h = harness(); h.init(); await h.resolve(h.requests[0], analytics()); h.page.destroy(); + h.container.innerHTML = '
'; + if (outcome === 'success') await h.resolve(h.requests[1], battery(3200)); + else await h.reject(h.requests[1], 'old battery error'); + assert.strictEqual(h.element('batteryStatusBadge').textContent, ''); + assert.strictEqual(h.element('batteryEmpty').textContent, ''); + assert.strictEqual(h.chartInstances.length, 5); + }); + } + await test('switching node keeps the newer response and chart ownership', async () => { + const h = harness(); h.init(); h.page.destroy(); h.init('bb'); + await h.resolve(h.requests[1], analytics('Node B')); + await h.resolve(h.requests[0], analytics('Node A')); + assert.match(h.container.innerHTML, /Node B/); + assert.doesNotMatch(h.container.innerHTML, /Node A/); + assert.strictEqual(h.chartInstances.length, 5); + assert.strictEqual(h.requests.length, 3); + }); + await test('disconnected main and battery targets do not render', async () => { + const h = harness(); h.init(); h.container.isConnected = false; + await h.resolve(h.requests[0], analytics()); + assert.strictEqual(h.chartInstances.length, 0); + const live = harness(); live.init(); await live.resolve(live.requests[0], analytics()); + live.container.isConnected = false; + await live.resolve(live.requests[1], battery()); + assert.strictEqual(live.chartInstances.length, 5); + }); + await test('old range callbacks cannot start requests or destroy current charts', async () => { + const h = harness(); h.init(); await h.resolve(h.requests[0], analytics()); + const oldClick = h.button(1).listeners.click; + h.button(30).listeners.click(); await h.resolve(h.requests[2], analytics('New range')); + const count = h.requests.length; + const activeCharts = h.chartInstances.filter(chart => !chart.destroyed); + oldClick(); await settle(); + assert.strictEqual(h.requests.length, count); + assert.ok(activeCharts.every(chart => !chart.destroyed)); + h.page.destroy(); oldClick(); await settle(); + assert.strictEqual(h.requests.length, count); + }); + await test('current charts and main/battery errors still render', async () => { + const h = harness(); h.init(); await h.resolve(h.requests[0], analytics()); + await h.resolve(h.requests[1], battery()); + assert.strictEqual(h.chartInstances.length, 6); + assert.strictEqual(h.hopLoads.length, 1); + assert.match(h.element('batteryStatusBadge').textContent, /3800 mV/); + h.page.destroy(); assert.ok(h.chartInstances.every(chart => chart.destroyed)); + const mainError = harness(); mainError.init(); await mainError.reject(mainError.requests[0], 'current error'); + assert.match(mainError.container.innerHTML, /Failed to load analytics: current error/); + const batteryError = harness(); batteryError.init(); await batteryError.resolve(batteryError.requests[0], analytics()); + await batteryError.reject(batteryError.requests[1], 'current battery error'); + assert.match(batteryError.element('batteryEmpty').textContent, /current battery error/); + }); + console.log(`\nNode Analytics lifecycle: ${passed} passed, ${failed} failed`); + process.exitCode = failed ? 1 : 0; +})(); From c24e5b20d0a5f203654479af3de79cfe57d7d5ec Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:55 -0400 Subject: [PATCH 43/67] fix: anchor resize handles to table headers --- public/app.js | 2 ++ tests/e2e/test-charts-fluid-1058-e2e.js | 42 +++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/public/app.js b/public/app.js index 822cc24ae..a37a910b2 100644 --- a/public/app.js +++ b/public/app.js @@ -2108,6 +2108,8 @@ function makeColumnsResizable(tableSelector, storageKey) { // Add resize handles ths.forEach((th, i) => { if (i === ths.length - 1) return; + // Anchor absolute handles to their column without replacing sticky headers. + if (getComputedStyle(th).position === 'static') th.style.position = 'relative'; const handle = document.createElement('div'); handle.className = 'col-resize-handle'; handle.addEventListener('mousedown', (e) => { diff --git a/tests/e2e/test-charts-fluid-1058-e2e.js b/tests/e2e/test-charts-fluid-1058-e2e.js index 1024f27ca..2f3189378 100644 --- a/tests/e2e/test-charts-fluid-1058-e2e.js +++ b/tests/e2e/test-charts-fluid-1058-e2e.js @@ -224,6 +224,48 @@ async function gatherRows(page) { await ctx.close(); }); + await step('analytics resize handles stay on their headers without page overflow', async () => { + const ctx = await browser.newContext({ viewport: { width: 1250, height: 900 } }); + const page = await ctx.newPage(); + try { + await page.goto(BASE + '/#/analytics?tab=nodes', { waitUntil: 'domcontentloaded' }); + await page.waitForSelector('.analytics-table .col-resize-handle'); + for (const width of [1250, 390]) { + await page.setViewportSize({ width, height: 900 }); + const layout = await page.evaluate(() => ({ + pageWidth: document.documentElement.clientWidth, + scrollWidth: document.documentElement.scrollWidth, + misplaced: Array.from(document.querySelectorAll('.analytics-table .col-resize-handle')) + .filter(handle => handle.offsetParent !== handle.parentElement).length, + })); + assert(layout.misplaced === 0, `${width}px: ${layout.misplaced} resize handles are not anchored to their header`); + assert(layout.scrollWidth <= layout.pageWidth + 1, + `${width}px: page scrollWidth=${layout.scrollWidth} exceeds clientWidth=${layout.pageWidth}`); + } + await page.setViewportSize({ width: 1250, height: 900 }); + await page.evaluate(() => window.scrollTo(0, 0)); + const header = page.locator('#analytics-tbl-nodes-0 th').first(); + const handle = header.locator('.col-resize-handle'); + await handle.hover(); + const before = await header.boundingBox(); + const grip = await handle.boundingBox(); + await page.mouse.move(grip.x + grip.width / 2, grip.y + grip.height / 2); + await page.mouse.down(); + await page.mouse.move(grip.x + grip.width / 2 + 40, grip.y + grip.height / 2, { steps: 4 }); + await page.mouse.up(); + const after = await header.boundingBox(); + assert(after.width > before.width + 20, `drag did not resize the column: ${before.width} -> ${after.width}`); + const saved = await page.evaluate(() => JSON.parse(localStorage.getItem('meshcore-analytics-nodes-0-col-widths'))); + assert(Array.isArray(saved) && saved.length > 1, 'drag did not save column widths'); + await page.goto(BASE + '/#/nodes', { waitUntil: 'domcontentloaded' }); + await page.waitForSelector('#nodesTable .col-resize-handle'); + const sticky = await page.$eval('#nodesTable th', th => getComputedStyle(th).position); + assert(sticky === 'sticky', `shared resizer changed sticky header positioning to ${sticky}`); + } finally { + await ctx.close(); + } + }); + await browser.close(); console.log(`\n=== #1058 fluid analytics charts E2E: ${passed} passed, ${failed} failed ===`); process.exit(failed ? 1 : 0); From 358904ab6198f2514318c93861234c3267fa131e Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:55 -0400 Subject: [PATCH 44/67] fix: fit priority navigation to available header width --- public/app.js | 75 ++++------------- public/style.css | 16 +++- tests/e2e/test-nav-fluid-1055-e2e.js | 117 +++++++++++++++++++++++++-- 3 files changed, 137 insertions(+), 71 deletions(-) diff --git a/public/app.js b/public/app.js index a37a910b2..33d7e7529 100644 --- a/public/app.js +++ b/public/app.js @@ -1501,14 +1501,8 @@ window.addEventListener('DOMContentLoaded', () => { navMoreBtn.classList.toggle('active', !!hasActiveMore); } - // #1105 MINOR 1: cached intrinsic width of the More button. Captured - // the first time `fits()` sees navMoreWrap rendered (display:flex). - // Falls back to MORE_BTN_RESERVE_PX (a conservative initial guess - // sized for "More ▾" at default font/padding) until that happens. - var cachedMoreW = 0; - var MORE_BTN_RESERVE_PX = 70; - function applyNavPriority() { + navTop.classList.remove('nav-hide-stats', 'nav-compact'); // Skip on mobile (<768px) — hamburger CSS owns that layout. if (window.innerWidth < 768) { allLinks.forEach(a => a.classList.remove('is-overflow')); @@ -1541,59 +1535,12 @@ window.addEventListener('DOMContentLoaded', () => { a.classList.add('is-overflow'); } }); - rebuildMoreMenu(); - return; } - // Iteratively hide low-priority links until the link strip fits. - // .top-nav has overflow:hidden and .nav-left has flex-shrink:1, so - // an overflowing strip silently clips rather than pushing - // nav-right out — bounding-rect math on .nav-left lies. Instead - // measure the *intrinsic* widths of the parts (independent of - // current clipping) and compare to the viewport. SAFETY absorbs - // the .top-nav side padding + nav-right inner gaps + sub-pixel - // rounding (the historic #1055 bug was a 6–20px overlap). - // - // #1105 MINOR 3: at the 1101px media-query flip `.nav-stats` - // toggles from display:none → flex (and vice-versa). The resize - // handler is rAF-debounced and runs *after* the layout flip, so - // navRightEl.scrollWidth measured here reflects the post-flip - // intrinsic width — not stale pre-flip width. - const navBrand = document.querySelector('.nav-brand'); - const SAFETY = 32; - // #1105 MINOR 1+2: read both gap values from CSS rather than a - // shared `GUTTER = 24` constant. Today `.nav-left` (gap between - // brand/links/more/right cells) and `.nav-links` (gap between - // individual link items) both resolve to --space-lg = 24px, but - // they're conceptually distinct gaps. If --space-lg or .nav-left's - // gap diverges in the future, the fit math must follow. - const navLeftGap = parseFloat(getComputedStyle(navLeft).columnGap || - getComputedStyle(navLeft).gap || '0') || 0; - // #1105 MINOR 1: compute the More-button reserve from its actual - // rendered width on first measure, instead of a hard-coded 70px - // fallback. Cached so we don't re-measure (offsetWidth is 0 when - // display:none; we capture the value the first time it's visible). + // Measure the strip's actual available space, including the current + // brand, account control, stats, gutters and More button. Its flex box + // can shrink below its children, which scrollWidth exposes directly. function fits() { - const visibleLinks = allLinks.filter(a => !a.classList.contains('is-overflow')); - let linkW = 0; - visibleLinks.forEach(a => { linkW += a.getBoundingClientRect().width; }); - const linkGapPx = parseFloat(getComputedStyle(linksContainer).columnGap || - getComputedStyle(linksContainer).gap || '0') || 0; - const linksGap = Math.max(0, visibleLinks.length - 1) * linkGapPx; - const brandW = navBrand ? navBrand.getBoundingClientRect().width : 0; - // Always reserve space for the More button if anything could - // overflow. Measure the live width when visible and cache it - // for use when the button is currently hidden (display:none → - // getBoundingClientRect() returns 0). MORE_BTN_RESERVE_PX is - // the conservative initial fallback used until we get a real - // measurement. - const moreVis = !navMoreWrap.classList.contains('is-hidden'); - const liveMoreW = moreVis ? navMoreWrap.getBoundingClientRect().width : 0; - if (liveMoreW > 0) cachedMoreW = liveMoreW; - const moreW = liveMoreW > 0 ? liveMoreW - : (cachedMoreW > 0 ? cachedMoreW : MORE_BTN_RESERVE_PX); - const rightW = navRightEl.scrollWidth; // intrinsic, ignores clipping - const needed = brandW + navLeftGap + linkW + linksGap + navLeftGap + moreW + navLeftGap + rightW + SAFETY; - return needed <= window.innerWidth; + return linksContainer.scrollWidth <= linksContainer.clientWidth; } let i = 0; // #1391: rebuild queue here so it reflects the CURRENT active @@ -1650,6 +1597,10 @@ window.addEventListener('DOMContentLoaded', () => { console.warn('[nav] More menu floor: overflowQueue exhausted with 1 item; cannot enforce >=2 floor'); } } + // Keep the primary links and active route. Optional statistics give + // way first; only then compact the surrounding desktop chrome. + if (!fits()) navTop.classList.add('nav-hide-stats'); + if (!fits()) navTop.classList.add('nav-compact'); rebuildMoreMenu(); } @@ -1661,15 +1612,17 @@ window.addEventListener('DOMContentLoaded', () => { document.fonts.ready.then(applyNavPriority); } let rafId = 0; - window.addEventListener('resize', function() { + function scheduleNavPriority() { if (rafId) cancelAnimationFrame(rafId); rafId = requestAnimationFrame(applyNavPriority); - }); + } + window.addEventListener('resize', scheduleNavPriority); + window.addEventListener('cs-auth-changed', scheduleNavPriority); // Re-apply on route change too: the active link gets bigger padding // (background pill), so which links fit can shift between pages. window.addEventListener('hashchange', function() { // Defer so the route handler's class toggles run first. - requestAnimationFrame(applyNavPriority); + scheduleNavPriority(); }); // #1406: position the fixed dropdown relative to the More button on each open. diff --git a/public/style.css b/public/style.css index 30f91f88c..34d3c3cdf 100644 --- a/public/style.css +++ b/public/style.css @@ -882,7 +882,7 @@ img.brand-logo { color: var(--nav-text-muted); text-decoration: none; padding: 14px clamp(8px, 0.6vw + 4px, 14px); font-size: var(--fs-sm); font-weight: var(--fw-medium); /* #1668 M3 — 12-14px fluid + 500 floor */ - border-bottom: 2px solid transparent; transition: all .15s; + border-bottom: 2px solid transparent; transition: color .15s, background-color .15s, border-color .15s; background: none; border-top: none; border-left: none; border-right: none; cursor: pointer; font-family: var(--font); white-space: nowrap; /* #1046: never wrap labels — wrapping makes the nav bar grow taller */ @@ -987,6 +987,16 @@ img.brand-logo { .nav-stats .stat-val { color: var(--nav-text); font-weight: 600; transition: color 0.3s ease; } .nav-stats .stat-val.updated { color: var(--link-color); } +/* Priority+ keeps primary links and the active page inline. Compact only + when the measured desktop strip cannot fit; mobile keeps its own layout. */ +.nav-hide-stats .nav-stats { display: none; } +.nav-compact .brand-text { display: none; } +.nav-compact .brand-logo { max-width: 80px; height: auto; max-height: 36px; } +.nav-compact .nav-left { gap: var(--space-xs); } +.nav-compact .nav-links .nav-link { padding-inline: 4px; } +.nav-compact .nav-more-btn { padding-inline: 6px; } +.nav-compact .nav-account-label { max-width: 5ch; } + /* === Layout === */ /* Default: body-scroll mode — content pushes beyond viewport, iOS status-bar tap-to-scroll works because is the scroll container. Pages that need @@ -2306,8 +2316,8 @@ button.ch-item:hover .ch-icon-btn { opacity: 1; } /* === Hamburger (hidden on desktop) === */ .hamburger { display: none; } /* "More" button (hidden on desktop) */ -.nav-more-wrap { display: none; position: relative; } -.nav-more-btn { display: inline-flex; } +.nav-more-wrap { display: none; position: relative; flex-shrink: 0; } +.nav-more-btn { display: inline-flex; transition-property: color, background-color, border-color; } /* #1406: position:fixed (not absolute) so the dropdown escapes .nav-more-wrap's layout box. When absolute, the dropdown's content extents inflated .nav-more-wrap.scrollHeight → bubbled into .nav-left flex line-height calc diff --git a/tests/e2e/test-nav-fluid-1055-e2e.js b/tests/e2e/test-nav-fluid-1055-e2e.js index d1ac152ac..beef9a04a 100644 --- a/tests/e2e/test-nav-fluid-1055-e2e.js +++ b/tests/e2e/test-nav-fluid-1055-e2e.js @@ -30,11 +30,12 @@ const BASE = process.env.BASE_URL || 'http://localhost:13581'; // #1061: bottom-nav activates at max-width:768px and hides the top-nav. // This test asserts top-nav layout stability — start at 769 to stay above // that breakpoint. Below 768 the top nav is intentionally display:none. -const VIEWPORTS = [769, 1024, 1280, 1440, 1920]; +const VIEWPORTS = [769, 900, 1024, 1101, 1250, 1280, 1440, 1920, 2560]; // Routes asserted at every viewport. The pre-#1097 version only checked // /#/home, but the bug reproduces on every top-level page since they -// all share the same .top-nav. Cover the four primary routes. -const ROUTES = ['/#/home', '/#/packets', '/#/nodes', '/#/map']; +// all share the same .top-nav. Include non-primary active routes too. +const ROUTES = ['/#/home', '/#/packets', '/#/nodes', '/#/map', '/#/analytics?tab=nodes', '/#/scope-audit']; +const HIGH_PRIORITY_HREFS = ['#/home', '#/packets', '#/map', '#/live', '#/nodes']; const HEIGHT = 900; // Whitespace tolerance (px) for the overflow/overlap assertions. // Browsers occasionally hand back layout coordinates with sub-pixel @@ -66,7 +67,25 @@ async function main() { let failures = 0; let passes = 0; - const context = await browser.newContext(); + // Include tablet touch targets while checking the desktop nav breakpoint. + const context = await browser.newContext({ hasTouch: true }); + // Account controls are optional in the fixture server but consume real nav + // space on enabled sites. Keep counts wide enough to exercise that layout. + await context.route('**/api/config/client', async route => { + const response = await route.fetch(); + const config = await response.json(); + config.userManagement = { enabled: true }; + await route.fulfill({ json: config }); + }); + await context.route('**/api/auth/me', route => route.fulfill({ + status: 401, json: { error: 'Not logged in' }, + })); + await context.route('**/api/stats', async route => { + const response = await route.fetch(); + const stats = await response.json(); + Object.assign(stats, { totalPackets: 30000, totalNodes: 2500, totalObservers: 213 }); + await route.fulfill({ json: stats }); + }); const page = await context.newPage(); page.setDefaultTimeout(15000); @@ -75,6 +94,8 @@ async function main() { await page.setViewportSize({ width: w, height: HEIGHT }); await page.goto(`${BASE}${route}`, { waitUntil: 'domcontentloaded' }); await page.waitForSelector('.top-nav .nav-right'); + await page.waitForSelector('#accountToggle'); + await page.waitForFunction(() => document.getElementById('navStats').textContent.includes('30000')); // Wait for fonts (which affect text measurement) AND for the nav // layout to settle: the .nav-right bounding box must hold steady // for two consecutive animation frames at the same coordinates. @@ -102,7 +123,6 @@ async function main() { const moreCs = more ? getComputedStyle(more) : null; const links = Array.from(document.querySelectorAll('.nav-links .nav-link')); const visible = links.filter(a => getComputedStyle(a).display !== 'none'); - const lastVisible = visible[visible.length - 1] || null; return { clientW: document.documentElement.clientWidth, navScroll: topNav.scrollWidth, @@ -110,8 +130,12 @@ async function main() { navRight: navRight.getBoundingClientRect().right, navRightL: navRight.getBoundingClientRect().left, navLeftR: navLeft.getBoundingClientRect().right, - lastLinkR: lastVisible ? lastVisible.getBoundingClientRect().right : -1, + lastLinkR: Math.max(...visible.map(a => a.getBoundingClientRect().right)), moreVisible: moreCs ? moreCs.display !== 'none' : false, + moreLeft: more ? more.getBoundingClientRect().left : 0, + moreRight: more ? more.getBoundingClientRect().right : 0, + visibleHrefs: visible.map(a => a.getAttribute('href')), + activeInline: visible.some(a => a.classList.contains('active')), visibleLinks: visible.length, totalLinks: links.length, }; @@ -134,6 +158,15 @@ async function main() { if (data.navScroll > data.navClient + SUBPIXEL_TOL) { reasons.push(`top-nav scrollWidth=${data.navScroll} > clientWidth=${data.navClient}`); } + if (data.moreVisible && data.lastLinkR > data.moreLeft + SUBPIXEL_TOL) { + reasons.push(`last visible link right=${data.lastLinkR.toFixed(1)} overlaps More left=${data.moreLeft.toFixed(1)}`); + } + if (data.moreVisible && data.moreRight > data.navRightL + SUBPIXEL_TOL) { + reasons.push(`More right=${data.moreRight.toFixed(1)} overlaps nav-right left=${data.navRightL.toFixed(1)}`); + } + if (!data.activeInline || HIGH_PRIORITY_HREFS.some(href => !data.visibleHrefs.includes(href))) { + reasons.push('active route or primary navigation was hidden'); + } if (reasons.length === 0) { passes++; @@ -146,9 +179,79 @@ async function main() { } } + // Auth changes happen without resize/hashchange. The new account label must + // trigger the same layout pass immediately, without waiting for stats polling. + await page.setViewportSize({ width: 1366, height: HEIGHT }); + // Scope Audit is the widest pinned non-primary route in the stock header. + await page.goto(`${BASE}/#/scope-audit`, { waitUntil: 'domcontentloaded' }); + await page.waitForSelector('#accountToggle'); + await page.waitForFunction(() => document.getElementById('navStats').textContent.includes('30000')); + await page.evaluate(() => window.CSAuth.ready()); + await page.evaluate(() => window.CSAuth.setUser({ displayName: 'Long account name', role: 'member' })); + await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))); + const accountLayout = await page.evaluate(() => { + const links = Array.from(document.querySelectorAll('.nav-links .nav-link')) + .filter(a => getComputedStyle(a).display !== 'none'); + return { + lastRight: Math.max(...links.map(a => a.getBoundingClientRect().right)), + moreLeft: document.querySelector('.nav-more-wrap').getBoundingClientRect().left, + name: document.querySelector('.nav-account-label').textContent, + }; + }); + if (accountLayout.lastRight <= accountLayout.moreLeft + SUBPIXEL_TOL && accountLayout.name === 'Long account name') { + passes++; + console.log(' ✅ account label change refits navigation and preserves its full name'); + } else { + failures++; + console.log(` ❌ account label change overlaps More: ${JSON.stringify(accountLayout)}`); + } + + const resizeWidths = [769, 1280, 900, 1440, 1101, 1920, 1250]; + for (const width of resizeWidths) { + await page.setViewportSize({ width, height: HEIGHT }); + await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))); + await page.evaluate(() => Promise.all(document.querySelector('.top-nav') + .getAnimations({ subtree: true }).map(animation => animation.finished.catch(() => {})))); + const layout = await page.evaluate(() => { + const visible = Array.from(document.querySelectorAll('.nav-links .nav-link')) + .filter(a => getComputedStyle(a).display !== 'none'); + const more = document.querySelector('.nav-more-wrap'); + return { + lastRight: Math.max(...visible.map(a => a.getBoundingClientRect().right)), + moreLeft: more.getBoundingClientRect().left, + moreVisible: getComputedStyle(more).display !== 'none', + rightLeft: document.querySelector('.nav-right').getBoundingClientRect().left, + controlsFit: Array.from(document.querySelectorAll('.nav-btn, .nav-account-btn')) + .filter(e => e.getBoundingClientRect().width > 0) + .every(e => e.getBoundingClientRect().width >= 48 && e.getBoundingClientRect().height >= 48), + }; + }); + const boundary = layout.moreVisible ? layout.moreLeft : layout.rightLeft; + const namedAccount = await page.getByRole('button', { name: 'Long account name', exact: true }).isVisible(); + if (layout.lastRight <= boundary + SUBPIXEL_TOL && layout.controlsFit && namedAccount) { + passes++; + console.log(` ✅ resize to ${width}px: links fit, controls retain their size and account name`); + } else { + failures++; + console.log(` ❌ resize to ${width}px: ${JSON.stringify(layout)}, namedAccount=${namedAccount}`); + } + } + await page.locator('#navMoreBtn').focus(); + await page.keyboard.press('Enter'); + const menuFocused = await page.evaluate(() => document.activeElement === document.querySelector('#navMoreMenu .nav-link')); + await page.keyboard.press('Escape'); + const menuClosed = await page.locator('#navMoreBtn').getAttribute('aria-expanded') === 'false'; + if (menuFocused && menuClosed) { + passes++; + console.log(' ✅ More retains keyboard focus and Escape dismissal'); + } else { + failures++; + console.log(` ❌ More keyboard interaction: focused=${menuFocused}, closed=${menuClosed}`); + } + await browser.close(); - const total = ROUTES.length * VIEWPORTS.length; + const total = ROUTES.length * VIEWPORTS.length + 2 + resizeWidths.length; console.log(`\ntest-nav-fluid-1055-e2e.js: ${failures === 0 ? 'OK' : 'FAIL'} — ${passes}/${total} passed`); process.exit(failures === 0 ? 0 : 1); } From 4e21970c1a0d3cb4d1737a70199ee3fe4a0ef873 Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:55 -0400 Subject: [PATCH 45/67] fix(account): discard views after route teardown --- public/account.js | 9 +++++- tests/e2e/test-e2e-playwright.js | 35 +++++++++++++++++++++ tests/unit/test-user-management-ui.js | 44 ++++++++++++++++++++++++++- 3 files changed, 86 insertions(+), 2 deletions(-) diff --git a/public/account.js b/public/account.js index e79f107ee..a956cd4f2 100644 --- a/public/account.js +++ b/public/account.js @@ -176,6 +176,7 @@ // The profile view on screen, so an auth change can redirect or re-render. var shown = { app: null, userId: null }; + var viewGeneration = 0; var views = { login: function (app) { @@ -401,9 +402,11 @@ }; function init(app, routeParam) { + var generation = ++viewGeneration; if (!window.CSAuth) { app.innerHTML = shell('Accounts', '

Accounts are not available.

'); return; } app.innerHTML = shell('Accounts', '

Loading…

'); CSAuth.ready().then(function () { + if (generation !== viewGeneration || !app.isConnected) return; if (!CSAuth.isEnabled()) { app.innerHTML = shell('Accounts', '

Accounts are not enabled on this instance.

'); return; } views[viewName(routeParam)](app); }); @@ -419,6 +422,10 @@ if (u.id !== shown.userId) views.profile(shown.app); }); - registerPage('account', { init: init, destroy: function () {} }); + registerPage('account', { init: init, destroy: function () { + viewGeneration++; + shown.app = null; + shown.userId = null; + } }); window.CSAccount = { _test: { profileHtml: profileHtml, sessionsHtml: sessionsHtml, companionsHtml: companionsHtml, renewLinkHtml: renewLinkHtml, views: views, checkMailHtml: checkMailHtml } }; })(); diff --git a/tests/e2e/test-e2e-playwright.js b/tests/e2e/test-e2e-playwright.js index 3a6b4bd08..b5f1cd05a 100644 --- a/tests/e2e/test-e2e-playwright.js +++ b/tests/e2e/test-e2e-playwright.js @@ -66,6 +66,41 @@ async function run() { console.log(`\nRunning E2E tests against ${BASE}\n`); + // Account readiness must not outlive the route that requested its view. + await test('Account readiness cannot replace Nodes after navigation', async () => { + const ctx = await browser.newContext(); + const p = await ctx.newPage(); + let release; + const held = new Promise(resolve => { release = resolve; }); + await p.route('**/api/config/client', async route => { + const response = await route.fetch(); + const config = await response.json(); + config.userManagement = { enabled: true }; + await route.fulfill({ json: config }); + }); + await p.route('**/api/auth/me', async route => { + await held; + await route.fulfill({ status: 401, json: { error: 'Not logged in' } }); + }); + const started = p.waitForRequest('**/api/auth/me', { timeout: 15000 }); + started.catch(() => {}); + try { + await p.goto(BASE + '/#/account/login', { waitUntil: 'domcontentloaded' }); + await started; + await p.goto(BASE + '/#/nodes', { waitUntil: 'domcontentloaded' }); + await p.waitForSelector('#nodesLeft[data-loaded="true"]'); + const response = p.waitForResponse('**/api/auth/me'); + release(); + await (await response).finished(); + await p.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))); + assert(await p.locator('#nodesLeft[data-loaded="true"]').isVisible(), 'account readiness replaced the current Nodes page'); + assert(await p.locator('#loginForm').count() === 0, 'login form appeared after leaving Accounts'); + } finally { + release(); + await ctx.close(); + } + }); + // #2218 Node Analytics lifecycle: delayed responses retain their own page/range. const analyticsKey = 'aa'.repeat(32); const otherAnalyticsKey = 'bb'.repeat(32); diff --git a/tests/unit/test-user-management-ui.js b/tests/unit/test-user-management-ui.js index 0bc08ce60..3b462f360 100644 --- a/tests/unit/test-user-management-ui.js +++ b/tests/unit/test-user-management-ui.js @@ -361,6 +361,7 @@ function loadAccount(hash, routes, extraWin) { Object.assign(CSAuth, loadAuthHelpers(ctx)); vm.runInContext(fs.readFileSync(path.join(ROOT, 'public/account.js'), 'utf8'), ctx); return { setRequest(f) { override = f; }, doc, t: ctx.window.CSAccount._test, els, calls, loc, user, pages, logouts, replaced, + setReady(promise) { CSAuth.ready = () => promise; }, setLogout(r) { logoutResult = r; }, fire(detail) { user.current = detail; if (listeners['cs-auth-changed']) listeners['cs-auth-changed']({ detail }); } }; } @@ -368,6 +369,47 @@ const submitForm = async (env, formId) => { await env.els[formId].handlers.submit({ preventDefault() {} }); }; +test('account readiness cannot render after leaving or detaching the page', async () => { + for (const detached of [false, true]) { + const env = loadAccount('#/account/login', () => ({})); + let ready; + env.setReady(new Promise(resolve => { ready = resolve; })); + const app = { isConnected: true, innerHTML: '' }; + env.pages.account.init(app, 'login'); + if (detached) app.isConnected = false; + else env.pages.account.destroy(); + app.innerHTML = 'Another page'; + env.loc.hash = '#/nodes'; + ready(); + await Promise.resolve(); + assert.strictEqual(app.innerHTML, 'Another page'); + assert.strictEqual(env.loc.hash, '#/nodes'); + } +}); + +test('an old profile readiness callback cannot redirect a newer registration view', async () => { + const env = loadAccount('#/account', () => ({})); + let ready; + env.setReady(new Promise(resolve => { ready = resolve; })); + const app = { isConnected: true, innerHTML: '' }; + env.pages.account.init(app); + env.pages.account.destroy(); + env.loc.hash = '#/account/register'; + env.pages.account.init(app, 'register'); + ready(); + await Promise.resolve(); + assert.strictEqual(env.loc.hash, '#/account/register'); + assert.match(app.innerHTML, /Create an account/); +}); + +test('current account readiness still renders the requested form', async () => { + const env = loadAccount('#/account/login', () => ({})); + const app = { isConnected: true, innerHTML: '' }; + env.pages.account.init(app, 'login'); + await Promise.resolve(); + assert.match(app.innerHTML, /id="loginForm"/); +}); + test('profile view escapes email and session fields', () => { const env = loadAccount('#/account', () => ({})); const payload = ''; @@ -721,7 +763,7 @@ test('auth changes on other views are ignored', () => { test('unknown and inherited view names fall back to the profile view', async () => { for (const name of ['constructor', '__proto__', 'toString', 'nope']) { const env = loadAccount('#/account/' + name, () => ({})); - env.pages.account.init({ set innerHTML(v) {} }, name); + env.pages.account.init({ isConnected: true, set innerHTML(v) {} }, name); await tick(); assert.strictEqual(env.loc.hash, '#/account/login', name + ' did not reach the profile view'); } From 77a1fb0f37fb5a1f0dd3599a90f290992b904907 Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:55 -0400 Subject: [PATCH 46/67] fix(packets): preserve desktop all-time filters --- public/packets.js | 29 +++++++++++---------- tests/e2e/test-e2e-playwright.js | 40 +++++++++++++++++++++++++++++ tests/unit/test-frontend-helpers.js | 18 ++++++++----- 3 files changed, 67 insertions(+), 20 deletions(-) diff --git a/public/packets.js b/public/packets.js index 8c5be8b93..ee250294f 100644 --- a/public/packets.js +++ b/public/packets.js @@ -753,9 +753,14 @@ function typeName(t) { return TYPE_NAMES[t] ?? `Type ${t}`; } const isMobile = window.innerWidth <= 1024; const PACKET_LIMIT = isMobile ? 1000 : 50000; - let savedTimeWindowMin = Number(localStorage.getItem('meshcore-time-window')); - if (!Number.isFinite(savedTimeWindowMin) || savedTimeWindowMin <= 0) savedTimeWindowMin = 15; - if (isMobile && savedTimeWindowMin > 180) savedTimeWindowMin = 15; + var DEFAULT_TIME_WINDOW = 15; + function normalizePacketTimeWindow(value) { + const minutes = value == null || String(value).trim() === '' ? NaN : Number(value); + if (!Number.isFinite(minutes) || minutes < 0 || + (isMobile && (minutes === 0 || minutes > 180))) return DEFAULT_TIME_WINDOW; + return minutes; + } + let savedTimeWindowMin = normalizePacketTimeWindow(localStorage.getItem('meshcore-time-window')); let totalCount = 0; let expandedHashes = new Set(); let hopNameCache = {}; @@ -769,11 +774,9 @@ let _colFit = null; // fitColumnsToContent controller for #pktTable var _pendingUrlRegion = null; - var DEFAULT_TIME_WINDOW = 15; - function buildPacketsQuery(timeWindowMin, regionParam, skipHash) { var parts = []; - if (timeWindowMin && timeWindowMin !== DEFAULT_TIME_WINDOW) parts.push('timeWindow=' + timeWindowMin); + if (timeWindowMin === 0 || (timeWindowMin && timeWindowMin !== DEFAULT_TIME_WINDOW)) parts.push('timeWindow=' + timeWindowMin); if (regionParam) parts.push('region=' + encodeURIComponent(regionParam)); if (!skipHash && filters.hash) parts.push('hash=' + encodeURIComponent(filters.hash)); if (filters.node) parts.push('node=' + encodeURIComponent(filters.node)); @@ -1323,10 +1326,10 @@ } else { showFullNames = localStorage.getItem('meshcore-full-names') === 'true'; } - var _urlTimeWindow = Number(_initUrlParams.get('timeWindow')); - if (Number.isFinite(_urlTimeWindow) && _urlTimeWindow > 0) { - savedTimeWindowMin = _urlTimeWindow; - localStorage.setItem('meshcore-time-window', String(_urlTimeWindow)); + var _urlTimeWindow = _initUrlParams.get('timeWindow'); + if (_urlTimeWindow !== null) { + savedTimeWindowMin = normalizePacketTimeWindow(_urlTimeWindow); + localStorage.setItem('meshcore-time-window', String(savedTimeWindowMin)); } var _urlRegion = _initUrlParams.get('region'); if (_urlRegion) _pendingUrlRegion = _urlRegion; @@ -2223,9 +2226,9 @@ const fTimeWindow = document.getElementById('fTimeWindow'); fTimeWindow.value = String(savedTimeWindowMin); fTimeWindow.addEventListener('change', () => { - savedTimeWindowMin = Number(fTimeWindow.value); - if (!Number.isFinite(savedTimeWindowMin) || savedTimeWindowMin <= 0) savedTimeWindowMin = 15; - localStorage.setItem('meshcore-time-window', fTimeWindow.value); + savedTimeWindowMin = normalizePacketTimeWindow(fTimeWindow.value); + fTimeWindow.value = String(savedTimeWindowMin); + localStorage.setItem('meshcore-time-window', String(savedTimeWindowMin)); updatePacketsUrl(); loadPackets(); }); diff --git a/tests/e2e/test-e2e-playwright.js b/tests/e2e/test-e2e-playwright.js index b5f1cd05a..6b426f774 100644 --- a/tests/e2e/test-e2e-playwright.js +++ b/tests/e2e/test-e2e-playwright.js @@ -883,6 +883,46 @@ async function run() { // --- Group: Packets page (test 4) --- + await test('Packets All time survives reload and a fresh shared link', async () => { + const ctx = await browser.newContext({ viewport: { width: 1280, height: 800 } }); + const p = await ctx.newPage(); + try { + await p.goto(BASE + '/#/packets', { waitUntil: 'domcontentloaded' }); + await p.waitForSelector('#fTimeWindow'); + assert(await p.locator('#fTimeWindow').inputValue() === '15', 'a fresh page must retain the default window'); + await p.selectOption('#fTimeWindow', '0'); + const saved = await p.evaluate(() => ({ + query: new URLSearchParams(location.hash.split('?')[1] || '').get('timeWindow'), + stored: localStorage.getItem('meshcore-time-window'), + })); + assert(saved.query === '0' && saved.stored === '0', 'All time was not saved/shared: ' + JSON.stringify(saved)); + await p.reload({ waitUntil: 'load' }); + assert(await p.locator('#fTimeWindow').inputValue() === '0', 'All time reset after reload'); + } finally { await ctx.close(); } + + const shared = await browser.newContext({ viewport: { width: 1280, height: 800 } }); + const fresh = await shared.newPage(); + try { + // The CI seed has three observations from May 15, outside the default window. + await fresh.goto(BASE + '/#/packets?hash=fae0c9e6d357a814&timeWindow=0', { waitUntil: 'domcontentloaded' }); + await fresh.waitForSelector('#fTimeWindow'); + assert(await fresh.locator('#fTimeWindow').inputValue() === '0', 'fresh shared link ignored All time'); + await fresh.waitForSelector('#pktBody tr[data-hash="fae0c9e6d357a814"][data-action="toggle-select"]'); + } finally { await shared.close(); } + }); + + await test('Packets mobile keeps its bounded time window for saved and shared All time', async () => { + const ctx = await browser.newContext({ viewport: { width: 390, height: 800 }, hasTouch: true }); + const p = await ctx.newPage(); + await p.addInitScript(() => localStorage.setItem('meshcore-time-window', '0')); + try { + await p.goto(BASE + '/#/packets?timeWindow=0', { waitUntil: 'domcontentloaded' }); + await p.waitForSelector('#fTimeWindow', { state: 'attached' }); + assert(await p.locator('#fTimeWindow').inputValue() === '15', 'mobile must retain its bounded default'); + assert(await p.locator('#fTimeWindow option[value="0"]').count() === 0, 'mobile unexpectedly exposes All time'); + } finally { await ctx.close(); } + }); + // Test 4: Packets page loads with filter await test('Packets page loads with filter', async () => { // Ensure desktop viewport and broad time window so fixture timestamps are included. diff --git a/tests/unit/test-frontend-helpers.js b/tests/unit/test-frontend-helpers.js index 43ea8fc26..e69ddbab0 100644 --- a/tests/unit/test-frontend-helpers.js +++ b/tests/unit/test-frontend-helpers.js @@ -3245,12 +3245,10 @@ console.log('\n=== packets.js: savedTimeWindowMin defaults ==='); assert.ok(deltaMin > 10 && deltaMin < 25, `expected default ~15m window, got ${deltaMin.toFixed(2)}m`); }); - test('savedTimeWindowMin defaults to 15 when localStorage returns "0"', async () => { + test('desktop preserves All time without adding a since filter', async () => { const r = await captureInitialPacketsRequest('0', 1366); - const since = r.params.get('since'); - assert.ok(since, 'initial packets request should include since parameter'); - const deltaMin = (Date.now() - Date.parse(since)) / 60000; - assert.ok(deltaMin > 10 && deltaMin < 25, `expected default ~15m window, got ${deltaMin.toFixed(2)}m`); + assert.strictEqual(r.params.get('since'), null, 'All time must not become a 15-minute request'); + assert.strictEqual(r.params.get('limit'), '50000', 'All time retains the existing packet limit'); }); test('savedTimeWindowMin preserves valid value (60)', async () => { @@ -5422,8 +5420,14 @@ console.log('\n=== packets.js: buildPacketsQuery ==='); test('buildPacketsQuery: timeWindow + region', () => { assert.strictEqual(buildPacketsQuery(30, 'US-SFO,US-LAX'), '?timeWindow=30®ion=US-SFO%2CUS-LAX'); }); - test('buildPacketsQuery: timeWindow=0 treated as default', () => { - assert.strictEqual(buildPacketsQuery(0, ''), ''); + test('buildPacketsQuery: All time is explicitly shareable', () => { + assert.strictEqual(buildPacketsQuery(0, ''), '?timeWindow=0'); + }); + test('buildPacketsQuery: All time combines with region filters', () => { + assert.strictEqual(buildPacketsQuery(0, 'US-SFO'), '?timeWindow=0®ion=US-SFO'); + }); + test('buildPacketsQuery: an omitted window remains omitted', () => { + assert.strictEqual(buildPacketsQuery(undefined, ''), ''); }); } else { console.log(' ⚠️ buildPacketsQuery not exposed — skipping'); From fda720ef7941d52e9485cdece6b790f111928325 Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:55 -0400 Subject: [PATCH 47/67] fix(analytics): render safe inline battery status icons --- public/node-analytics.js | 11 +++-- public/style.css | 8 ++-- tests/e2e/test-issue-1648-m2-icons-e2e.js | 51 +++++++++++++++++++++ tests/unit/test-node-analytics-lifecycle.js | 4 +- 4 files changed, 63 insertions(+), 11 deletions(-) diff --git a/public/node-analytics.js b/public/node-analytics.js index 0c63ccc70..6b5e4a272 100644 --- a/public/node-analytics.js +++ b/public/node-analytics.js @@ -327,13 +327,14 @@ const thr = (data && data.thresholds) || { low_mv: 3300, critical_mv: 3000 }; if (badge) { - const STATUS_COLOR = { ok: '#51cf66', low: '#fcc419', critical: '#ff6b6b', unknown: 'var(--text-muted)' }; - const label = data && data.status === 'ok' ? ' OK' - : data && data.status === 'low' ? ' Low' - : data && data.status === 'critical' ? ' Critical' + const STATUS_COLOR = { ok: 'var(--status-green-text)', low: 'var(--status-yellow-text)', critical: 'var(--status-red-text)', unknown: 'var(--text-muted)' }; + const label = data && data.status === 'ok' ? ' OK' + : data && data.status === 'low' ? ' Low' + : data && data.status === 'critical' ? ' Critical' : 'No data'; const mv = data && data.latest_mv ? ' · ' + data.latest_mv + ' mV' : ''; - badge.textContent = label + mv; + badge.innerHTML = label; // Only the fixed status markup is parsed as HTML. + badge.appendChild(document.createTextNode(mv)); badge.style.color = STATUS_COLOR[(data && data.status) || 'unknown']; } diff --git a/public/style.css b/public/style.css index 34d3c3cdf..dc0d4a676 100644 --- a/public/style.css +++ b/public/style.css @@ -3521,11 +3521,9 @@ tr[data-hops]:hover { background: rgba(59,130,246,0.1); } @container (max-width: 480px) { .dumbbell-row, .dumbbell-axis { --relay-airtime-columns: 70px minmax(20px, 1fr) 90px; } } -/* Constrain chart media inside the card (svg/canvas at any depth). The - `.analytics-chart-card svg, .analytics-chart-card canvas` descendant - selector is robust to wrapper elements (legends, tooltips, axis - groups) being added between the card and the chart media. */ -.analytics-chart-card svg, +/* Constrain chart media at any depth, preserving inline Phosphor icons. + Descendant selectors allow wrappers between the card and chart media. */ +.analytics-chart-card svg:not(.ph-icon), .analytics-chart-card canvas { max-width: 100%; height: auto; display: block; } .analytics-chart-card h4 { font-size: 11px; text-transform: uppercase; letter-spacing: .5px; color: var(--text-muted); margin-bottom: 4px; } .analytics-chart-desc { font-size: 10px; color: var(--text-muted); margin-bottom: 8px; font-style: italic; } diff --git a/tests/e2e/test-issue-1648-m2-icons-e2e.js b/tests/e2e/test-issue-1648-m2-icons-e2e.js index 064d6dd9d..a721d8c5b 100644 --- a/tests/e2e/test-issue-1648-m2-icons-e2e.js +++ b/tests/e2e/test-issue-1648-m2-icons-e2e.js @@ -133,6 +133,56 @@ async function checkConfiguredScope(page) { } } +async function checkBatteryStatus(page) { + const key = 'b1'.repeat(32); + const cases = [ + ['ok', 4000, 'OK', 'ph-battery-high'], + ['low', 3300, 'Low', 'ph-warning'], + ['critical', 3100, 'Critical', 'ph-battery-low'], + ['unknown', null, 'No data', null], + ['ok', '', 'OK', 'ph-battery-high'], + ]; + let battery; + const urls = [ + '**/api/nodes/' + key + '/analytics*', + '**/api/nodes/' + key + '/battery*', + '**/api/nodes/' + key + '/hop_analytics*', + ]; + await page.route(urls[0], route => route.fulfill({ json: { + node: { public_key: key, name: 'Battery fixture', role: 'repeater' }, + computedStats: {}, peerInteractions: [], activityTimeline: [], snrTrend: [], + packetTypeBreakdown: [], observerCoverage: [], hopDistribution: [], uptimeHeatmap: [], + } })); + await page.route(urls[1], route => route.fulfill({ json: battery })); + await page.route(urls[2], route => route.fulfill({ json: { packets: [], ambiguous: 0 } })); + try { + for (const theme of ['light', 'dark']) { + for (const [status, mv, label, icon] of cases) { + battery = { status, latest_mv: mv, samples: [] }; + await page.goto('about:blank'); + await page.goto(`${BASE}/#/nodes/${key}/analytics`, { waitUntil: 'domcontentloaded' }); + await page.waitForFunction(() => window.__scopeThemeReady, null, { timeout: 8000 }); + if (await page.locator('html').getAttribute('data-theme') !== theme) await page.locator('#darkModeToggle').click(); + assert.strictEqual(await page.locator('html').getAttribute('data-theme'), theme); + await page.waitForFunction(() => document.getElementById('batteryStatusBadge')?.textContent.trim()); + const badge = page.locator('#batteryStatusBadge'); + assert.strictEqual((await badge.textContent()).trim(), label + (mv ? ' · ' + mv + ' mV' : ''), 'battery status must contain readable text'); + assert.strictEqual(await badge.locator('svg.ph-icon').count(), icon ? 1 : 0); + assert.strictEqual(await badge.locator('img').count(), 0, 'voltage text must never be parsed as HTML'); + if (icon) { + assert.strictEqual(await badge.locator('use').getAttribute('href'), '/icons/phosphor-sprite.svg#' + icon); + assert.strictEqual(await badge.locator('svg').getAttribute('aria-hidden'), 'true'); + const bounds = await badge.locator('svg').boundingBox(); + assert.ok(bounds.width <= 24 && bounds.height <= 24, 'battery icon was stretched by chart media styles'); + } + pass('battery status: ' + status + ' is readable and safe in ' + theme + ' theme'); + } + } + } finally { + for (const url of urls) await page.unroute(url); + } +} + async function main() { const requireChromium = process.env.CHROMIUM_REQUIRE === '1'; let browser; @@ -158,6 +208,7 @@ async function main() { const page = await ctx.newPage(); await checkConfiguredScope(page); + await checkBatteryStatus(page); // (a) /analytics await checkHeading(page, '/analytics', /Mesh Analytics/, '(a) /analytics'); diff --git a/tests/unit/test-node-analytics-lifecycle.js b/tests/unit/test-node-analytics-lifecycle.js index c5716d749..a03a5305f 100644 --- a/tests/unit/test-node-analytics-lifecycle.js +++ b/tests/unit/test-node-analytics-lifecycle.js @@ -29,6 +29,7 @@ function harness() { id, innerHTML: '', textContent: '', style: {}, dataset: {}, listeners: {}, get isConnected() { return container.isConnected && elements[id] === this; }, addEventListener(name, fn) { this.listeners[name] = fn; }, + appendChild(node) { this.textContent += node.textContent; }, }; } let html = ''; @@ -53,7 +54,8 @@ function harness() { Chart.defaults = {}; const sandbox = { console, Chart, Date, CLIENT_TTL: { nodeAnalytics: 1000 }, - document: { documentElement: {}, getElementById: id => elements[id] || null }, + document: { documentElement: {}, getElementById: id => elements[id] || null, + createTextNode: text => ({ textContent: String(text) }) }, getComputedStyle: () => ({ getPropertyValue: () => '' }), escapeHtml: value => String(value), timeAgo: () => 'now', NodeHopAnalytics: { load: (...args) => hopLoads.push(args), destroy() {} }, From 70ca320ce042138299fa8989ca61b6802594b035 Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:56 -0400 Subject: [PATCH 48/67] fix(live): show relay-aware node activity --- public/live.js | 7 +++-- tests/e2e/test-issue-1648-m2-icons-e2e.js | 35 +++++++++++++++++++++ tests/unit/test-live.js | 37 +++++++++++++++++++++++ 3 files changed, 76 insertions(+), 3 deletions(-) diff --git a/public/live.js b/public/live.js index abee9316c..35edc292d 100644 --- a/public/live.js +++ b/public/live.js @@ -2609,9 +2609,10 @@ const roleColor = ROLE_COLORS[n.role] || '#6b7280'; const roleLabel = (ROLE_LABELS[n.role] || n.role || 'unknown').replace(/s$/, ''); const hasLoc = n.lat != null && n.lon != null; - const lastSeen = formatLiveTimestampHtml(n.last_seen); + const heardMs = getEffectiveHeardMs({ ...n, _lastHeard: stats.lastHeard || n.last_heard || n.last_seen }); + const lastActivity = formatLiveTimestampHtml(Number.isFinite(heardMs) ? new Date(heardMs).toISOString() : null); const thresholds = window.getHealthThresholds ? getHealthThresholds(n.role) : { degradedMs: 3600000, silentMs: 86400000 }; - const ageMs = n.last_seen ? Date.now() - new Date(n.last_seen).getTime() : Infinity; + const ageMs = Number.isFinite(heardMs) ? Date.now() - heardMs : Infinity; const statusDot = ageMs < thresholds.degradedMs ? 'health-green' : ageMs < thresholds.silentMs ? 'health-yellow' : 'health-red'; const statusLabel = ageMs < thresholds.degradedMs ? 'Online' : ageMs < thresholds.silentMs ? 'Degraded' : 'Offline'; @@ -2629,7 +2630,7 @@ ${escapeHtml(n.public_key)} - + ${'configured_scope' in n && n.configured_scope !== null ? `` : ''} ${'default_scope' in n ? `
Last Seen${lastSeen}
Last Activity${lastActivity}
Adverts${n.advert_count || 0}
Configured scope ${n.configured_scope === '' ? 'none configured' : `${escapeHtml(n.configured_scope)}`}
Scope${n.default_scope === null ? '—' diff --git a/tests/e2e/test-issue-1648-m2-icons-e2e.js b/tests/e2e/test-issue-1648-m2-icons-e2e.js index a721d8c5b..31cd73b78 100644 --- a/tests/e2e/test-issue-1648-m2-icons-e2e.js +++ b/tests/e2e/test-issue-1648-m2-icons-e2e.js @@ -183,6 +183,40 @@ async function checkBatteryStatus(page) { } } +async function checkLiveActivityStatus(page) { + const key = 'c2'.repeat(32); + const recent = new Date(Date.now() - 60000).toISOString(); + const old = new Date(Date.now() - 100 * 3600000).toISOString(); + let node; + const listUrl = /\/api\/nodes(?:\?.*)?$/; + const urls = ['**/api/nodes/' + key, '**/api/nodes/' + key + '/health', '**/api/nodes/' + key + '/paths']; + await page.route(listUrl, route => route.fulfill({ json: { nodes: [{ ...node, lat: 1, lon: 1 }], total: 1 } })); + await page.route(urls[0], route => route.fulfill({ json: { node, recentAdverts: [] } })); + await page.route(urls[1], route => route.fulfill({ json: {} })); + await page.route(urls[2], route => route.fulfill({ json: { paths: [] } })); + try { + for (const evidence of [ + { role: 'repeater', last_relayed: recent, relay_active: true }, + { role: 'companion', last_heard: recent }, + ]) { + node = { public_key: key, name: 'Recently active ' + evidence.role, last_seen: old, ...evidence }; + await page.goto('about:blank'); + await page.goto(`${BASE}/#/live?lat=1&lon=1&zoom=10`, { waitUntil: 'domcontentloaded' }); + await page.waitForFunction(() => window.__scopeThemeReady, null, { timeout: 8000 }); + await page.locator('#liveMap .live-node-marker').click(); + await page.locator('#nodeDetailContent h3').filter({ hasText: node.name }).waitFor({ state: 'visible' }); + const detail = await page.locator('#nodeDetailContent').textContent(); + assert.match(detail, /\bOnline\b/, 'recent activity must keep the Live detail online'); + assert.doesNotMatch(detail, /\bOffline\b/); + assert.strictEqual(await page.locator('#nodeDetailContent tr').filter({ hasText: 'Last Activity' }).count(), 1); + pass('Live detail recognizes recent ' + evidence.role + ' activity'); + } + } finally { + await page.unroute(listUrl); + for (const url of urls) await page.unroute(url); + } +} + async function main() { const requireChromium = process.env.CHROMIUM_REQUIRE === '1'; let browser; @@ -209,6 +243,7 @@ async function main() { await checkConfiguredScope(page); await checkBatteryStatus(page); + await checkLiveActivityStatus(page); // (a) /analytics await checkHeading(page, '/analytics', /Mesh Analytics/, '(a) /analytics'); diff --git a/tests/unit/test-live.js b/tests/unit/test-live.js index 7b104319d..2701c18f0 100644 --- a/tests/unit/test-live.js +++ b/tests/unit/test-live.js @@ -1155,6 +1155,43 @@ console.log('\n=== live.js: clickable paths ==='); }); } +// The detail pane must agree with the shared relay-aware freshness rules. +{ + const source = fs.readFileSync('public/live.js', 'utf8'); + const start = source.indexOf(' async function showNodeDetail(pubkey)'); + const end = source.indexOf(' async function loadNodes(beforeTs)', start); + assert.ok(start >= 0 && end > start, 'Live detail function markers moved'); + const recent = new Date(Date.now() - 60000).toISOString(); + const old = new Date(Date.now() - 100 * 3600000).toISOString(); + const cases = [ + ['active repeater relay', { role: 'repeater', last_seen: old, last_relayed: recent }, {}, 'Online'], + ['active room relay', { role: 'room', last_seen: old, last_relayed: recent }, {}, 'Online'], + ['recently heard companion', { role: 'companion', last_seen: old, last_heard: recent }, {}, 'Online'], + ['recent health evidence', { role: 'companion', last_seen: old }, { lastHeard: recent }, 'Online'], + ['silent repeater', { role: 'repeater', last_seen: old }, {}, 'Offline'], + ['degraded companion', { role: 'companion', last_seen: new Date(Date.now() - 2 * 3600000).toISOString() }, {}, 'Degraded'], + ['unknown recency', { role: 'companion', last_seen: 'invalid' }, {}, 'Offline'], + ]; + for (const [name, node, stats, expected] of cases) { + test('Live detail freshness: ' + name, async () => { + const ctx = makeLiveSandbox({ withAppJs: true }); + const els = Object.fromEntries(['liveNodeDetail', 'nodeDetailContent', 'nodeDetailBackdrop', 'liveNodePaths'] + .map(id => [id, ctx.document.createElement('div')])); + ctx.document.getElementById = id => els[id] || null; + ctx.formatLiveTimestampHtml = value => value || 'unknown'; + ctx.activeNodeDetailKey = null; + ctx.api = async url => url.endsWith('/health') ? { stats } + : url.endsWith('/paths') ? { paths: [] } + : { node: { public_key: 'ab'.repeat(32), name, ...node } }; + vm.runInContext(source.slice(start, end), ctx); + await ctx.showNodeDetail('ab'.repeat(32)); + assert.match(els.nodeDetailContent.innerHTML, new RegExp('>' + expected + '')); + assert.match(els.nodeDetailContent.innerHTML, /Last Activity/); + if (expected === 'Online') assert.ok(els.nodeDetailContent.innerHTML.includes(recent)); + }); + } +} + // ===== SUMMARY ===== Promise.allSettled(pendingTests).then(() => { console.log(`\n${'═'.repeat(40)}`); From 5d2367b2b000a2a978015c5d2c969d5f8547fe74 Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:56 -0400 Subject: [PATCH 49/67] fix(operators): normalize CRLF after logging setup --- manage.sh | 14 ++++++------ test-all.sh | 1 + tests/helpers/manage-shell.js | 33 +++++++++++++++++++++++++++++ tests/unit/test-manage-bootstrap.js | 28 ++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 7 deletions(-) create mode 100644 tests/helpers/manage-shell.js create mode 100644 tests/unit/test-manage-bootstrap.js diff --git a/manage.sh b/manage.sh index 5bd2cd0fd..698368452 100755 --- a/manage.sh +++ b/manage.sh @@ -28,13 +28,6 @@ if [ -f .env ]; then set +a fi -# Auto-fix CRLF in .env if detected -if [ -f .env ] && grep -qP '\r' .env 2>/dev/null; then - warn ".env has Windows line endings (CRLF) — fixing automatically..." - sed -i 's/\r$//' .env - log ".env converted to Unix line endings." -fi - # Resolved paths for prod/staging data (must match docker-compose.yml) PROD_DATA="${PROD_DATA_DIR:-$HOME/meshcore-data}" STAGING_DATA="${STAGING_DATA_DIR:-$HOME/meshcore-staging-data}" @@ -69,6 +62,13 @@ err() { printf '%b\n' "${RED}✗${NC} $1" >&2; } info() { printf '%b\n' "${CYAN}→${NC} $1"; } step() { printf '%b\n' "\n${BOLD}[$1/$TOTAL_STEPS] $2${NC}"; } +# Auto-fix CRLF only after the logging helpers are available. +if [ -f .env ] && grep -qP '\r' .env 2>/dev/null; then + warn ".env has Windows line endings (CRLF) — fixing automatically..." + sed -i 's/\r$//' .env + log ".env converted to Unix line endings." +fi + is_true() { case "${1:-}" in 1|true|TRUE|yes|YES|y|Y|on|ON) return 0 ;; diff --git a/test-all.sh b/test-all.sh index b0a3e0ce6..79eb25741 100755 --- a/test-all.sh +++ b/test-all.sh @@ -155,6 +155,7 @@ node tests/unit/test-live-legend-helper.js node tests/unit/test-live-multibyte-filter.js node tests/unit/test-live-region-filter.js node tests/unit/test-live.js +node tests/unit/test-manage-bootstrap.js node tests/unit/test-map-clustering.js node tests/unit/test-mqtt-status-panel.js node tests/unit/test-my-repeaters-dashboard.js diff --git a/tests/helpers/manage-shell.js b/tests/helpers/manage-shell.js new file mode 100644 index 000000000..a66518e3d --- /dev/null +++ b/tests/helpers/manage-shell.js @@ -0,0 +1,33 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); + +const root = path.resolve(__dirname, '../..'); +const bash = require('../../scripts/bash-path')(); +const quote = value => "'" + value.replace(/'/g, "'\\''") + "'"; +const posix = value => value.replace(/\\/g, '/'); + +// Source the complete production script from its real path. Only external +// commands supplied by each test are replaced; management functions stay real. +module.exports = function manageFixture() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'corescope-manage-test-')); + return { + dir, + run(before = '', after = '') { + return spawnSync(bash, ['--noprofile', '--norc'], { + input: `set -e\n${before}\nsource ${quote(posix(path.join(root, 'manage.sh')))} help >/dev/null\n${after}\n`, + cwd: dir, + env: { ...process.env, PROD_DATA_DIR: posix(path.join(dir, 'data')), STAGING_DATA_DIR: posix(path.join(dir, 'staging')) }, + encoding: 'utf8', + timeout: 20000, + }); + }, + cleanup() { + assert(path.resolve(dir).startsWith(path.resolve(os.tmpdir()) + path.sep)); + assert(path.basename(dir).startsWith('corescope-manage-test-')); + fs.rmSync(dir, { recursive: true }); + }, + }; +}; diff --git a/tests/unit/test-manage-bootstrap.js b/tests/unit/test-manage-bootstrap.js new file mode 100644 index 000000000..9d1885242 --- /dev/null +++ b/tests/unit/test-manage-bootstrap.js @@ -0,0 +1,28 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const manageFixture = require('../helpers/manage-shell'); + +for (const endings of ['\r\n', '\n', null]) { + const fixture = manageFixture(); + try { + if (endings !== null) { + fs.writeFileSync(path.join(fixture.dir, '.env'), ['PROD_HTTP_PORT=18080', 'DISABLE_MOSQUITTO=true', ''].join(endings)); + } + const result = fixture.run(` +docker(){ [ "$*" = 'compose version' ]; } +git(){ printf 'test-revision\\n'; } +# Git Bash grep otherwise strips CRLF before matching. Binary mode reproduces +# the Linux host's CR detection without changing the management code under test. +grep(){ command grep -U "$@"; } +`, `printf 'port=%s mqtt=%s\\n' "\${PROD_HTTP_PORT:-unset}" "\${DISABLE_MOSQUITTO:-unset}"`); + assert.equal(result.status, 0, result.stderr || result.error?.message); + if (endings !== null) { + assert.match(result.stdout, /port=18080 mqtt=true/); + assert(!fs.readFileSync(path.join(fixture.dir, '.env'), 'utf8').includes('\r'), 'CRLF was not normalized'); + } + } finally { + fixture.cleanup(); + } +} +console.log('PASS actual management bootstrap with CRLF, LF, and absent environment file'); From 451ab334f688c067cdfdafcfc470d2f319e6ba2e Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:56 -0400 Subject: [PATCH 50/67] fix(operators): refresh build identity after update checkout --- manage.sh | 10 +++++-- test-all.sh | 1 + tests/unit/test-manage-update-metadata.js | 36 +++++++++++++++++++++++ tests/unit/test-storage-manage.js | 2 +- 4 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 tests/unit/test-manage-update-metadata.js diff --git a/manage.sh b/manage.sh index 698368452..fe40c7153 100755 --- a/manage.sh +++ b/manage.sh @@ -34,9 +34,12 @@ STAGING_DATA="${STAGING_DATA_DIR:-$HOME/meshcore-staging-data}" STAGING_COMPOSE_FILE="docker-compose.staging.yml" # Build metadata — exported so docker compose build picks them up via args -export APP_VERSION=$(git describe --tags --match "v*" 2>/dev/null || echo "unknown") -export GIT_COMMIT=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown") -export BUILD_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ) +refresh_build_metadata() { + export APP_VERSION=$(git describe --tags --match "v*" 2>/dev/null || echo "unknown") + export GIT_COMMIT=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown") + export BUILD_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ) +} +refresh_build_metadata # Docker Compose — detect v2 plugin vs v1 standalone if docker compose version &>/dev/null 2>&1; then @@ -1600,6 +1603,7 @@ cmd_update() { git checkout "$version" || { err "Failed to checkout '${version}'."; exit 1; } fi + refresh_build_metadata migrate_config auto info "Rebuilding image..." diff --git a/test-all.sh b/test-all.sh index 79eb25741..a3d3ffcae 100755 --- a/test-all.sh +++ b/test-all.sh @@ -156,6 +156,7 @@ node tests/unit/test-live-multibyte-filter.js node tests/unit/test-live-region-filter.js node tests/unit/test-live.js node tests/unit/test-manage-bootstrap.js +node tests/unit/test-manage-update-metadata.js node tests/unit/test-map-clustering.js node tests/unit/test-mqtt-status-panel.js node tests/unit/test-my-repeaters-dashboard.js diff --git a/tests/unit/test-manage-update-metadata.js b/tests/unit/test-manage-update-metadata.js new file mode 100644 index 000000000..db0a29240 --- /dev/null +++ b/tests/unit/test-manage-update-metadata.js @@ -0,0 +1,36 @@ +const assert = require('node:assert/strict'); +const manageFixture = require('../helpers/manage-shell'); + +for (const target of ['latest', 'v-new', '']) { + const fixture = manageFixture(); + try { + const result = fixture.run(` +task_ref=old +git(){ + case "$1" in + describe) printf 'v-%s\\n' "$task_ref" ;; + rev-parse) printf '%s-sha\\n' "$task_ref" ;; + checkout) task_ref=new ;; + tag) printf 'v-new\\n' ;; + fetch) : ;; + *) return 1 ;; + esac +} +docker(){ + case "$*" in + 'compose version') : ;; + *'build prod') printf 'source=%s version=%s commit=%s\\n' "$task_ref" "$APP_VERSION" "$GIT_COMMIT" ;; + *'/app/storage.sh prod field state') printf 'ready\\n' ;; + *'/app/storage.sh prod field backend') printf 'sqlite\\n' ;; + *'up -d --force-recreate prod') : ;; + *) printf 'unexpected docker call\\n' >&2; return 1 ;; + esac +} +`, `cmd_update '${target}'`); + assert.equal(result.status, 0, result.stderr || result.error?.message); + assert.match(result.stdout, /source=new version=v-new commit=new-sha/, 'new image was labelled with the previous checkout'); + } finally { + fixture.cleanup(); + } +} +console.log('PASS update build identity follows latest, explicit, and default release checkout'); diff --git a/tests/unit/test-storage-manage.js b/tests/unit/test-storage-manage.js index 8396a2f04..e9fe037ba 100644 --- a/tests/unit/test-storage-manage.js +++ b/tests/unit/test-storage-manage.js @@ -8,7 +8,7 @@ const bash=require('../../scripts/bash-path')(); const temp=fs.mkdtempSync(path.join(os.tmpdir(),'corescope-storage-manage-')); const quote=value=>"'"+value.replace(/'/g,"'\\''")+"'"; const posix=value=>value.replace(/\\/g,'/'); -const names=['is_true','dc_prod_base','dc_staging_base','dc_postgres','storage_at','storage_action','storage_field','selected_backend','require_managed_postgres','choose_storage','dc_prod','dc_staging','cmd_start','cmd_update','cmd_stop','cmd_storage']; +const names=['refresh_build_metadata','is_true','dc_prod_base','dc_staging_base','dc_postgres','storage_at','storage_action','storage_field','selected_backend','require_managed_postgres','choose_storage','dc_prod','dc_staging','cmd_start','cmd_update','cmd_stop','cmd_storage']; const functions=names.map(name=>source.match(new RegExp(`^${name}\\(\\)\\s*\\{[\\s\\S]*?^}`,'m'))?.[0]||'').join('\n'); function run(command,backend='sqlite',state='ready',input=''){ const log=path.join(temp,'calls');fs.writeFileSync(log,''); From e623f082dbbeb5a2a3a92db433475a425ec7a260 Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:56 -0400 Subject: [PATCH 51/67] fix(operators): verify readiness before reporting healthy --- docker-compose.example.yml | 2 +- docker-compose.no-mosquitto.yml | 2 +- docker-compose.staging.no-mosquitto.yml | 2 +- docker-compose.staging.yml | 2 +- docker-compose.yml | 2 +- manage.sh | 12 +++---- test-all.sh | 1 + tests/unit/test-deployment-readiness.js | 48 +++++++++++++++++++++++++ 8 files changed, 60 insertions(+), 11 deletions(-) create mode 100644 tests/unit/test-deployment-readiness.js diff --git a/docker-compose.example.yml b/docker-compose.example.yml index 3eec371f0..54df15b75 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -30,7 +30,7 @@ services: - DISABLE_MOSQUITTO=${DISABLE_MOSQUITTO:-false} restart: unless-stopped healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/stats"] + test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/healthz"] interval: 30s timeout: 5s retries: 3 diff --git a/docker-compose.no-mosquitto.yml b/docker-compose.no-mosquitto.yml index 39c04a640..1d6f8a3f1 100644 --- a/docker-compose.no-mosquitto.yml +++ b/docker-compose.no-mosquitto.yml @@ -43,7 +43,7 @@ services: - NODE_ENV=production - DISABLE_MOSQUITTO=true healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/stats"] + test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/healthz"] interval: 30s timeout: 5s retries: 3 diff --git a/docker-compose.staging.no-mosquitto.yml b/docker-compose.staging.no-mosquitto.yml index e58dd11c5..402c3f6ef 100644 --- a/docker-compose.staging.no-mosquitto.yml +++ b/docker-compose.staging.no-mosquitto.yml @@ -48,7 +48,7 @@ services: - ENABLE_PPROF=true - DISABLE_MOSQUITTO=true healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/stats"] + test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/healthz"] interval: 30s timeout: 10s retries: 5 diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml index 30e4f5af5..18d77a424 100644 --- a/docker-compose.staging.yml +++ b/docker-compose.staging.yml @@ -76,7 +76,7 @@ services: # via docker DNS without going through the host. meshcore-net: {} healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/stats"] + test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/healthz"] interval: 30s timeout: 10s retries: 5 diff --git a/docker-compose.yml b/docker-compose.yml index ed00d9860..754a9fc48 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,7 +35,7 @@ services: - DISABLE_MOSQUITTO=${DISABLE_MOSQUITTO:-false} - DISABLE_CADDY=${DISABLE_CADDY:-false} healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/stats"] + test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/healthz"] interval: 30s timeout: 5s retries: 3 diff --git a/manage.sh b/manage.sh index fe40c7153..99502f4b2 100755 --- a/manage.sh +++ b/manage.sh @@ -649,11 +649,11 @@ verify_health() { fi fi - # Wait for /api/stats response (Go backend loads packets into memory — may take 60s+) - info "Waiting for server to respond..." + # Wait for initialization, not just the HTTP listener (loading can take 60s+). + info "Waiting for server initialization..." local healthy=false for i in $(seq 1 45); do - if docker exec "$container" wget -qO- http://localhost:3000/api/stats &>/dev/null; then + if docker exec "$container" wget -qO- http://localhost:3000/api/healthz &>/dev/null; then healthy=true break fi @@ -661,11 +661,11 @@ verify_health() { done if ! $healthy; then - err "Server did not respond after 90 seconds." + err "Server did not become ready after 90 seconds." warn "Check logs: ./manage.sh logs" return 1 fi - log "Server is responding." + log "Server is ready." # Check for MQTT errors in recent logs local mqtt_errors @@ -679,7 +679,7 @@ verify_health() { if $use_https; then info "Checking HTTPS for ${caddyfile_domain}..." if command -v curl &>/dev/null; then - if curl -sf --connect-timeout 5 "https://${caddyfile_domain}/api/stats" &>/dev/null; then + if curl -sf --connect-timeout 5 "https://${caddyfile_domain}/api/healthz" &>/dev/null; then log "HTTPS is working: https://${caddyfile_domain}" else warn "HTTPS not reachable yet for ${caddyfile_domain}" diff --git a/test-all.sh b/test-all.sh index a3d3ffcae..838c9c8e7 100755 --- a/test-all.sh +++ b/test-all.sh @@ -210,6 +210,7 @@ node tests/unit/test-ws-stale-watchdog-1074.js node tests/unit/test-xss-escape-sinks.js node tests/unit/test-postgres-ops.js +node tests/unit/test-deployment-readiness.js node tests/unit/test-storage-entrypoint.js node tests/unit/test-storage-manage.js diff --git a/tests/unit/test-deployment-readiness.js b/tests/unit/test-deployment-readiness.js new file mode 100644 index 000000000..144b1f296 --- /dev/null +++ b/tests/unit/test-deployment-readiness.js @@ -0,0 +1,48 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const manageFixture = require('../helpers/manage-shell'); +const root = path.resolve(__dirname, '../..'); + +const bases = ['docker-compose.yml', 'docker-compose.no-mosquitto.yml', 'docker-compose.example.yml', + 'docker-compose.staging.yml', 'docker-compose.staging.no-mosquitto.yml']; +for (const file of bases) { + const source = fs.readFileSync(path.join(root, file), 'utf8'); + const command = source.match(/healthcheck:\s*\n\s*test:\s*(\[[^\n]+\])/); + assert(command, `${file} has no health command`); + assert.equal(JSON.parse(command[1]).at(-1), 'http://localhost:3000/api/healthz', `${file} accepts a partially loaded server`); +} + +for (const becomesReady of [true, false]) { + const fixture = manageFixture(); + try { + fs.mkdirSync(path.join(fixture.dir, 'caddy-config')); + fs.writeFileSync(path.join(fixture.dir, 'caddy-config/Caddyfile'), 'scope.example.org {\n}\n'); + const result = fixture.run(` +git(){ printf 'test-revision\\n'; } +docker(){ + case "$*" in + 'compose version') : ;; + *'/api/stats') printf 'stats\\n' >> probes; return 0 ;; + *'/api/healthz') + printf 'ready\\n' >> probes + [ ${becomesReady} = true ] && [ "$(wc -l < probes)" -ge 3 ] ;; + logs*) : ;; + *) return 1 ;; + esac +} +sleep(){ :; } +curl(){ printf '%s\\n' "$*" > https-probe; } +`, 'verify_health'); + assert.equal(result.status, becomesReady ? 0 : 1, result.stderr || result.stdout); + const probes = fs.readFileSync(path.join(fixture.dir, 'probes'), 'utf8').trim().split('\n'); + assert.equal(probes.length, becomesReady ? 3 : 45, 'readiness was not polled to success or timeout'); + assert(probes.every(p => p === 'ready'), 'verification used statistics as readiness'); + if (becomesReady) { + assert.match(fs.readFileSync(path.join(fixture.dir, 'https-probe'), 'utf8'), /https:\/\/scope\.example\.org\/api\/healthz/); + } + } finally { + fixture.cleanup(); + } +} +console.log('PASS all Compose healthchecks and management verification wait for readiness'); From f949b7d2e2d90666a40b2e8371d5af8e058117c5 Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:56 -0400 Subject: [PATCH 52/67] test(frontend): register portable regression suites --- .github/workflows/deploy.yml | 1 + scripts/non-unit-tests.json | 1 + test-all.sh | 6 ++++++ 3 files changed, 8 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 12329d0d1..459aa4ca0 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -821,6 +821,7 @@ jobs: suite 3 test-live-fullscreen-1572-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B suite 3 test-issue-1599-replay-freeze-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B suite 2 test-issue-1648-m1-icons-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-analytics-loading-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B suite 3 test-issue-1648-m2-icons-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B suite 2 test-issue-1648-m3-icons-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B suite 1 test-issue-1648-m4-icons-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B diff --git a/scripts/non-unit-tests.json b/scripts/non-unit-tests.json index 54454465b..2943af78d 100644 --- a/scripts/non-unit-tests.json +++ b/scripts/non-unit-tests.json @@ -71,6 +71,7 @@ "tests/e2e/test-issue-661-airtime-e2e.js", "tests/e2e/test-issue-1640-compare-discovery-e2e.js", "tests/e2e/test-issue-1648-m1-icons-e2e.js", + "tests/e2e/test-analytics-loading-e2e.js", "tests/e2e/test-issue-1648-m2-icons-e2e.js", "tests/e2e/test-issue-1648-m3-icons-e2e.js", "tests/e2e/test-issue-1648-m4-icons-e2e.js", diff --git a/test-all.sh b/test-all.sh index 838c9c8e7..da4367b70 100755 --- a/test-all.sh +++ b/test-all.sh @@ -214,4 +214,10 @@ node tests/unit/test-deployment-readiness.js node tests/unit/test-storage-entrypoint.js node tests/unit/test-storage-manage.js +node tests/unit/test-analytics-loading.js + +node tests/unit/test-api-inflight-cleanup.js + +node tests/unit/test-node-analytics-lifecycle.js + echo "All standalone frontend suites passed" From 2408dcf2a71acde1cdc88d4817d905d59113e15d Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:40:56 -0400 Subject: [PATCH 53/67] docs(storage): explain backend sizing and application cache limits --- docs/storage.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/storage.md b/docs/storage.md index 3d7b59760..9f83996c9 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -4,6 +4,14 @@ CoreScope uses **SQLite by default**. PostgreSQL 18.6 is optional. Both keep tel Use the [measured backend comparison](performance/optional-backend-b5-2026-10-09.md) to compare latency, resource use and conversion overhead before choosing PostgreSQL. +Choose for the hardware and workload, not the age of the installation. SQLite +keeps the deployment small; PostgreSQL separates database resources and can help +with concurrent writes and maintenance, but adds a service and its own memory, +storage and backup costs. Neither backend is universally faster. The hot packet +cache now accounts for SQLite observation fan-out when admitting history: under +the same memory budget it may retain a shorter hot window. Omitted history stays +in SQL; a smaller cache does not change database retention. + ## Install Keep a complete checkout pinned to the same release as the image. For the usual single-host installation, preserve an existing private `.env` or copy `.env.example`, set `DATA_DIR` to the intended persistent directory, then run: @@ -63,6 +71,17 @@ Before the first upgrade from an older deployment, stop that old stack using its Plan a maintenance window and measure disk headroom on a representative copy. Keep source data, recovery snapshots and the new target together until validation is complete. Data/index/WAL and process memory costs differ between engines; one synthetic benchmark does not size every installation. +The published B5 experiment observed a PostgreSQL DB/cluster plus WAL peak about +**2.12 times** the SQLite database plus sidecars (about **112% more**). This is an +observation, not a sizing multiplier or guarantee. Using that ratio only as an +illustration, source + one recovery copy + PostgreSQL alone is `1 + 1 + 2.12 = +4.12` times the source: a 16.8 GB source would total about **69.2 GB** before +additional working space. The current forward importer also retains a normalized +working snapshot. Allow separately for that copy, account data, index building, +temporary work and additional WAL. The measured PostgreSQL peak already included +1 GiB of WAL; it does not bound WAL on another installation. Rehearse the actual +conversion and keep its observed free-space margin. + For the managed production deployment: ```sh @@ -120,3 +139,8 @@ Restoring accounts can revive deleted users, old passwords, revoked sessions and | PostgreSQL telemetry/accounts | Telemetry readiness version2, with explicit offline owner upgrade from version1; account readiness version1 and logical schema version6 | Unknown/future columns, hidden generated columns, unsupported objects or conflicting data refuse conversion. Preserve a raw recovery snapshot before an explicit repair/upgrade of a disposable working copy. Deleted legacy integer IDs without allocator or retained-reference evidence have no recoverable historical high-water mark. Do not infer one or discard references to make validation pass. + +The large-instance preparation work tracked by [#2214](https://github.com/Kpa-clawbot/CoreScope/issues/2214) +is still pending. This update does not add a snapshot bypass or accept arbitrary +triggers. An unsupported schema remains a stop condition; retain the original +and recovery copies rather than deleting objects merely to pass validation. From 30a812f3c6cd06e4e0f340239b4bbb4362bedd61 Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:50:14 -0400 Subject: [PATCH 54/67] test(analytics): wait for initial theme before resizing --- tests/e2e/test-charts-fluid-1058-e2e.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/e2e/test-charts-fluid-1058-e2e.js b/tests/e2e/test-charts-fluid-1058-e2e.js index 2f3189378..8be1e1129 100644 --- a/tests/e2e/test-charts-fluid-1058-e2e.js +++ b/tests/e2e/test-charts-fluid-1058-e2e.js @@ -226,18 +226,29 @@ async function gatherRows(page) { await step('analytics resize handles stay on their headers without page overflow', async () => { const ctx = await browser.newContext({ viewport: { width: 1250, height: 900 } }); + await ctx.addInitScript(() => { + window.addEventListener('theme-refresh', () => { window.__chartsThemeReady = true; }, { once: true }); + }); const page = await ctx.newPage(); try { await page.goto(BASE + '/#/analytics?tab=nodes', { waitUntil: 'domcontentloaded' }); - await page.waitForSelector('.analytics-table .col-resize-handle'); + // Initial theme loading rebuilds this table. Exercise the settled table, + // rather than starting a drag that bootstrap replaces before mouseup. + await page.waitForFunction(() => window.__chartsThemeReady && + document.querySelector('#analytics-tbl-nodes-0[data-resizable="1"] .col-resize-handle')); + const table = page.locator('#analytics-tbl-nodes-0'); + await table.waitFor({ state: 'visible' }); + const settledTable = await table.elementHandle(); for (const width of [1250, 390]) { await page.setViewportSize({ width, height: 900 }); const layout = await page.evaluate(() => ({ pageWidth: document.documentElement.clientWidth, scrollWidth: document.documentElement.scrollWidth, + handleCount: document.querySelectorAll('.analytics-table .col-resize-handle').length, misplaced: Array.from(document.querySelectorAll('.analytics-table .col-resize-handle')) .filter(handle => handle.offsetParent !== handle.parentElement).length, })); + assert(layout.handleCount > 0, `${width}px: no resize handles were rendered`); assert(layout.misplaced === 0, `${width}px: ${layout.misplaced} resize handles are not anchored to their header`); assert(layout.scrollWidth <= layout.pageWidth + 1, `${width}px: page scrollWidth=${layout.scrollWidth} exceeds clientWidth=${layout.pageWidth}`); @@ -249,11 +260,14 @@ async function gatherRows(page) { await handle.hover(); const before = await header.boundingBox(); const grip = await handle.boundingBox(); + assert(before && grip, 'the settled column header and resize handle must be visible'); await page.mouse.move(grip.x + grip.width / 2, grip.y + grip.height / 2); await page.mouse.down(); await page.mouse.move(grip.x + grip.width / 2 + 40, grip.y + grip.height / 2, { steps: 4 }); await page.mouse.up(); + assert(await settledTable.evaluate(el => el.isConnected), 'the table was replaced during the resize'); const after = await header.boundingBox(); + assert(after, 'the resized column header must remain visible'); assert(after.width > before.width + 20, `drag did not resize the column: ${before.width} -> ${after.width}`); const saved = await page.evaluate(() => JSON.parse(localStorage.getItem('meshcore-analytics-nodes-0-col-widths'))); assert(Array.isArray(saved) && saved.length > 1, 'drag did not save column widths'); From d132e582e6260897cde43c8041f104ba53bde0ff Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 17:55:09 -0400 Subject: [PATCH 55/67] chore(lint): declare shared node activity helper --- .eslintrc.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintrc.json b/.eslintrc.json index deb317e8d..3ea75fab5 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -227,6 +227,7 @@ "formatTimestampCustom": "readonly", "formatTimestampWithTooltip": "readonly", "getDistanceUnit": "readonly", + "getEffectiveHeardMs": "readonly", "getFavorites": "readonly", "getHashParams": "readonly", "getHealthThresholds": "readonly", From b101c933634ea449a489b5ea4e4fb4604db3b532 Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 20 Oct 2026 08:42:31 -0400 Subject: [PATCH 56/67] test(storage): expose readiness version and upgrade diagnostic failures --- cmd/migrate/recorded_upgrade_test.go | 52 +++++++++++++++++++++++++++ qa/scripts/test-blacklist-postgres.sh | 22 ++++++++++++ tests/unit/test-storage-entrypoint.js | 44 +++++++++++++++++++---- 3 files changed, 111 insertions(+), 7 deletions(-) diff --git a/cmd/migrate/recorded_upgrade_test.go b/cmd/migrate/recorded_upgrade_test.go index c89f3c5ae..cac9e04b0 100644 --- a/cmd/migrate/recorded_upgrade_test.go +++ b/cmd/migrate/recorded_upgrade_test.go @@ -3,10 +3,13 @@ package main import ( "bytes" "context" + "encoding/json" "errors" + "log" "net/url" "os" "path/filepath" + "strings" "testing" "github.com/meshcore-analyzer/dbconfig" @@ -14,6 +17,55 @@ import ( "github.com/meshcore-analyzer/users" ) +func TestRecordedPostgresUpgradeDiagnosticsKeepStatusJSON(t *testing.T) { + o, _ := recordedOldPostgres(t, true) + var diagnostic, status bytes.Buffer + old := log.Writer() + log.SetOutput(&diagnostic) + t.Cleanup(func() { log.SetOutput(old) }) + if err := runStorageAction(context.Background(), o, &status); err != nil { + t.Fatal(err) + } + if !json.Valid(status.Bytes()) { + t.Fatal("upgrade messages contaminated machine-readable status", status.String()) + } + for _, message := range []string{ + "PostgreSQL telemetry schema upgrade required: 1 -> 2", + "PostgreSQL accounts schema upgrade required: 5 -> 6", + "upgraded recorded PostgreSQL telemetry schema 1 -> 2", + "upgraded recorded PostgreSQL accounts schema 5 -> 6", + } { + if !strings.Contains(diagnostic.String(), message) { + t.Errorf("missing supported upgrade diagnostic %q", message) + } + } + diagnostic.Reset() + status.Reset() + if err := runStorageAction(context.Background(), o, &status); err != nil { + t.Fatal(err) + } + if strings.Contains(diagnostic.String(), "upgrade required") || strings.Contains(diagnostic.String(), "upgraded recorded") { + t.Fatal("unchanged current schemas reported a new upgrade") + } +} + +func TestRecordedPostgresUpgradeDoesNotAnnounceUnknownLayout(t *testing.T) { + o, _ := recordedOldPostgres(t, true) + if _, err := openImportDB(t, o.UsersOwnerURL).Exec(`ALTER TABLE sessions ADD COLUMN unknown_extension TEXT`); err != nil { + t.Fatal(err) + } + var diagnostic bytes.Buffer + old := log.Writer() + log.SetOutput(&diagnostic) + t.Cleanup(func() { log.SetOutput(old) }) + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err == nil { + t.Fatal("unknown account layout accepted") + } + if strings.Contains(diagnostic.String(), "upgrade required") || strings.Contains(diagnostic.String(), "upgraded recorded") { + t.Fatal("unknown layout was announced as a supported upgrade") + } +} + func recordedOldPostgres(t *testing.T, accounts bool) (storageOptions, dbconfig.Selection) { t.Helper() o, _ := switchFixture(t) diff --git a/qa/scripts/test-blacklist-postgres.sh b/qa/scripts/test-blacklist-postgres.sh index fec09b285..0fd3fd386 100644 --- a/qa/scripts/test-blacklist-postgres.sh +++ b/qa/scripts/test-blacklist-postgres.sh @@ -129,6 +129,28 @@ out=$(count "$PK_A" "$READER_URL"); rc=$? assert_eq 'legitimate pubkey has three rows' 3 "$out" assert_eq 'legitimate pubkey returns success' 0 "$rc" assert_eq 'native parameter capability probe round-trips' "$POSTGRES_PROBE_TOKEN" "$(postgres_probe_sql | run_local "$READER_URL")" +# Derive the expected version from the shipping migrator's fresh schema. This +# catches a stale QA guard whenever the telemetry schema advances. +current_version=$(run_local "$READER_URL" <<<"SELECT version FROM corescope_schema WHERE kind='telemetry' AND ready;") +assert_true 'shipping schema has a current version' bash -c '[[ "$1" =~ ^[1-9][0-9]*$ ]]' _ "$current_version" +for spec in "unfinished|$current_version|false" "old|1|true" "future|$((current_version+1))|true" "invalid|0|true"; do + IFS='|' read -r label version ready <<<"$spec" + admin_sql >/dev/null <<<"UPDATE $SCHEMA.corescope_schema SET version=$version,ready=$ready WHERE kind='telemetry';" || exit 1 + out=$(count "$PK_A" "$READER_URL" 2>"$FIXTURE_DIR/ready.err"); rc=$? + assert_true "$label schema returns nonzero" test "$rc" -ne 0 + assert_true "$label schema reports not-ready SQLSTATE" contains "$(cat "$FIXTURE_DIR/ready.err")" '55000' + assert_eq "$label schema produces no count" '' "$out" + out=$(postgres_probe_sql | run_local "$READER_URL" 2>"$FIXTURE_DIR/probe-ready.err"); rc=$? + assert_true "$label schema capability probe returns nonzero" test "$rc" -ne 0 + assert_eq "$label schema capability probe produces no token" '' "$out" +done +admin_sql >/dev/null <<<"DELETE FROM $SCHEMA.corescope_schema WHERE kind='telemetry';" || exit 1 +out=$(count "$PK_A" "$READER_URL" 2>"$FIXTURE_DIR/missing-ready.err"); rc=$? +assert_true 'missing readiness marker returns nonzero' test "$rc" -ne 0 +assert_true 'missing readiness marker reports not-ready SQLSTATE' contains "$(cat "$FIXTURE_DIR/missing-ready.err")" '55000' +assert_eq 'missing readiness marker produces no count' '' "$out" +admin_sql >/dev/null <<<"INSERT INTO $SCHEMA.corescope_schema(kind,version,ready) VALUES('telemetry',$current_version,true);" || exit 1 +assert_eq 'current ready schema is accepted after rejected states' 3 "$(count "$PK_A" "$READER_URL")" COUNT_SQL=$(transmission_count_sql "$PK_A") assert_true 'query uses the real attribution column' contains "$COUNT_SQL" 'WHERE from_pubkey = lower(convert_from(decode($1' assert_true 'query has no invented from_node column' lacks "$COUNT_SQL" from_node diff --git a/tests/unit/test-storage-entrypoint.js b/tests/unit/test-storage-entrypoint.js index 70ef873dc..2b59610be 100644 --- a/tests/unit/test-storage-entrypoint.js +++ b/tests/unit/test-storage-entrypoint.js @@ -26,7 +26,18 @@ const field=flag=>args.find(v=>v.startsWith(flag+'='))?.slice(flag.length+1)||ar if(name==='corescope-migrate'){ if(args.includes('-check-ready')){ fs.appendFileSync(calls,JSON.stringify(['readiness-accounts',Boolean(process.env.CORESCOPE_USERS_DATABASE_URL)])+'\\n'); - if(process.env.SCENARIO==='unsupported-schema'||(process.env.SCENARIO==='old-schema'&&!fs.existsSync(calls+'.upgraded')))process.exit(1); + const scenario=process.env.SCENARIO; + const failure={ + 'unsupported-schema':'unsupported telemetry schema version 99 (binary supports 2)', + 'missing-schema':'telemetry schema is not initialized', + 'corrupt-schema':'invalid telemetry schema layout' + }[scenario]; + if(failure){process.stderr.write(failure+'\\n');process.exit(1)} + if(['old-schema','failed-upgrade','failed-recheck'].includes(scenario)&&!fs.existsSync(calls+'.upgraded')){ + process.stderr.write('unsupported telemetry schema version 1 (binary supports 2)\\n');process.exit(1); + } + if(scenario==='failed-recheck'){process.stderr.write('telemetry schema verification failed after upgrade\\n');process.exit(1)} + console.log('telemetry PostgreSQL schema is ready'); process.exit(0); } const selection=field('-selection-file');const action=field('-storage-action'); @@ -43,9 +54,14 @@ if(name==='corescope-migrate'){ fs.writeFileSync(selection,JSON.stringify({version:1,state:'ready',backend,generation:'one',job_id:''}));console.log(JSON.stringify({verified:true})); }else if(action==='upgrade'){ const report=JSON.parse(fs.readFileSync(selection,'utf8')); - if(report.state!=='ready'||report.backend!=='postgres'||process.env.SCENARIO==='unsupported-schema')process.exit(1); + if(report.state!=='ready'||report.backend!=='postgres'||['unsupported-schema','missing-schema','corrupt-schema'].includes(process.env.SCENARIO)){ + process.stderr.write('recorded target cannot be upgraded\\n');process.exit(1); + } + process.stderr.write('[storage] PostgreSQL telemetry schema upgrade required: 1 -> 2\\n'); + if(process.env.SCENARIO==='failed-upgrade'){process.stderr.write('owner upgrade failed\\n');process.exit(1)} fs.appendFileSync(calls,JSON.stringify(['upgraded-stores',report.accounts?['telemetry','accounts']:['telemetry']])+'\\n'); fs.writeFileSync(calls+'.upgraded','verified'); + process.stderr.write('[storage] upgraded recorded PostgreSQL telemetry schema 1 -> 2\\n'); }else{process.stderr.write('unexpected migration action');process.exit(2)} }else if(name==='su'){ if(process.env.SCENARIO==='fail-user-switch')process.exit(1); @@ -96,7 +112,9 @@ function run(extra = {}, action = [], bootstrap = false) { const inherited = env.PATH || env.Path || ''; delete env.Path; env.PATH = path.join(temp, 'bin') + path.delimiter + inherited; - const result = spawnSync(bash, ['--noprofile','--norc',posix(bootstrap ? path.join(app,'postgres-bootstrap.sh') : action.length ? path.join(app,'storage.sh') : path.join(temp,'entrypoint.sh')), ...action], { cwd: app, env, encoding: 'utf8', timeout: 15000 }); + // A run launches several native stub processes through the shell. Allow + // Windows process-start overhead without changing any product deadline. + const result = spawnSync(bash, ['--noprofile','--norc',posix(bootstrap ? path.join(app,'postgres-bootstrap.sh') : action.length ? path.join(app,'storage.sh') : path.join(temp,'entrypoint.sh')), ...action], { cwd: app, env, encoding: 'utf8', timeout: 30000 }); result.calls = fs.readFileSync(calls,'utf8').trim().split('\n').filter(Boolean).map(JSON.parse); return result; } @@ -153,9 +171,12 @@ try { assert.deepEqual(started.calls.find(call=>call[0]==='readiness-accounts'),['readiness-accounts',accounts!==null]); assert(!started.calls.some(call=>call.includes('-storage-action=setup')),'ready PostgreSQL was implicitly initialized'); assert(!started.calls.some(call=>call.includes('-storage-action=upgrade')),'current schema was unnecessarily upgraded'); + assert.match(started.stdout,/telemetry PostgreSQL schema is ready/,'current readiness output was hidden'); const originalSelection=fs.readFileSync(selectionFile,'utf8'); const upgraded=run({SCENARIO:'old-schema',CORESCOPE_OWNER_PASSWORD:'synthetic-owner',CORESCOPE_USERS_OWNER_DATABASE_URL:'synthetic-owner-url',CORESCOPE_USERS_DATABASE_URL:'runtime-account-url'},[],true); assert.equal(upgraded.status,0,upgraded.stderr||upgraded.error?.message); + assert.match(upgraded.stderr,/upgrade required: 1 -> 2/,'supported upgrade did not report stored and current versions'); + assert(!upgraded.stderr.includes('unsupported telemetry'),'successful supported upgrade emitted a provisional failure'); const upgrade=upgraded.calls.filter(call=>call.includes('-storage-action=upgrade')); assert.equal(upgrade.length,1,'old recorded schema did not invoke exactly one owner upgrade'); assert(upgrade[0].includes('-offline'),'schema upgrade was not explicitly offline'); @@ -166,10 +187,19 @@ try { assert(upgraded.calls.findIndex(call=>call[0]==='psql')>finalReady,'runtime grants preceded upgrade verification'); assert(!upgraded.calls.some(call=>call.includes('-storage-action=setup')),'recorded upgrade could initialize a replacement or absent account target'); assert.equal(fs.readFileSync(selectionFile,'utf8'),originalSelection,'upgrade changed the selection'); - const unsupported=run({SCENARIO:'unsupported-schema',CORESCOPE_OWNER_PASSWORD:'synthetic-owner',CORESCOPE_USERS_OWNER_DATABASE_URL:'synthetic-owner-url'},[],true); - assert.notEqual(unsupported.status,0,'unsupported schema passed bootstrap'); - assert(!unsupported.calls.some(call=>call[0]==='psql'),'failed schema upgrade reached grants'); - assert.equal(fs.readFileSync(selectionFile,'utf8'),originalSelection,'failed upgrade changed the selection'); + for(const [scenario,diagnostic] of [ + ['unsupported-schema','unsupported telemetry schema version 99'], + ['missing-schema','telemetry schema is not initialized'], + ['corrupt-schema','invalid telemetry schema layout'], + ['failed-upgrade','owner upgrade failed'], + ['failed-recheck','telemetry schema verification failed after upgrade'] + ]){ + const unsupported=run({SCENARIO:scenario,CORESCOPE_OWNER_PASSWORD:'synthetic-owner',CORESCOPE_USERS_OWNER_DATABASE_URL:'synthetic-owner-url'},[],true); + assert.notEqual(unsupported.status,0,scenario+' passed bootstrap'); + assert(!unsupported.calls.some(call=>call[0]==='psql'),scenario+' reached grants'); + assert.match(unsupported.stderr,new RegExp(diagnostic),scenario+' diagnostic was hidden'); + assert.equal(fs.readFileSync(selectionFile,'utf8'),originalSelection,'failed upgrade changed the selection'); + } } for(const report of [{...pgReport,telemetry:target('unselected_telemetry')},{...pgReport,accounts:target('unselected_accounts')},{...pgReport,accounts:{sqlite_path:'/app/data/users.db'}}]){ fs.writeFileSync(selectionFile,JSON.stringify(report)); From 122f8b5f9f404b541e55d6ccac0c95b76eff4ded Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 20 Oct 2026 08:42:33 -0400 Subject: [PATCH 57/67] fix(storage): accept current schemas and report supported upgrades clearly --- cmd/migrate/recorded_upgrade.go | 14 ++++++++++++-- docker/postgres-bootstrap.sh | 17 ++++++++++++++--- qa/scripts/blacklist-test.sh | 4 +++- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/cmd/migrate/recorded_upgrade.go b/cmd/migrate/recorded_upgrade.go index 581e60f5e..dec7f9751 100644 --- a/cmd/migrate/recorded_upgrade.go +++ b/cmd/migrate/recorded_upgrade.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/json" "errors" + "log" "slices" "github.com/meshcore-analyzer/dbconfig" @@ -72,6 +73,8 @@ func upgradeRecordedPostgres(ctx context.Context, o storageOptions, expected dbc if !plan.needed { continue } + previous, current := recordedPostgresVersions(plan.kind) + log.Printf("[storage] PostgreSQL %s schema upgrade required: %d -> %d", plan.kind, previous, current) if plan.kind == "telemetry" { err = dbschema.ApplyPostgres(plan.db, nil) } else { @@ -80,6 +83,7 @@ func upgradeRecordedPostgres(ctx context.Context, o storageOptions, expected dbc if err != nil { return err } + log.Printf("[storage] upgraded recorded PostgreSQL %s schema %d -> %d", plan.kind, previous, current) } if err := grantRuntimeTargets(ctx, selected, owners, raw); err != nil { return err @@ -100,7 +104,7 @@ func validateRecordedPostgresUpgrade(ctx context.Context, db *sql.DB, kind strin if err := db.QueryRowContext(ctx, `SELECT count(*),min(kind),min(version),bool_and(ready) FROM corescope_schema`).Scan(&count, &recordedKind, &version, &ready); err != nil || count != 1 || recordedKind != kind || !ready { return false, errors.New("recorded PostgreSQL store is missing, unsupported or incomplete; it was not initialized") } - previous, current := 1, dbschema.Version + previous, current := recordedPostgresVersions(kind) if kind == "accounts" { if version != users.PostgresSchemaVersion { return false, errors.New("unsupported PostgreSQL account readiness version") @@ -108,7 +112,6 @@ func validateRecordedPostgresUpgrade(ctx context.Context, db *sql.DB, kind strin if err := db.QueryRowContext(ctx, `SELECT count(*),coalesce(max(version),0) FROM schema_version`).Scan(&count, &version); err != nil || count != 1 { return false, errors.New("recorded PostgreSQL account schema is incomplete") } - previous, current = 5, users.CurrentSchemaVersion } if version != previous && version != current { return false, errors.New("unsupported PostgreSQL schema version; upgrade was not attempted") @@ -173,6 +176,13 @@ func validateRecordedPostgresUpgrade(ctx context.Context, db *sql.DB, kind strin return true, nil } +func recordedPostgresVersions(kind string) (previous, current int) { + if kind == "accounts" { + return 5, users.CurrentSchemaVersion + } + return 1, dbschema.Version +} + func upgradeAddedRelation(kind, name string) bool { if kind == "telemetry" { return name == "tx_rewrite_feed" || name == "key_backfill_state" || name == "key_backfill_done" diff --git a/docker/postgres-bootstrap.sh b/docker/postgres-bootstrap.sh index d15f458bd..675a02ef6 100755 --- a/docker/postgres-bootstrap.sh +++ b/docker/postgres-bootstrap.sh @@ -17,12 +17,23 @@ case $(/app/storage.sh field has_accounts) in false) ;; *) echo 'Cannot read the recorded account target.' >&2; exit 1 ;; esac -if ! CORESCOPE_USERS_DATABASE_URL="$account_owner" /app/corescope-migrate -check-ready; then +if readiness=$(CORESCOPE_USERS_DATABASE_URL="$account_owner" /app/corescope-migrate -check-ready 2>&1); then + printf '%s\n' "$readiness" +else # Only a recorded, existing PostgreSQL target can be upgraded. Unlike setup, # this action cannot create a missing database or add an absent account store. # Its offline lease refuses active writers; failed upgrades block startup. - /app/storage.sh upgrade - CORESCOPE_USERS_DATABASE_URL="$account_owner" /app/corescope-migrate -check-ready + # Supported old versions are expected here. The owner upgrade reports their + # version transition; retain provisional diagnostics only if it fails. + if ! /app/storage.sh upgrade; then + printf '%s\n' "$readiness" >&2 + echo 'PostgreSQL schema upgrade failed; runtime services remain stopped.' >&2 + exit 1 + fi + if ! CORESCOPE_USERS_DATABASE_URL="$account_owner" /app/corescope-migrate -check-ready; then + echo 'PostgreSQL schema verification failed; runtime services remain stopped.' >&2 + exit 1 + fi fi export PGHOST=postgres PGPORT=5432 PGSSLMODE=disable diff --git a/qa/scripts/blacklist-test.sh b/qa/scripts/blacklist-test.sh index 277f21ef8..14467cc00 100755 --- a/qa/scripts/blacklist-test.sh +++ b/qa/scripts/blacklist-test.sh @@ -197,7 +197,9 @@ BEGIN WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema' AND CASE WHEN c.relkind='S' THEN has_sequence_privilege(c.oid,'USAGE,UPDATE') ELSE false END) THEN RAISE EXCEPTION 'QA requires a restricted PostgreSQL reader role' USING ERRCODE='42501'; END IF; - IF (SELECT count(*) FROM corescope_schema WHERE kind='telemetry' AND version=1 AND ready) <> 1 + -- Keep the runtime QA reader at dbschema.Version; the native regression uses + -- the shipping migrator's schema and rejects old, future and unfinished markers. + IF (SELECT count(*) FROM corescope_schema WHERE kind='telemetry' AND version=2 AND ready) <> 1 THEN RAISE EXCEPTION 'telemetry schema is not ready' USING ERRCODE='55000'; END IF; END $qa$; SQL From a2b417b6d7f00911bf2b65d5f1c18d1cf7e7b8c1 Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 20 Oct 2026 08:42:34 -0400 Subject: [PATCH 58/67] test(migrate): reject loss of supported SQLite hop-index sources --- cmd/migrate/observation_hops_test.go | 173 +++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 cmd/migrate/observation_hops_test.go diff --git a/cmd/migrate/observation_hops_test.go b/cmd/migrate/observation_hops_test.go new file mode 100644 index 000000000..a743dc61c --- /dev/null +++ b/cmd/migrate/observation_hops_test.go @@ -0,0 +1,173 @@ +package main + +import ( + "context" + "database/sql" + "path/filepath" + "reflect" + "testing" + + "github.com/meshcore-analyzer/dbschema" +) + +// The accelerator can be absent, complete, or midway through its independently +// committed backfill. None of those states may change the canonical rows. +func hopIndexSource(t *testing.T, state string) string { + t.Helper() + path := telemetrySource(t) + db, err := openSQLite(path, "rw") + if err != nil { + t.Fatal(err) + } + defer db.Close() + for _, stmt := range []string{ + `UPDATE observations SET path_json=' {malformed',score=0.75 WHERE id=90`, + `UPDATE observations SET path_json=' ["aB","Cd","aB",17,null] ' WHERE id=91`, + `INSERT INTO observations(id,transmission_id,observer_idx,path_json,timestamp) VALUES(92,80,NULL,'["Ef"]',3)`, + } { + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } + } + if state == "absent" { + return path + } + if err := dbschema.EnsureObservationHopsSchema(db, nil); err != nil { + t.Fatal(err) + } + floor := int64(0) + if state == "partial" { + floor = 3 + } + if _, err := dbschema.IndexObservationHops(db, floor, 4); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`UPDATE observation_hops_coverage SET since=? WHERE id=1`, floor); err != nil { + t.Fatal(err) + } + return path +} + +type observationTuple struct { + ID, Timestamp int64 + Path sql.NullString + Score sql.NullFloat64 +} + +func observationTuples(t *testing.T, db *sql.DB) []observationTuple { + t.Helper() + rows, err := db.Query(`SELECT id,timestamp,path_json,score FROM observations ORDER BY id`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var out []observationTuple + for rows.Next() { + var row observationTuple + if err := rows.Scan(&row.ID, &row.Timestamp, &row.Path, &row.Score); err != nil { + t.Fatal(err) + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return out +} + +func TestPrepareHopIndexSourcePreservesOriginalAndRecovery(t *testing.T) { + for _, state := range []string{"absent", "complete", "partial"} { + t.Run(state, func(t *testing.T) { + ctx := context.Background() + source := hopIndexSource(t, state) + before, err := fingerprint(ctx, source) + if err != nil { + t.Fatal(err) + } + o := importOptions{Source: source, StateDir: t.TempDir(), Kind: "telemetry", targetDigest: "fixture"} + tables, err := layouts(o.Kind) + if err != nil { + t.Fatal(err) + } + manifest, normalized, err := prepareSource(ctx, o, tables, true) + if err != nil { + t.Fatalf("known %s hop-index source rejected: %v", state, err) + } + if after, err := fingerprint(ctx, source); err != nil || after != before { + t.Fatal("preparation modified source", err) + } + var original []observationTuple + for i, path := range []string{source, filepath.Join(o.StateDir, o.Kind, "recovery.sqlite"), normalized} { + db, err := openSQLite(path, "ro") + if err != nil { + t.Fatal(err) + } + got := observationTuples(t, db) + if i == 0 { + original = got + } else if !reflect.DeepEqual(got, original) { + t.Fatal("canonical observations changed", got, original) + } + var objects int + err = db.QueryRow(`SELECT count(*) FROM sqlite_schema WHERE name LIKE 'observation_hops%' OR name LIKE 'trg_observation_hops%'`).Scan(&objects) + db.Close() + want := 5 + if state == "absent" || i == 2 { + want = 0 + } + if err != nil || objects != want { + t.Fatalf("snapshot %d has %d hop objects, want %d: %v", i, objects, want, err) + } + } + o.Resume = true + resumed, resumedPath, err := prepareSource(ctx, o, tables, false) + if err != nil || !reflect.DeepEqual(resumed, manifest) || resumedPath != normalized { + t.Fatal("prepared snapshots cannot resume unchanged", err) + } + }) + } +} + +func TestPrepareHopIndexRejectsUnknownOrChangedObjects(t *testing.T) { + for name, statements := range map[string][]string{ + "missing trigger": {`DROP TRIGGER trg_observation_hops_ad`}, + "changed trigger": {`DROP TRIGGER trg_observation_hops_ai`, `CREATE TRIGGER trg_observation_hops_ai AFTER INSERT ON observations BEGIN DELETE FROM observations; END`}, + "changed column": {`ALTER TABLE observation_hops ADD COLUMN private_payload TEXT`}, + "changed constraint": {`DROP TABLE observation_hops_coverage`, `CREATE TABLE observation_hops_coverage(id INTEGER PRIMARY KEY,since INTEGER NOT NULL)`}, + "changed primary key": {`DROP TABLE observation_hops`, `CREATE TABLE observation_hops(hop TEXT NOT NULL,timestamp INTEGER NOT NULL,observation_id INTEGER NOT NULL,PRIMARY KEY(observation_id,hop,timestamp)) WITHOUT ROWID`}, + "unknown index": {`CREATE INDEX surprise_hop_index ON observation_hops(observation_id)`}, + "unknown table": {`CREATE TABLE private_hops(payload TEXT)`}, + "unknown trigger": {`CREATE TRIGGER unexpected_hops AFTER INSERT ON observations BEGIN DELETE FROM observation_hops; END`}, + "unknown view": {`CREATE VIEW extra_hops AS SELECT * FROM observation_hops`}, + } { + t.Run(name, func(t *testing.T) { + source := hopIndexSource(t, "complete") + db, err := openSQLite(source, "rw") + if err != nil { + t.Fatal(err) + } + for _, stmt := range statements { + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } + } + db.Close() + ctx := context.Background() + before, err := fingerprint(ctx, source) + if err != nil { + t.Fatal(err) + } + o := importOptions{Source: source, StateDir: t.TempDir(), Kind: "telemetry", targetDigest: "fixture"} + tables, err := layouts(o.Kind) + if err != nil { + t.Fatal(err) + } + if _, _, err := prepareSource(ctx, o, tables, true); err == nil { + t.Fatal("unknown or altered source layout was accepted") + } + if after, err := fingerprint(ctx, source); err != nil || after != before { + t.Fatal("refusal modified source", err) + } + }) + } +} From 15eb2b6bd84f22eec72f4d6fe517abf7710a6fef Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 20 Oct 2026 08:42:36 -0400 Subject: [PATCH 59/67] fix(migrate): recognize and exclude the exact derived hop-index layout --- cmd/migrate/import.go | 39 ++++--- cmd/migrate/layout.go | 19 ++++ cmd/migrate/observation_hops_test.go | 145 +++++++++++++++++++++++++++ cmd/migrate/sqlite_derived.go | 108 ++++++++++++++++++++ cmd/migrate/sqlite_hop_index_v1.json | 35 +++++++ 5 files changed, 333 insertions(+), 13 deletions(-) create mode 100644 cmd/migrate/sqlite_derived.go create mode 100644 cmd/migrate/sqlite_hop_index_v1.json diff --git a/cmd/migrate/import.go b/cmd/migrate/import.go index 8dda84c09..76484286e 100644 --- a/cmd/migrate/import.go +++ b/cmd/migrate/import.go @@ -37,18 +37,20 @@ type tableReport struct { SHA256 string `json:"sha256"` } type importReport struct { - Kind string `json:"kind"` - Verified bool `json:"verified"` - Tables []tableReport `json:"tables"` + Kind string `json:"kind"` + Verified bool `json:"verified"` + Tables []tableReport `json:"tables"` + ExcludedDerived []derivedTableReport `json:"excluded_derived,omitempty"` } type sourceManifest struct { - Version int `json:"version"` - RunID string `json:"run_id"` - Kind string `json:"kind"` - SourceDigest string `json:"source_digest"` - RawDigest string `json:"raw_digest"` - NormalizedDigest string `json:"normalized_digest"` - TargetDigest string `json:"target_digest"` + Version int `json:"version"` + RunID string `json:"run_id"` + Kind string `json:"kind"` + SourceDigest string `json:"source_digest"` + RawDigest string `json:"raw_digest"` + NormalizedDigest string `json:"normalized_digest"` + TargetDigest string `json:"target_digest"` + ExcludedDerived []derivedTableReport `json:"excluded_derived,omitempty"` } const importerLock int64 = 7289041410 @@ -96,6 +98,13 @@ func prepareSource(ctx context.Context, o importOptions, tables []importTable, a return manifest, "", errors.New("migration recovery or normalized snapshot changed; resume refused") } } + excluded, err := recoveryDerivedExclusions(raw, o.Kind) + if err != nil { + return manifest, "", err + } + if !slices.Equal(excluded, manifest.ExcludedDerived) { + return manifest, "", errors.New("migration derived-schema report differs from immutable recovery; resume refused") + } return manifest, normalized, nil } if !errors.Is(err, os.ErrNotExist) { @@ -125,8 +134,11 @@ func prepareSource(ctx context.Context, o importOptions, tables []importTable, a return manifest, "", err } db.SetMaxOpenConns(1) + var excluded []derivedTableReport if _, err = db.ExecContext(ctx, `PRAGMA trusted_schema=OFF`); err == nil { - err = normalizeSource(db, o.Kind, raw, tables) + if excluded, err = sqliteDerivedExclusions(db, o.Kind); err == nil { + err = normalizeSource(db, o.Kind, raw, tables) + } } closeErr := db.Close() if err != nil { @@ -135,7 +147,7 @@ func prepareSource(ctx context.Context, o importOptions, tables []importTable, a if closeErr != nil { return manifest, "", closeErr } - manifest = sourceManifest{Version: 1, Kind: o.Kind, SourceDigest: digest, TargetDigest: o.targetDigest} + manifest = sourceManifest{Version: 1, Kind: o.Kind, SourceDigest: digest, TargetDigest: o.targetDigest, ExcludedDerived: excluded} var random [16]byte if _, err := rand.Read(random[:]); err != nil { return manifest, "", err @@ -240,6 +252,7 @@ func importSQLite(ctx context.Context, o importOptions) (importReport, error) { if err != nil { return report, err } + report.ExcludedDerived = manifest.ExcludedDerived source, err := openSQLite(path, "ro") if err != nil { return report, err @@ -369,7 +382,7 @@ func importSQLite(ctx context.Context, o importOptions) (importReport, error) { } report.Verified = true if ready { - if !slices.Equal(report.Tables, recordedReport.Tables) { + if !slices.Equal(report.Tables, recordedReport.Tables) || !slices.Equal(report.ExcludedDerived, recordedReport.ExcludedDerived) { return report, errors.New("ready destination differs from its verified import report; resume refused") } return report, nil diff --git a/cmd/migrate/layout.go b/cmd/migrate/layout.go index 689a473a2..43874e298 100644 --- a/cmd/migrate/layout.go +++ b/cmd/migrate/layout.go @@ -100,6 +100,16 @@ func sqliteColumns(db *sql.DB, table string) ([]importColumn, error) { } func validateSource(db *sql.DB, kind string, tables []importTable) error { + derived := map[string]bool{} + if kind == "telemetry" { + objects, err := knownSQLiteHopIndex(db) + if err != nil { + return err + } + for _, object := range objects { + derived[object.Name] = true + } + } var accountVersion int if kind == "accounts" { var n int @@ -138,6 +148,9 @@ func validateSource(db *sql.DB, kind string, tables []importTable) error { if o.kind == "table" && internalTables[o.name] && ordinary { continue } + if derived[o.name] { + continue // Complete definitions were checked above, not names alone. + } objects = append(objects, o) if len(objects) > len(allowed)+1 { rows.Close() @@ -199,6 +212,12 @@ func normalizeSource(db *sql.DB, kind, rawPath string, tables []importTable) err return users.ApplySQLite(db) } + // This is the private normalized snapshot. Keep every source/recovery + // object intact, but remove the validated derived cache before canonical + // normalization so its SQLite JSON triggers cannot execute during import. + if _, err := dbschema.DropObservationHopsSchema(db); err != nil { + return err + } if err := legacy.Normalize(db, nil); err != nil { return err } diff --git a/cmd/migrate/observation_hops_test.go b/cmd/migrate/observation_hops_test.go index a743dc61c..732ecdb54 100644 --- a/cmd/migrate/observation_hops_test.go +++ b/cmd/migrate/observation_hops_test.go @@ -3,9 +3,13 @@ package main import ( "context" "database/sql" + "encoding/json" + "errors" + "os" "path/filepath" "reflect" "testing" + "time" "github.com/meshcore-analyzer/dbschema" ) @@ -171,3 +175,144 @@ func TestPrepareHopIndexRejectsUnknownOrChangedObjects(t *testing.T) { }) } } + +func TestHopIndexResumeRejectsChangedExclusionReport(t *testing.T) { + ctx := context.Background() + o := importOptions{Source: hopIndexSource(t, "partial"), StateDir: t.TempDir(), Kind: "telemetry", targetDigest: "fixture"} + tables, err := layouts(o.Kind) + if err != nil { + t.Fatal(err) + } + manifest, _, err := prepareSource(ctx, o, tables, true) + if err != nil { + t.Fatal(err) + } + manifest.ExcludedDerived = nil + data, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(o.StateDir, o.Kind, "manifest.json"), data, 0600); err != nil { + t.Fatal(err) + } + o.Resume = true + if _, _, err := prepareSource(ctx, o, tables, false); err == nil { + t.Fatal("altered exclusion report was trusted on resume") + } +} + +func TestHopIndexNativeRoundTripAndInterruptedImport(t *testing.T) { + requirePostgres(t) + for _, state := range []string{"absent", "complete", "partial"} { + t.Run(state, func(t *testing.T) { + ctx := context.Background() + source := hopIndexSource(t, state) + before, err := fingerprint(ctx, source) + if err != nil { + t.Fatal(err) + } + o := importOptions{Source: source, DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 1, afterBatch: func() error { return context.Canceled }} + if _, err := importSQLite(ctx, o); !errors.Is(err, context.Canceled) { + t.Fatal("expected committed batch interruption", err) + } + o.Resume, o.afterBatch = true, nil + report, err := importSQLite(ctx, o) + if err != nil || !report.Verified { + t.Fatal("hop-index import did not resume", err) + } + for _, table := range report.Tables { + if table.Table == "observation_hops" || table.Table == "observation_hops_coverage" { + t.Fatal("derived rows included in canonical report") + } + } + var decoded struct { + ExcludedDerived []struct { + Table string `json:"table"` + SchemaVersion int `json:"schema_version"` + Reason string `json:"reason"` + } `json:"excluded_derived"` + } + data, err := json.Marshal(report) + if err != nil || json.Unmarshal(data, &decoded) != nil { + t.Fatal("invalid import report", err) + } + wantExclusions := 2 + if state == "absent" { + wantExclusions = 0 + } + if len(decoded.ExcludedDerived) != wantExclusions { + t.Fatal("report did not explicitly identify excluded derived rows", string(data)) + } + for i, table := range decoded.ExcludedDerived { + if table.SchemaVersion != 1 || table.Reason == "" || table.Table != []string{"observation_hops", "observation_hops_coverage"}[i] { + t.Fatal("exclusion report lacks its exact known layout", table) + } + } + target := openImportDB(t, o.DatabaseURL) + var count int + if err := target.QueryRow(`SELECT count(*) FROM pg_class WHERE relnamespace=current_schema()::regnamespace AND relname IN ('observation_hops','observation_hops_coverage')`).Scan(&count); err != nil || count != 0 { + t.Fatal("derived SQLite cache copied to PostgreSQL", count, err) + } + if _, err := finalizeImport(ctx, o.DatabaseURL, o.Kind); err != nil { + t.Fatal(err) + } + resumed, err := importSQLite(ctx, o) + if err != nil || !reflect.DeepEqual(resumed, report) { + t.Fatal("finalized resume changed report", err) + } + reverse := reverseOptions{DatabaseURL: o.DatabaseURL, Destination: filepath.Join(t.TempDir(), "reverse.sqlite"), StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 1, afterBatch: func() error { return context.Canceled }} + if _, err := reversePostgres(ctx, reverse); !errors.Is(err, context.Canceled) { + t.Fatal("expected reverse interruption", err) + } + reverse.Resume, reverse.afterBatch = true, nil + back, err := reversePostgres(ctx, reverse) + if err != nil || !back.Verified { + t.Fatal("hop-index reverse did not resume", err) + } + if !reflect.DeepEqual(back.Tables, report.Tables) { + t.Fatal("canonical table counts or digests changed in the round trip") + } + recovered, err := openSQLite(reverse.Destination, "rw") + if err != nil { + t.Fatal(err) + } + defer recovered.Close() + if err := recovered.QueryRow(`SELECT count(*) FROM sqlite_schema WHERE name LIKE 'observation_hops%' OR name LIKE 'trg_observation_hops%'`).Scan(&count); err != nil || count != 0 { + t.Fatal("reverse created false hop coverage", count, err) + } + if got, want := observationTuples(t, recovered), observationTuples(t, target); !reflect.DeepEqual(got, want) { + t.Fatal("raw paths, IDs, or fractional scores changed in reverse", got, want) + } + // The owner-controlled SQLite ingestor installs this exact schema. + // Installing it must not inherit source coverage or imply old rows + // have been rebuilt; its existing bounded backfill does that later. + now := time.Now().Unix() + if err := dbschema.EnsureObservationHopsSchema(recovered, nil); err != nil { + t.Fatal(err) + } + var since int64 + if err := recovered.QueryRow(`SELECT since FROM observation_hops_coverage WHERE id=1`).Scan(&since); err != nil || since < now { + t.Fatal("new installation inherited old coverage", since, err) + } + if err := recovered.QueryRow(`SELECT count(*) FROM observation_hops`).Scan(&count); err != nil || count != 0 { + t.Fatal("unrebuilt cache unexpectedly populated", count, err) + } + if after, err := fingerprint(ctx, source); err != nil || after != before { + t.Fatal("round trip changed original source", err) + } + if state == "partial" { + report.ExcludedDerived = nil + data, err := json.Marshal(report) + if err != nil { + t.Fatal(err) + } + if _, err := target.Exec(`UPDATE corescope_import SET report=$1 WHERE id=1`, string(data)); err != nil { + t.Fatal(err) + } + if _, err := importSQLite(ctx, o); err == nil { + t.Fatal("finalized resume trusted changed exclusion report") + } + } + }) + } +} diff --git a/cmd/migrate/sqlite_derived.go b/cmd/migrate/sqlite_derived.go new file mode 100644 index 000000000..b03fdbb64 --- /dev/null +++ b/cmd/migrate/sqlite_derived.go @@ -0,0 +1,108 @@ +package main + +import ( + "database/sql" + _ "embed" + "encoding/json" + "errors" + "fmt" + "slices" +) + +// Frozen SQLite layout installed by #2214. Do not regenerate this allowlist +// when runtime DDL changes: a new supported layout needs a separate version +// and migration tests. Exact SQL preserves literals, column types, NOT NULL, +// CHECK, primary-key order, WITHOUT ROWID, and trigger bodies. +// +//go:embed sqlite_hop_index_v1.json +var sqliteHopIndexV1JSON []byte + +type sqliteDerivedObject struct { + Type string `json:"type"` + Name string `json:"name"` + Table string `json:"table"` + SQL string `json:"sql"` +} + +type derivedTableReport struct { + Table string `json:"table"` + SchemaVersion int `json:"schema_version"` + Reason string `json:"reason"` +} + +// knownSQLiteHopIndex recognizes the complete schema even when its data +// backfill is partial. No derived content is trusted or promoted to canonical +// data. Unknown indexes on either table also fail this exact-object comparison; +// v1 has only the implicit WITHOUT ROWID primary key, no standalone indexes. +func knownSQLiteHopIndex(db *sql.DB) ([]sqliteDerivedObject, error) { + var known struct { + Version int `json:"version"` + Objects []sqliteDerivedObject `json:"objects"` + } + if err := json.Unmarshal(sqliteHopIndexV1JSON, &known); err != nil || known.Version != 1 || len(known.Objects) != 5 { + return nil, errors.New("invalid built-in SQLite hop-index layout") + } + rows, err := db.Query(`SELECT type,name,tbl_name,COALESCE(sql,'') FROM sqlite_schema + WHERE name LIKE 'observation_hops%' OR name LIKE 'trg_observation_hops%' + OR tbl_name IN ('observation_hops','observation_hops_coverage') + ORDER BY name`) + if err != nil { + return nil, err + } + defer rows.Close() + var got []sqliteDerivedObject + for rows.Next() { + var o sqliteDerivedObject + if err := rows.Scan(&o.Type, &o.Name, &o.Table, &o.SQL); err != nil { + return nil, err + } + got = append(got, o) + if len(got) > len(known.Objects) { + return nil, errors.New("unsupported SQLite hop-index objects; source data was not discarded") + } + } + if err := rows.Err(); err != nil { + return nil, err + } + if len(got) > 0 && !slices.Equal(got, known.Objects) { + return nil, errors.New("unsupported SQLite hop-index layout (expected complete v1 tables, constraints, indexes and triggers); source data was not discarded") + } + return got, nil +} + +func sqliteDerivedExclusions(db *sql.DB, kind string) ([]derivedTableReport, error) { + if kind != "telemetry" { + return nil, nil + } + objects, err := knownSQLiteHopIndex(db) + if err != nil { + return nil, err + } + var reports []derivedTableReport + for _, o := range objects { + if o.Type == "table" { + reports = append(reports, derivedTableReport{ + Table: o.Name, SchemaVersion: 1, + Reason: "derived SQLite hop-index rows and coverage excluded; rebuild through the SQLite ingestor", + }) + } + } + return reports, nil +} + +func recoveryDerivedExclusions(rawPath, kind string) ([]derivedTableReport, error) { + db, err := openSQLite(rawPath, "ro") + if err != nil { + return nil, err + } + defer db.Close() + db.SetMaxOpenConns(1) + if _, err := db.Exec(`PRAGMA trusted_schema=OFF`); err != nil { + return nil, err + } + reports, err := sqliteDerivedExclusions(db, kind) + if err != nil { + return nil, fmt.Errorf("validate recovery derived schema: %w", err) + } + return reports, nil +} diff --git a/cmd/migrate/sqlite_hop_index_v1.json b/cmd/migrate/sqlite_hop_index_v1.json new file mode 100644 index 000000000..2a572bb14 --- /dev/null +++ b/cmd/migrate/sqlite_hop_index_v1.json @@ -0,0 +1,35 @@ +{ + "version": 1, + "objects": [ + { + "type": "table", + "name": "observation_hops", + "table": "observation_hops", + "sql": "CREATE TABLE observation_hops (\n\t\t\thop TEXT NOT NULL,\n\t\t\ttimestamp INTEGER NOT NULL,\n\t\t\tobservation_id INTEGER NOT NULL,\n\t\t\tPRIMARY KEY (hop, timestamp, observation_id)\n\t\t) WITHOUT ROWID" + }, + { + "type": "table", + "name": "observation_hops_coverage", + "table": "observation_hops_coverage", + "sql": "CREATE TABLE observation_hops_coverage (\n\t\t\tid INTEGER PRIMARY KEY CHECK (id = 1),\n\t\t\tsince INTEGER NOT NULL\n\t\t)" + }, + { + "type": "trigger", + "name": "trg_observation_hops_ad", + "table": "observations", + "sql": "CREATE TRIGGER trg_observation_hops_ad\n\t\tAFTER DELETE ON observations\n\t\tWHEN OLD.path_json IS NOT NULL AND json_valid(OLD.path_json)\n\t\t AND OLD.timestamp \u003e= (SELECT since FROM observation_hops_coverage WHERE id = 1)\n\t\tBEGIN\n\t\t\tDELETE FROM observation_hops\n\t\t\tWHERE observation_id = OLD.id AND timestamp = OLD.timestamp\n\t\t\t AND hop IN (SELECT UPPER(value) FROM json_each(OLD.path_json) WHERE type = 'text');\n\t\tEND" + }, + { + "type": "trigger", + "name": "trg_observation_hops_ai", + "table": "observations", + "sql": "CREATE TRIGGER trg_observation_hops_ai\n\t\tAFTER INSERT ON observations\n\t\tWHEN NEW.path_json IS NOT NULL AND json_valid(NEW.path_json)\n\t\t AND NEW.timestamp \u003e= (SELECT since FROM observation_hops_coverage WHERE id = 1)\n\t\tBEGIN\n\t\t\tINSERT OR IGNORE INTO observation_hops (hop, timestamp, observation_id)\n\t\t\tSELECT DISTINCT UPPER(value), NEW.timestamp, NEW.id FROM json_each(NEW.path_json) WHERE type = 'text';\n\t\tEND" + }, + { + "type": "trigger", + "name": "trg_observation_hops_au", + "table": "observations", + "sql": "CREATE TRIGGER trg_observation_hops_au\n\t\tAFTER UPDATE OF path_json ON observations\n\t\tWHEN IFNULL(OLD.path_json, '') != IFNULL(NEW.path_json, '')\n\t\tBEGIN\n\t\t\tDELETE FROM observation_hops\n\t\t\tWHERE observation_id = OLD.id AND timestamp = OLD.timestamp\n\t\t\t AND json_valid(OLD.path_json)\n\t\t\t AND OLD.timestamp \u003e= (SELECT since FROM observation_hops_coverage WHERE id = 1)\n\t\t\t AND hop IN (SELECT UPPER(value) FROM json_each(OLD.path_json) WHERE type = 'text');\n\t\t\tINSERT OR IGNORE INTO observation_hops (hop, timestamp, observation_id)\n\t\t\tSELECT DISTINCT UPPER(value), NEW.timestamp, NEW.id FROM json_each(NEW.path_json)\n\t\t\tWHERE json_valid(NEW.path_json) AND type = 'text'\n\t\t\t AND NEW.timestamp \u003e= (SELECT since FROM observation_hops_coverage WHERE id = 1);\n\t\tEND" + } + ] +} From 7e7c23520b74eaf565be1dc3fee6bc425aa2488c Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 20 Oct 2026 08:42:37 -0400 Subject: [PATCH 60/67] test(server): require a terminal startup summary and honest coverage --- cmd/server/startup_admission_summary_test.go | 127 +++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 cmd/server/startup_admission_summary_test.go diff --git a/cmd/server/startup_admission_summary_test.go b/cmd/server/startup_admission_summary_test.go new file mode 100644 index 000000000..405700688 --- /dev/null +++ b/cmd/server/startup_admission_summary_test.go @@ -0,0 +1,127 @@ +package main + +import ( + "fmt" + "strings" + "testing" + "time" +) + +func TestStartupAdmissionSummaryReheardOldHashKeepsCoverageFloor(t *testing.T) { + dsn := postgresTestDSN(t) + writer, err := openFixtureSQL(dsn) + if err != nil { + t.Fatal(err) + } + defer writer.Close() + if err := applyTestSchema(t, writer); err != nil { + t.Fatal(err) + } + now := time.Now().UTC().Truncate(time.Second) + reheardFirst := now.Add(-30 * 24 * time.Hour).Format(time.RFC3339) + inactive := now.Add(-48 * time.Hour) + if _, err := writer.Exec(`INSERT INTO observers(rowid,id,name) VALUES(1,'summary-observer','Summary fixture')`); err != nil { + t.Fatal(err) + } + if _, err := writer.Exec(`INSERT INTO transmissions(id,raw_hex,hash,first_seen,last_seen,route_type,payload_type,decoded_json) + VALUES(1,'AA','summary-reheard',$1,$2,0,4,'{}'), + (2,'BB','summary-inactive',$3,$4,0,4,'{}')`, + reheardFirst, now.Unix(), inactive.Format(time.RFC3339), inactive.Unix()); err != nil { + t.Fatal(err) + } + if _, err := writer.Exec(`INSERT INTO observations(id,transmission_id,observer_idx,path_json,timestamp) + VALUES(1,1,1,'[]',$1),(2,2,1,'[]',$2)`, now.Unix(), inactive.Unix()); err != nil { + t.Fatal(err) + } + db, err := openFixtureReader(t, dsn) + if err != nil { + t.Fatal(err) + } + defer db.conn.Close() + s := NewPacketStore(db, &PacketStoreConfig{RetentionHours: 24, HotStartupHours: 0}) + started := time.Now().UTC() + logs := captureLog(func() { + if err := s.RunStartupLoad(1); err != nil { + t.Fatal(err) + } + }) + finished := time.Now().UTC() + if len(s.packets) != 1 || s.byTxID[1] == nil || s.byTxID[2] != nil { + t.Fatal("fixture must retain the re-heard hash and omit the inactive hash") + } + if !strings.Contains(logs, fmt.Sprintf("firstSeenRange=[%q,%q]", reheardFirst, reheardFirst)) { + t.Fatal("summary lost the actual retained first-seen range", logs) + } + _, rest, found := strings.Cut(logs, `coverageFrom="`) + coverage, _, closed := strings.Cut(rest, `"`) + if !found || !closed { + t.Fatal("summary has no quoted coverage floor", logs) + } + floor, err := time.Parse(time.RFC3339, coverage) + if err != nil || floor.Before(started.Add(-24*time.Hour).Truncate(time.Second)) || floor.After(finished.Add(-24*time.Hour)) { + t.Fatalf("summary claims coverageFrom=%q outside the complete 24-hour window; retained first_seen=%q: %v", coverage, reheardFirst, err) + } +} + +func TestStartupAdmissionSummary(t *testing.T) { + for _, tc := range []struct { + name string + transmissions int + observations int + hotHours float64 + budgetMB int + closeDB bool + reason string + }{ + {"empty", 0, 0, 0, 0, false, "window_complete"}, + {"full", 4, 2, 0, 0, false, "window_complete"}, + {"background", 4, 2, 1, 0, false, "window_complete"}, + {"budget", 40, 300, 0, 1, false, "memory_budget"}, + {"background_budget", 40, 300, 1, 1, false, "memory_budget"}, + {"error", 4, 2, 1, 1, true, "error"}, + } { + t.Run(tc.name, func(t *testing.T) { + db := startupMemoryFixture(t, tc.transmissions, tc.observations, tc.hotHours == 0) + s := NewPacketStore(db, &PacketStoreConfig{ + MaxMemoryMB: tc.budgetMB, RetentionHours: 24, HotStartupHours: tc.hotHours, + }) + if tc.closeDB { + db.conn.Close() + } + logs := captureLog(func() { + err := s.RunStartupLoad(2) + if (err != nil) != tc.closeDB { + t.Fatalf("startup error=%v, want error=%v", err, tc.closeDB) + } + }) + var summaries []string + for _, line := range strings.Split(logs, "\n") { + if strings.Contains(line, "[store] startup admission:") { + summaries = append(summaries, line) + } + } + if len(summaries) != 1 { + t.Fatalf("want one terminal admission summary, got %d:\n%s", len(summaries), logs) + } + first, last := "", "" + if len(s.packets) > 0 { + first, last = s.packets[0].FirstSeen, s.packets[len(s.packets)-1].FirstSeen + } + for _, want := range []string{ + fmt.Sprintf("budgetMB=%d", tc.budgetMB), + fmt.Sprintf("transmissions=%d observations=%d", len(s.packets), s.totalObs), + fmt.Sprintf("firstSeenRange=[%q,%q]", first, last), + fmt.Sprintf("coverageFrom=%q", s.oldestLoaded), + "stop=" + tc.reason, + "formerPolicyTransmissionEstimate=", + } { + if !strings.Contains(summaries[0], want) { + t.Errorf("summary missing %q: %s", want, summaries[0]) + } + } + if tc.name == "background" && len(s.packets) != tc.transmissions { + t.Fatalf("summary preceded background fill: %d/%d packets", len(s.packets), tc.transmissions) + } + }) + } +} From 0abec137cba660da8b154472fa106647538f9f0c Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 20 Oct 2026 08:42:39 -0400 Subject: [PATCH 61/67] fix(server): report startup admission with the complete coverage floor --- cmd/server/chunked_load.go | 36 ++++++++++++++++++++ cmd/server/startup_admission_summary_test.go | 22 ++++++++++-- 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/cmd/server/chunked_load.go b/cmd/server/chunked_load.go index b6043c03a..8ab7d648f 100644 --- a/cmd/server/chunked_load.go +++ b/cmd/server/chunked_load.go @@ -199,6 +199,7 @@ func (s *PacketStore) fireChunkCallbacks(rowsThisChunk, totalRows int) { // bg loader starts. func (s *PacketStore) RunStartupLoad(chunkSize int) error { defer s.signalStartupLoadDone() + defer s.logStartupAdmission() // Clear any stale error from a previous invocation (single-call // invariant — see godoc above). Production never re-enters but // test fixtures may construct fresh stores that share no state; @@ -264,6 +265,41 @@ func (s *PacketStore) RunStartupLoad(chunkSize int) error { return nil } +// logStartupAdmission reports the terminal cache state after both startup phases. +// The first_seen endpoints describe retained rows, not complete time coverage: +// a long-lived hash may be older than coverageFrom. All inputs are existing +// counters or sorted-slice endpoints; this adds no database query or store scan. +func (s *PacketStore) logStartupAdmission() { + s.mu.RLock() + txCount, obsCount, trackedBytes := len(s.packets), s.totalObs, s.trackedBytes + budgetMB, limited := s.maxMemoryMB, s.startupMemoryLimited + coverageFrom := "" + if floor, _ := s.coverageStart(time.Now().UTC()); !floor.IsZero() { + coverageFrom = floor.Format(time.RFC3339) + } + first, last := "", "" + if txCount > 0 { + first, last = s.packets[0].FirstSeen, s.packets[txCount-1].FirstSeen + } + s.mu.RUnlock() + reason := "window_complete" + if s.backgroundLoadFailed.Load() { + reason = "error" + } else if limited { + reason = "memory_budget" + } + // This is the former count-admission policy, not measured memory usage. + // It assumed ten observations per transmission and always admitted at + // least 1,000 transmissions, even when their real fanout exceeded budget. + formerEstimate := "unlimited" + if budgetMB > 0 { + typicalBytes := max(int64(1000), estimateStoreTxBytesTypical(10)) + formerEstimate = fmt.Sprint(max(int64(1000), int64(budgetMB)*1048576/typicalBytes)) + } + log.Printf("[store] startup admission: budgetMB=%d transmissions=%d observations=%d trackedMB=%.2f firstSeenRange=[%q,%q] coverageFrom=%q stop=%s formerPolicyTransmissionEstimate=%s (assumedObservations=10 minimumTransmissions=1000)", + budgetMB, txCount, obsCount, float64(trackedBytes)/1048576, first, last, coverageFrom, reason, formerEstimate) +} + // LoadChunked streams transmissions + observations from the selected backend into // the in-memory store in bounded chunks of `chunkSize` rows. Pass // 0 to use the default (10000). diff --git a/cmd/server/startup_admission_summary_test.go b/cmd/server/startup_admission_summary_test.go index 405700688..a24b361f6 100644 --- a/cmd/server/startup_admission_summary_test.go +++ b/cmd/server/startup_admission_summary_test.go @@ -52,15 +52,24 @@ func TestStartupAdmissionSummaryReheardOldHashKeepsCoverageFloor(t *testing.T) { if !strings.Contains(logs, fmt.Sprintf("firstSeenRange=[%q,%q]", reheardFirst, reheardFirst)) { t.Fatal("summary lost the actual retained first-seen range", logs) } + floor := loggedStartupCoverage(t, logs) + if floor.Before(started.Add(-24*time.Hour).Truncate(time.Second)) || floor.After(finished.Add(-24*time.Hour)) { + t.Fatalf("summary claims coverageFrom=%q outside the complete 24-hour window; retained first_seen=%q", floor.Format(time.RFC3339), reheardFirst) + } +} + +func loggedStartupCoverage(t *testing.T, logs string) time.Time { + t.Helper() _, rest, found := strings.Cut(logs, `coverageFrom="`) coverage, _, closed := strings.Cut(rest, `"`) if !found || !closed { t.Fatal("summary has no quoted coverage floor", logs) } floor, err := time.Parse(time.RFC3339, coverage) - if err != nil || floor.Before(started.Add(-24*time.Hour).Truncate(time.Second)) || floor.After(finished.Add(-24*time.Hour)) { - t.Fatalf("summary claims coverageFrom=%q outside the complete 24-hour window; retained first_seen=%q: %v", coverage, reheardFirst, err) + if err != nil { + t.Fatal("summary has an invalid coverage floor", coverage, err) } + return floor } func TestStartupAdmissionSummary(t *testing.T) { @@ -88,12 +97,14 @@ func TestStartupAdmissionSummary(t *testing.T) { if tc.closeDB { db.conn.Close() } + started := time.Now().UTC() logs := captureLog(func() { err := s.RunStartupLoad(2) if (err != nil) != tc.closeDB { t.Fatalf("startup error=%v, want error=%v", err, tc.closeDB) } }) + finished := time.Now().UTC() var summaries []string for _, line := range strings.Split(logs, "\n") { if strings.Contains(line, "[store] startup admission:") { @@ -111,7 +122,6 @@ func TestStartupAdmissionSummary(t *testing.T) { fmt.Sprintf("budgetMB=%d", tc.budgetMB), fmt.Sprintf("transmissions=%d observations=%d", len(s.packets), s.totalObs), fmt.Sprintf("firstSeenRange=[%q,%q]", first, last), - fmt.Sprintf("coverageFrom=%q", s.oldestLoaded), "stop=" + tc.reason, "formerPolicyTransmissionEstimate=", } { @@ -122,6 +132,12 @@ func TestStartupAdmissionSummary(t *testing.T) { if tc.name == "background" && len(s.packets) != tc.transmissions { t.Fatalf("summary preceded background fill: %d/%d packets", len(s.packets), tc.transmissions) } + floor := loggedStartupCoverage(t, summaries[0]) + earliest, _ := s.coverageStart(started) + latest, _ := s.coverageStart(finished) + if floor.Before(earliest.Truncate(time.Second)) || floor.After(latest) { + t.Fatalf("summary coverage floor=%s, want complete coverage between %s and %s", floor, earliest, latest) + } }) } } From 93bf7f8cca256834fd33d2e0691c3e7bf5fa0df5 Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 20 Oct 2026 08:42:41 -0400 Subject: [PATCH 62/67] test(server): reproduce decoded cache races during rewrite publication --- cmd/server/decoded_rewrite_race_test.go | 90 +++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 cmd/server/decoded_rewrite_race_test.go diff --git a/cmd/server/decoded_rewrite_race_test.go b/cmd/server/decoded_rewrite_race_test.go new file mode 100644 index 000000000..8c36583b2 --- /dev/null +++ b/cmd/server/decoded_rewrite_race_test.go @@ -0,0 +1,90 @@ +package main + +import ( + "fmt" + "sync" + "testing" +) + +func TestTxRewriteFeedConcurrentDecodedReaders(t *testing.T) { + db, writer := txRewriteFixture(t, true) + s := NewPacketStore(db, &PacketStoreConfig{}) + if err := s.Load(); err != nil { + t.Fatal(err) + } + tx := s.byTxID[1] + original := `{"type":"CHAN","channel":"#initial","flags":{"revision":"initial"}}` + backfillRewrite(t, writer, original, "#initial") + if err := s.pollTxRewrites(500); err != nil { + t.Fatal(err) + } + retained := tx.ParsedDecoded() + stop := make(chan struct{}) + var ready, readers sync.WaitGroup + ready.Add(4) + readers.Add(4) + for range 4 { + go func() { + defer readers.Done() + tx.ParsedDecoded() + ready.Done() + for { + select { + case <-stop: + return + default: + d := tx.ParsedDecoded() + if d == nil || d["type"] != "CHAN" { + t.Error("rewrite exposed an empty or partially parsed cache") + return + } + if flags, ok := d["flags"].(map[string]interface{}); !ok || flags["revision"] == nil { + t.Error("rewrite exposed a partially parsed nested map") + return + } + } + } + }() + } + ready.Wait() + defer func() { + close(stop) + readers.Wait() + }() + for revision := 0; revision < 24; revision++ { + decoded := fmt.Sprintf(`{"type":"CHAN","channel":"#revision-%d","flags":{"revision":"%d"}}`, revision, revision) + backfillRewrite(t, writer, decoded, "#updated") + if err := s.pollTxRewrites(500); err != nil { + t.Fatal(err) + } + want := fmt.Sprintf("#revision-%d", revision) + if got := tx.ParsedDecoded()["channel"]; got != want { + t.Fatalf("rewrite cache stale: got %v, want %s", got, want) + } + if retained["channel"] != "#initial" || retained["flags"].(map[string]interface{})["revision"] != "initial" { + t.Fatal("rewrite mutated a map retained by an earlier reader") + } + } +} + +func BenchmarkParsedDecodedWarm(b *testing.B) { + tx := &StoreTx{DecodedJSON: decodedCacheFixture} + tx.ParsedDecoded() + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + if tx.ParsedDecoded()["type"] != "ADVERT" { + b.Fatal("cached decoded payload changed") + } + } +} + +func BenchmarkParsedDecodedCold(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + tx := &StoreTx{DecodedJSON: decodedCacheFixture} + if tx.ParsedDecoded()["type"] != "ADVERT" { + b.Fatal("decoded payload changed") + } + } +} From 7794756249db3aa2b38a3f0c2e240d5d0797b351 Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 20 Oct 2026 08:42:42 -0400 Subject: [PATCH 63/67] fix(server): publish immutable decoded cache generations Fixes #2252. --- cmd/server/decoded_rewrite_race_test.go | 106 +++++++++++++++++++++--- cmd/server/eviction_test.go | 3 +- cmd/server/store.go | 72 ++++++++++++---- cmd/server/tx_rewrite_feed.go | 8 +- 4 files changed, 152 insertions(+), 37 deletions(-) diff --git a/cmd/server/decoded_rewrite_race_test.go b/cmd/server/decoded_rewrite_race_test.go index 8c36583b2..9bba9567e 100644 --- a/cmd/server/decoded_rewrite_race_test.go +++ b/cmd/server/decoded_rewrite_race_test.go @@ -1,9 +1,12 @@ package main import ( + "encoding/json" "fmt" + "runtime" "sync" "testing" + "time" ) func TestTxRewriteFeedConcurrentDecodedReaders(t *testing.T) { @@ -42,6 +45,7 @@ func TestTxRewriteFeedConcurrentDecodedReaders(t *testing.T) { t.Error("rewrite exposed a partially parsed nested map") return } + runtime.Gosched() } } }() @@ -49,21 +53,99 @@ func TestTxRewriteFeedConcurrentDecodedReaders(t *testing.T) { ready.Wait() defer func() { close(stop) - readers.Wait() - }() - for revision := 0; revision < 24; revision++ { - decoded := fmt.Sprintf(`{"type":"CHAN","channel":"#revision-%d","flags":{"revision":"%d"}}`, revision, revision) - backfillRewrite(t, writer, decoded, "#updated") - if err := s.pollTxRewrites(500); err != nil { - t.Fatal(err) + done := make(chan struct{}) + go func() { readers.Wait(); close(done) }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Error("decoded readers blocked after rewrite") } - want := fmt.Sprintf("#revision-%d", revision) - if got := tx.ParsedDecoded()["channel"]; got != want { - t.Fatalf("rewrite cache stale: got %v, want %s", got, want) + }() + rewritten := make(chan struct{}) + go func() { + defer close(rewritten) + for revision := 0; revision < 24; revision++ { + decoded := fmt.Sprintf(`{"type":"CHAN","channel":"#revision-%d","flags":{"revision":"%d"}}`, revision, revision) + backfillRewrite(t, writer, decoded, "#updated") + if err := s.pollTxRewrites(500); err != nil { + t.Error(err) + return + } + want := fmt.Sprintf("#revision-%d", revision) + if got := tx.ParsedDecoded()["channel"]; got != want { + t.Errorf("rewrite cache stale: got %v, want %s", got, want) + return + } + if retained["channel"] != "#initial" || retained["flags"].(map[string]interface{})["revision"] != "initial" { + t.Error("rewrite mutated a map retained by an earlier reader") + return + } } - if retained["channel"] != "#initial" || retained["flags"].(map[string]interface{})["revision"] != "initial" { - t.Fatal("rewrite mutated a map retained by an earlier reader") + }() + select { + case <-rewritten: + case <-time.After(10 * time.Second): + t.Fatal("rewrite/cache reader did not finish") + } +} + +// Model a reader already parsing an old generation. A writer holding store.mu +// must not wait for it, and its eventual completion must not replace new data. +func TestDecodedRewriteDoesNotWaitForOldParse(t *testing.T) { + const oldJSON = `{"type":"old"}` + tx := &StoreTx{DecodedJSON: oldJSON} + old := &txDecodedCache{source: oldJSON} + tx.decodedCache.Store(old) + entered, release, parsed := make(chan struct{}), make(chan struct{}), make(chan struct{}) + go func() { + defer close(parsed) + old.once.Do(func() { + close(entered) + <-release + json.Unmarshal([]byte(old.source), &old.parsed) + }) + }() + <-entered + var releaseOnce sync.Once + finishParse := func() { releaseOnce.Do(func() { close(release) }); <-parsed } + defer finishParse() + rewritten := make(chan struct{}) + go func() { + tx.setDecodedJSON(`{"type":"new"}`) + close(rewritten) + }() + select { + case <-rewritten: + case <-time.After(time.Second): + t.Fatal("rewrite waited for an old generation's JSON parse") + } + current := tx.ParsedDecoded() + if current["type"] != "new" { + t.Fatal("new generation did not parse the rewritten source") + } + cache := tx.decodedCache.Load() + if tx.setDecodedJSON(`{"type":"new"}`) || tx.decodedCache.Load() != cache { + t.Fatal("unchanged JSON discarded its warm cache") + } + finishParse() + if old.parsed["type"] != "old" || tx.ParsedDecoded()["type"] != "new" { + t.Fatal("old parser completion replaced or mutated the new generation") + } +} + +func TestDecodedGenerationIncludedInMemoryBudget(t *testing.T) { + tx := &StoreTx{} + before := estimateStoreTxBytes(tx) + if before < int64(storeTxBaseBytes+numIndexesPerTx*indexEntryBytes+decodedCacheGenerationBytes) { + t.Fatal("cache generation is not charged before parsing") + } + allocs := testing.AllocsPerRun(100, func() { + if tx.ParsedDecoded() != nil { + t.Fatal("empty source must stay empty") } + }) + if allocs != 0 { + t.Fatalf("warm decoded cache allocates: %v", allocs) } } diff --git a/cmd/server/eviction_test.go b/cmd/server/eviction_test.go index 46761c859..1d6fd5aae 100644 --- a/cmd/server/eviction_test.go +++ b/cmd/server/eviction_test.go @@ -560,10 +560,11 @@ func TestEstimateStoreTxBytes(t *testing.T) { PathJSON: `["aa","bb"]`, } est := estimateStoreTxBytes(tx) - // Manual calculation: base + string lengths + index entries + perTxMaps + path hops + // Manual calculation: base + string lengths + index entries + decoded generation + path hops // (subpaths are counted per request, so they cost the store nothing) hops := int64(len(txGetParsedPath(tx))) manualCalc := int64(storeTxBaseBytes) + int64(len(tx.RawHex)+len(tx.Hash)+len(tx.DecodedJSON)+len(tx.PathJSON)) + int64(numIndexesPerTx*indexEntryBytes) + manualCalc += decodedCacheGenerationBytes manualCalc += int64(decodedCacheFactor * len(tx.DecodedJSON)) manualCalc += hops * perPathHopBytes if est != manualCalc { diff --git a/cmd/server/store.go b/cmd/server/store.go index 6efcde3df..a8f76fb1f 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -61,11 +61,11 @@ type StoreTx struct { LatestSeen string // max observation timestamp (or FirstSeen if no observations) UniqueObserverCount int // cached count of distinct observer IDs // Cached parsed fields (set once, read many) - parsedPath []string // cached parsePathJSON result - pathParsed bool // whether parsedPath has been set - AdvertRouteEvidence uint8 // union of known flood/direct-empty-path evidence - decodedOnce sync.Once // guards parsedDecoded - parsedDecoded map[string]interface{} // cached json.Unmarshal of DecodedJSON + parsedPath []string // cached parsePathJSON result + pathParsed bool // whether parsedPath has been set + AdvertRouteEvidence uint8 // union of known flood/direct-empty-path evidence + decodedMu sync.Mutex + decodedCache atomic.Pointer[txDecodedCache] // Dedup maps, built only once a transmission passes obsIndexThreshold // observations (see obs_dedup.go); nil below it. obsKeys map[obsDedupKey]struct{} @@ -171,18 +171,51 @@ func broadcastPayload(tx *StoreTx) json.RawMessage { return json.RawMessage(tx.DecodedJSON) } -// ParsedDecoded returns the parsed DecodedJSON map, caching the result. -// Thread-safe via sync.Once — the first call parses, subsequent calls return cached. +// Each decoded source generation owns its once and immutable parsed map. +// A rewrite retires the whole generation; readers can finish using the old one. +type txDecodedCache struct { + source string + once sync.Once + parsed map[string]interface{} +} + +// ParsedDecoded returns a cached, read-only map. Callers must not mutate it or +// its nested values. The publication mutex never covers JSON parsing: a rewrite +// holding store.mu can publish a new generation without waiting for old readers. func (tx *StoreTx) ParsedDecoded() map[string]interface{} { - tx.decodedOnce.Do(func() { - if tx.DecodedJSON != "" { - json.Unmarshal([]byte(tx.DecodedJSON), &tx.parsedDecoded) + cache := tx.decodedCache.Load() + if cache == nil { + tx.decodedMu.Lock() + cache = tx.decodedCache.Load() + if cache == nil { + cache = &txDecodedCache{source: tx.DecodedJSON} + tx.decodedCache.Store(cache) + } + tx.decodedMu.Unlock() + } + cache.once.Do(func() { + if cache.source != "" { + json.Unmarshal([]byte(cache.source), &cache.parsed) for _, k := range decodedCacheDroppedKeys { - delete(tx.parsedDecoded, k) + delete(cache.parsed, k) } } }) - return tx.parsedDecoded + return cache.parsed +} + +// setDecodedJSON is the only mutation of a published transmission's decoded +// source. The caller holds store.mu to protect ordinary field readers; decodedMu +// also protects cache initialization by analytics that already released store.mu. +func (tx *StoreTx) setDecodedJSON(decoded string) bool { + tx.decodedMu.Lock() + defer tx.decodedMu.Unlock() + if tx.DecodedJSON == decoded { + return false + } + tx.DecodedJSON = decoded + tx.decodedCache.Store(nil) + return true } // PacketStore holds all transmissions in memory with indexes for fast queries. @@ -4615,7 +4648,7 @@ func (s *PacketStore) buildDistanceIndex() { // // Note: ResolvedPath per-obs overhead eliminated by #800 refactor. const ( - storeTxBaseBytes = 384 // StoreTx struct fields + map headers + sync.Once + string headers + storeTxBaseBytes = 384 // StoreTx struct fields + map headers + cache guards + string headers storeObsBaseBytes = 192 // StoreObs struct fields + string headers strHdr = 16 // Go string header (ptr + len) on a 64-bit build; used by GetStoreMemoryBreakdown indexEntryBytes = 48 // average cost of one index map entry (key + pointer + bucket overhead) @@ -4630,6 +4663,10 @@ const ( // production heap profile. Analytics touch every tx, so it is charged // up front rather than when the cache fills. decodedCacheFactor = 4 + // One current txDecodedCache allocation (40 bytes, rounded to the 48-byte + // size class on 64-bit builds). Charge it before the first parse, like the + // decoded map. Retired generations live only while readers retain them. + decodedCacheGenerationBytes = 48 ) // estimateStoreTxBytes returns the estimated memory cost of a StoreTx (excluding observations). @@ -4639,6 +4676,7 @@ func estimateStoreTxBytes(tx *StoreTx) int64 { base += int64(len(tx.RawHex) + len(tx.Hash) + len(tx.DecodedJSON) + len(tx.PathJSON)) base += int64(numIndexesPerTx * indexEntryBytes) base += int64(decodedCacheFactor * len(tx.DecodedJSON)) + base += decodedCacheGenerationBytes // Path-dependent costs hops := int64(len(txGetParsedPath(tx))) @@ -4647,9 +4685,9 @@ func estimateStoreTxBytes(tx *StoreTx) int64 { return base } -// estimateStoreTxBytesTypical returns the estimated memory cost of a typical -// transmission with the given number of observations. Used for budget -// calculation during bounded cold load (no actual StoreTx needed). +// estimateStoreTxBytesTypical preserves the former admission policy's typical +// transmission estimate for the startup diagnostic. Actual admission uses each +// transmission's observation-aware charge, including the decoded cache generation. func estimateStoreTxBytesTypical(numObs int) int64 { // Typical tx: ~64 byte hash, ~200 byte decoded JSON, ~40 byte path, 3 hops base := int64(storeTxBaseBytes) + 64 + 200 + 40 @@ -5139,7 +5177,7 @@ func buildHopContextPubkeys(tx *StoreTx, pm *prefixMap) []string { } // Sender / originator pubkey from decoded payload. Use the cached - // ParsedDecoded() (sync.Once-gated) instead of re-unmarshaling — the + // ParsedDecoded() (once per source generation) instead of re-unmarshaling — the // helper is hot (3 distance sites + analytics topology, all 30k+ tx // loops). See #1197 (carmack/adversarial r1). if dec := tx.ParsedDecoded(); dec != nil { diff --git a/cmd/server/tx_rewrite_feed.go b/cmd/server/tx_rewrite_feed.go index c7515923d..72f6adb39 100644 --- a/cmd/server/tx_rewrite_feed.go +++ b/cmd/server/tx_rewrite_feed.go @@ -3,7 +3,6 @@ package main import ( "database/sql" "log" - "sync" ) // tx_rewrite_feed (#2107) is the ingestor's change feed for transmissions it @@ -87,12 +86,7 @@ func (s *PacketStore) pollTxRewrites(limit int) error { if tx == nil { continue } - if r.decoded.Valid && r.decoded.String != tx.DecodedJSON { - tx.DecodedJSON = r.decoded.String - // ParsedDecoded caches behind a sync.Once; reset both under the - // write lock so the next reader parses the new JSON. - tx.decodedOnce = sync.Once{} - tx.parsedDecoded = nil + if r.decoded.Valid && tx.setDecodedJSON(r.decoded.String) { s.trackedBytes += rechargeTx(tx) if tx.PayloadType != nil && *tx.PayloadType == PayloadGRP_TXT { channelChanged = true From d3f6134c174f6e24f32eeab6ff6b98cdeeeb8523 Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 20 Oct 2026 08:42:44 -0400 Subject: [PATCH 64/67] docs(storage): explain verified upgrades and derived-cache behavior --- docs/storage.md | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/storage.md b/docs/storage.md index 9f83996c9..3cbcd4638 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -12,6 +12,18 @@ cache now accounts for SQLite observation fan-out when admitting history: under the same memory budget it may retain a shorter hot window. Omitted history stays in SQL; a smaller cache does not change database retention. +After startup loading finishes, the server emits one `[store] startup admission` +summary with the configured memory budget, admitted transmission and observation +counts, tracked memory estimate and retained `firstSeenRange`. `coverageFrom` is +the conservative complete-coverage boundary; an older first-seen timestamp can +belong to a long-lived hash and does not establish complete history coverage. +`stop=memory_budget` means the bounded cache is ready, `window_complete` means the +requested window finished, and `error` leaves startup unready. +`formerPolicyTransmissionEstimate` describes the previous admission formula +(ten assumed observations per transmission and a 1,000-transmission minimum); +it is not a measured previous memory footprint. Use `/api/healthz` for readiness +and SQL-backed views to inspect history outside the admitted cache. + ## Install Keep a complete checkout pinned to the same release as the image. For the usual single-host installation, preserve an existing private `.env` or copy `.env.example`, set `DATA_DIR` to the intended persistent directory, then run: @@ -45,7 +57,12 @@ stores, including disabled accounts, and preserve the matching state/configurati and previous image. Stop both application writers for the update. Managed bootstrap checks readiness and, when necessary, invokes the offline owner upgrade for the existing recorded targets before runtime -starts. It cannot replace missing initialized databases or add an absent account +starts. A supported older schema is reported as `upgrade required` with its +stored and target versions, followed by the verified upgrade result. The +provisional readiness error is not printed after a successful upgrade. +Missing, corrupt or future schemas still fail; their diagnostics remain visible +and runtime services stay stopped. Bootstrap rechecks readiness before granting +runtime access. It cannot replace missing initialized databases or add an absent account store. Previously initialized accounts are upgraded even while their UI is disabled. Native deployments use the same operation with owner credentials in private environment configuration: @@ -94,6 +111,22 @@ The switch asks for confirmation, stops the owned application processes, starts The converter snapshots/normalizes supported source data, copies bounded batches, verifies table counts, typed logical digests and reference/identity high-water marks, applies narrow destination grants, and checks runtime connections. Both requested stores must verify before the single selection replacement. The state directory itself does not move. +The optional SQLite hop index is derived data, not a telemetry source. Conversion +accepts only the known v1 layout: `observation_hops`, +`observation_hops_coverage`, and all three `trg_observation_hops_ai`, `_ad`, `_au` +triggers with their expected definitions. Partial layouts, changed definitions +and extra indexes on these tables are refused. The converter removes these +objects only from its private normalized snapshot; the original source and +recovery copy retain them. Reports list both tables in `excluded_derived` with +their schema version and rebuild reason, outside the canonical row/digest +comparison. + +`retention.hopIndexDays` controls this cache on SQLite only. PostgreSQL uses its +native reach query and logs that the SQLite cache option is inactive; no +PostgreSQL hop-index schema is installed. After conversion back to SQLite, +the cache is absent until the ingestor normally installs and rebuilds it when +`hopIndexDays` is positive. + On success, inspect the credential-free report and recorded targets. The management command leaves writers stopped: ```sh From 8457c8c6a3a0334893e4235aac05170cec3bc158 Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 20 Oct 2026 08:42:45 -0400 Subject: [PATCH 65/67] test(e2e): wait for theme initialization before checking home state The initial theme request replaced the test sentinel before the assertion. Wait for initialization and retain the original equality check. --- tests/e2e/test-e2e-playwright.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/e2e/test-e2e-playwright.js b/tests/e2e/test-e2e-playwright.js index f5c6dafae..88a1f574f 100644 --- a/tests/e2e/test-e2e-playwright.js +++ b/tests/e2e/test-e2e-playwright.js @@ -369,6 +369,9 @@ async function run() { // TODO: requires running server with full customize/home wiring await page.goto(BASE, { waitUntil: 'domcontentloaded' }); await page.waitForSelector('nav, .navbar, .nav, [class*="nav"]'); + // The initial theme response installs SITE_CONFIG asynchronously. Set the + // sentinel only after that completes, so this tests opening the customizer. + await page.waitForFunction(() => window._customizerV2 && window._customizerV2.initDone); await page.evaluate(() => { localStorage.removeItem('cs-theme-overrides'); window.SITE_CONFIG = window.SITE_CONFIG || {}; From 81be14d51055f8777419c6e9aac7a7c2261706a7 Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 20 Oct 2026 09:28:40 -0400 Subject: [PATCH 66/67] fix(users): bind pending registration updates for native backends --- internal/users/users.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/users/users.go b/internal/users/users.go index c408f557d..9d7864edf 100644 --- a/internal/users/users.go +++ b/internal/users/users.go @@ -86,8 +86,8 @@ func (s *Store) CreatePending(email, displayName, passwordHash string) (*User, e // account is still pending. A caller's earlier lookup cannot authorize a write // after activation, disablement or deletion has committed. func (s *Store) UpdatePendingRegistration(id int64, passwordHash, displayName string) error { - err := expectOne(s.db.Exec(`UPDATE users SET password_hash = ?, display_name = ? - WHERE id = ? AND status = 'pending'`, passwordHash, displayName, id)) + err := expectOne(s.db.Exec(`UPDATE users SET password_hash = `+s.p(1)+`, display_name = `+s.p(2)+` + WHERE id = `+s.p(3)+` AND status = 'pending'`, passwordHash, displayName, id)) if errors.Is(err, ErrNotFound) { return ErrAccountChanged } From 0c6595cffeb8ac7b8aa611462fc06bf0b532729f Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 20 Oct 2026 09:28:40 -0400 Subject: [PATCH 67/67] test(e2e): resolve hop fixture using canonical API identity --- tests/e2e/test-hop-path-inference-e2e.js | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/e2e/test-hop-path-inference-e2e.js b/tests/e2e/test-hop-path-inference-e2e.js index 99bfaf360..762cd63ab 100644 --- a/tests/e2e/test-hop-path-inference-e2e.js +++ b/tests/e2e/test-hop-path-inference-e2e.js @@ -2,7 +2,7 @@ * E2E: the packet detail pane infers an observation's hops by position, and * the map route view draws the same hops. * - * Fixture: packet 57224a4a01c9c682, observation 147 — a 22-hop flood whose + * Fixture: transmission 147, observation 147 — a 22-hop flood whose * path carries the 1-byte prefix "10" three times and "D6" twice, so a * per-prefix resolution would give each repeat the same answer. * @@ -12,7 +12,7 @@ const { chromium } = require('playwright'); const BASE = process.env.BASE_URL || 'http://localhost:13581'; -const HASH = '57224a4a01c9c682'; +const TRANSMISSION_ID = '147'; const OBS_ID = '147'; let passed = 0, failed = 0; @@ -36,9 +36,22 @@ function assert(c, m) { if (!c) throw new Error(m || 'assertion failed'); } console.log(`\n=== Hop path inference E2E against ${BASE} ===`); + // Offline fixture preparation normalizes legacy hashes. Resolve the preserved + // transmission ID through the real API, then exercise its canonical hash links. + const fixtureResponse = await ctx.request.get(`${BASE}/api/packets/${TRANSMISSION_ID}`); + assert(fixtureResponse.ok(), `fixture transmission returned HTTP ${fixtureResponse.status()}`); + const fixture = await fixtureResponse.json(); + const hash = fixture.packet.hash; + assert(/^[a-f0-9]{16}$/.test(hash), 'fixture has no canonical packet hash'); + const observation = fixture.observations.find(o => String(o.id) === OBS_ID); + assert(observation, 'fixture observation is missing'); + const path = JSON.parse(observation.path_json); + assert(path.length === 22 && path.filter(h => h === '10').length === 3 && + path.filter(h => h === 'D6').length === 2, 'fixture no longer has the repeated-prefix route'); + let detailHops = []; await step('the detail pane renders the observation\'s 22 hops', async () => { - await page.goto(`${BASE}/#/packets/${HASH}?obs=${OBS_ID}`, { waitUntil: 'domcontentloaded' }); + await page.goto(`${BASE}/#/packets/${hash}?obs=${OBS_ID}`, { waitUntil: 'domcontentloaded' }); await page.waitForFunction(() => { const dt = [...document.querySelectorAll('.detail-meta dt')].find(d => d.textContent === 'Path'); return dt && dt.nextElementSibling.querySelectorAll('.hop').length > 0; @@ -53,6 +66,7 @@ function assert(c, m) { if (!c) throw new Error(m || 'assertion failed'); } }); await step('every hop is an inferred entry: a likelihood or an unlisted reason', async () => { + assert(detailHops.length === 22, 'expected the complete observation path'); const bad = detailHops.filter(h => !/% likely|no listed node|no known node|no GPS/.test(h.title)); assert(!bad.length, 'hops without an inferred title: ' + JSON.stringify(bad.slice(0, 3))); }); @@ -75,7 +89,7 @@ function assert(c, m) { if (!c) throw new Error(m || 'assertion failed'); } }); await step('the map route view draws the same hops, unlisted ones as placeholders', async () => { - await page.goto(`${BASE}/#/map?packet=${HASH}&obs=${OBS_ID}`, { waitUntil: 'domcontentloaded' }); + await page.goto(`${BASE}/#/map?packet=${hash}&obs=${OBS_ID}`, { waitUntil: 'domcontentloaded' }); await page.waitForFunction(() => document.querySelectorAll('.mc-rt-row').length >= 22, null, { timeout: 15000 }); const rows = await page.evaluate(() => [...document.querySelectorAll('.mc-rt-row')].map(r => r.innerText)); for (let i = 0; i < detailHops.length; i++) {