From c5e2ce4465b0e4f11b558dec0cd1db8a8613866f Mon Sep 17 00:00:00 2001 From: kvlonge <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 02:42:09 +0000 Subject: [PATCH 01/10] feat!: bring the large-project server work onto main Integrates the large-project branch: the code location runtime, warehouse history sync, plan previews over the synced warehouse state, job pools, ignored audit failures, job-group waits, the in-memory node list and node state cache, and the UI for them. - Locations sync warehouse history in bounded pages (2,000 rows, 100 statistics relations) and report each page's health. A time-out or memory failure halves the page; other read failures back off 30, 60, 120 and 240 s, and the fifth in a row halts the sync until a run, a request, a publication or a due safety sync. The health report carries the closed cause, the redacted error and the halted state, and the UI shows it. - Plan previews read the held warehouse state instead of an emulating synced-copy adapter. After each publication and completed sync a location plans "Build what changed" and reports it; a failed plan is reported with its own error. - The server keeps each target's node list and node states in memory and pushes the changed states on the change stream. A listener resync pushes only real differences, since clients refetch on `all`. - Job-group waits use the nearest-member rule inside the group wait graph. - Ignored audit failures are stored in real columns (`sqb_runs.ignored_audit_failures`, `sqb_warehouse_audit_results.ignored`); the Recall `audit_results` view gains `ignored` (Recall schema version 7). - A SQLite control store gives freed pages back with incremental vacuum until it finishes. Job and job group files declare `pools` instead of `concurrency_groups`, and server.toml launchers lower them with `pool_caps` instead of `concurrency_limits`; the old names are now unknown keys. The wide node-listing guard allows 12 fixed statements instead of 9: a cold node-list cache authorizes in its own transaction, which adds a clock read and a second current-snapshot read, and reads the target's changed build. Builds are still read 100 nodes per statement. Code this change duplicated is shared instead: one adapter read and one state-history read on WarehouseReads, a listing-read helper, shared thousands grouping and row-count estimates. The Python bindings crate cap rises to 51,200 code lines. BREAKING CHANGE: the control store schema changes: runs record ignored audit failures in `sqb_runs.ignored_audit_failures` instead of a reserved node-count key, and stored check results record them in `sqb_warehouse_audit_results.ignored` instead of the result metadata. Job and job group files rename `concurrency_groups` to `pools`, and server.toml launchers rename `concurrency_limits` to `pool_caps`. --- .github/workflows/verify.yml | 43 + Cargo.lock | 4 + Makefile | 11 +- benchmarks/duckshop-10k/README.md | 25 +- benchmarks/duckshop-10k/synced_state.py | 374 ++++++ .../duckshop-10k/tests/test_synced_state.py | 137 +++ .../orchestration/check_identity.json | 1 + .../orchestration/group_shared_limit.json | 6 +- .../jobs/annual_orders_job.toml | 2 +- .../edge_cases/edge_inventory_job_group.toml | 2 +- .../edge_cases/edge_products_job_group.toml | 2 +- .../edge_snapshot_inventory_job.toml | 2 +- .../edge_snapshot_products_job.toml | 2 +- .../duckshop/project/sqlbuild_project.toml | 7 + clients/typescript/src/schema.d.ts | 1026 ++++++++++++++++- .../src/graph/_helpers/selectors.rs | 4 +- .../sqlbuild-analysis/src/graph/constants.rs | 6 +- .../src/graph/tests/helpers.rs | 13 + .../graph/tests/test_selector_scenarios.rs | 18 +- .../src/python_plan/tests/helpers.rs | 3 +- .../src/adhoc/_helpers/evaluation.rs | 294 +++++ crates/sqlbuild-cli/src/adhoc/_helpers/mod.rs | 2 + .../src/adhoc/_helpers/planner.rs | 355 +++--- .../src/adhoc/_helpers/publication.rs | 188 +++ crates/sqlbuild-cli/src/adhoc/constants.rs | 19 +- .../src/adhoc/main/adhoc_planner.rs | 2 +- .../src/adhoc/main/command_settings.rs | 12 + crates/sqlbuild-cli/src/adhoc/main/mod.rs | 1 + crates/sqlbuild-cli/src/adhoc/mod.rs | 1 + crates/sqlbuild-cli/src/adhoc/models.rs | 32 + crates/sqlbuild-cli/src/adhoc/tests.rs | 2 + .../sqlbuild-cli/src/adhoc/tests/helpers.rs | 76 ++ .../src/adhoc/tests/test_adhoc_planner.rs | 12 +- .../adhoc/tests/test_selection_evaluation.rs | 262 +++++ .../src/adhoc/tests/test_types.rs | 40 + .../sqlbuild-cli/src/argv/command_table.json | 74 ++ crates/sqlbuild-cli/src/argv/constants.rs | 7 +- .../src/argv/main/job_settings_arguments.rs | 10 +- .../argv/tests/test_parse_cli_arguments.rs | 20 + .../src/dev_session/main/run_dev_session.rs | 2 +- .../src/diff_report/_helpers/numbers.rs | 16 +- crates/sqlbuild-cli/src/dispatch/constants.rs | 7 +- .../src/dispatch/main/command_dispatch.rs | 11 +- crates/sqlbuild-cli/src/dispatch/models.rs | 2 + .../dispatch/tests/test_command_dispatch.rs | 16 +- .../_helpers/{ => command}/argv_host.rs | 0 .../_helpers/{ => command}/decision.rs | 16 +- .../_helpers/{ => command}/init.rs | 6 +- .../native_commands/_helpers/command/mod.rs | 9 + .../_helpers/{ => command}/package.rs | 0 .../_helpers/{ => command}/paths.rs | 1 + .../_helpers/{ => command}/project_facts.rs | 13 + .../_helpers/{ => command}/rules.rs | 2 +- .../_helpers/{ => command}/skills.rs | 0 .../_helpers/{ => command}/terminal.rs | 30 +- .../_helpers/location/changed_build.rs | 40 + .../_helpers/{ => location}/execution.rs | 639 +++++----- .../_helpers/location/held_copy.rs | 106 ++ .../native_commands/_helpers/location/mod.rs | 8 + .../_helpers/location/plan_document.rs | 47 + .../_helpers/location/plan_request.rs | 89 ++ .../_helpers/location/sync_outcomes.rs | 143 +++ .../_helpers/location/sync_report.rs | 218 ++++ .../native_commands/_helpers/location/work.rs | 42 + .../src/native_commands/_helpers/mod.rs | 12 +- .../src/native_commands/constants.rs | 96 ++ .../main/_cost_report_options.rs | 4 +- .../main/_location_changed_build.rs | 106 ++ .../main/_location_compiler.rs | 145 ++- .../main/_location_metadata.rs | 92 +- .../native_commands/main/_location_plan.rs | 151 +-- .../native_commands/main/_location_sensor.rs | 2 +- .../native_commands/main/_location_work.rs | 69 +- .../native_commands/main/_plan_document.rs | 34 - .../src/native_commands/main/_run_admin.rs | 14 +- .../src/native_commands/main/_run_control.rs | 10 +- .../src/native_commands/main/_run_dev.rs | 8 +- .../src/native_commands/main/_run_location.rs | 77 +- .../main/_run_warehouse_sync.rs | 4 +- .../native_commands/main/effective_target.rs | 2 +- .../src/native_commands/main/mod.rs | 2 +- .../main/parse_native_invocation.rs | 6 +- .../main/run_control_invocation.rs | 6 +- .../main/run_location_invocation.rs | 8 +- .../main/run_login_invocation.rs | 4 +- .../main/run_native_invocation.rs | 4 +- .../main/run_person_invocation.rs | 4 +- .../src/native_commands/models.rs | 226 +++- .../sqlbuild-cli/src/native_commands/tests.rs | 4 + .../src/native_commands/tests/helpers.rs | 4 +- .../native_commands/tests/test_argv_host.rs | 2 +- .../tests/test_changed_build.rs | 123 ++ .../native_commands/tests/test_decision.rs | 2 +- .../src/native_commands/tests/test_init.rs | 2 +- .../tests/test_location_metadata.rs | 610 +++++++++- .../tests/test_location_plan.rs | 81 +- .../src/native_commands/tests/test_paths.rs | 2 +- .../tests/test_project_facts.rs | 43 +- .../src/native_commands/tests/test_rules.rs | 2 +- .../tests/test_sync_schedule.rs | 113 ++ .../native_commands/tests/test_terminal.rs | 38 +- .../src/native_commands/tests/test_types.rs | 136 +++ .../sqlbuild-cli/src/native_commands/types.rs | 25 +- .../sqlbuild-cli/src/server_config/models.rs | 3 +- .../src/workers/_helpers/configuration.rs | 24 +- .../src/workers/_helpers/recovery.rs | 2 +- .../src/workers/_helpers/shutdown.rs | 2 +- .../src/workers/_helpers/supervisor.rs | 12 +- crates/sqlbuild-cli/src/workers/constants.rs | 6 +- .../workers/main/open_development_workers.rs | 2 +- crates/sqlbuild-cli/src/workers/models.rs | 9 +- .../tests/native_commands/admin/test_admin.rs | 2 +- .../native_commands/test_location_health.rs | 2 +- .../tests/native_commands/worker/helpers.rs | 6 +- .../src/effective/constants.rs | 7 + .../main/effective_ignore_audit_failures.rs | 12 + .../sqlbuild-config/src/effective/main/mod.rs | 1 + .../sqlbuild-config/src/effective/models.rs | 11 + .../src/effective/tests/test_effective.rs | 69 +- .../src/effective/tests/test_types.rs | 8 + .../src/project/_helpers/documents.rs | 69 +- .../src/project/_helpers/sections.rs | 62 +- .../src/project/_helpers/targets.rs | 17 +- .../sqlbuild-config/src/project/constants.rs | 13 +- .../src/project/main/load_pool_limits.rs | 9 + .../project/main/load_project_setting_keys.rs | 9 + .../sqlbuild-config/src/project/main/mod.rs | 2 + crates/sqlbuild-config/src/project/models.rs | 15 + .../src/project/tests/test_load_config.rs | 99 +- .../src/project/tests/test_types.rs | 14 + .../src/build_aggregate/constants.rs | 2 + .../main/aggregate_build_counts.rs | 21 +- .../sqlbuild-core/src/build_aggregate/mod.rs | 2 +- .../src/build_aggregate/models.rs | 2 + .../src/build_command/_helpers/text.rs | 13 - .../build_command/main/render_run_context.rs | 9 +- .../src/build_progress/_helpers/footer.rs | 40 + .../src/build_progress/_helpers/text.rs | 20 +- .../main/format_build_footer.rs | 29 +- .../main/render_resource_result.rs | 6 +- .../src/build_progress/models.rs | 2 + .../src/cli_text/_helpers/plan_document.rs | 23 +- .../sqlbuild-core/src/cli_text/constants.rs | 2 + crates/sqlbuild-core/src/cli_text/models.rs | 2 + .../src/cli_text/tests/helpers.rs | 1 + .../src/cli_text/tests/test_cli_text.rs | 9 + crates/sqlbuild-core/src/lib.rs | 1 + .../src/lifecycle_events/constants.rs | 45 + .../src/load_command/_helpers/padding.rs | 17 - .../load_command/main/render_load_result.rs | 5 +- .../src/number_text/main/format_thousands.rs | 18 + .../sqlbuild-core/src/number_text/main/mod.rs | 1 + crates/sqlbuild-core/src/number_text/mod.rs | 3 + .../src/plan_display/_helpers/diff_lines.rs | 43 + .../src/plan_display/_helpers/mod.rs | 3 + .../src/plan_display/_helpers/plan_pages.rs | 381 ++++++ .../plan_display/_helpers/plan_sections.rs | 153 +++ .../src/plan_display/constants.rs | 33 +- .../plan_display/main/detach_model_diffs.rs | 33 + .../src/plan_display/main/mod.rs | 4 + .../src/plan_display/main/plan_model_diff.rs | 20 + .../plan_display/main/plan_section_page.rs | 31 + .../src/plan_display/main/plan_summary.rs | 21 + .../sqlbuild-core/src/plan_display/models.rs | 65 ++ .../sqlbuild-core/src/plan_display/tests.rs | 2 + .../src/plan_display/tests/helpers.rs | 121 ++ .../src/plan_display/tests/test_plan_pages.rs | 594 ++++++++++ .../src/plan_display/tests/test_types.rs | 67 +- .../_helpers/dependencies.rs | 113 ++ .../src/orchestration_files/_helpers/mod.rs | 2 + .../src/orchestration_files/_helpers/pools.rs | 78 ++ .../src/orchestration_files/_helpers/raw.rs | 15 +- .../_helpers/read_group_file.rs | 3 + .../orchestration_files/_helpers/reading.rs | 201 +++- .../src/orchestration_files/constants.rs | 33 +- .../orchestration_files/main/check_pools.rs | 38 + .../src/orchestration_files/main/mod.rs | 1 + .../src/orchestration_files/models.rs | 58 +- .../src/orchestration_files/tests.rs | 2 + .../src/orchestration_files/tests/helpers.rs | 20 + .../orchestration_files/tests/test_pools.rs | 96 ++ .../tests/test_read_files.rs | 241 +++- .../orchestration_files/tests/test_types.rs | 20 +- .../src/python_paths/constants.rs | 6 +- .../python_paths/main/python_test_paths.rs | 2 +- .../tests/test_python_test_paths.rs | 4 +- crates/sqlbuild-host/src/models.rs | 3 + .../authentication/password_sessions.rs | 28 + .../api/_helpers/resources/administrators.rs | 3 +- .../src/api/_helpers/resources/application.rs | 54 + .../api/_helpers/resources/changed_builds.rs | 31 + .../src/api/_helpers/resources/changes.rs | 76 +- .../api/_helpers/resources/location_health.rs | 4 +- .../api/_helpers/resources/location_work.rs | 6 +- .../src/api/_helpers/resources/mod.rs | 3 + .../resources/person_location_work.rs | 4 +- .../api/_helpers/resources/person_node_sql.rs | 2 +- .../_helpers/resources/person_operations.rs | 8 +- .../api/_helpers/resources/person_plans.rs | 92 +- .../api/_helpers/resources/person_reads.rs | 146 ++- .../_helpers/resources/person_selection.rs | 72 ++ .../src/api/_helpers/resources/router.rs | 12 +- .../src/api/_helpers/resources/run_reads.rs | 4 +- .../src/api/_helpers/resources/runs.rs | 10 +- .../src/api/_helpers/resources/snapshots.rs | 8 +- .../_helpers/resources/warehouse_metadata.rs | 22 +- .../src/api/constants.rs | 2 + .../src/api/main/serve_api.rs | 59 +- .../sqlbuild-orchestration/src/api/models.rs | 35 +- .../authorization/_helpers/role_overview.rs | 2 +- .../authorization/main/person_holds_target.rs | 2 +- .../authorization/main/read_person_access.rs | 2 +- .../main/locations/development_location.rs | 26 +- .../src/declarations/_helpers/documents.rs | 65 +- .../src/declarations/_helpers/group_waits.rs | 73 +- .../src/declarations/_helpers/job_detail.rs | 93 +- .../src/declarations/_helpers/reads.rs | 47 +- .../declarations/_helpers/trigger_scope.rs | 4 +- .../src/declarations/constants.rs | 17 + .../src/declarations/main/current_jobs.rs | 22 + .../declarations/main/frozen_definitions.rs | 2 +- .../declarations/main/group_dependencies.rs | 12 +- .../declarations/main/group_wait_reasons.rs | 60 + .../src/declarations/main/list_person_jobs.rs | 11 +- .../src/declarations/main/mod.rs | 21 + .../src/declarations/main/read_person_job.rs | 20 +- .../main/read_person_providers.rs | 27 +- .../declarations/main/snapshot_definitions.rs | 3 +- .../src/declarations/mod.rs | 3 + .../src/declarations/models.rs | 144 ++- crates/sqlbuild-orchestration/src/lib.rs | 18 +- .../src/node_states/_helpers/authorized.rs | 33 + .../src/node_states/_helpers/bodies.rs | 73 ++ .../src/node_states/_helpers/mod.rs | 4 + .../src/node_states/_helpers/reads.rs | 182 +++ .../src/node_states/_helpers/views.rs | 180 +++ .../src/node_states/constants.rs | 12 + .../main/list_person_catalog_nodes.rs | 18 + .../src/node_states/main/mod.rs | 5 + .../main/read_person_node_states.rs | 18 + .../node_states/main/report_changed_build.rs | 69 ++ .../src/node_states/main/start_node_states.rs | 87 ++ .../src/node_states/main/watch_node_states.rs | 17 + .../src/node_states/mod.rs | 7 + .../src/node_states/models.rs | 128 ++ .../src/node_states/types.rs | 11 + .../_helpers/{ => health}/event_history.rs | 0 .../src/platform/_helpers/health/mod.rs | 7 + .../_helpers/{ => health}/observations.rs | 0 .../_helpers/health/pool_cap_warnings.rs | 64 + .../_helpers/{ => health}/problems.rs | 0 .../_helpers/{ => health}/recent_health.rs | 0 .../{ => location}/connection_tests.rs | 0 .../_helpers/{ => location}/liveness.rs | 2 +- .../{ => location}/location_heartbeat.rs | 0 .../src/platform/_helpers/location/mod.rs | 9 + .../_helpers/location/published_generation.rs | 40 + .../_helpers/location/reported_generation.rs | 33 + .../_helpers/location/rule_findings.rs | 29 + .../_helpers/{ => location}/work_client.rs | 83 +- .../src/platform/_helpers/maintenance/mod.rs | 4 + .../_helpers/{ => maintenance}/retention.rs | 57 +- .../_helpers/{ => maintenance}/watchdog.rs | 6 +- .../src/platform/_helpers/mod.rs | 14 +- .../src/platform/_helpers/plans/mod.rs | 4 + .../plans/plan_access.rs} | 18 +- .../src/platform/_helpers/plans/plan_diffs.rs | 80 ++ .../src/platform/constants.rs | 13 + .../main/{ => health}/location_health.rs | 2 +- .../src/platform/main/health/mod.rs | 5 + .../read_person_connection_test.rs | 4 +- .../health/read_person_location_health.rs | 45 + .../{ => health}/report_location_health.rs | 47 +- .../report_remote_location_health.rs | 2 +- .../src/platform/main/installation/mod.rs | 4 + .../{ => installation}/read_installation.rs | 2 +- .../read_platform_events.rs | 0 .../{ => installation}/require_instance.rs | 0 .../{ => installation}/run_maintenance.rs | 13 +- .../src/platform/main/mod.rs | 24 +- .../list_person_platform_events.rs | 2 +- .../src/platform/main/person_reads/mod.rs | 7 + .../read_person_mirrored_totals.rs | 63 + .../main/person_reads/read_person_plan.rs | 20 + .../person_reads/read_person_plan_diff.rs | 45 + .../person_reads/read_person_plan_rows.rs | 78 ++ .../person_reads/read_person_plan_summary.rs | 56 + .../read_person_problems.rs | 2 +- .../main/read_person_location_health.rs | 30 - .../main/{ => work}/_read_target_work.rs | 2 +- .../main/{ => work}/claim_location_work.rs | 4 +- .../main/{ => work}/complete_location_work.rs | 20 +- .../main/{ => work}/connect_location_work.rs | 0 .../main/{ => work}/enqueue_location_work.rs | 0 .../platform/main/{ => work}/location_work.rs | 2 +- .../src/platform/main/work/mod.rs | 8 + .../main/{ => work}/read_location_work.rs | 0 .../main/{ => work}/request_location_work.rs | 6 +- .../src/platform/models.rs | 252 +++- .../src/runs/_helpers/groups/admission.rs | 39 +- .../src/runs/_helpers/groups/context.rs | 15 +- .../src/runs/_helpers/groups/lifecycle.rs | 160 ++- .../src/runs/_helpers/groups/selection.rs | 73 +- .../src/runs/_helpers/lifecycle/changes.rs | 4 +- .../src/runs/_helpers/lifecycle/claims.rs | 93 +- .../src/runs/_helpers/lifecycle/mod.rs | 1 + .../src/runs/_helpers/lifecycle/narrowing.rs | 12 + .../src/runs/_helpers/lifecycle/projection.rs | 11 + .../runs/_helpers/lifecycle/publications.rs | 280 +++++ .../runs/_helpers/lifecycle/publish_change.rs | 27 +- .../src/runs/_helpers/lifecycle/reads.rs | 2 +- .../src/runs/_helpers/lifecycle/retries.rs | 203 +++- .../runs/_helpers/lifecycle/run_failure.rs | 2 +- .../src/runs/_helpers/lifecycle/waits.rs | 32 +- .../src/runs/_helpers/requests/adhoc.rs | 167 +-- .../src/runs/_helpers/requests/job.rs | 4 +- .../src/runs/_helpers/requests/manual_job.rs | 2 +- .../src/runs/_helpers/requests/plan.rs | 4 +- .../src/runs/_helpers/requests/publication.rs | 2 +- .../src/runs/_helpers/requests/rerun.rs | 2 +- .../src/runs/constants.rs | 15 + .../main/{ => execution}/advance_groups.rs | 0 .../runs/main/{ => execution}/change_run.rs | 0 .../main/{ => execution}/claim_next_run.rs | 0 .../main/{ => execution}/heartbeat_run.rs | 0 .../src/runs/main/execution/mod.rs | 9 + .../runs/main/{ => execution}/monitor_runs.rs | 0 .../runs/main/{ => execution}/observe_run.rs | 0 .../{ => execution}/publish_run_output.rs | 0 .../{ => execution}/publish_run_recall.rs | 0 .../main/{ => execution}/run_execution.rs | 0 .../{ => listings}/list_person_run_nodes.rs | 0 .../main/{ => listings}/list_person_runs.rs | 0 .../main/{ => listings}/list_run_claims.rs | 0 .../main/{ => listings}/list_run_nodes.rs | 0 .../src/runs/main/{ => listings}/list_runs.rs | 0 .../src/runs/main/listings/mod.rs | 5 + .../src/runs/main/mod.rs | 23 +- .../main/{ => requests}/check_adhoc_run.rs | 37 +- .../runs/main/requests/evaluate_selection.rs | 74 ++ .../main/{ => requests}/frozen_job_request.rs | 13 +- .../src/runs/main/requests/mod.rs | 8 + .../src/runs/main/{ => requests}/queue_run.rs | 0 .../requests/read_selection_vocabulary.rs | 34 + .../main/{ => requests}/request_job_run.rs | 0 .../main/{ => requests}/request_person_run.rs | 0 .../runs/main/{ => requests}/request_rerun.rs | 0 .../sqlbuild-orchestration/src/runs/models.rs | 214 +++- .../sqlbuild-orchestration/src/runs/types.rs | 31 +- .../src/scheduler/_helpers/persistence.rs | 7 +- .../src/scheduler/main/_scheduler_cycle.rs | 2 +- .../src/sensors/_helpers/custom.rs | 2 +- .../src/sensors/_helpers/nodes.rs | 2 +- .../src/sensors/_helpers/platform.rs | 2 +- .../src/sensors/_helpers/requests.rs | 4 +- .../src/sensors/_helpers/runs.rs | 23 +- .../src/sensors/models.rs | 3 + .../snapshots/_helpers/connection_settings.rs | 39 + .../src/snapshots/_helpers/mod.rs | 2 + .../snapshots/_helpers/published_listing.rs | 81 ++ .../src/snapshots/_helpers/reading.rs | 11 +- .../src/snapshots/_helpers/write_set.rs | 65 +- .../src/snapshots/constants.rs | 36 +- .../snapshots/main/list_person_snapshots.rs | 87 -- .../src/snapshots/main/mod.rs | 21 +- .../{ => publication}/_publish_snapshot.rs | 0 .../src/snapshots/main/publication/mod.rs | 7 + .../next_remote_generation.rs | 0 .../main/{ => publication}/prune_snapshots.rs | 0 .../publish_development_snapshot.rs | 2 +- .../publish_remote_snapshot.rs | 4 + .../{ => publication}/publish_snapshot.rs | 2 +- .../{ => publication}/verify_execution.rs | 2 +- .../main/reads/connection_details.rs | 32 + .../main/{ => reads}/current_snapshot.rs | 6 +- .../{ => reads}/current_snapshot_record.rs | 2 +- .../main/reads/list_person_snapshots.rs | 58 + .../src/snapshots/main/reads/listing_read.rs | 33 + .../src/snapshots/main/reads/mod.rs | 12 + .../snapshots/main/{ => reads}/node_hashes.rs | 0 .../main/{ => reads}/read_current_record.rs | 0 .../snapshots/main/reads/read_kind_nodes.rs | 31 + .../main/{ => reads}/read_person_node_sql.rs | 6 +- .../{ => reads}/read_snapshot_documents.rs | 0 .../main/{ => reads}/read_snapshot_keys.rs | 0 .../main/{ => reads}/read_snapshot_nodes.rs | 0 .../main/write_sets/held_write_set.rs | 26 + .../holds_on_current_nodes.rs | 27 +- .../main/write_sets/holds_on_nodes.rs | 28 + .../main/{ => write_sets}/job_write_set.rs | 21 +- .../src/snapshots/main/write_sets/mod.rs | 5 + .../snapshots/main/write_sets/writer_nodes.rs | 14 + .../src/snapshots/mod.rs | 2 +- .../src/snapshots/models.rs | 31 + .../src/store/_helpers/mod.rs | 1 + .../src/store/_helpers/open_sqlite.rs | 2 +- .../src/store/_helpers/writer_gate.rs | 33 + .../src/store/indexes.sql | 4 + .../src/store/main/begin.rs | 6 + .../src/store/main/decompress.rs | 30 +- .../src/store/models.rs | 2 + .../src/store/schema.sql | 27 +- .../src/warehouse_history/_helpers/checks.rs | 8 +- .../warehouse_history/_helpers/run_recall.rs | 131 ++- .../main/read_person_check.rs | 2 +- .../main/read_person_checks.rs | 20 +- .../main/read_person_run_node_error.rs | 2 +- .../src/warehouse_history/models.rs | 6 +- .../_helpers/catalog_nodes.rs | 2 +- .../src/warehouse_metadata/_helpers/events.rs | 11 +- .../_helpers/fingerprints.rs | 60 +- .../warehouse_metadata/_helpers/ingestion.rs | 55 +- .../_helpers/mirror_pages.rs | 107 +- .../_helpers/node_status.rs | 2 +- .../src/warehouse_metadata/constants.rs | 35 + .../main/ingest/begin_ingestion.rs | 17 + .../main/ingest/ingest_audit_results.rs | 6 +- .../main/ingest/ingest_catalog.rs | 10 +- .../main/ingest/ingest_fingerprints.rs | 66 +- .../main/ingest/ingest_source_freshness.rs | 10 +- .../src/warehouse_metadata/main/ingest/mod.rs | 2 + .../main/ingest/prune_state_history.rs | 64 + .../main/location_reads/mod.rs | 1 + .../location_reads/preview_state_parts.rs | 84 ++ .../main/location_reads/read_preview_copy.rs | 25 +- .../src/warehouse_metadata/main/mod.rs | 1 + .../main/node_reads/catalog_node.rs | 20 + .../main/node_reads/definition_evidences.rs | 16 + .../main/node_reads/destination.rs | 11 + .../main/node_reads/last_outcome.rs | 13 + .../main/node_reads/latest_attempts.rs | 19 + .../main/node_reads/latest_fingerprints.rs | 23 + .../warehouse_metadata/main/node_reads/mod.rs | 6 + .../person_reads/list_person_catalog_nodes.rs | 93 -- .../main/person_reads/mod.rs | 1 - .../read_person_column_lineage.rs | 6 +- .../person_reads/read_person_node_status.rs | 4 +- .../src/warehouse_metadata/mod.rs | 4 + .../src/warehouse_metadata/models.rs | 62 +- .../src/webhooks/_helpers/events.rs | 2 +- .../src/worker/main/start_worker.rs | 2 +- .../tests/api/changes/helpers.rs | 11 +- .../tests/api/fingerprint_reads/helpers.rs | 2 +- .../tests/api/helpers.rs | 19 + .../tests/api/node_sql/helpers.rs | 2 +- .../tests/api/node_states/helpers.rs | 88 ++ .../tests/api/node_status/helpers.rs | 35 +- .../tests/api/plans/helpers.rs | 4 +- .../tests/api/state_history/helpers.rs | 116 ++ .../tests/api/test_acknowledgements.rs | 4 +- .../tests/api/test_catalog_reads.rs | 4 +- .../tests/api/test_changes.rs | 4 +- .../tests/api/test_checks.rs | 11 +- .../tests/api/test_column_lineage.rs | 2 +- .../tests/api/test_contract.rs | 12 +- .../tests/api/test_contracts.rs | 10 + .../tests/api/test_denials.rs | 4 +- .../tests/api/test_development.rs | 54 +- .../tests/api/test_ingestion_turns.rs | 71 ++ .../tests/api/test_location_health.rs | 186 ++- .../tests/api/test_location_work.rs | 10 +- .../tests/api/test_maintenance.rs | 44 +- .../tests/api/test_node_states.rs | 529 +++++++++ .../tests/api/test_person_live.rs | 4 +- .../tests/api/test_person_location_work.rs | 8 +- .../tests/api/test_person_plans.rs | 327 +++++- .../tests/api/test_person_sessions.rs | 12 +- .../tests/api/test_providers.rs | 6 +- .../tests/api/test_publish_encoding.rs | 2 +- .../tests/api/test_reads.rs | 34 +- .../tests/api/test_run_ids.rs | 4 +- .../tests/api/test_state_history_retention.rs | 176 +++ .../tests/api/test_statistics_recovery.rs | 2 +- .../tests/api/test_trigger_upcoming.rs | 2 +- .../tests/api/test_types.rs | 55 + .../tests/api/test_warehouse_metadata.rs | 9 +- .../tests/api/test_worker.rs | 8 +- .../tests/api/warehouse_history/helpers.rs | 4 +- .../api/warehouse_metadata/events/helpers.rs | 3 +- .../tests/api/warehouse_metadata/helpers.rs | 3 +- .../preview_copy/helpers.rs | 75 +- .../tests/credentials/roles/helpers.rs | 4 +- .../tests/credentials/test_audit_retention.rs | 7 +- .../tests/credentials/test_contract.rs | 4 +- .../tests/credentials/test_custom_roles.rs | 2 +- .../tests/credentials/test_project_roles.rs | 2 +- .../tests/declarations/helpers.rs | 63 + .../tests/declarations/test_contracts.rs | 5 + .../declarations/test_group_dependencies.rs | 190 +++ .../declarations/test_group_wait_graph.rs | 4 +- .../tests/declarations/test_types.rs | 18 + .../tests/platform/helpers.rs | 4 +- .../tests/platform/test_compile_failure.rs | 99 ++ .../tests/platform/test_contracts.rs | 2 + .../tests/platform/test_installation.rs | 2 +- .../tests/platform/test_problems.rs | 10 +- .../tests/platform/test_types.rs | 8 + .../tests/platform/test_watchdog.rs | 10 +- .../tests/platform/test_webhook.rs | 4 +- .../tests/runs/claims/helpers.rs | 242 +++- .../tests/runs/helpers.rs | 91 +- .../tests/runs/monitor/helpers.rs | 4 +- .../tests/runs/termination/helpers.rs | 2 +- .../tests/runs/test_adhoc.rs | 4 +- .../tests/runs/test_checks.rs | 6 +- .../tests/runs/test_claims.rs | 40 +- .../tests/runs/test_coalesce.rs | 6 +- .../tests/runs/test_contracts.rs | 16 +- .../tests/runs/test_faults.rs | 14 +- .../tests/runs/test_group_admission.rs | 36 +- .../tests/runs/test_group_declarations.rs | 102 ++ .../tests/runs/test_group_dependencies.rs | 177 +++ .../tests/runs/test_group_retries.rs | 257 +++++ .../tests/runs/test_group_round_trips.rs | 172 +++ .../tests/runs/test_groups.rs | 177 ++- .../tests/runs/test_job_detail.rs | 49 +- .../tests/runs/test_job_subsets.rs | 18 +- .../tests/runs/test_lifecycle.rs | 12 +- .../tests/runs/test_monitor.rs | 12 +- .../tests/runs/test_node_errors.rs | 6 +- .../tests/runs/test_permissions.rs | 9 +- .../tests/runs/test_project_limits.rs | 204 ++++ .../tests/runs/test_publication_cache.rs | 232 ++++ .../tests/runs/test_requests.rs | 21 +- .../tests/runs/test_reruns.rs | 13 +- .../tests/runs/test_retries.rs | 7 +- .../tests/runs/test_run_logs.rs | 4 +- .../tests/runs/test_selection.rs | 154 +++ .../tests/runs/test_types.rs | 223 +++- .../tests/scheduler/test_sensor_fairness.rs | 2 +- .../tests/scheduler/ticks/helpers.rs | 4 +- .../tests/scheduler/ticks/test_loop.rs | 13 +- .../tests/scheduler/ticks/test_precision.rs | 2 +- .../tests/scheduler/ticks/test_progress.rs | 2 +- .../tests/scheduler/ticks/test_retention.rs | 4 +- .../tests/scheduler/ticks/test_triggers.rs | 4 +- .../tests/sensors/helpers.rs | 44 +- .../tests/sensors/test_contracts.rs | 2 + .../tests/sensors/test_custom.rs | 8 +- .../tests/sensors/test_delivery.rs | 6 +- .../tests/sensors/test_execution_bounds.rs | 6 +- .../tests/sensors/test_group_members.rs | 566 +++++++++ .../tests/sensors/test_nodes.rs | 6 +- .../tests/sensors/test_person_state.rs | 2 +- .../tests/sensors/test_recovery.rs | 6 +- .../tests/sensors/test_runs.rs | 6 +- .../tests/sensors/test_sources.rs | 6 +- .../tests/sensors/test_transitions.rs | 4 +- .../tests/sensors/test_types.rs | 23 +- .../tests/snapshots/helpers.rs | 4 +- .../snapshots/test_connection_details.rs | 67 ++ .../tests/snapshots/test_contracts.rs | 2 + .../tests/snapshots/test_retention.rs | 4 +- .../tests/snapshots/test_types.rs | 8 + .../tests/store/schema/test_crash_recovery.rs | 2 +- .../tests/warehouse_metadata/helpers.rs | 54 + .../warehouse_metadata/test_contracts.rs | 8 + .../warehouse_metadata/test_preview_state.rs | 62 + .../tests/warehouse_metadata/test_types.rs | 14 + crates/sqlbuild-python-base/Cargo.toml | 1 + .../src/native_host/_helpers/dialect.rs | 48 + .../src/native_host/_helpers/host.rs | 13 +- .../src/native_host/_helpers/mod.rs | 5 + .../src/native_host/_helpers/results.rs | 4 +- .../src/native_host/models.rs | 61 + .../src/native_host/types.rs | 6 + .../native_threads/_helpers/compile_pool.rs | 19 + .../src/native_threads/_helpers/mod.rs | 1 + .../src/native_threads/constants.rs | 5 + .../src/native_threads/main/mod.rs | 2 + .../src/native_threads/main/pooled.rs | 13 + .../src/native_threads/main/spawned.rs | 11 + .../_helpers/orchestration_declarations.rs | 51 +- .../main/dependency_failures.rs | 16 + .../src/orchestration_reads/main/mod.rs | 1 + .../_helpers/compiler_records/constants.rs | 1 + .../_helpers/contract_records/constants.rs | 3 + .../records/_helpers/native_classes/fields.rs | 4 + .../_helpers/planner_records/constants.rs | 4 + .../src/records/constants.rs | 3 + .../src/warehouse_access/_helpers/mod.rs | 1 - .../warehouse_access/_helpers/synced_copy.rs | 109 -- .../_helpers/warehouse_catalog.rs | 8 +- .../_helpers/warehouse_statistics.rs | 46 +- .../main/is_warehouse_access_refused.rs | 11 - .../src/warehouse_access/main/mod.rs | 2 - .../main/register_synced_copy.rs | 12 - .../src/warehouse_access/mod.rs | 2 +- .../src/warehouse_access/models.rs | 484 -------- .../_helpers/stages/analysis.rs | 108 +- .../_helpers/model/model_loop_host.rs | 12 +- .../_helpers/model_loop/model_loop_opt_out.rs | 2 +- .../_helpers/model_loop/model_loop_prepass.rs | 42 +- .../_helpers/resources/audit_loop.rs | 6 +- .../_helpers/resources/function_loop.rs | 3 +- .../_helpers/resources/resource_attachment.rs | 17 +- .../resources/resource_diagnostics.rs | 10 +- .../_helpers/resources/source_loop.rs | 3 +- .../_helpers/resources/sql_test_loop.rs | 6 +- .../src/input_loops/main/mod.rs | 1 - .../src/input_loops/main/pooled.rs | 9 - .../main/syntax_error_messages.rs | 2 +- .../src/command_runtime/main/mod.rs | 1 + .../main/published_concurrency.rs | 44 + .../_helpers/project/project_config.rs | 16 +- .../_helpers/project/project_validation.rs | 40 +- .../sources/orchestration_declarations.rs | 5 + .../_helpers/model_nodes/contract_classes.rs | 4 +- .../native_host/_helpers/consumption_io.rs | 187 +++ .../native_host/_helpers/consumption_state.rs | 160 +-- .../src/native_host/_helpers/mod.rs | 1 + .../main/append_replaced_consumption.rs | 2 +- .../src/native_host/main/execute.rs | 16 +- .../src/native_host/mod.rs | 1 + .../src/native_host/models.rs | 42 +- .../src/native_host/types.rs | 19 + .../build_progress/_helpers/result_text.rs | 1 + .../_helpers/results/aggregation.rs | 48 +- crates/sqlbuild-python-planner/Cargo.toml | 1 + .../src/consumption/_helpers/facts.rs | 83 +- .../src/consumption/_helpers/recorder.rs | 13 +- .../consumption/_helpers/relation_reads.rs | 34 +- .../src/consumption/main/preview_changed.rs | 10 +- .../freshness/_helpers/freshness_observe.rs | 59 +- .../src/freshness/_helpers/observation.rs | 116 +- .../src/freshness/_helpers/planning.rs | 62 +- .../src/freshness/_helpers/state_io.rs | 5 +- .../src/freshness/_helpers/support.rs | 5 +- .../src/freshness/models.rs | 8 +- crates/sqlbuild-python-planner/src/lib.rs | 1 + .../lifecycle_events/_helpers/envelopes.rs | 10 +- .../lifecycle_events/_helpers/validation.rs | 7 +- .../src/live_reads/_helpers/bounded.rs | 172 +++ .../src/live_reads/_helpers/mod.rs | 1 + .../src/live_reads/constants.rs | 20 + .../src/live_reads/main/_live_reads.rs | 23 + .../src/live_reads/main/bound_task.rs | 14 + .../src/live_reads/main/mod.rs | 3 + .../live_reads/main/register_live_reads.rs | 12 + .../src/live_reads/mod.rs | 4 + .../src/live_reads/models.rs | 479 ++++++++ .../src/microbatches/_helpers/projection.rs | 4 +- .../src/microbatches/_helpers/statements.rs | 3 +- .../src/observations/_helpers/publication.rs | 7 +- .../src/plan_output/_helpers/migrations.rs | 10 +- .../src/plan_output/_helpers/plan_json.rs | 2 + .../src/plan_output/_helpers/plan_text.rs | 23 +- .../src/plan_output/main/format_plan_json.rs | 4 + .../src/planner_changes/_helpers/detect.rs | 34 +- .../_helpers/entries/contexts.rs | 6 +- .../_helpers/entries/getters.rs | 6 +- .../_helpers/entries/model_entries.rs | 38 +- .../_helpers/entries/relations.rs | 43 +- .../_helpers/plans/clone_plans.rs | 59 +- .../_helpers/plans/command_plans.rs | 78 +- .../_helpers/plans/display_plan.rs | 14 +- .../_helpers/plans/execution_plan.rs | 47 +- .../_helpers/plans/plan_output.rs | 36 +- .../_helpers/plans/test_planning.rs | 9 +- .../main/entries/plan_model.rs | 5 +- .../main/plans/build_audit_command_plan.rs | 7 +- .../main/plans/build_execution_plan.rs | 34 +- .../plans/read_selected_seed_fingerprints.rs | 6 +- .../plans/resolve_static_relation_context.rs | 8 +- .../src/planner_entries/models.rs | 61 +- .../planner_identity/_helpers/identities.rs | 3 +- .../_helpers/column/column_candidates.rs | 8 +- .../_helpers/column/column_entries.rs | 34 +- .../_helpers/column/column_overlay.rs | 11 +- .../_helpers/column/column_planning.rs | 39 +- .../_helpers/column/column_warnings.rs | 8 +- .../_helpers/discovery/discovery_requests.rs | 10 +- .../_helpers/discovery/model_discovery.rs | 17 +- .../_helpers/discovery/state_inspection.rs | 5 +- .../_helpers/model/model_entries.rs | 14 +- .../_helpers/model/model_handover.rs | 3 +- .../_helpers/model/model_overlay.rs | 3 +- .../_helpers/model/model_planning.rs | 19 +- .../_helpers/model/model_warnings.rs | 5 +- .../_helpers/old_name/old_name_entries.rs | 24 +- .../_helpers/old_name/old_name_objects.rs | 7 +- .../_helpers/old_name/old_name_planning.rs | 3 +- .../_helpers/old_name/old_name_references.rs | 3 +- .../_helpers/planning/compatibility.rs | 3 +- .../_helpers/planning/matching.rs | 3 +- .../_helpers/planning/planning_boundary.rs | 12 +- .../_helpers/planning/planning_execution.rs | 18 +- .../_helpers/planning/planning_origin.rs | 4 +- .../src/planner_migrations/constants.rs | 6 - .../src/planner_migrations/models.rs | 339 +++--- .../planner_phases/_helpers/inputs/getters.rs | 10 +- .../planner_phases/_helpers/inputs/runtime.rs | 33 +- .../_helpers/inputs/warehouse_state.rs | 16 +- .../src/planner_phases/_helpers/mod.rs | 1 + .../_helpers/outputs/buildability.rs | 7 +- .../_helpers/outputs/output_assembly.rs | 37 +- .../_helpers/rebuilds/deferred_origins.rs | 40 +- .../_helpers/rebuilds/execution_changes.rs | 29 +- .../_helpers/rebuilds/retention.rs | 84 +- .../src/planner_phases/_helpers/timing/mod.rs | 1 + .../planner_phases/_helpers/timing/timings.rs | 34 + .../src/planner_phases/constants.rs | 40 + .../src/planner_phases/main/inputs/mod.rs | 2 + .../main/inputs/runtime_dialect.rs | 11 + .../main/inputs/runtime_reads.rs | 11 + .../src/planner_phases/main/mod.rs | 1 + .../main/outputs/add_plan_metadata.rs | 6 + .../rebuilds/build_planner_entry_results.rs | 7 +- .../rebuilds/defer_to_migration_origins.rs | 10 +- .../src/planner_phases/main/timing/mod.rs | 2 + .../planner_phases/main/timing/record_step.rs | 9 + .../src/planner_phases/main/timing/timed.rs | 12 + .../src/planner_phases/models.rs | 108 +- .../_helpers/cursor/cursor_model.rs | 5 +- .../_helpers/cursor/cursor_overrides.rs | 10 +- .../_helpers/cursor/cursor_ranges.rs | 9 +- .../_helpers/references/orchestration.rs | 23 +- .../_helpers/references/refs.rs | 43 +- .../_helpers/references/sources.rs | 73 +- .../_helpers/references/values.rs | 23 +- .../_helpers/safety/future_safety.rs | 15 +- .../_helpers/safety/maximum_safety.rs | 22 +- .../main/references/resolve_function_sql.rs | 5 +- .../src/planner_resolution/models.rs | 21 +- .../planner_scenario/_helpers/artifacts.rs | 10 +- .../src/planner_scenario/_helpers/cli.rs | 12 +- .../src/planner_scenario/_helpers/entries.rs | 5 +- .../planner_scenario/_helpers/execution.rs | 41 +- .../src/planner_scenario/_helpers/fixtures.rs | 5 +- .../planner_scenario/_helpers/helper_scope.rs | 7 +- .../planner_scenario/_helpers/relations.rs | 8 +- .../src/planner_scenario/_helpers/sql.rs | 9 +- .../src/planner_scenario/models.rs | 3 +- .../_helpers/audit_entries.rs | 12 +- .../_helpers/cursor_inputs.rs | 28 +- .../planner_strategy/_helpers/cursor_types.rs | 3 +- .../src/planner_strategy/_helpers/lineage.rs | 3 +- .../_helpers/microbatch_limits.rs | 6 +- .../_helpers/model_entries.rs | 61 +- .../src/planner_strategy/_helpers/strategy.rs | 4 +- .../build_selected_audit_entries.rs | 3 +- .../build_cursor_input_relations.rs | 5 +- .../main/model_entries/column_info.rs | 12 + .../main/model_entries/mod.rs | 1 + .../_helpers/inspection/cursors.rs | 359 +++--- .../_helpers/inspection/fingerprint_cache.rs | 6 +- .../_helpers/inspection/snapshot.rs | 237 ++-- .../_helpers/inspection/watermarks.rs | 6 +- .../src/planner_warehouse/_helpers/mod.rs | 1 + .../_helpers/sources/semantic_sources.rs | 53 +- .../_helpers/sources/source_columns.rs | 46 +- .../_helpers/sources/source_deferral.rs | 3 +- .../_helpers/sources/source_freshness.rs | 28 +- .../_helpers/sources/source_tables.rs | 77 +- .../_helpers/synced/exception_types.rs | 11 + .../planner_warehouse/_helpers/synced/mod.rs | 2 + .../_helpers/synced/synced_reads.rs | 90 ++ .../src/planner_warehouse/constants.rs | 27 +- .../main/inspection/adapter_cursor_sql.rs | 38 - .../main/inspection/bound_task.rs | 27 - .../main/inspection/columns_for.rs | 14 + .../main/inspection/dialect_cursor_sql.rs | 15 + .../main/inspection/list_relations.rs | 16 + .../planner_warehouse/main/inspection/mod.rs | 5 +- .../src/planner_warehouse/main/mod.rs | 1 + .../planner_warehouse/main/reads/_unknown.rs | 15 + .../main/reads/is_refused_access.rs | 10 + .../src/planner_warehouse/main/reads/mod.rs | 4 + .../main/reads/refused_access.rs | 10 + .../main/reads/register_exception_types.rs | 12 + .../build_planner_source_freshness_result.rs | 6 +- .../check_selected_source_tables_exist.rs | 11 +- .../main/sources/gather_source_columns.rs | 8 +- .../main/sources/gather_source_inspection.rs | 7 +- .../src/planner_warehouse/mod.rs | 1 + .../src/planner_warehouse/models.rs | 581 +++++++++- .../src/planner_warehouse/types.rs | 158 +++ .../src/analysis/_helpers/catalog.rs | 3 +- .../src/analysis/_helpers/relation_names.rs | 50 +- .../main/extract_sql_relation_names.rs | 22 +- .../src/analysis/main/mod.rs | 1 + .../main/prefetch_sql_relation_names.rs | 34 + .../src/analysis/types.rs | 3 + .../_helpers/query_columns.rs | 6 +- .../_helpers/stage_functions.rs | 20 +- .../compiled_project/_helpers/job_catalog.rs | 6 +- .../_helpers/project/getters.rs | 12 +- .../_helpers/contract_diagnostics.rs | 5 +- .../src/fingerprints/_helpers/storage_io.rs | 10 +- .../src/fingerprints/main/mod.rs | 1 + .../src/fingerprints/main/selected_set.rs | 36 + .../src/planner_identity/_helpers/hashing.rs | 38 +- .../main/baselined_model_query_hash.rs | 4 +- .../_helpers/fixtures/fixture_completion.rs | 15 +- .../_helpers/fixtures/fixture_context.rs | 19 +- .../_helpers/fixtures/fixture_metadata.rs | 3 +- .../_helpers/fixtures/fixture_preparation.rs | 11 +- .../_helpers/fixtures/fixture_reads.rs | 22 +- .../_helpers/fixtures/fixture_selection.rs | 4 +- .../_helpers/fixtures/fixture_sql.rs | 5 +- .../_helpers/fixtures/fixture_types.rs | 6 +- .../main/complete_empty_fixture_sql.rs | 11 +- .../main/complete_fixtures.rs | 11 +- .../main/complete_typed_null_columns.rs | 5 +- .../src/planner_strategy/models.rs | 5 +- .../_helpers/snapshot/snapshot_boundary.rs | 3 +- .../snapshot/snapshot_declarations.rs | 63 +- .../_helpers/snapshot/snapshot_models.rs | 6 +- .../_helpers/snapshot/snapshot_resources.rs | 12 +- .../main/compile_declarations.rs | 8 +- .../src/project_snapshot/models.rs | 72 +- .../_helpers/explicit_references.rs | 3 +- .../src/semantic_shapes/_helpers/shapes.rs | 3 +- .../_helpers/source_bindings.rs | 7 +- .../_helpers/inputs/cursor_window.rs | 14 +- .../_helpers/inputs/fixture_validation.rs | 12 +- .../_helpers/plan/plan_assembly.rs | 3 +- .../_helpers/plan/plan_driver.rs | 5 +- .../_helpers/plan/plan_entries.rs | 3 +- .../_helpers/plan/plan_rendering.rs | 9 +- .../_helpers/audit_command/command.rs | 3 +- .../_helpers/build_command/completion.rs | 14 +- .../_helpers/build_command/execution.rs | 20 +- .../_helpers/build_command/planning.rs | 15 +- .../_helpers/build_command/registration.rs | 4 +- .../_helpers/change_selection/base_cache.rs | 6 +- .../change_selection/definition_snapshot.rs | 10 +- .../bindings/_helpers/check/check_command.rs | 14 +- .../bindings/_helpers/check/registration.rs | 4 +- .../src/bindings/_helpers/cli/dispatch.rs | 87 +- .../bindings/_helpers/clone/clone_pipeline.rs | 15 +- .../bindings/_helpers/clone/registration.rs | 3 +- .../command_runtime/audit_failures.rs | 42 + .../bindings/_helpers/command_runtime/mod.rs | 8 + .../bindings/_helpers/compile_driver/mod.rs | 5 + .../_helpers/compile_driver/plan_pipeline.rs | 166 ++- .../_helpers/compile_driver/plan_result.rs | 9 +- .../_helpers/compile_driver/rules_at_load.rs | 32 + .../_helpers/compile_driver/static_context.rs | 3 +- .../_helpers/cost_command/registration.rs | 4 +- .../_helpers/cost_report/registration.rs | 4 +- .../dagster_invocation/registration.rs | 5 +- .../bindings/_helpers/dbt_command/debug.rs | 5 +- .../_helpers/dbt_command/init_project.rs | 5 +- .../_helpers/dbt_command/registration.rs | 4 +- .../_helpers/dbt_interop/pipeline_phases.rs | 3 +- .../_helpers/dbt_interop/plan_output.rs | 17 +- .../_helpers/dbt_interop/registration.rs | 10 +- .../_helpers/debug_command/run_debug.rs | 22 +- .../bindings/_helpers/diff_report/query.rs | 8 +- .../_helpers/diff_report/registration.rs | 8 +- .../_helpers/format_command/registration.rs | 3 +- .../_helpers/freshness/freshness_command.rs | 6 +- .../_helpers/janitor/janitor_command.rs | 19 +- .../_helpers/janitor/janitor_planning.rs | 42 +- .../_helpers/janitor/janitor_usage.rs | 5 +- .../bindings/_helpers/janitor/registration.rs | 11 +- .../_helpers/location_command/command.rs | 80 +- .../_helpers/location_command/constants.rs | 4 + .../_helpers/location_command/load_rules.rs | 81 ++ .../bindings/_helpers/location_command/mod.rs | 4 + .../_helpers/location_command/runtime.rs | 31 +- .../_helpers/location_command/runtime_plan.rs | 120 ++ .../_helpers/location_command/snapshot.rs | 21 +- .../location_command/snapshot_connection.rs | 64 + .../_helpers/location_command/types.rs | 6 + .../_helpers/location_command/worker.rs | 25 +- .../_helpers/native_host/registration.rs | 8 +- .../native_host/warehouse_metadata.rs | 120 +- .../bindings/_helpers/plan_command/plan.rs | 312 +++-- .../_helpers/plan_command/registration.rs | 3 +- .../planner_registration/registration.rs | 17 +- .../_helpers/query_command/registration.rs | 3 +- .../_helpers/review_command/execution.rs | 3 +- .../_helpers/review_command/support.rs | 2 +- .../_helpers/scenario_command/registration.rs | 3 +- .../scenario_command/snapshot_sync.rs | 24 +- .../bindings/_helpers/seed_command/command.rs | 13 +- .../_helpers/sql_test_command/python_tests.rs | 6 +- .../_helpers/sql_test_command/registration.rs | 3 +- .../_helpers/sql_test_command/test_command.rs | 33 +- .../src/bindings/tests/test_failure_cause.rs | 3 +- .../src/schema/_helpers/clickhouse_views.rs | 3 +- .../src/schema/_helpers/relational_views.rs | 6 +- .../sqlbuild-recall/src/schema/constants.rs | 2 +- .../src/schema/tests/test_ddl.rs | 4 +- .../src/store/tests/test_views.rs | 47 +- .../src/python_rules/constants.rs | 4 +- .../sqlbuild-rules/src/python_rules/models.rs | 2 +- .../tests/test_python_test_rules.rs | 24 + crates/sqlbuild-state/Cargo.toml | 2 + crates/sqlbuild-state/src/lib.rs | 3 +- .../src/state_tables/_helpers/history_sql.rs | 89 ++ .../src/state_tables/_helpers/mod.rs | 1 + .../src/state_tables/constants.rs | 8 + .../src/state_tables/main/read_history_sql.rs | 64 +- crates/sqlbuild-state/src/state_tables/mod.rs | 1 + .../sqlbuild-state/src/state_tables/models.rs | 11 + .../src/state_tables/tests/test_sql.rs | 62 +- .../src/state_tables/tests/test_types.rs | 6 + .../src/warehouse_state/_helpers/indexes.rs | 46 + .../src/warehouse_state/_helpers/mod.rs | 2 + .../src/warehouse_state/_helpers/queries.rs | 218 ++++ .../src/warehouse_state/constants.rs | 4 + .../src/warehouse_state/main/mod.rs | 1 + .../warehouse_state/main/warehouse_state.rs | 9 + .../sqlbuild-state/src/warehouse_state/mod.rs | 7 + .../src/warehouse_state/models.rs | 150 +++ .../src/warehouse_state/tests.rs | 6 + .../src/warehouse_state/tests/helpers.rs | 143 +++ .../src/warehouse_state/tests/test_lookups.rs | 240 ++++ .../src/warehouse_state/tests/test_types.rs | 50 + fensu.toml | 112 +- pyproject.toml | 1 + scripts/fensu_policy/_helpers/unused_state.py | 71 +- scripts/fensu_policy/constants.py | 27 +- .../rules/observation_boundary.py | 4 +- scripts/fensu_policy/rules/planner_reads.py | 35 + .../sqlbuild/references/docs/CONTENTS.md | 1 + .../sqlbuild/references/docs/cli/build.md | 2 + .../sqlbuild/references/docs/cli/dev.md | 19 +- .../sqlbuild/references/docs/cli/janitor.md | 15 +- .../sqlbuild/references/docs/cli/location.md | 13 + .../sqlbuild/references/docs/cli/plan.md | 23 + .../sqlbuild/references/docs/cli/server.md | 142 ++- .../sqlbuild/references/docs/cli/test.md | 11 +- .../references/docs/concepts/audits.md | 56 +- .../references/docs/concepts/concurrency.md | 176 +++ .../references/docs/concepts/orchestration.md | 74 +- .../docs/concepts/project-configuration.md | 30 +- .../references/docs/concepts/rules.md | 5 + .../custom-rules/testing-and-determinism.md | 6 +- .../docs/concepts/rules/python-rules.md | 7 +- .../references/docs/concepts/selectors.md | 3 +- .../references/docs/concepts/testing.md | 4 +- .../sqlbuild/references/investigating-runs.md | 1 + src/sqlbuild/_native.pyi | 136 ++- .../adapter/contract/classes/base_adapter.py | 14 + src/sqlbuild/adapter/contract/classes/diff.py | 7 + .../contract/classes/duckdb_backed_adapter.py | 13 + .../contract/main/estimates_by_name.py | 20 + .../contract/main/estimates_one_at_a_time.py | 28 + .../bigquery/classes/bigquery_adapter.py | 13 + .../clickhouse/classes/clickhouse_adapter.py | 43 +- .../databricks/classes/databricks_adapter.py | 14 + .../motherduck/classes/motherduck_adapter.py | 1 + .../postgres/classes/postgres_adapter.py | 33 + .../snowflake/classes/snowflake_adapter.py | 64 + .../adapters/sqlite/classes/sqlite_adapter.py | 13 + .../sqlserver/classes/sqlserver_adapter.py | 40 +- .../executor/audit_results/constants.py | 2 + .../executor/auditing/_helpers/execution.py | 1 + .../auditing/_helpers/result_projection.py | 9 +- src/sqlbuild/executor/auditing/models.py | 7 + .../executor/build/_helpers/source_audits.py | 2 +- .../executor/build/classes/build_scheduler.py | 2 +- src/sqlbuild/executor/build/models.py | 1 + .../executor/pipeline/_helpers/scenario.py | 5 +- .../run/_helpers/execution/final_audits.py | 6 +- .../run/_helpers/materializations/view.py | 4 +- .../orchestration_declarations/types.py | 1 + .../cli/commands/main/build/_test_types.py | 79 +- .../cli/commands/main/build/helpers.py | 186 ++- .../main/build/test_ignored_audit_failures.py | 239 ++++ .../cli/commands/main/compile/_test_types.py | 14 +- .../cli/commands/main/plan/_test_types.py | 29 + .../cli/commands/main/plan/conftest.py | 36 + .../cli/commands/main/plan/helpers.py | 263 +++++ .../test_duckshop_10k_preview_performance.py | 142 +++ .../commands/main/plan/test_plan_timings.py | 67 ++ .../commands/main/test/test_python_tests.py | 19 + .../scripts/fensu_policy/_test_types.py | 9 + .../scripts/fensu_policy/helpers.py | 36 +- .../fensu_policy/test_observation_boundary.py | 8 +- .../fensu_policy/test_planner_reads.py | 92 ++ .../compiler/compiled_project/_test_types.py | 19 + .../compiler/compiled_project/helpers.py | 6 + .../publication/_test_types.py | 81 +- .../compiled_project/publication/helpers.py | 273 ++++- .../publication/test_calendar.py | 12 +- .../publication/test_dev_terminal.py | 6 +- .../publication/test_local_recovery.py | 6 +- .../publication/test_plan_preview.py | 357 ++++++ .../publication/test_publication.py | 19 +- .../publication/test_reload.py | 36 +- .../publication/test_rules_at_load.py | 135 +++ .../publication/test_warehouse_catalog.py | 2 +- .../publication/test_warehouse_metadata.py | 147 ++- .../compiled_project/publication/test_work.py | 12 +- .../publication/test_worker.py | 25 +- .../test_orchestration_snapshot.py | 241 +++- .../_helpers/fixtures/test_type_scope.py | 3 +- .../compiler/planner/_helpers/helpers.py | 7 +- .../planner/_helpers/resolve/helpers.py | 3 +- .../resolve/test_adapter_overrides.py | 4 +- .../_helpers/resolve/test_orchestration.py | 7 +- .../planner/_helpers/test_audit_entry.py | 3 +- .../planner/_helpers/test_fixture_planning.py | 9 +- .../_helpers/test_warehouse_cursor_reads.py | 13 +- .../_helpers/test_warehouse_snapshot.py | 64 +- .../sqlbuild/compiler/planner/main/helpers.py | 6 + .../scenario/test_snapshot_fingerprint.py | 3 +- tests/native_field_fallbacks.txt | 2 + .../unit/src/sqlbuild/adapters/_test_types.py | 19 + .../adapters/clickhouse/test_final_reads.py | 11 +- tests/unit/src/sqlbuild/adapters/helpers.py | 19 + .../sqlbuild/adapters/test_history_compute.py | 39 + .../test_relation_row_count_estimate.py | 80 ++ .../compiler/compile/_helpers/helpers.py | 3 +- .../compiler/migrations/main/_test_types.py | 8 + .../compiler/migrations/main/helpers.py | 20 + .../main/test_named_fingerprints.py | 89 ++ .../main/test_old_name_view_events.py | 3 +- .../test_column_migration_candidates.py | 6 +- .../planner/_helpers/planning/helpers.py | 6 +- .../_helpers/planning/test_retention.py | 4 +- .../resolve/test_cursor_intrinsics.py | 5 +- .../planner/_helpers/resolve/test_hooks.py | 3 +- .../planner/_helpers/resolve/test_refs.py | 15 +- .../planner/_helpers/resolve/test_sources.py | 22 +- .../sql_test_assembly/test_plan_test.py | 9 +- .../planner/_helpers/test_audit_entry.py | 7 +- .../_helpers/test_fixture_planning_work.py | 5 +- .../planner/_helpers/test_scenario_cli.py | 3 +- .../_helpers/test_scenario_relations.py | 17 +- .../planner/_helpers/test_source_columns.py | 16 +- .../planner/_helpers/test_source_freshness.py | 6 +- .../planner/_helpers/warehouse/helpers.py | 6 +- .../warehouse/test_semantic_sources.py | 10 +- .../compiler/planner/classes/_test_types.py | 14 + .../compiler/planner/classes/test_dialect.py | 68 ++ .../planner/classes/test_sql_test_planning.py | 7 +- .../classes/test_sql_test_planning_exit.py | 2 +- .../sqlbuild/compiler/planner/main/helpers.py | 6 + .../main/test_static_command_planning.py | 15 +- .../test_observation.py | 14 +- .../main/test_adapter_observation.py | 10 +- .../source_freshness/main/test_observation.py | 14 +- .../source_freshness/main/test_planning.py | 42 +- .../executor/auditing/main/_test_types.py | 8 + .../auditing/main/test_project_results.py | 72 +- .../presentation/classes/_test_types.py | 7 + .../test_transient_line_coordinator.py | 44 +- ui/.gitignore | 2 + ui/COPY.md | 88 ++ ui/PERF.md | 36 + ui/README.md | 10 + ui/e2e/choice-filter.spec.ts | 127 ++ ui/e2e/compact-shots.spec.ts | 78 ++ ui/e2e/compact.spec.ts | 131 +++ ui/e2e/concurrency.spec.ts | 67 ++ ui/e2e/edges.spec.ts | 67 ++ ui/e2e/ignored-audits.spec.ts | 78 ++ ui/e2e/layout.spec.ts | 362 ++++++ ui/e2e/perf.spec.ts | 189 +++ ui/e2e/plan-browser.spec.ts | 172 +++ ui/e2e/plan-timings.spec.ts | 35 + ui/e2e/routes.spec.ts | 50 + ui/e2e/screens.spec.ts | 32 + ui/e2e/selection.spec.ts | 127 ++ ui/e2e/small-graph.spec.ts | 29 + ui/e2e/zoom.ts | 22 + ui/package-lock.json | 46 + ui/package.json | 4 +- ui/playwright.config.ts | 26 + ui/src/lib/api/client.ts | 31 +- ui/src/lib/api/http-catalog.test.ts | 42 +- ui/src/lib/api/http-catalog.ts | 91 +- ui/src/lib/api/http-changes.ts | 2 +- ui/src/lib/api/http-checks.ts | 3 +- ui/src/lib/api/http-client.test.ts | 72 ++ ui/src/lib/api/http-client.ts | 130 ++- ui/src/lib/api/http-jobs.test.ts | 23 +- ui/src/lib/api/http-jobs.ts | 14 +- ui/src/lib/api/http-locations.test.ts | 116 +- ui/src/lib/api/http-locations.ts | 126 +- ui/src/lib/api/http-overview.ts | 4 +- ui/src/lib/api/http-people.test.ts | 3 +- ui/src/lib/api/http-people.ts | 1 + ui/src/lib/api/http-providers.test.ts | 11 +- ui/src/lib/api/http-providers.ts | 4 +- ui/src/lib/api/http-runs.test.ts | 64 +- ui/src/lib/api/http-runs.ts | 75 +- ui/src/lib/api/http-selection.test.ts | 78 ++ ui/src/lib/api/http-selection.ts | 43 + ui/src/lib/api/index.ts | 10 +- ui/src/lib/api/mock/awkward.test.ts | 23 + ui/src/lib/api/mock/awkward.ts | 44 + ui/src/lib/api/mock/big-group.ts | 72 ++ ui/src/lib/api/mock/build-fixtures.ts | 6 +- ui/src/lib/api/mock/build.ts | 10 +- ui/src/lib/api/mock/large-plan.ts | 211 ++++ ui/src/lib/api/mock/large-project.ts | 87 ++ ui/src/lib/api/mock/mock-client.ts | 77 +- ui/src/lib/api/mock/mock-node-states.test.ts | 26 + .../api/mock/operate-graph-selection.test.ts | 33 + ui/src/lib/api/mock/operate-graph.ts | 79 ++ ui/src/lib/api/mock/operate-store.ts | 109 +- ui/src/lib/api/mock/operate.ts | 166 ++- ui/src/lib/api/mock/plan-rows-log.ts | 20 + ui/src/lib/api/mock/platform.ts | 51 +- ui/src/lib/api/mock/run-detail.ts | 10 +- ui/src/lib/api/node-states.test.ts | 76 ++ ui/src/lib/api/node-states.ts | 105 ++ ui/src/lib/api/signals.ts | 13 +- ui/src/lib/api/types.ts | 4 + ui/src/lib/api/types/build.ts | 31 + ui/src/lib/api/types/operate.ts | 201 +++- ui/src/lib/api/types/platform.ts | 53 +- ui/src/lib/build-area/attention.test.ts | 59 +- ui/src/lib/build-area/attention.ts | 62 +- ui/src/lib/build-area/changes.test.ts | 4 +- ui/src/lib/build-area/changes.ts | 29 +- ui/src/lib/build-area/kind-tabs.test.ts | 22 + ui/src/lib/build-area/labels.ts | 75 +- ui/src/lib/build-area/overview-panels.test.ts | 84 ++ ui/src/lib/build-area/overview-panels.ts | 85 ++ .../components/admin/access-checker.svelte | 18 +- .../components/admin/assignment-list.svelte | 40 +- .../admin/effective-access-grid.svelte | 9 +- ui/src/lib/components/admin/role-cell.svelte | 8 +- ui/src/lib/components/app-sidebar.svelte | 3 +- ui/src/lib/components/app-topbar.svelte | 20 +- .../components/build-area/stale-badges.svelte | 5 +- .../lib/components/code/code-preview.svelte | 61 +- ui/src/lib/components/code/sql-block.svelte | 36 +- ui/src/lib/components/code/sql-viewer.svelte | 119 +- ui/src/lib/components/copy-button.svelte | 5 +- .../lib/components/disclosure-button.svelte | 56 + .../lib/components/graph/compact-face.svelte | 134 +++ .../components/graph/compact-tails.test.ts | 41 + ui/src/lib/components/graph/compact-tails.ts | 50 + ui/src/lib/components/graph/compact-title.ts | 59 + ui/src/lib/components/graph/fonts.svelte.ts | 9 + .../lib/components/graph/graph-canvas.svelte | 212 ++++ .../lib/components/graph/graph-toolbar.svelte | 91 ++ .../lib/components/graph/no-raw-graph.test.ts | 38 + ui/src/lib/components/help-tip.svelte | 62 + .../lib/components/lineage/asset-node.svelte | 49 +- .../lib/components/lineage/folder-node.svelte | 34 +- .../components/lineage/inspector-head.svelte | 54 + .../lineage/inspector-section.svelte | 26 +- .../lineage/node-fields-popover.svelte | 14 +- .../lib/components/lineage/stub-node.svelte | 56 + .../components/lineage/too-many-card.svelte | 56 + .../components/lineage/view-summary.svelte | 103 ++ .../lib/components/list/choice-filter.svelte | 85 +- ui/src/lib/components/list/choice-list.svelte | 255 ++++ ui/src/lib/components/list/data-table.svelte | 178 +++ .../lib/components/list/list-toolbar.svelte | 67 +- ui/src/lib/components/list/load-row.svelte | 7 +- .../lib/components/list/no-raw-tables.test.ts | 57 + .../components/list/segmented-filter.svelte | 24 +- .../lib/components/list/table-columns.test.ts | 27 + ui/src/lib/components/list/table-columns.ts | 49 + .../components/list/virtual-rows.svelte.ts | 81 ++ .../lib/components/list/window-range.test.ts | 24 + ui/src/lib/components/list/window-range.ts | 19 + .../components/operate/gated-button.svelte | 7 +- .../components/operate/run-status-pill.svelte | 20 +- ui/src/lib/components/page-body.svelte | 41 + ui/src/lib/components/page-body.test.ts | 27 + .../platform/connection-test.svelte | 15 +- .../components/platform/gated-button.svelte | 2 +- .../components/platform/issued-secret.svelte | 16 +- .../lib/components/platform/key-values.svelte | 9 +- .../platform/location-tokens.svelte | 4 +- .../lib/components/platform/page-tabs.svelte | 20 +- ui/src/lib/components/platform/panel.svelte | 16 +- .../components/platform/problem-list.svelte | 38 +- .../components/platform/syncing-note.svelte | 29 + .../platform/warehouse-sync-now.svelte | 7 +- ui/src/lib/components/refresh-button.svelte | 19 + .../components/selection/selection-bar.svelte | 253 ++++ .../selection/selection-chip.svelte | 188 +++ .../selection/selection-chips.svelte | 72 ++ .../selection/selection-field.svelte | 380 ++++++ .../components/selection/selector-icon.svelte | 28 + .../shell/platform-event-banner.svelte | 6 +- .../shell/warehouse-sync-status.svelte | 6 +- .../components/timeline/run-timeline.svelte | 94 +- .../components/ui/button/button-size.test.ts | 10 + ui/src/lib/components/ui/button/button.svelte | 56 +- .../ui/button/no-adhoc-buttons.test.ts | 115 ++ ui/src/lib/config/config-form.svelte | 17 +- ui/src/lib/copy-style/baseline.txt | 1 + ui/src/lib/copy-style/check.test.ts | 57 + ui/src/lib/copy-style/check.ts | 131 +++ ui/src/lib/copy-style/copy-style.test.ts | 47 + ui/src/lib/format/plural.test.ts | 21 + ui/src/lib/format/plural.ts | 49 + ui/src/lib/lineage/catalog-graph.test.ts | 40 +- ui/src/lib/lineage/catalog-graph.ts | 64 +- ui/src/lib/lineage/column-lineage.svelte | 30 +- ui/src/lib/lineage/count-edge.svelte | 22 + ui/src/lib/lineage/details-panel.test.ts | 28 + ui/src/lib/lineage/details-panel.ts | 24 + ui/src/lib/lineage/edge-routes.test.ts | 50 + ui/src/lib/lineage/edge-routes.ts | 161 +++ ui/src/lib/lineage/fit-controls.svelte | 20 + ui/src/lib/lineage/fit.test.ts | 146 +++ ui/src/lib/lineage/fit.ts | 143 +++ ui/src/lib/lineage/flow-controller.svelte | 62 +- ui/src/lib/lineage/folder-graph.test.ts | 132 +++ ui/src/lib/lineage/folder-graph.ts | 290 +++++ ui/src/lib/lineage/folder-selection.test.ts | 165 +++ ui/src/lib/lineage/folder-selection.ts | 181 +++ ui/src/lib/lineage/hit-area.test.ts | 11 + ui/src/lib/lineage/hit-area.ts | 7 + ui/src/lib/lineage/job-box-node.svelte | 62 +- ui/src/lib/lineage/job-graph.test.ts | 10 +- ui/src/lib/lineage/job-graph.ts | 5 +- ui/src/lib/lineage/member-box-node.svelte | 125 ++ ui/src/lib/lineage/member-expand.ts | 13 + ui/src/lib/lineage/member-folders.test.ts | 44 + ui/src/lib/lineage/member-folders.ts | 113 ++ ui/src/lib/lineage/member-graph.svelte | 316 +++++ ui/src/lib/lineage/member-graph.test.ts | 289 +++++ ui/src/lib/lineage/member-graph.ts | 443 +++++++ ui/src/lib/lineage/member-node.svelte | 184 +++ ui/src/lib/lineage/scope-graph.svelte | 146 ++- ui/src/lib/lineage/scoped-lineage.svelte | 78 +- ui/src/lib/lineage/status-line.test.ts | 26 + ui/src/lib/lineage/status-line.ts | 37 + ui/src/lib/lineage/trunk-path.test.ts | 15 + ui/src/lib/lineage/trunk-path.ts | 18 + ui/src/lib/lineage/view-summary.test.ts | 40 + ui/src/lib/lineage/view-summary.ts | 33 + ui/src/lib/lineage/visible-edges.test.ts | 59 + ui/src/lib/lineage/visible-edges.ts | 59 + ui/src/lib/list/choice.test.ts | 105 ++ ui/src/lib/list/choice.ts | 119 ++ ui/src/lib/list/count-by.test.ts | 21 + ui/src/lib/list/count-by.ts | 14 + ui/src/lib/list/no-filter-counts.test.ts | 62 + ui/src/lib/operate/labels.test.ts | 65 ++ ui/src/lib/operate/labels.ts | 117 +- ui/src/lib/platform/access.ts | 9 +- ui/src/lib/platform/connections.test.ts | 80 ++ ui/src/lib/platform/connections.ts | 97 ++ ui/src/lib/platform/labels.ts | 19 +- ui/src/lib/platform/maintenance.ts | 8 +- ui/src/lib/run/command-table.generated.json | 18 + ui/src/lib/run/command.test.ts | 36 +- ui/src/lib/run/command.ts | 34 +- ui/src/lib/run/data-range.svelte | 87 +- ui/src/lib/run/ignored-audits.test.ts | 19 + ui/src/lib/run/ignored-audits.ts | 11 + ui/src/lib/run/plan-browser.svelte | 479 ++++++++ ui/src/lib/run/plan-index.test.ts | 214 ++++ ui/src/lib/run/plan-index.ts | 441 +++++++ ui/src/lib/run/plan-report.svelte | 218 ---- ui/src/lib/run/plan-text.test.ts | 24 +- ui/src/lib/run/plan-text.ts | 78 +- ui/src/lib/run/plan-timings.svelte | 101 ++ ui/src/lib/run/plan-timings.test.ts | 31 + ui/src/lib/run/plan-timings.ts | 26 + ui/src/lib/run/range-calendar.svelte | 17 +- ui/src/lib/selection/command.ts | 80 ++ ui/src/lib/selection/field.ts | 82 ++ ui/src/lib/selection/folders.test.ts | 34 + ui/src/lib/selection/folders.ts | 65 ++ ui/src/lib/selection/picker.ts | 51 + ui/src/lib/selection/reading.ts | 102 ++ ui/src/lib/selection/selection.test.ts | 185 +++ ui/src/lib/selection/suggest.ts | 108 ++ ui/src/lib/selection/syntax.ts | 58 + ui/src/lib/state/account-table.svelte.ts | 14 +- ui/src/lib/state/catalog-feed.test.ts | 229 ++++ ui/src/lib/state/catalog-feed.ts | 212 ++++ ui/src/lib/state/catalog.svelte.ts | 55 + ui/src/lib/state/details-panel.svelte.ts | 20 + ui/src/lib/state/prefetch.ts | 11 +- ui/src/lib/state/selection.svelte.ts | 215 ++++ ui/src/lib/status.ts | 6 + ui/src/routes/+layout.svelte | 7 +- ui/src/routes/+layout.ts | 18 +- ui/src/routes/+page.svelte | 130 ++- ui/src/routes/admin/audit/+page.svelte | 59 +- ui/src/routes/admin/check-access/+page.svelte | 14 +- ui/src/routes/admin/people/+page.svelte | 42 +- .../routes/admin/people/[name]/+page.svelte | 23 +- ui/src/routes/admin/roles/+page.svelte | 9 +- ui/src/routes/admin/roles/[name]/+page.svelte | 68 +- ui/src/routes/admin/roles/roles-tab.svelte | 34 +- ui/src/routes/admin/roles/sso-tab.svelte | 39 +- ui/src/routes/admin/roles/utils.ts | 4 +- ui/src/routes/catalog/+page.svelte | 124 +- .../routes/catalog/[name]/checks-tab.svelte | 30 +- ui/src/routes/catalog/[name]/drift-tab.svelte | 4 +- .../routes/catalog/[name]/history-tab.svelte | 45 +- ui/src/routes/catalog/[name]/node-view.svelte | 94 +- .../routes/catalog/[name]/overview-tab.svelte | 25 +- ui/src/routes/catalog/[name]/runs-tab.svelte | 28 +- .../catalog/[name]/warehouse-tab.svelte | 30 +- ui/src/routes/changes/+page.svelte | 180 +-- ui/src/routes/changes/utils.test.ts | 14 + ui/src/routes/changes/utils.ts | 7 +- ui/src/routes/checks/+page.svelte | 118 +- ui/src/routes/checks/[id]/check-view.svelte | 83 +- ui/src/routes/checks/utils.test.ts | 14 + ui/src/routes/checks/utils.ts | 17 +- ui/src/routes/connections/+page.svelte | 181 ++- ui/src/routes/jobs/+page.svelte | 187 ++- ui/src/routes/jobs/[name]/job-view.svelte | 409 ++++--- ui/src/routes/jobs/utils.test.ts | 80 ++ ui/src/routes/jobs/utils.ts | 80 ++ ui/src/routes/layout.css | 54 + ui/src/routes/lineage/+page.svelte | 977 +++++++++------- ui/src/routes/locations/+page.svelte | 54 +- ui/src/routes/locations/[id]/+page.svelte | 51 +- ui/src/routes/locations/server-status.svelte | 21 +- ui/src/routes/locations/utils.ts | 2 +- ui/src/routes/login/+page.svelte | 3 +- ui/src/routes/maintenance/+page.svelte | 123 +- ui/src/routes/overview-cost.svelte | 39 +- ui/src/routes/plan/+page.svelte | 277 +++-- ui/src/routes/plan/rerun/+page.svelte | 165 +-- ui/src/routes/providers/+page.svelte | 35 +- ui/src/routes/providers/[name]/+page.svelte | 69 +- .../providers/[name]/used-by-graph.svelte | 265 +++++ ui/src/routes/providers/utils.test.ts | 77 ++ ui/src/routes/providers/utils.ts | 100 +- ui/src/routes/runs/+page.svelte | 7 +- ui/src/routes/runs/[id]/cost-tab.svelte | 96 +- ui/src/routes/runs/[id]/failures-panel.svelte | 35 +- .../runs/[id]/ignored-audits-panel.svelte | 115 ++ .../routes/runs/[id]/live-statements.svelte | 3 +- ui/src/routes/runs/[id]/log-tab.svelte | 127 +- ui/src/routes/runs/[id]/node-panel.svelte | 33 +- ui/src/routes/runs/[id]/related-runs.svelte | 44 +- ui/src/routes/runs/[id]/rerun-button.svelte | 51 +- ui/src/routes/runs/[id]/run-gantt.svelte | 45 +- ui/src/routes/runs/[id]/run-graph.svelte | 9 +- ui/src/routes/runs/[id]/run-header.svelte | 48 +- ui/src/routes/runs/[id]/run-tabs.svelte | 55 +- ui/src/routes/runs/[id]/run-view.svelte | 26 +- ui/src/routes/runs/[id]/statements-tab.svelte | 77 +- ui/src/routes/runs/[id]/utils/events.ts | 11 +- ui/src/routes/runs/[id]/utils/index.ts | 1 + ui/src/routes/runs/[id]/utils/members.test.ts | 24 + ui/src/routes/runs/[id]/utils/members.ts | 16 + ui/src/routes/runs/[id]/utils/nodes.ts | 6 +- ui/src/routes/runs/[id]/utils/rerun.ts | 6 +- ui/src/routes/runs/bulk-bar.svelte | 20 +- ui/src/routes/runs/run-list.svelte | 166 +-- ui/src/routes/runs/run-queue.svelte | 44 +- ui/src/routes/runs/runs-toolbar.svelte | 89 +- ui/src/routes/runs/state.svelte.ts | 7 +- ui/src/routes/runs/utils/status-line.test.ts | 18 + ui/src/routes/runs/utils/status-line.ts | 10 +- ui/src/routes/settings/+page.svelte | 40 +- ui/src/routes/triggers/+page.svelte | 181 ++- .../routes/triggers/[id]/sensor-panel.svelte | 114 +- ui/src/routes/triggers/[id]/tick-log.svelte | 104 +- .../routes/triggers/[id]/trigger-view.svelte | 38 +- ui/src/routes/triggers/set-in-ui-pin.svelte | 64 + ui/src/routes/triggers/utils.test.ts | 33 + ui/src/routes/triggers/utils.ts | 24 + website/src/content/docs/docs/cli/build.mdx | 2 + website/src/content/docs/docs/cli/dev.mdx | 19 +- website/src/content/docs/docs/cli/janitor.mdx | 15 +- .../src/content/docs/docs/cli/location.mdx | 13 + website/src/content/docs/docs/cli/plan.mdx | 22 + website/src/content/docs/docs/cli/server.mdx | 141 ++- website/src/content/docs/docs/cli/test.mdx | 11 +- .../src/content/docs/docs/concepts/audits.mdx | 55 +- .../docs/docs/concepts/concurrency.mdx | 165 +++ .../docs/docs/concepts/orchestration.mdx | 74 +- .../docs/concepts/project-configuration.mdx | 29 +- .../src/content/docs/docs/concepts/rules.mdx | 5 + .../custom-rules/testing-and-determinism.mdx | 6 +- .../docs/docs/concepts/rules/python-rules.mdx | 7 +- .../content/docs/docs/concepts/selectors.mdx | 3 +- .../content/docs/docs/concepts/testing.mdx | 4 +- website/src/sidebar.json | 3 +- 1376 files changed, 50662 insertions(+), 10520 deletions(-) create mode 100644 benchmarks/duckshop-10k/synced_state.py create mode 100644 benchmarks/duckshop-10k/tests/test_synced_state.py create mode 100644 crates/sqlbuild-cli/src/adhoc/_helpers/evaluation.rs create mode 100644 crates/sqlbuild-cli/src/adhoc/_helpers/publication.rs create mode 100644 crates/sqlbuild-cli/src/adhoc/main/command_settings.rs create mode 100644 crates/sqlbuild-cli/src/adhoc/models.rs create mode 100644 crates/sqlbuild-cli/src/adhoc/tests/test_selection_evaluation.rs rename crates/sqlbuild-cli/src/native_commands/_helpers/{ => command}/argv_host.rs (100%) rename crates/sqlbuild-cli/src/native_commands/_helpers/{ => command}/decision.rs (97%) rename crates/sqlbuild-cli/src/native_commands/_helpers/{ => command}/init.rs (96%) create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/command/mod.rs rename crates/sqlbuild-cli/src/native_commands/_helpers/{ => command}/package.rs (100%) rename crates/sqlbuild-cli/src/native_commands/_helpers/{ => command}/paths.rs (99%) rename crates/sqlbuild-cli/src/native_commands/_helpers/{ => command}/project_facts.rs (85%) rename crates/sqlbuild-cli/src/native_commands/_helpers/{ => command}/rules.rs (98%) rename crates/sqlbuild-cli/src/native_commands/_helpers/{ => command}/skills.rs (100%) rename crates/sqlbuild-cli/src/native_commands/_helpers/{ => command}/terminal.rs (89%) create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/location/changed_build.rs rename crates/sqlbuild-cli/src/native_commands/_helpers/{ => location}/execution.rs (83%) create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/location/held_copy.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/location/mod.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/location/plan_document.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/location/plan_request.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/location/sync_outcomes.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/location/sync_report.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/location/work.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/main/_location_changed_build.rs delete mode 100644 crates/sqlbuild-cli/src/native_commands/main/_plan_document.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_changed_build.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_sync_schedule.rs create mode 100644 crates/sqlbuild-config/src/effective/main/effective_ignore_audit_failures.rs create mode 100644 crates/sqlbuild-config/src/project/main/load_pool_limits.rs create mode 100644 crates/sqlbuild-config/src/project/main/load_project_setting_keys.rs create mode 100644 crates/sqlbuild-core/src/number_text/main/format_thousands.rs create mode 100644 crates/sqlbuild-core/src/number_text/main/mod.rs create mode 100644 crates/sqlbuild-core/src/number_text/mod.rs create mode 100644 crates/sqlbuild-core/src/plan_display/_helpers/diff_lines.rs create mode 100644 crates/sqlbuild-core/src/plan_display/_helpers/plan_pages.rs create mode 100644 crates/sqlbuild-core/src/plan_display/_helpers/plan_sections.rs create mode 100644 crates/sqlbuild-core/src/plan_display/main/detach_model_diffs.rs create mode 100644 crates/sqlbuild-core/src/plan_display/main/plan_model_diff.rs create mode 100644 crates/sqlbuild-core/src/plan_display/main/plan_section_page.rs create mode 100644 crates/sqlbuild-core/src/plan_display/main/plan_summary.rs create mode 100644 crates/sqlbuild-core/src/plan_display/tests/test_plan_pages.rs create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/_helpers/dependencies.rs create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/_helpers/pools.rs create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/main/check_pools.rs create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/tests/test_pools.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/resources/application.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/resources/changed_builds.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/resources/person_selection.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/current_jobs.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/group_wait_reasons.rs create mode 100644 crates/sqlbuild-orchestration/src/node_states/_helpers/authorized.rs create mode 100644 crates/sqlbuild-orchestration/src/node_states/_helpers/bodies.rs create mode 100644 crates/sqlbuild-orchestration/src/node_states/_helpers/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/node_states/_helpers/reads.rs create mode 100644 crates/sqlbuild-orchestration/src/node_states/_helpers/views.rs create mode 100644 crates/sqlbuild-orchestration/src/node_states/constants.rs create mode 100644 crates/sqlbuild-orchestration/src/node_states/main/list_person_catalog_nodes.rs create mode 100644 crates/sqlbuild-orchestration/src/node_states/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/node_states/main/read_person_node_states.rs create mode 100644 crates/sqlbuild-orchestration/src/node_states/main/report_changed_build.rs create mode 100644 crates/sqlbuild-orchestration/src/node_states/main/start_node_states.rs create mode 100644 crates/sqlbuild-orchestration/src/node_states/main/watch_node_states.rs create mode 100644 crates/sqlbuild-orchestration/src/node_states/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/node_states/models.rs create mode 100644 crates/sqlbuild-orchestration/src/node_states/types.rs rename crates/sqlbuild-orchestration/src/platform/_helpers/{ => health}/event_history.rs (100%) create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/health/mod.rs rename crates/sqlbuild-orchestration/src/platform/_helpers/{ => health}/observations.rs (100%) create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/health/pool_cap_warnings.rs rename crates/sqlbuild-orchestration/src/platform/_helpers/{ => health}/problems.rs (100%) rename crates/sqlbuild-orchestration/src/platform/_helpers/{ => health}/recent_health.rs (100%) rename crates/sqlbuild-orchestration/src/platform/_helpers/{ => location}/connection_tests.rs (100%) rename crates/sqlbuild-orchestration/src/platform/_helpers/{ => location}/liveness.rs (97%) rename crates/sqlbuild-orchestration/src/platform/_helpers/{ => location}/location_heartbeat.rs (100%) create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/location/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/location/published_generation.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/location/reported_generation.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/location/rule_findings.rs rename crates/sqlbuild-orchestration/src/platform/_helpers/{ => location}/work_client.rs (68%) create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/maintenance/mod.rs rename crates/sqlbuild-orchestration/src/platform/_helpers/{ => maintenance}/retention.rs (80%) rename crates/sqlbuild-orchestration/src/platform/_helpers/{ => maintenance}/watchdog.rs (96%) create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/plans/mod.rs rename crates/sqlbuild-orchestration/src/platform/{main/read_person_plan.rs => _helpers/plans/plan_access.rs} (76%) create mode 100644 crates/sqlbuild-orchestration/src/platform/_helpers/plans/plan_diffs.rs rename crates/sqlbuild-orchestration/src/platform/main/{ => health}/location_health.rs (91%) create mode 100644 crates/sqlbuild-orchestration/src/platform/main/health/mod.rs rename crates/sqlbuild-orchestration/src/platform/main/{ => health}/read_person_connection_test.rs (87%) create mode 100644 crates/sqlbuild-orchestration/src/platform/main/health/read_person_location_health.rs rename crates/sqlbuild-orchestration/src/platform/main/{ => health}/report_location_health.rs (59%) rename crates/sqlbuild-orchestration/src/platform/main/{ => health}/report_remote_location_health.rs (85%) create mode 100644 crates/sqlbuild-orchestration/src/platform/main/installation/mod.rs rename crates/sqlbuild-orchestration/src/platform/main/{ => installation}/read_installation.rs (96%) rename crates/sqlbuild-orchestration/src/platform/main/{ => installation}/read_platform_events.rs (100%) rename crates/sqlbuild-orchestration/src/platform/main/{ => installation}/require_instance.rs (100%) rename crates/sqlbuild-orchestration/src/platform/main/{ => installation}/run_maintenance.rs (80%) rename crates/sqlbuild-orchestration/src/platform/main/{ => person_reads}/list_person_platform_events.rs (96%) create mode 100644 crates/sqlbuild-orchestration/src/platform/main/person_reads/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_mirrored_totals.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_plan.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_plan_diff.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_plan_rows.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_plan_summary.rs rename crates/sqlbuild-orchestration/src/platform/main/{ => person_reads}/read_person_problems.rs (96%) delete mode 100644 crates/sqlbuild-orchestration/src/platform/main/read_person_location_health.rs rename crates/sqlbuild-orchestration/src/platform/main/{ => work}/_read_target_work.rs (91%) rename crates/sqlbuild-orchestration/src/platform/main/{ => work}/claim_location_work.rs (96%) rename crates/sqlbuild-orchestration/src/platform/main/{ => work}/complete_location_work.rs (87%) rename crates/sqlbuild-orchestration/src/platform/main/{ => work}/connect_location_work.rs (100%) rename crates/sqlbuild-orchestration/src/platform/main/{ => work}/enqueue_location_work.rs (100%) rename crates/sqlbuild-orchestration/src/platform/main/{ => work}/location_work.rs (90%) create mode 100644 crates/sqlbuild-orchestration/src/platform/main/work/mod.rs rename crates/sqlbuild-orchestration/src/platform/main/{ => work}/read_location_work.rs (100%) rename crates/sqlbuild-orchestration/src/platform/main/{ => work}/request_location_work.rs (92%) create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publications.rs rename crates/sqlbuild-orchestration/src/runs/main/{ => execution}/advance_groups.rs (100%) rename crates/sqlbuild-orchestration/src/runs/main/{ => execution}/change_run.rs (100%) rename crates/sqlbuild-orchestration/src/runs/main/{ => execution}/claim_next_run.rs (100%) rename crates/sqlbuild-orchestration/src/runs/main/{ => execution}/heartbeat_run.rs (100%) create mode 100644 crates/sqlbuild-orchestration/src/runs/main/execution/mod.rs rename crates/sqlbuild-orchestration/src/runs/main/{ => execution}/monitor_runs.rs (100%) rename crates/sqlbuild-orchestration/src/runs/main/{ => execution}/observe_run.rs (100%) rename crates/sqlbuild-orchestration/src/runs/main/{ => execution}/publish_run_output.rs (100%) rename crates/sqlbuild-orchestration/src/runs/main/{ => execution}/publish_run_recall.rs (100%) rename crates/sqlbuild-orchestration/src/runs/main/{ => execution}/run_execution.rs (100%) rename crates/sqlbuild-orchestration/src/runs/main/{ => listings}/list_person_run_nodes.rs (100%) rename crates/sqlbuild-orchestration/src/runs/main/{ => listings}/list_person_runs.rs (100%) rename crates/sqlbuild-orchestration/src/runs/main/{ => listings}/list_run_claims.rs (100%) rename crates/sqlbuild-orchestration/src/runs/main/{ => listings}/list_run_nodes.rs (100%) rename crates/sqlbuild-orchestration/src/runs/main/{ => listings}/list_runs.rs (100%) create mode 100644 crates/sqlbuild-orchestration/src/runs/main/listings/mod.rs rename crates/sqlbuild-orchestration/src/runs/main/{ => requests}/check_adhoc_run.rs (64%) create mode 100644 crates/sqlbuild-orchestration/src/runs/main/requests/evaluate_selection.rs rename crates/sqlbuild-orchestration/src/runs/main/{ => requests}/frozen_job_request.rs (94%) create mode 100644 crates/sqlbuild-orchestration/src/runs/main/requests/mod.rs rename crates/sqlbuild-orchestration/src/runs/main/{ => requests}/queue_run.rs (100%) create mode 100644 crates/sqlbuild-orchestration/src/runs/main/requests/read_selection_vocabulary.rs rename crates/sqlbuild-orchestration/src/runs/main/{ => requests}/request_job_run.rs (100%) rename crates/sqlbuild-orchestration/src/runs/main/{ => requests}/request_person_run.rs (100%) rename crates/sqlbuild-orchestration/src/runs/main/{ => requests}/request_rerun.rs (100%) create mode 100644 crates/sqlbuild-orchestration/src/snapshots/_helpers/connection_settings.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/_helpers/published_listing.rs delete mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/list_person_snapshots.rs rename crates/sqlbuild-orchestration/src/snapshots/main/{ => publication}/_publish_snapshot.rs (100%) create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/publication/mod.rs rename crates/sqlbuild-orchestration/src/snapshots/main/{ => publication}/next_remote_generation.rs (100%) rename crates/sqlbuild-orchestration/src/snapshots/main/{ => publication}/prune_snapshots.rs (100%) rename crates/sqlbuild-orchestration/src/snapshots/main/{ => publication}/publish_development_snapshot.rs (82%) rename crates/sqlbuild-orchestration/src/snapshots/main/{ => publication}/publish_remote_snapshot.rs (92%) rename crates/sqlbuild-orchestration/src/snapshots/main/{ => publication}/publish_snapshot.rs (83%) rename crates/sqlbuild-orchestration/src/snapshots/main/{ => publication}/verify_execution.rs (96%) create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/reads/connection_details.rs rename crates/sqlbuild-orchestration/src/snapshots/main/{ => reads}/current_snapshot.rs (89%) rename crates/sqlbuild-orchestration/src/snapshots/main/{ => reads}/current_snapshot_record.rs (89%) create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/reads/list_person_snapshots.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/reads/listing_read.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/reads/mod.rs rename crates/sqlbuild-orchestration/src/snapshots/main/{ => reads}/node_hashes.rs (100%) rename crates/sqlbuild-orchestration/src/snapshots/main/{ => reads}/read_current_record.rs (100%) create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/reads/read_kind_nodes.rs rename crates/sqlbuild-orchestration/src/snapshots/main/{ => reads}/read_person_node_sql.rs (94%) rename crates/sqlbuild-orchestration/src/snapshots/main/{ => reads}/read_snapshot_documents.rs (100%) rename crates/sqlbuild-orchestration/src/snapshots/main/{ => reads}/read_snapshot_keys.rs (100%) rename crates/sqlbuild-orchestration/src/snapshots/main/{ => reads}/read_snapshot_nodes.rs (100%) create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/write_sets/held_write_set.rs rename crates/sqlbuild-orchestration/src/snapshots/main/{ => write_sets}/holds_on_current_nodes.rs (60%) create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/write_sets/holds_on_nodes.rs rename crates/sqlbuild-orchestration/src/snapshots/main/{ => write_sets}/job_write_set.rs (56%) create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/write_sets/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/write_sets/writer_nodes.rs create mode 100644 crates/sqlbuild-orchestration/src/store/_helpers/writer_gate.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/begin_ingestion.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/prune_state_history.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/preview_state_parts.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/catalog_node.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/definition_evidences.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/destination.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/last_outcome.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/latest_attempts.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/latest_fingerprints.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/mod.rs delete mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/list_person_catalog_nodes.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/node_states/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/state_history/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_ingestion_turns.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_node_states.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_state_history_retention.rs create mode 100644 crates/sqlbuild-orchestration/tests/declarations/test_group_dependencies.rs create mode 100644 crates/sqlbuild-orchestration/tests/platform/test_compile_failure.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_group_declarations.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_group_dependencies.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_group_retries.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_group_round_trips.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_project_limits.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_publication_cache.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_selection.rs create mode 100644 crates/sqlbuild-orchestration/tests/sensors/test_group_members.rs create mode 100644 crates/sqlbuild-orchestration/tests/snapshots/test_connection_details.rs create mode 100644 crates/sqlbuild-orchestration/tests/warehouse_metadata/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/warehouse_metadata/test_contracts.rs create mode 100644 crates/sqlbuild-orchestration/tests/warehouse_metadata/test_preview_state.rs create mode 100644 crates/sqlbuild-orchestration/tests/warehouse_metadata/test_types.rs create mode 100644 crates/sqlbuild-python-base/src/native_host/_helpers/dialect.rs create mode 100644 crates/sqlbuild-python-base/src/native_threads/_helpers/compile_pool.rs create mode 100644 crates/sqlbuild-python-base/src/native_threads/main/pooled.rs create mode 100644 crates/sqlbuild-python-base/src/native_threads/main/spawned.rs create mode 100644 crates/sqlbuild-python-base/src/orchestration_reads/main/dependency_failures.rs delete mode 100644 crates/sqlbuild-python-base/src/warehouse_access/_helpers/synced_copy.rs delete mode 100644 crates/sqlbuild-python-base/src/warehouse_access/main/is_warehouse_access_refused.rs delete mode 100644 crates/sqlbuild-python-base/src/warehouse_access/main/register_synced_copy.rs delete mode 100644 crates/sqlbuild-python-compile/src/input_loops/main/pooled.rs create mode 100644 crates/sqlbuild-python-front/src/command_runtime/main/published_concurrency.rs create mode 100644 crates/sqlbuild-python-front/src/native_host/_helpers/consumption_io.rs create mode 100644 crates/sqlbuild-python-front/src/native_host/types.rs create mode 100644 crates/sqlbuild-python-planner/src/live_reads/_helpers/bounded.rs create mode 100644 crates/sqlbuild-python-planner/src/live_reads/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/live_reads/constants.rs create mode 100644 crates/sqlbuild-python-planner/src/live_reads/main/_live_reads.rs create mode 100644 crates/sqlbuild-python-planner/src/live_reads/main/bound_task.rs create mode 100644 crates/sqlbuild-python-planner/src/live_reads/main/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/live_reads/main/register_live_reads.rs create mode 100644 crates/sqlbuild-python-planner/src/live_reads/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/live_reads/models.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/_helpers/timing/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/_helpers/timing/timings.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/inputs/runtime_dialect.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/inputs/runtime_reads.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/timing/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/timing/record_step.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/timing/timed.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/column_info.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/synced/exception_types.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/synced/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/synced/synced_reads.rs delete mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/adapter_cursor_sql.rs delete mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/bound_task.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/columns_for.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/dialect_cursor_sql.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/list_relations.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/reads/_unknown.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/reads/is_refused_access.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/reads/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/reads/refused_access.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/reads/register_exception_types.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/types.rs create mode 100644 crates/sqlbuild-python-project/src/analysis/main/prefetch_sql_relation_names.rs create mode 100644 crates/sqlbuild-python-project/src/fingerprints/main/selected_set.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/command_runtime/audit_failures.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_driver/rules_at_load.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/location_command/load_rules.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime_plan.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_connection.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/location_command/types.rs create mode 100644 crates/sqlbuild-state/src/state_tables/_helpers/history_sql.rs create mode 100644 crates/sqlbuild-state/src/state_tables/_helpers/mod.rs create mode 100644 crates/sqlbuild-state/src/warehouse_state/_helpers/indexes.rs create mode 100644 crates/sqlbuild-state/src/warehouse_state/_helpers/mod.rs create mode 100644 crates/sqlbuild-state/src/warehouse_state/_helpers/queries.rs create mode 100644 crates/sqlbuild-state/src/warehouse_state/constants.rs create mode 100644 crates/sqlbuild-state/src/warehouse_state/main/mod.rs create mode 100644 crates/sqlbuild-state/src/warehouse_state/main/warehouse_state.rs create mode 100644 crates/sqlbuild-state/src/warehouse_state/mod.rs create mode 100644 crates/sqlbuild-state/src/warehouse_state/models.rs create mode 100644 crates/sqlbuild-state/src/warehouse_state/tests.rs create mode 100644 crates/sqlbuild-state/src/warehouse_state/tests/helpers.rs create mode 100644 crates/sqlbuild-state/src/warehouse_state/tests/test_lookups.rs create mode 100644 crates/sqlbuild-state/src/warehouse_state/tests/test_types.rs create mode 100644 scripts/fensu_policy/rules/planner_reads.py create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/concurrency.md create mode 100644 src/sqlbuild/adapter/contract/main/estimates_by_name.py create mode 100644 src/sqlbuild/adapter/contract/main/estimates_one_at_a_time.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/build/test_ignored_audit_failures.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/plan/conftest.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/plan/test_plan_timings.py create mode 100644 tests/integration/scripts/fensu_policy/test_planner_reads.py create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rules_at_load.py create mode 100644 tests/unit/src/sqlbuild/adapters/test_history_compute.py create mode 100644 tests/unit/src/sqlbuild/compiler/migrations/main/test_named_fingerprints.py create mode 100644 tests/unit/src/sqlbuild/compiler/planner/classes/test_dialect.py create mode 100644 ui/COPY.md create mode 100644 ui/PERF.md create mode 100644 ui/e2e/choice-filter.spec.ts create mode 100644 ui/e2e/compact-shots.spec.ts create mode 100644 ui/e2e/compact.spec.ts create mode 100644 ui/e2e/concurrency.spec.ts create mode 100644 ui/e2e/edges.spec.ts create mode 100644 ui/e2e/ignored-audits.spec.ts create mode 100644 ui/e2e/layout.spec.ts create mode 100644 ui/e2e/perf.spec.ts create mode 100644 ui/e2e/plan-browser.spec.ts create mode 100644 ui/e2e/plan-timings.spec.ts create mode 100644 ui/e2e/routes.spec.ts create mode 100644 ui/e2e/screens.spec.ts create mode 100644 ui/e2e/selection.spec.ts create mode 100644 ui/e2e/small-graph.spec.ts create mode 100644 ui/e2e/zoom.ts create mode 100644 ui/playwright.config.ts create mode 100644 ui/src/lib/api/http-selection.test.ts create mode 100644 ui/src/lib/api/http-selection.ts create mode 100644 ui/src/lib/api/mock/awkward.test.ts create mode 100644 ui/src/lib/api/mock/awkward.ts create mode 100644 ui/src/lib/api/mock/big-group.ts create mode 100644 ui/src/lib/api/mock/large-plan.ts create mode 100644 ui/src/lib/api/mock/large-project.ts create mode 100644 ui/src/lib/api/mock/mock-node-states.test.ts create mode 100644 ui/src/lib/api/mock/operate-graph-selection.test.ts create mode 100644 ui/src/lib/api/mock/plan-rows-log.ts create mode 100644 ui/src/lib/api/node-states.test.ts create mode 100644 ui/src/lib/api/node-states.ts create mode 100644 ui/src/lib/build-area/kind-tabs.test.ts create mode 100644 ui/src/lib/build-area/overview-panels.test.ts create mode 100644 ui/src/lib/build-area/overview-panels.ts create mode 100644 ui/src/lib/components/disclosure-button.svelte create mode 100644 ui/src/lib/components/graph/compact-face.svelte create mode 100644 ui/src/lib/components/graph/compact-tails.test.ts create mode 100644 ui/src/lib/components/graph/compact-tails.ts create mode 100644 ui/src/lib/components/graph/compact-title.ts create mode 100644 ui/src/lib/components/graph/fonts.svelte.ts create mode 100644 ui/src/lib/components/graph/graph-canvas.svelte create mode 100644 ui/src/lib/components/graph/graph-toolbar.svelte create mode 100644 ui/src/lib/components/graph/no-raw-graph.test.ts create mode 100644 ui/src/lib/components/help-tip.svelte create mode 100644 ui/src/lib/components/lineage/inspector-head.svelte create mode 100644 ui/src/lib/components/lineage/stub-node.svelte create mode 100644 ui/src/lib/components/lineage/too-many-card.svelte create mode 100644 ui/src/lib/components/lineage/view-summary.svelte create mode 100644 ui/src/lib/components/list/choice-list.svelte create mode 100644 ui/src/lib/components/list/data-table.svelte create mode 100644 ui/src/lib/components/list/no-raw-tables.test.ts create mode 100644 ui/src/lib/components/list/table-columns.test.ts create mode 100644 ui/src/lib/components/list/table-columns.ts create mode 100644 ui/src/lib/components/list/virtual-rows.svelte.ts create mode 100644 ui/src/lib/components/list/window-range.test.ts create mode 100644 ui/src/lib/components/list/window-range.ts create mode 100644 ui/src/lib/components/page-body.svelte create mode 100644 ui/src/lib/components/page-body.test.ts create mode 100644 ui/src/lib/components/platform/syncing-note.svelte create mode 100644 ui/src/lib/components/refresh-button.svelte create mode 100644 ui/src/lib/components/selection/selection-bar.svelte create mode 100644 ui/src/lib/components/selection/selection-chip.svelte create mode 100644 ui/src/lib/components/selection/selection-chips.svelte create mode 100644 ui/src/lib/components/selection/selection-field.svelte create mode 100644 ui/src/lib/components/selection/selector-icon.svelte create mode 100644 ui/src/lib/components/ui/button/button-size.test.ts create mode 100644 ui/src/lib/components/ui/button/no-adhoc-buttons.test.ts create mode 100644 ui/src/lib/copy-style/baseline.txt create mode 100644 ui/src/lib/copy-style/check.test.ts create mode 100644 ui/src/lib/copy-style/check.ts create mode 100644 ui/src/lib/copy-style/copy-style.test.ts create mode 100644 ui/src/lib/format/plural.test.ts create mode 100644 ui/src/lib/format/plural.ts create mode 100644 ui/src/lib/lineage/count-edge.svelte create mode 100644 ui/src/lib/lineage/details-panel.test.ts create mode 100644 ui/src/lib/lineage/details-panel.ts create mode 100644 ui/src/lib/lineage/edge-routes.test.ts create mode 100644 ui/src/lib/lineage/edge-routes.ts create mode 100644 ui/src/lib/lineage/fit-controls.svelte create mode 100644 ui/src/lib/lineage/fit.test.ts create mode 100644 ui/src/lib/lineage/fit.ts create mode 100644 ui/src/lib/lineage/folder-graph.test.ts create mode 100644 ui/src/lib/lineage/folder-graph.ts create mode 100644 ui/src/lib/lineage/folder-selection.test.ts create mode 100644 ui/src/lib/lineage/folder-selection.ts create mode 100644 ui/src/lib/lineage/hit-area.test.ts create mode 100644 ui/src/lib/lineage/hit-area.ts create mode 100644 ui/src/lib/lineage/member-box-node.svelte create mode 100644 ui/src/lib/lineage/member-expand.ts create mode 100644 ui/src/lib/lineage/member-folders.test.ts create mode 100644 ui/src/lib/lineage/member-folders.ts create mode 100644 ui/src/lib/lineage/member-graph.svelte create mode 100644 ui/src/lib/lineage/member-graph.test.ts create mode 100644 ui/src/lib/lineage/member-graph.ts create mode 100644 ui/src/lib/lineage/member-node.svelte create mode 100644 ui/src/lib/lineage/status-line.test.ts create mode 100644 ui/src/lib/lineage/status-line.ts create mode 100644 ui/src/lib/lineage/trunk-path.test.ts create mode 100644 ui/src/lib/lineage/trunk-path.ts create mode 100644 ui/src/lib/lineage/view-summary.test.ts create mode 100644 ui/src/lib/lineage/view-summary.ts create mode 100644 ui/src/lib/lineage/visible-edges.test.ts create mode 100644 ui/src/lib/lineage/visible-edges.ts create mode 100644 ui/src/lib/list/choice.test.ts create mode 100644 ui/src/lib/list/choice.ts create mode 100644 ui/src/lib/list/count-by.test.ts create mode 100644 ui/src/lib/list/count-by.ts create mode 100644 ui/src/lib/list/no-filter-counts.test.ts create mode 100644 ui/src/lib/operate/labels.test.ts create mode 100644 ui/src/lib/platform/connections.test.ts create mode 100644 ui/src/lib/platform/connections.ts create mode 100644 ui/src/lib/run/ignored-audits.test.ts create mode 100644 ui/src/lib/run/ignored-audits.ts create mode 100644 ui/src/lib/run/plan-browser.svelte create mode 100644 ui/src/lib/run/plan-index.test.ts create mode 100644 ui/src/lib/run/plan-index.ts delete mode 100644 ui/src/lib/run/plan-report.svelte create mode 100644 ui/src/lib/run/plan-timings.svelte create mode 100644 ui/src/lib/run/plan-timings.test.ts create mode 100644 ui/src/lib/run/plan-timings.ts create mode 100644 ui/src/lib/selection/command.ts create mode 100644 ui/src/lib/selection/field.ts create mode 100644 ui/src/lib/selection/folders.test.ts create mode 100644 ui/src/lib/selection/folders.ts create mode 100644 ui/src/lib/selection/picker.ts create mode 100644 ui/src/lib/selection/reading.ts create mode 100644 ui/src/lib/selection/selection.test.ts create mode 100644 ui/src/lib/selection/suggest.ts create mode 100644 ui/src/lib/selection/syntax.ts create mode 100644 ui/src/lib/state/catalog-feed.test.ts create mode 100644 ui/src/lib/state/catalog-feed.ts create mode 100644 ui/src/lib/state/catalog.svelte.ts create mode 100644 ui/src/lib/state/details-panel.svelte.ts create mode 100644 ui/src/lib/state/selection.svelte.ts create mode 100644 ui/src/routes/changes/utils.test.ts create mode 100644 ui/src/routes/checks/utils.test.ts create mode 100644 ui/src/routes/jobs/utils.test.ts create mode 100644 ui/src/routes/jobs/utils.ts create mode 100644 ui/src/routes/providers/[name]/used-by-graph.svelte create mode 100644 ui/src/routes/providers/utils.test.ts create mode 100644 ui/src/routes/runs/[id]/ignored-audits-panel.svelte create mode 100644 ui/src/routes/runs/[id]/utils/members.test.ts create mode 100644 ui/src/routes/runs/[id]/utils/members.ts create mode 100644 ui/src/routes/runs/utils/status-line.test.ts create mode 100644 ui/src/routes/triggers/set-in-ui-pin.svelte create mode 100644 ui/src/routes/triggers/utils.test.ts create mode 100644 ui/src/routes/triggers/utils.ts create mode 100644 website/src/content/docs/docs/concepts/concurrency.mdx diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 34416bc183..2f67639bcd 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -1430,6 +1430,49 @@ jobs: run: npm test - name: Build the static bundle sqb server embeds run: npm run build + - name: Install the layout test's browser + run: npx playwright install --with-deps chromium + - name: Layout test (every page with awkward mock data at 1440 and 1920 px) + run: npx playwright test --project=layout + + ui_timings: + # The timing guards in ui/e2e/perf.spec.ts, with the CPU slowed 6x. They go red over budget and + # report their numbers, but stay out of Verify's needs: a slow hosted runner must not block a merge. + name: UI work timings (not required) + needs: changes + if: needs.changes.outputs.ui == 'true' + runs-on: ubuntu-24.04 + defaults: + run: + working-directory: ui + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + cache-dependency-path: ui/package-lock.json + - name: Install dependencies + run: npm ci + - name: Build the static bundle sqb server embeds + run: npm run build + - name: Install the timing test's browser + run: npx playwright install --with-deps chromium + - name: Time the slow paths (graph gestures, Checks reopen, plan sections) + shell: bash + run: | + set -o pipefail + npx playwright test --project=perf --no-deps --reporter=list 2>&1 | tee perf.log + - name: Report the timings + if: always() + shell: bash + run: | + { + echo "### UI work timings" + echo '```' + grep -E 'CPU slowed|second visit' perf.log || echo "No timings were recorded." + echo '```' + } >> "$GITHUB_STEP_SUMMARY" verify: # The required Verify of a Release Please pull request is the commit status that the trusted diff --git a/Cargo.lock b/Cargo.lock index f26f85692f..69d1b839c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4474,6 +4474,7 @@ dependencies = [ "chrono", "mimalloc", "pyo3", + "rayon", "rusqlite", "rustc-hash", "serde", @@ -4575,6 +4576,7 @@ dependencies = [ name = "sqlbuild-python-planner" version = "0.0.0" dependencies = [ + "chrono", "pyo3", "serde_json", "sqlbuild-analysis", @@ -4726,6 +4728,8 @@ dependencies = [ name = "sqlbuild-state" version = "0.0.0" dependencies = [ + "chrono", + "serde", "serde_json", "sha2 0.10.9", "sqlbuild-core", diff --git a/Makefile b/Makefile index 0a7d58ea32..d45aa2ba11 100644 --- a/Makefile +++ b/Makefile @@ -139,6 +139,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_execution_limits.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_expression_sources.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_ignored_audit_failures.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_lifecycle_commands.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_manifest_artifact_gating.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_materialization_type_change.py \ @@ -340,7 +341,7 @@ test-e2e-performance: @log="/tmp/opencode/test-e2e-performance-$$(date +%Y%m%d-%H%M%S).log"; \ echo "Logging to $$log"; \ env PYTHONUNBUFFERED=1 SQLBUILD_CONCURRENCY=$(SQLBUILD_CONCURRENCY) uv run pytest tests/e2e \ - -m "performance and not cold_compile_performance and not cache_compile_performance and not real_warehouse and not dbt" \ + -m "performance and not cold_compile_performance and not cache_compile_performance and not preview_performance and not real_warehouse and not dbt" \ -vv --log-cli-level=INFO --color=yes 2>&1 | tee "$$log"; \ status=$${PIPESTATUS[0]}; \ echo "TEST_E2E_PERFORMANCE_EXIT=$$status (log: $$log)" | tee -a "$$log"; \ @@ -370,13 +371,17 @@ test-e2e-dense-compile-performance: -n auto --dist loadfile \ -m cold_compile_performance -k "models_$(SQLBUILD_BENCHMARK_MODELS)_" \ -vv -rP --log-level=INFO --log-cli-level=INFO --color=yes -# DuckShop 10K runs after the dense guards, alone, so its compile never overlaps theirs. +# DuckShop 10K runs after the dense guards, alone, so its compile never overlaps theirs; then its +# plan preview guard, which serves the project on a location. if [ "$(SQLBUILD_BENCHMARK_MODELS)" = "10000" ]; then \ env PYTHONUNBUFFERED=1 uv run pytest benchmarks/duckshop-10k/tests -q --color=yes && \ env PYTHONUNBUFFERED=1 SQLBUILD_CONCURRENCY=4 uv run pytest \ tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_10k_compile.py \ -m cold_compile_performance -k "models_$(SQLBUILD_BENCHMARK_MODELS)_" \ - -vv -rP --log-level=INFO --log-cli-level=INFO --color=yes; \ + -vv -rP --log-level=INFO --log-cli-level=INFO --color=yes && \ + env PYTHONUNBUFFERED=1 uv run pytest \ + tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py \ + -m preview_performance -vv -rP --log-level=INFO --log-cli-level=INFO --color=yes; \ fi test-e2e-cache-compile-performance: diff --git a/benchmarks/duckshop-10k/README.md b/benchmarks/duckshop-10k/README.md index c90c334e42..d9d990e1c2 100644 --- a/benchmarks/duckshop-10k/README.md +++ b/benchmarks/duckshop-10k/README.md @@ -7,7 +7,7 @@ marketplace partners, carriers, warehouses, retail stores, suppliers, payment ga networks, messaging, affiliates, support tools, ledgers, wholesale desks, returns portals and a product catalogue. Every name and value is invented. -Only the generator, its tests and `checksum.txt` are committed. The generated project is written +Only the generators, their tests and `checksum.txt` are committed. The generated project is written to a directory you choose. ## Usage @@ -64,3 +64,26 @@ because their results depend on the engine or the session. CI runs both in the 10,000-model cell of the dense compile job, under its 4 GiB memory limit, after the dense guards have finished, so the DuckShop compile never shares the limit with them. + +## Synced warehouse state + +`synced_state.py` writes the warehouse state a location syncs after building the project: about 13K +relations, a fingerprint and two `--changed` executions per built model, an execution per seed and +cursor observations, as the metadata pages a location sends the server. No warehouse runs the +project, so the hashes come from the planner itself: a `SQLBUILD_PLANNER_CAPTURE_DIR` capture of +one plan of the published project gives each model's identity metadata and query, and an unedited +model then plans as unchanged. `--changed-share` of the built models record a definition one line +older, and the models outside `--built-share` were never built. + +```bash +python benchmarks/duckshop-10k/synced_state.py control.sqlite3 001-plan_output.json state.json \ + --built-share 0.95 +``` + +`tests/e2e/.../plan/test_duckshop_10k_preview_performance.py` serves the project on a location, +mirrors that state into the server, and times a warm `--changed` plan preview of every node, with +every model built and with one in twenty unbuilt. It fails when the warm preview takes more than 0.45 +of the same location's publish, a linear 10K compile on the same machine, so a lookup over every +model or relation shows however fast the runner is; it also checks that only the edited and unbuilt +models report their own change, and times the ad-hoc command check. It runs in the 10,000-model +cell after the DuckShop compile guard. diff --git a/benchmarks/duckshop-10k/synced_state.py b/benchmarks/duckshop-10k/synced_state.py new file mode 100644 index 0000000000..383ccd9185 --- /dev/null +++ b/benchmarks/duckshop-10k/synced_state.py @@ -0,0 +1,374 @@ +"""The warehouse state a location syncs after building the DuckShop 10K project, as mirror batches. + +usage: synced_state.py + [--built-share 1.0] [--changed-share 0.01] [--seed 20261116] + +The plan preview guard plans the generated project from a synced copy of a warehouse that built it. +No warehouse runs the project, so this writes what one would have recorded: a relation, a +fingerprint and two `--changed` executions per built model, an execution per seed, cursor +observations of incremental models, the source tables and a few stray relations. + +The fingerprints and executions carry the hashes the planner computes for the project as published, +read from a `SQLBUILD_PLANNER_CAPTURE_DIR` capture of one plan of it, so an unedited model plans as +unchanged and `--changed` finds only the edits: `--changed-share` of the built models record a +definition one line older, and the models outside `--built-share` were never built. +""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import json +import random +import sqlite3 +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +STATE_DATABASE: str = "DUCKSHOP_10K" +DIALECT: str = "snowflake" +STATE_SCHEMA: str = "DEV" +SYNCED_AT: datetime = datetime(2026, 11, 16, 6, 0, tzinfo=UTC) +NODE_KINDS: tuple[str, ...] = ("model", "source", "seed", "udf", "table_fn") +FUNCTION_KINDS: frozenset[str] = frozenset({"udf", "table_fn"}) +STATE_TABLES: tuple[str, ...] = ( + "_sqlbuild_fingerprints", + "_sqlbuild_consumption", + "_sqlbuild_observations", + "_sqlbuild_microbatches", + "_sqlbuild_source_freshness", +) +STRAY_RELATIONS: int = 150 +EXECUTIONS: int = 2 +RUN_IDS: tuple[str, ...] = tuple( + f"01a1d{index:03x}-0000-7000-8000-{index:012x}" for index in range(40) +) + + +def iso(moment: datetime) -> str: + return moment.isoformat().replace("+00:00", "Z") + + +def digest(*parts: str) -> str: + return hashlib.sha256("\x1f".join(parts).encode()).hexdigest() + + +def canonical(value: dict[str, Any]) -> str: + """JSON with sorted keys and no spaces, as the planner hashes identity metadata.""" + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def published_nodes(control: Path) -> dict[str, dict[str, Any]]: + """Every model, source, seed and function of the current snapshot, by node key.""" + with sqlite3.connect(control) as connection: + rows = connection.execute( + "SELECT n.node_key, d.body FROM sqb_snapshot_nodes n " + "JOIN sqb_current_snapshots c ON c.snapshot_id=n.snapshot_id " + "JOIN sqb_node_definitions d ON d.node_hash=n.node_hash " + f"WHERE n.node_kind IN ({','.join('?' * len(NODE_KINDS))})", + NODE_KINDS, + ).fetchall() + return {key: json.loads(gzip.decompress(body))["definition"] for key, body in rows} + + +def planned_identities(capture: Path) -> dict[str, dict[str, str]]: + """Each planned model's and seed's fingerprint fields and `--changed` version hash, by node key. + + A model's `--changed` version hash is its local identity hash: the query fingerprint and the + identity metadata without the declared columns, which change detection adds back. A plan of a + project never built records no query fingerprint, so it is computed as the planner does. + """ + from sqlbuild._native import compute_query_hash + + document: dict[str, Any] = json.loads(capture.read_text()) + shared: dict[str, Any] = document["__shared_nodes__"] + + def resolved(value: Any) -> Any: + while isinstance(value, dict) and set(value) == {"__shared__"}: + value = shared[value["__shared__"]] + return value + + found: dict[str, dict[str, str]] = {} + pending: list[Any] = [document] + while pending: + value: Any = resolved(pending.pop()) + if isinstance(value, list): + pending.extend(value) + continue + if not isinstance(value, dict): + continue + entry_type: str = str(value.get("__type__", "")) + if entry_type.endswith((":ModelPlanEntry", ":SeedPlanEntry")) and "name" in value: + kind: str = "model" if entry_type.endswith(":ModelPlanEntry") else "seed" + metadata_json: str = resolved(value["fingerprint_metadata_json"]) + definition: str = str( + resolved( + value.get( + "fingerprint_query_sql" if kind == "model" else "fingerprint_definition" + ) + ) + or "" + ) + definition_hash: str = str(resolved(value.get("fingerprint_definition_hash")) or "") + if kind == "model" and not definition_hash: + definition_hash = compute_query_hash(query_sql=definition, dialect=DIALECT) + identity: dict[str, Any] = json.loads(metadata_json) + identity.pop("declared_columns_hash", None) + local: str = hashlib.sha256( + f"{definition_hash}\n{canonical(identity)}".encode() + ).hexdigest() + version_hash: str = resolved(value["fingerprint_version_hash"]) + definition_hash = definition_hash or version_hash + found[f"{kind}:{resolved(value['name'])}"] = { + "definition_hash": definition_hash, + "version_hash": version_hash, + "metadata_json": metadata_json, + "definition": definition, + "local_hash": local if kind == "model" else version_hash, + } + continue + pending.extend(value.values()) + return found + + +def inputs(node: dict[str, Any], nodes: dict[str, dict[str, Any]]) -> list[tuple[str, str]]: + """The inputs `--changed` reads for a node: upstream nodes, sources, and through functions.""" + found: list[tuple[str, str]] = [] + seen: set[str] = set() + pending: list[str] = list(node["deps"]) + while pending: + dep: str = pending.pop(0) + if dep in seen: + continue + seen.add(dep) + kind, name = dep.split(":", 1) + if kind in ("model", "seed"): + found.append(("node", dep)) + elif kind == "source": + found.append(("source", name)) + elif kind in FUNCTION_KINDS and dep in nodes: + pending.extend(nodes[dep]["deps"]) + else: + found.append(("opaque", dep)) + return sorted(found) + + +def upstream_first(keys: list[str], nodes: dict[str, dict[str, Any]]) -> list[str]: + """`keys` with every node after the built nodes it reads, so each execution names theirs.""" + wanted: set[str] = set(keys) + ordered: list[str] = [] + seen: set[str] = set() + for start in sorted(keys): + stack: list[tuple[str, bool]] = [(start, False)] + while stack: + key, done = stack.pop() + if done: + ordered.append(key) + continue + if key in seen: + continue + seen.add(key) + stack.append((key, True)) + stack.extend( + (dep, False) for dep in nodes[key]["deps"] if dep in wanted and dep not in seen + ) + return ordered + + +def relation( + name: str, relation_type: str, columns: list[dict[str, Any]], location: tuple[str, str] +) -> dict[str, Any]: + database, schema = location + return { + "state_database": database, + "state_schema": schema, + "name": name, + "relation_type": relation_type, + "columns": [ + {"name": column["name"].upper(), "data_type": column["type"] or "VARCHAR"} + for column in columns + ], + } + + +def mirror_batches( + nodes: dict[str, dict[str, Any]], + identities: dict[str, dict[str, str]], + *, + built_share: float, + changed_share: float, + seed: int, +) -> dict[str, list[dict[str, Any]]]: + """The catalog, fingerprint, consumption and observation records of the built warehouse.""" + rng: random.Random = random.Random(seed) + state: tuple[str, str] = (STATE_DATABASE, STATE_SCHEMA) + models: list[str] = sorted(key for key in nodes if key.startswith("model:")) + built: list[str] = [key for key in models if rng.random() < built_share] + seeds: list[str] = sorted(key for key in nodes if key.startswith("seed:")) + catalog: list[dict[str, Any]] = [] + fingerprints: list[dict[str, Any]] = [] + consumption: list[dict[str, Any]] = [] + observations: list[dict[str, Any]] = [] + for key in sorted(nodes): + kind, name = key.split(":", 1) + node: dict[str, Any] = nodes[key] + if kind == "source": + location = (node["database"] or STATE_DATABASE, node["schema"] or STATE_SCHEMA) + catalog.append( + relation((node["table"] or name).upper(), "TABLE", node["columns"], location) + ) + elif kind == "seed": + catalog.append(relation(name.upper(), "TABLE", node["columns"], state)) + latest_execution: dict[str, str] = {} + for position, key in enumerate([*seeds, *upstream_first(built, nodes)]): + kind, name = key.split(":", 1) + node = nodes[key] + identity: dict[str, str] = identities[key] + run_id: str = RUN_IDS[position % len(RUN_IDS)] + built_at: datetime = SYNCED_AT - timedelta(hours=2) + timedelta(milliseconds=position * 50) + edited: bool = kind == "model" and rng.random() < changed_share + local_hash: str = digest("edited", name) if edited else identity["local_hash"] + fingerprints.append( + { + "event_id": digest("event", key)[:32], + "state_database": STATE_DATABASE, + "state_schema": STATE_SCHEMA, + "node_type": kind, + "node_name": name, + "target_database": STATE_DATABASE, + "target_schema": STATE_SCHEMA, + "target_name": name, + "run_id": run_id, + "definition_hash": digest("edited definition", name) + if edited + else identity["definition_hash"], + "version_hash": digest("edited version", name) + if edited + else identity["version_hash"], + "schema_fingerprint": digest( + "schema", name, *[column["name"] for column in node["columns"]] + ), + "definition": identity["definition"].replace("SELECT", "SELECT DISTINCT", 1) + if edited + else identity["definition"], + "metadata_json": identity["metadata_json"], + "ts": iso(built_at), + } + ) + if kind == "model": + materialization: str = node["materialization"] + catalog.append( + relation( + name.upper(), + "VIEW" if materialization == "view" else "TABLE", + node["columns"], + state, + ) + ) + if materialization == "incremental": + observations.append( + { + "state_database": STATE_DATABASE, + "state_schema": STATE_SCHEMA, + "observation_id": digest("cursor", name)[:32], + "kind": "cursor_max", + "node": name, + "cursor_type": node["config"].get("cursor_type", "timestamp"), + "value": iso(built_at - timedelta(minutes=5)), + "run_id": run_id, + "observed_at": iso(built_at), + } + ) + read: list[tuple[str, str]] = inputs(node, nodes) if kind == "model" else [] + for execution in reversed(range(EXECUTIONS)): + recorded_at: datetime = built_at - timedelta(days=execution) + row: dict[str, Any] = { + "state_database": STATE_DATABASE, + "state_schema": STATE_SCHEMA, + "execution_id": digest("execution", key, str(execution))[:32], + "run_id": run_id, + "node_type": kind, + "node_name": name, + "target_database": STATE_DATABASE, + "target_schema": STATE_SCHEMA, + "target_name": name, + "version_hash": local_hash, + "input_count": len(read), + } + consumption.append( + row + | { + "phase": "started", + "input_kind": None, + "input_name": None, + "input_version": None, + "recorded_at": iso(recorded_at), + } + ) + succeeded_at: str = iso(recorded_at + timedelta(milliseconds=20)) + rows: list[tuple[str | None, str | None]] = [(kind, name) for kind, name in read] + if not rows: + rows.append((None, None)) + for input_kind, input_name in rows: + version: str | None = None + if input_kind is not None and input_name is not None: + version = ( + latest_execution.get(input_name, "unknown") + if input_kind == "node" + else "unknown" + ) + consumption.append( + row + | { + "phase": "succeeded", + "input_kind": input_kind, + "input_name": input_name, + "input_version": version, + "recorded_at": succeeded_at, + } + ) + latest_execution[key] = digest("execution", key, "0")[:32] + for table in STATE_TABLES: + catalog.append(relation(table, "TABLE", [{"name": "run_id", "type": "VARCHAR"}], state)) + for index in range(STRAY_RELATIONS): + catalog.append( + relation( + f"LEGACY_REPORT_{index:04d}", + "VIEW", + [{"name": "id", "type": "NUMBER(38,0)"}, {"name": "value", "type": "VARCHAR"}], + state, + ) + ) + return { + "catalog": catalog, + "fingerprints": fingerprints, + "consumption": consumption, + "observations": observations, + } + + +def main() -> None: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("control", type=Path) + parser.add_argument("capture", type=Path) + parser.add_argument("out", type=Path) + parser.add_argument("--built-share", type=float, default=1.0) + parser.add_argument("--changed-share", type=float, default=0.01) + parser.add_argument("--seed", type=int, default=20261116) + options = parser.parse_args() + batches = mirror_batches( + published_nodes(options.control), + planned_identities(options.capture), + built_share=options.built_share, + changed_share=options.changed_share, + seed=options.seed, + ) + options.out.write_text(json.dumps(batches)) + print(json.dumps({kind: len(records) for kind, records in batches.items()})) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/duckshop-10k/tests/test_synced_state.py b/benchmarks/duckshop-10k/tests/test_synced_state.py new file mode 100644 index 0000000000..6deb60fd37 --- /dev/null +++ b/benchmarks/duckshop-10k/tests/test_synced_state.py @@ -0,0 +1,137 @@ +"""Offline checks of the synced warehouse state the plan preview guard mirrors.""" + +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path +from typing import Any + +import pytest + +HERE = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(HERE)) + +from synced_state import inputs, mirror_batches, planned_identities # noqa: E402 + +_METADATA: str = '{"declared_columns_hash":"columns","model_name":"orders"}' + +NODES: dict[str, dict[str, Any]] = { + "seed:countries": {"deps": [], "columns": [{"name": "code", "type": "VARCHAR"}]}, + "source:raw_orders": { + "deps": [], + "database": None, + "schema": "RAW", + "table": "orders", + "columns": [{"name": "id", "type": "NUMBER"}], + }, + "udf:to_cents": {"deps": ["seed:countries"]}, + "model:stg_orders": { + "deps": ["source:raw_orders"], + "materialization": "view", + "config": {}, + "columns": [{"name": "id", "type": "NUMBER"}], + }, + "model:orders": { + "deps": ["model:stg_orders", "udf:to_cents"], + "materialization": "incremental", + "config": {"cursor_type": "timestamp"}, + "columns": [{"name": "id", "type": "NUMBER"}], + }, +} + + +def identity(name: str) -> dict[str, str]: + return { + "definition_hash": f"definition-{name}", + "version_hash": f"version-{name}", + "metadata_json": "{}", + "definition": f"SELECT 1 AS {name}", + "local_hash": f"local-{name}", + } + + +IDENTITIES: dict[str, dict[str, str]] = { + key: identity(key.split(":", 1)[1]) + for key in ("seed:countries", "model:stg_orders", "model:orders") +} + + +def test_given_a_capture_when_reading_identities_then_a_model_hashes_metadata_without_its_columns( + tmp_path: Path, +) -> None: + capture: Path = tmp_path / "001-plan_output.json" + capture.write_text( + json.dumps( + { + "__shared_nodes__": {"metadata": _METADATA}, + "__capture__": [ + { + "__type__": "sqlbuild.compiler.planner.models:ModelPlanEntry", + "name": "orders", + "fingerprint_metadata_json": {"__shared__": "metadata"}, + "fingerprint_definition_hash": "query", + "fingerprint_version_hash": "version", + "fingerprint_query_sql": "SELECT 1", + } + ], + } + ) + ) + + found: dict[str, dict[str, str]] = planned_identities(capture) + + assert ( + found["model:orders"]["local_hash"] + == hashlib.sha256(b'query\n{"model_name":"orders"}').hexdigest() + ) + assert found["model:orders"]["metadata_json"] == _METADATA + + +@pytest.mark.parametrize( + ("description", "node", "expected_inputs"), + [ + ("a source and an upstream model", "model:stg_orders", [("source", "raw_orders")]), + ( + "a function's own inputs, not the function", + "model:orders", + [("node", "model:stg_orders"), ("node", "seed:countries")], + ), + ], +) +def test_given_a_node_when_listing_its_inputs_then_functions_expand_to_what_they_read( + description: str, node: str, expected_inputs: list[tuple[str, str]] +) -> None: + assert inputs(NODES[node], NODES) == expected_inputs, description + + +@pytest.mark.parametrize( + ("description", "built_share", "expected_fingerprints", "expected_relations"), + [ + ("every model built", 1.0, ["countries", "orders", "stg_orders"], 159), + ("no model built", 0.0, ["countries"], 157), + ], +) +def test_given_built_models_when_mirroring_then_each_executes_after_what_it_reads_with_its_hashes( + description: str, built_share: float, expected_fingerprints: list[str], expected_relations: int +) -> None: + batches = mirror_batches(NODES, IDENTITIES, built_share=built_share, changed_share=0.0, seed=1) + + succeeded: list[dict[str, Any]] = [ + row for row in batches["consumption"] if row["phase"] == "succeeded" + ] + executions: dict[str, str] = {row["node_name"]: row["execution_id"] for row in succeeded} + reads: dict[tuple[str, str | None], str | None] = { + (row["node_name"], row["input_name"]): row["input_version"] for row in succeeded + } + assert sorted(row["node_name"] for row in batches["fingerprints"]) == expected_fingerprints, ( + description + ) + assert len(batches["catalog"]) == expected_relations, description + assert {row["version_hash"] for row in succeeded} == { + f"local-{name}" for name in expected_fingerprints + } + if built_share: + assert reads[("orders", "model:stg_orders")] == executions["stg_orders"], description + assert reads[("orders", "seed:countries")] == executions["countries"], description diff --git a/benchmarks/duckshop/edge_cases/orchestration/check_identity.json b/benchmarks/duckshop/edge_cases/orchestration/check_identity.json index 8ba0ca5e42..fca1c77bfb 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/check_identity.json +++ b/benchmarks/duckshop/edge_cases/orchestration/check_identity.json @@ -42,6 +42,7 @@ "result_id": "customers-not-null-1", "occurred_at": "2026-02-03T05:05:06Z", "outcome": "error", + "metadata_json": "{\"ignored\": true}", "violation_count": null, "executed_sql": "SELECT customer_id FROM customers WHERE customer_id IS NULL", "execution_error": "Binder Error: Referenced column \"customer_id\" not found\n\nLINE 1: SELECT customer_id FROM customers WHERE email = 'private@example.com'" diff --git a/benchmarks/duckshop/edge_cases/orchestration/group_shared_limit.json b/benchmarks/duckshop/edge_cases/orchestration/group_shared_limit.json index ce32db3db1..cbca3111da 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/group_shared_limit.json +++ b/benchmarks/duckshop/edge_cases/orchestration/group_shared_limit.json @@ -1,8 +1,8 @@ { - "description": "A daily group whose two load jobs share a warehouse limit of one. The group holds the warehouse slot and its children use it one at a time.", + "description": "A daily group whose two load jobs share a warehouse pool of one slot. The group holds the warehouse slot and its children use it one at a time.", "jobs": { - "load_customers": {"description": "Load customers", "selection": ["customers"], "concurrency_groups": ["warehouse"]}, - "load_inventory": {"description": "Load inventory", "selection": ["inventory"], "concurrency_groups": ["warehouse"]} + "load_customers": {"description": "Load customers", "selection": ["customers"], "pools": {"warehouse": {"amount": 1, "limit": 1}}}, + "load_inventory": {"description": "Load inventory", "selection": ["inventory"], "pools": {"warehouse": {"amount": 1, "limit": 1}}} }, "groups": { "daily": {"description": "Daily loads", "selection": ["customers", "inventory"], "jobs": {"load_customers": [], "load_inventory": []}} diff --git a/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders/jobs/annual_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders/jobs/annual_orders_job.toml index 7f3e260dae..8b7f9cded8 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders/jobs/annual_orders_job.toml +++ b/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders/jobs/annual_orders_job.toml @@ -2,5 +2,5 @@ name = "annual_orders_job" description = "Build orders under a shared concurrency limit." select = "orders" -concurrency_groups = ["orders_limit"] +pools = { orders_limit = 1 } schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" } diff --git a/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_inventory_job_group.toml b/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_inventory_job_group.toml index 91d5adf5bb..a602bf4a0a 100644 --- a/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_inventory_job_group.toml +++ b/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_inventory_job_group.toml @@ -1,4 +1,4 @@ name = "edge_inventory_job_group" description = "Complete inventory without reserving its child's product limit." jobs = ["edge_snapshot_inventory_job"] -concurrency_groups = ["edge_inventory_limit"] +pools = { edge_inventory_limit = 1 } diff --git a/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_products_job_group.toml b/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_products_job_group.toml index 8177a64029..95a3d2c1f2 100644 --- a/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_products_job_group.toml +++ b/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_products_job_group.toml @@ -1,4 +1,4 @@ name = "edge_products_job_group" description = "Wait for safe admission when both installation limits are one." jobs = ["edge_snapshot_products_job"] -concurrency_groups = ["edge_products_limit"] +pools = { edge_products_limit = 1 } diff --git a/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_inventory_job.toml b/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_inventory_job.toml index 4b624fe69f..fdc4768949 100644 --- a/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_inventory_job.toml +++ b/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_inventory_job.toml @@ -1,4 +1,4 @@ name = "edge_snapshot_inventory_job" description = "Exercise a child limit independent of its parent." select = "tag:edge_case" -concurrency_groups = ["edge_products_limit"] +pools = { edge_products_limit = 1 } diff --git a/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_products_job.toml b/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_products_job.toml index 68c81fe2ee..40a52a5013 100644 --- a/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_products_job.toml +++ b/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_products_job.toml @@ -1,4 +1,4 @@ name = "edge_snapshot_products_job" description = "Exercise the opposing parent and child acquisition order." select = "tag:edge_case" -concurrency_groups = ["edge_inventory_limit"] +pools = { edge_inventory_limit = 1 } diff --git a/benchmarks/duckshop/project/sqlbuild_project.toml b/benchmarks/duckshop/project/sqlbuild_project.toml index 4b9a37a0bc..8304ad9919 100644 --- a/benchmarks/duckshop/project/sqlbuild_project.toml +++ b/benchmarks/duckshop/project/sqlbuild_project.toml @@ -60,6 +60,13 @@ database = "DUCKSHOP" schema = "EDGE" # <<< edge-case area +# >>> edge-case area (scripts/core_project.py leaves this block out of the core copy) +# The edge-case job groups each take the other's pool, so admission must hold both. +[pools] +edge_inventory_limit = 1 +edge_products_limit = 1 +# <<< edge-case area + [settings] default_audit_severity = "error" unit_test_max_real_models = 6 diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index ef9ad3ebf4..6a1859dfb7 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -530,6 +530,38 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/locations/{location_id}/targets/{target}/mirrored-totals": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPersonMirroredTotals"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/locations/{location_id}/targets/{target}/node-states": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPersonNodeStates"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/locations/{location_id}/targets/{target}/node-status": { parameters: { query?: never; @@ -642,6 +674,54 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/locations/{location_id}/targets/{target}/plans/{request_id}/diff": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPlanDiff"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/locations/{location_id}/targets/{target}/plans/{request_id}/rows": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPlanRows"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/locations/{location_id}/targets/{target}/plans/{request_id}/summary": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPlanSummary"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/locations/{location_id}/targets/{target}/providers": { parameters: { query?: never; @@ -786,6 +866,38 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/locations/{location_id}/targets/{target}/selection": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["evaluateSelection"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/locations/{location_id}/targets/{target}/selection/vocabulary": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readSelectionVocabulary"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/locations/{location_id}/targets/{target}/snapshots": { parameters: { query?: never; @@ -1042,6 +1154,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/locations/{location_id}/targets/{target}/changed-build": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["report_changed_build"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/locations/{location_id}/targets/{target}/changes": { parameters: { query?: never; @@ -1090,6 +1218,23 @@ export interface paths { patch?: never; trace?: never; }; + "/api/locations/{location_id}/targets/{target}/metadata/retention": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** How much state history a full read delivers; older servers answer 404 and the location reads it all. */ + get: operations["read_metadata_retention"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/locations/{location_id}/targets/{target}/metadata/statistics-due": { parameters: { query?: never; @@ -1563,7 +1708,53 @@ export interface components { }; /** @description SSE invalidation topics; `all` means refetch the authorized REST resources. */ ChangeNotice: { + /** @description `node_status` only: too many nodes changed, or the snapshot or changed build is new; refetch node states. */ + full?: boolean | null; + /** + * Format: int64 + * @description `node_status` only: the version this change applies to; refetch node states when it is not yours. + */ + previous_version?: number | null; + /** @description `node_status` only: the nodes whose state changed, absent when `full` is set. */ + states?: { + [key: string]: components["schemas"]["NodeState"]; + } | null; topic: string; + /** + * Format: int64 + * @description `node_status` only: the state version after this change. + */ + version?: number | null; + }; + /** @description The nodes the default `sqb build --changed` runs, as the location last planned it from the synced copy. */ + ChangedBuild: { + /** @description Nodes that run for a change of their own, as `kind:name`. */ + changed: string[]; + /** @description Nodes that run only because an upstream they read runs. */ + downstream: string[]; + /** @description Why the location's plan failed, with both lists empty; absent when it planned. */ + failing?: string | null; + /** + * Format: int64 + * @description The publication generation it was planned for. + */ + generation: number; + /** + * Format: date-time + * @description When the server received it. + */ + reported_at: string; + }; + /** @description A location's plan of the default `sqb build --changed`, fenced to its reader session and generation. */ + ChangedBuildReport: { + changed: string[]; + downstream: string[]; + /** @description Why the plan failed, sent with empty lists; only newer servers accept it. */ + failing?: string | null; + /** Format: int64 */ + generation: number; + /** Format: uuid */ + session_id: string; }; /** @description One result's stored evidence rows; `view.evidence` reads them, audited. */ CheckEvidence: { @@ -1593,6 +1784,8 @@ export interface components { /** Format: int64 */ evidence_count?: number | null; evidence_truncated?: string | null; + /** @description An error-severity failure its build let through instead of holding anything back. */ + ignored?: boolean; invocation_id: string; measured_value?: string | null; /** Format: int64 */ @@ -1653,6 +1846,11 @@ export interface components { /** Format: uuid */ snapshot_id?: string | null; }; + /** + * @description The layer that set a run's concurrency. + * @enum {string} + */ + ConcurrencySource: "job" | "target" | "local" | "project" | "default"; /** @description A row whose key was already stored with different content; the first row is kept. */ ConflictEvidence: { existing_sha256: string; @@ -1660,6 +1858,26 @@ export interface components { rejected_sha256: string; table: components["schemas"]["Table"]; }; + /** + * @description When a location checks its warehouse connection on its own. + * @enum {string} + */ + ConnectionCheckTrigger: "startup" | "publish"; + /** @description A target's connection settings as its location publishes them: names only, never a secret. */ + ConnectionDetails: { + account?: string | null; + /** @description The adapter, such as `snowflake`, `postgres` or `duckdb`. */ + adapter: string; + /** @description How the location signs in, by name only, such as `key_pair`, `password` or `oauth`. */ + auth_method?: string | null; + /** @description The database the connection opens by default. */ + database?: string | null; + /** @description The server host, or for a file database its path. */ + host?: string | null; + role?: string | null; + user?: string | null; + warehouse?: string | null; + }; /** @description What the UI needs first: how people sign in, and what this person may see and do. */ ControlBootstrap: { access: components["schemas"]["PersonAccess"]; @@ -1824,7 +2042,11 @@ export interface components { }; /** @description One job or group as declared, for a person who may list the target; free-form setting values are withheld. */ JobDetailRecord: { - concurrency_groups: string[]; + /** + * Format: int64 + * @description The job's own `concurrency` setting, models building at once in a run; absent uses the project's. + */ + concurrency?: number | null; /** @description Selected nodes whose failure stops a group. */ critical_nodes: string[]; /** @description The cursor type every selected cursor node shares; absent when none has a cursor or types differ. */ @@ -1832,6 +2054,8 @@ export interface components { declared_at?: null | components["schemas"]["DeclaredAt"]; description: string; exclude?: string | null; + /** @description The job's own `ignore_audit_failures` setting; absent uses the target's, then the project's. */ + ignore_audit_failures?: boolean | null; job_key: string; kind: components["schemas"]["RunKind"]; /** @description The groups this job belongs to. */ @@ -1841,6 +2065,9 @@ export interface components { on_failure?: null | components["schemas"]["OnFailure"]; overlap: components["schemas"]["Overlap"]; owner?: string | null; + /** @description Each pool the job takes, by pool name. */ + pools: components["schemas"]["PoolUse"][]; + resolved_concurrency?: null | components["schemas"]["ResolvedConcurrency"]; select?: string | null; selection: string[]; /** @description Command settings; `vars` and a `defer_*` value that is not a target name show as `[REDACTED]`, and text values pass through secret redaction. */ @@ -1854,20 +2081,31 @@ export interface components { }; JobMember: { job_key: string; + /** @description Node by node, why it waits for each of `waits_for`; a wait with none follows lineage through another member. */ + reasons: components["schemas"]["WaitReason"][]; /** @description Members this one waits for, from the lineage between their nodes. */ waits_for: string[]; }; /** @description A job or group of the current published snapshot. Settings are left out of person reads. */ JobRecord: { - concurrency_groups: string[]; + /** + * Format: int64 + * @description The job's own `concurrency` setting, models building at once in a run; absent uses the project's. + */ + concurrency?: number | null; declared_at?: null | components["schemas"]["DeclaredAt"]; description: string; + /** @description The job's own `ignore_audit_failures` setting; absent uses the target's, then the project's. */ + ignore_audit_failures?: boolean | null; job_key: string; /** @description `job` or `group`; a group's selection is the union of its member jobs' selections. */ kind: components["schemas"]["RunKind"]; on_failure?: null | components["schemas"]["OnFailure"]; overlap: components["schemas"]["Overlap"]; owner?: string | null; + /** @description Each pool the job takes, by pool name. */ + pools: components["schemas"]["PoolUse"][]; + resolved_concurrency?: null | components["schemas"]["ResolvedConcurrency"]; /** @description A job's selectors as written; absent for a group and for snapshots published before they were recorded. */ select?: string | null; selection: string[]; @@ -1923,12 +2161,19 @@ export interface components { generation: number; /** @enum {string} */ kind: "compile_failed"; + rules?: null | components["schemas"]["LocationRuleFailure"]; } | { + /** + * Format: int64 + * @description How long the check took; absent from older locations. + */ + elapsed_millis?: number | null; /** Format: int64 */ generation: number; healthy: boolean; /** @enum {string} */ kind: "connection"; + trigger?: null | components["schemas"]["ConnectionCheckTrigger"]; } | { /** Format: int64 */ generation: number; @@ -1936,6 +2181,12 @@ export interface components { kind: "provider_secrets"; /** Format: int64 */ missing: number; + } | { + /** Format: int64 */ + generation: number; + jobs: components["schemas"]["PoolCapWarning"][]; + /** @enum {string} */ + kind: "pool_capped"; } | { /** Format: int64 */ elapsed_millis: number; @@ -1947,6 +2198,18 @@ export interface components { /** @enum {string} */ kind: "warehouse_sync"; location: components["schemas"]["WarehouseStateLocation"]; + /** + * Format: date-time + * @description On the report that completes or stops a sync, when the next safety sync is due unless a run ends first; absent otherwise, where no timed sync follows, and from older locations. + */ + next_sync_at?: string | null; + /** + * Format: int64 + * @description Rows, or statistics relations, the failed read asked for. + */ + page_limit?: number | null; + progress?: null | components["schemas"]["WarehouseSyncProgress"]; + read_failure?: null | components["schemas"]["WarehouseReadFailure"]; /** Format: int64 */ records_sent: number; table: components["schemas"]["WarehouseHistoryKind"]; @@ -1965,6 +2228,21 @@ export interface components { session_id: string; target: string; }; + /** @description Configured rules that failed a location's load: how many findings, and the first of them. */ + LocationRuleFailure: { + /** Format: int64 */ + finding_count: number; + /** @description At most `MAX_REPORTED_RULE_FINDINGS`, in the rules' order. */ + findings: components["schemas"]["LocationRuleFinding"][]; + }; + /** @description One rule finding as `sqb compile` prints it. */ + LocationRuleFinding: { + code: string; + /** Format: int64 */ + line?: number | null; + message: string; + path?: string | null; + }; /** @description Only named location operations cross this boundary, never arbitrary SQL or Python. */ LocationWork: { /** @enum {string} */ @@ -2003,6 +2281,10 @@ export interface components { kind: "sensor_evaluation"; outcome: components["schemas"]["CustomSensorOutcome"]; } | { + /** @description Model diff text the document left out to stay under its cap; stored apart, never read back here. */ + diffs?: { + [key: string]: components["schemas"]["PlanModelDiff"]; + }; document: Record; /** @enum {string} */ kind: "plan"; @@ -2148,6 +2430,12 @@ export interface components { physical_generation_id: string; target_name: string; }; + /** @description How many records the server holds for one mirrored state table of a target. */ + MirroredTotal: { + /** Format: int64 */ + records: number; + table: components["schemas"]["WarehouseHistoryKind"]; + }; MutationResponse: { applied: boolean; }; @@ -2201,6 +2489,24 @@ export interface components { /** Format: uuid */ snapshot_id: string; }; + /** @description The state fields of a `CatalogNodeRecord`, with the same meaning. */ + NodeState: { + checks: components["schemas"]["CatalogNodeChecks"]; + definition_changed: boolean; + last_build?: null | components["schemas"]["CatalogNodeBuild"]; + last_outcome?: null | components["schemas"]["CatalogNodeOutcome"]; + }; + /** @description Every current node's state at one version; `snapshot_id` is null before the first publication. */ + NodeStatesRecord: { + changed_build?: null | components["schemas"]["ChangedBuild"]; + /** Format: uuid */ + snapshot_id?: string | null; + states: { + [key: string]: components["schemas"]["NodeState"]; + }; + /** Format: int64 */ + version: number; + }; NodeStatus: { build?: null | components["schemas"]["FingerprintObservation"]; definition: components["schemas"]["DefinitionEvidence"]; @@ -2443,12 +2749,56 @@ export interface components { /** @enum {string} */ status: "unavailable"; }; + /** @description A planned model's diff text, from the plan document or, where the document left it out, apart from it. */ + PlanDiffRead: components["schemas"]["PlanModelDiff"] & { + model: string; + }; + /** @description One planned model's diff text, as `sqb plan` prints it. */ + PlanModelDiff: { + config_diff?: string[] | null; + query_diff?: string[] | null; + }; /** @description A person's build command to plan on the location; the server freezes it like an ad-hoc run. */ PlanRequest: { command: string[]; /** Format: uuid */ request_id: string; }; + /** @description A page of a plan section with the `--changed` runs, warnings and cursor window starts of its models. */ + PlanRows: { + cursor_window_starts: { + [key: string]: string; + }; + /** @description Model rows carry `diff_stats` in place of their diff text, which `plans/{request_id}/diff` serves. */ + rows: Record[]; + runs: Record[]; + /** @description Rows in the section, filter or not. */ + section_total: number; + /** @description Rows matching the name filter; the section's size without one. */ + total: number; + warnings: Record[]; + }; + /** @description A plan request's state with its summary: changed models in full, large sections as counts to page. */ + PlanSummaryRead: { + /** Format: date-time */ + expires_at: string; + outcome?: null | components["schemas"]["LocationWorkOutcome"]; + timings?: null | components["schemas"]["PlanTimings"]; + }; + /** @description One planning step and its wall time. */ + PlanTimingStep: { + name: string; + /** Format: double */ + seconds: number; + }; + /** @description How long one plan took: its steps in pipeline order with `Other` last, and what it read. */ + PlanTimings: { + /** @description `synced_state` for a preview over the synced warehouse state; `live` when it read the warehouse. */ + planned_from: string; + steps: components["schemas"]["PlanTimingStep"][]; + /** Format: double */ + total_seconds: number; + }; PlatformEvent: { /** @enum {string} */ kind: "heartbeat"; @@ -2525,6 +2875,52 @@ export interface components { */ scheduler_lease_expires_at?: string | null; }; + /** @description A job or group asking for more of a pool than this server's cap, so it runs alone in that pool. */ + PoolCapWarning: { + /** Format: int32 */ + amount: number; + /** Format: int32 */ + cap: number; + job_key: string; + kind: components["schemas"]["RunKind"]; + /** @description Such as "job X takes 3 of pool warehouse_heavy; this server caps it at 1, so it runs alone in that pool". */ + message: string; + pool: string; + }; + /** @description How many slots of a pool a job takes, and the pool's limit on the job's target. */ + PoolSlots: { + /** Format: int32 */ + amount: number; + /** Format: int32 */ + limit: number; + }; + /** @description A pool a job takes: its slots, the pool's limit, and the other jobs and groups taking it. */ + PoolUse: { + /** + * Format: int32 + * @description The slots the job asks for. + */ + amount: number; + /** + * Format: int32 + * @description This server's `pool_caps` entry for the location and target; absent when it sets none. + */ + cap?: number | null; + /** @description The job asks for more slots than `server_limit`, so on this server it takes the whole pool. */ + clamped: boolean; + /** + * Format: int32 + * @description The project's limit on the target. + */ + limit: number; + pool: string; + /** + * Format: int32 + * @description The limit on this server: the project's limit, lowered to the cap. + */ + server_limit: number; + shared_with: string[]; + }; /** @description What the mirror knows as of `synced_at`; `None` when a required table never synced cleanly. */ PreviewCopy: { /** @description Every mirrored consumption row when asked for; omitted otherwise, as older locations reject it. */ @@ -2535,6 +2931,10 @@ export interface components { relations: components["schemas"]["WarehouseCatalogRecord"][]; /** Format: date-time */ synced_at?: string | null; + /** @description Set when the mirror is still at the read's known version: the records are left out. */ + unchanged?: boolean; + /** @description The mirror version these records were read at; sent only to a versioned read. */ + version?: string | null; }; /** @description A location asks for the mirrored copy a plan preview plans from, as its fenced reader. */ PreviewCopyRead: { @@ -2542,8 +2942,12 @@ export interface components { consumption?: boolean; /** Format: int64 */ generation: number; + /** @description The mirror version of the copy the location holds; an unchanged mirror returns no records. */ + known_version?: string | null; /** Format: uuid */ session_id: string; + /** @description Whether the location holds copies by mirror version; sent only when set, as older servers reject it. */ + versioned?: boolean; }; /** @description The newest successful build of a node, which a `cursor_max` observation must come from. */ PreviewLatestRun: { @@ -2564,6 +2968,10 @@ export interface components { class: string; /** @description The class docstring. */ description: string; + /** @description Each of those jobs with the provider's nodes it selects, in node order. */ + job_nodes?: { + [key: string]: string[]; + }; /** @description Jobs whose selection includes one of those nodes. */ jobs?: string[]; name: string; @@ -2604,6 +3012,7 @@ export interface components { PublishedSnapshot: { /** @description Added, changed and removed nodes and roles, in publication order. */ changes: components["schemas"]["PublishedChange"][]; + connection?: null | components["schemas"]["ConnectionDetails"]; /** @description Digest of the lock file, Python and `sqb` versions; a change means dependencies changed. */ environment_hash: string; /** Format: int64 */ @@ -2624,6 +3033,11 @@ export interface components { /** Format: uuid */ run_id: string; } | { + /** + * Format: int32 + * @description The upstream job's attempt when the group is retrying it; absent for its first attempt. + */ + attempt?: number | null; job_key: string; /** @enum {string} */ kind: "upstream_job"; @@ -2643,16 +3057,14 @@ export interface components { /** Format: uuid */ run_id?: string | null; } | { - /** Format: int32 */ - elsewhere: number; holders: string[]; /** @enum {string} */ - kind: "concurrency_limit"; - limit: string; + kind: "pool_full"; + pool: string; } | { /** @enum {string} */ kind: "earlier_run"; - limit: string; + pool: string; /** Format: uuid */ run_id?: string | null; } | { @@ -2713,6 +3125,12 @@ export interface components { /** Format: uuid */ source_run_id: string; }; + /** @description The models a run builds at once on its target, and the layer that set it. */ + ResolvedConcurrency: { + /** Format: int64 */ + limit: number; + source: components["schemas"]["ConcurrencySource"]; + }; RoleAssignmentRequest: { /** Format: uuid */ location_id?: string | null; @@ -2820,6 +3238,11 @@ export interface components { kind: "finished"; outcome: components["schemas"]["RunOutcome"]; reason: string; + } | { + /** Format: int32 */ + count: number; + /** @enum {string} */ + kind: "audit_failures_ignored"; }; RunChangeRequest: { change: components["schemas"]["RunChange"]; @@ -2848,7 +3271,7 @@ export interface components { occurred_at?: string | null; }; /** @enum {string} */ - RunEventKind: "completed" | "success" | "warn" | "partial" | "failed" | "cancelled"; + RunEventKind: "completed" | "success" | "warn" | "partial" | "failed" | "cancelled" | "skipped"; /** @description Recall event metadata of a run. Payloads are left out: they can carry SQL, request settings and error text. */ RunEventRecord: { /** Format: int64 */ @@ -2860,6 +3283,8 @@ export interface components { event_id: string; event_type: string; invocation_id: string; + /** @description A summary the scheduler writes for a group run's member events, such as a member retry; never error or SQL text. */ + message?: string | null; occurred_at: string; outcome?: string | null; producer?: string | null; @@ -2921,7 +3346,7 @@ export interface components { * @description Terminal outcomes cannot be confused with a request to change an active status. * @enum {string} */ - RunOutcome: "success" | "warn" | "partial" | "failed" | "cancelled"; + RunOutcome: "success" | "warn" | "partial" | "failed" | "cancelled" | "skipped"; /** @description Captured output and error text of a run's invocations; both can quote warehouse data. */ RunOutput: { /** @description Invocation failures and error diagnostics, oldest first. */ @@ -2954,6 +3379,11 @@ export interface components { ended_at?: string | null; git_dirty?: boolean | null; git_revision?: string | null; + /** + * Format: int32 + * @description Error-severity audit failures the run let through; a group sums its members'. + */ + ignored_audit_failures?: number; /** @description The deployment image the run is pinned to, when its location publishes one. */ image_id?: string | null; job_key?: string | null; @@ -2964,6 +3394,11 @@ export interface components { launched_at?: string | null; /** Format: uuid */ location_id: string; + /** + * Format: int32 + * @description Which attempt of its member job a group child is, once the group retried it; absent for a first attempt and for every other run. + */ + member_attempt?: number | null; narrowing?: null | components["schemas"]["RunNarrowing"]; node_counts_json: { [key: string]: number; @@ -3012,16 +3447,24 @@ export interface components { /** @description The authorized selection and pinned execution inputs are immutable once requested. */ RunRequest: { command?: components["schemas"]["RunCommand"]; - concurrency_groups: string[]; image_id?: string | null; job_key?: string | null; kind: components["schemas"]["RunKind"]; /** Format: uuid */ location_id: string; + /** + * Format: int32 + * @description Which attempt of its member job a group child is, from the second on; the first leaves it unset. + */ + member_attempt?: number | null; narrowing?: null | components["schemas"]["RunNarrowing"]; origin: components["schemas"]["RunOrigin"]; /** Format: uuid */ parent_run_id?: string | null; + /** @description Each pool the run takes, its amount and the pool's limit frozen with it; the server's cap may lower the limit. */ + pools?: { + [key: string]: components["schemas"]["PoolSlots"]; + }; /** Format: int64 */ priority: number; requested_by: string; @@ -3046,7 +3489,7 @@ export interface components { window?: null | components["schemas"]["RunWindow"]; write_set: string[]; }; - /** @description Which retry of the job's `on_failure` policy a run is, and when it may start. */ + /** @description Which retry of the job's `[retry]` policy a run is, and when it may start. */ RunRetry: { /** Format: int32 */ attempt: number; @@ -3084,7 +3527,7 @@ export interface components { statement_id: string; }; /** @enum {string} */ - RunStatus: "queued" | "launched" | "running" | "unresponsive" | "success" | "warn" | "partial" | "failed" | "cancelled"; + RunStatus: "queued" | "launched" | "running" | "unresponsive" | "success" | "warn" | "partial" | "failed" | "cancelled" | "skipped"; /** @description Inclusive typed CLI cursor bounds; the native planner derives exclusive execution ends. */ RunWindow: { /** Format: date-time */ @@ -3133,8 +3576,89 @@ export interface components { /** Format: int64 */ storage_id: number; }; - SensorCondition: { - /** @enum {string} */ + /** @description The nodes `sqb build --select … --exclude …` selects in the current publication, or why it refuses the selectors. */ + SelectionEvaluation: { + /** + * Format: int64 + * @description Every selected node, across all pages. + */ + count: number; + error?: null | components["schemas"]["SelectionProblem"]; + /** @description One page of the selected `kind:name` keys, sorted. */ + keys: string[]; + /** + * Format: int32 + * @description The offset of the next page, when there is one. + */ + next_offset?: number | null; + /** + * Format: uuid + * @description The publication evaluated; `None` before the first one. + */ + snapshot_id?: string | null; + }; + /** @enum {string} */ + SelectionField: "select" | "exclude"; + /** @description One selector token in the request's `select` or `exclude` text. */ + SelectionLocation: { + /** Format: int32 */ + end: number; + field: components["schemas"]["SelectionField"]; + /** + * Format: int32 + * @description Where the token starts and ends in its field, in UTF-16 code units, as JavaScript indexes strings. + */ + start: number; + token: string; + }; + /** @description Why `sqb` refuses a selection, where, and what might have been meant. */ + SelectionProblem: { + /** @description The selector error code, such as `S007`; `None` when the selection as a whole is refused. */ + code?: string | null; + /** @description `sqb`'s own help line, such as "did you mean 'orders'?". */ + help?: string | null; + location?: null | components["schemas"]["SelectionLocation"]; + message: string; + refused?: null | components["schemas"]["SelectionRefusal"]; + /** @description Replacements for that whole token, closest first. */ + suggestions: string[]; + }; + /** @description One refused comma part: its exact text and position in the token, its selector type and value. */ + SelectionRefusal: { + /** + * Format: int32 + * @description Which comma part of the token it is, from 0. + */ + index: number; + /** @description The part as typed, such as `+tag:finanse`. */ + part: string; + /** @description `tag`, `path`, `job`, `source`, `seed`, `kind`, `test`, `name`, `glob`, or a Python node type such as `task`. */ + type: string; + /** @description The value the part names, without its type prefix or `+` markers, such as `finanse`. */ + value: string; + }; + /** @description The values each selector type takes in the current publication, with how many nodes each one selects. */ + SelectionVocabulary: { + /** @description `job:` names: the nodes each job or job group selects. */ + jobs: components["schemas"]["VocabularyEntry"][]; + /** @description `kind:` values: the nodes of each kind. */ + kinds: components["schemas"]["VocabularyEntry"][]; + /** @description `path:` folders such as `models/staging`: the nodes below each, at any depth. */ + paths: components["schemas"]["VocabularyEntry"][]; + /** @description `seed:` names, one node each. */ + seeds: components["schemas"]["VocabularyEntry"][]; + /** + * Format: uuid + * @description The publication described; `None` before the first one. + */ + snapshot_id?: string | null; + /** @description `source:` names, one node each. */ + sources: components["schemas"]["VocabularyEntry"][]; + /** @description `tag:` values: the SQL resources and Python nodes with each tag. */ + tags: components["schemas"]["VocabularyEntry"][]; + }; + SensorCondition: { + /** @enum {string} */ kind: "platform_event"; kinds: components["schemas"]["PlatformEventKind"][]; } | { @@ -3202,6 +3726,11 @@ export interface components { * @description Publications kept per location target. */ snapshots_kept: number; + /** + * Format: int64 + * @description Builds of each node, and results of each check, the store mirrors from the warehouse. + */ + state_history_versions: number; /** Format: int64 */ statistics_days: number; }; @@ -3400,6 +3929,18 @@ export interface components { }; /** @enum {string} */ TriggerStatus: "running" | "paused"; + /** @description One selector value and the number of nodes it selects. */ + VocabularyEntry: { + /** Format: int64 */ + count: number; + value: string; + }; + /** @description One reason a member waits: `from_node` of `upstream_job` feeds its `to_node`, directly or through nodes no member selects. */ + WaitReason: { + from_node: string; + to_node: string; + upstream_job: string; + }; WarehouseCatalogColumn: { data_type: string; name: string; @@ -3518,6 +4059,19 @@ export interface components { state_schema: string; value: string; }; + /** @description What went wrong reading one page, without the warehouse's message, which may name hosts or users. */ + WarehouseReadFailure: { + /** @enum {string} */ + kind: "reader_start"; + } | { + /** @enum {string} */ + kind: "read_error"; + } | { + /** @enum {string} */ + kind: "timed_out"; + /** Format: int64 */ + seconds: number; + }; /** @description One estimated query cost observation of a run, resource and warehouse; refreshes append. */ WarehouseResourceCostRecord: { /** Format: int64 */ @@ -3555,6 +4109,11 @@ export interface components { warehouse_size: string; warehouse_type?: string | null; }; + /** @description The newest versions per node or check a full read of state history needs to deliver. */ + WarehouseRetention: { + /** Format: int32 */ + state_history_versions: number; + }; /** @description Physical namespace of SQLBuild's source-freshness state table, never a source query. */ WarehouseStateLocation: { database?: string | null; @@ -3593,6 +4152,33 @@ export interface components { * @enum {string} */ WarehouseSyncFailure: "watermark" | "read" | "delivery" | "busy"; + /** @description How far one synchronization has got, so people see it running and when it completed. */ + WarehouseSyncProgress: { + /** @description This report's page was the synchronization's last, so it has completed. */ + completed: boolean; + /** + * Format: int64 + * @description Milliseconds since the synchronization began. + */ + elapsed_millis: number; + /** @description Failed reads in a row stopped the sync until a run, a request, a publication or `next_sync_at`. */ + halted?: boolean; + /** + * Format: int64 + * @description Kinds the synchronization reads: 14 for each state schema. + */ + kinds: number; + /** + * Format: int64 + * @description The reported kind's position in the synchronization, from 1. + */ + position: number; + /** + * Format: int64 + * @description Rows the synchronization has delivered, this report's included. + */ + rows: number; + }; WarehouseSyncResponse: { applied: boolean; }; @@ -4349,7 +4935,7 @@ export interface operations { }; requestBody?: never; responses: { - /** @description changed events: all, runs, ticks, snapshot:, node_status:. Refetch REST on all/reconnect; no durable cursor. */ + /** @description changed events: all, runs, ticks, snapshot:, node_status:. Refetch REST on all/reconnect; no durable cursor. node_status carries version, previous_version and the changed states (empty when none changed), or full: refetch node states when full is set or previous_version is not the version held. */ 200: { headers: { [name: string]: unknown; @@ -4959,6 +5545,82 @@ export interface operations { }; }; }; + readPersonMirroredTotals: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Records the server mirrors of each state table of this target, in sync order */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MirroredTotal"][]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + readPersonNodeStates: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Every current node's state at the state version node_status events continue from */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["NodeStatesRecord"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; readPersonNodeStatus: { parameters: { query: { @@ -5009,7 +5671,10 @@ export interface operations { listPersonCatalogNodes: { parameters: { query?: never; - header?: never; + header?: { + /** @description An ETag from an earlier response; 304 when the snapshot and state version are unchanged */ + "If-None-Match"?: string | null; + }; path: { location_id: string; target: string; @@ -5021,12 +5686,22 @@ export interface operations { /** @description Every node of the current snapshot; empty before the first publication */ 200: { headers: { + /** @description ":"; absent before the first publication */ + ETag?: string; [name: string]: unknown; }; content: { "application/json": components["schemas"]["CatalogNodeRecord"][]; }; }; + /** @description Unchanged since the If-None-Match ETag */ + 304: { + headers: { + ETag?: string; + [name: string]: unknown; + }; + content?: never; + }; /** @description Session rejected */ 401: { headers: { @@ -5257,6 +5932,139 @@ export interface operations { }; }; }; + readPlanDiff: { + parameters: { + query: { + /** @description The planned model whose diff to read */ + model: string; + }; + header?: never; + path: { + location_id: string; + target: string; + request_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["PlanDiffRead"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description No diff of that model, or no view.sql on every current node */ + 404: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + readPlanRows: { + parameters: { + query: { + /** @description The section to page */ + section: "full_refresh" | "query_changed" | "function_changed" | "config_changed" | "schema_changed" | "first_run" | "renamed" | "routine_models" | "migrations" | "column_migrations" | "table_type_conversions" | "retention" | "loader_reloads" | "loader_loads" | "source_reloads" | "source_loads" | "changed_functions" | "functions" | "seeds" | "python_nodes" | "held_back" | "unchanged" | "warnings"; + /** @description Rows to skip; 0 by default */ + offset?: number; + /** @description At most this many rows, capped at 1000 */ + limit?: number; + /** @description Only rows whose name contains this, ignoring case */ + name?: string; + }; + header?: never; + path: { + location_id: string; + target: string; + request_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": null | components["schemas"]["PlanRows"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + readPlanSummary: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + request_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": null | components["schemas"]["PlanSummaryRead"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; readPersonProviders: { parameters: { query?: never; @@ -5681,6 +6489,98 @@ export interface operations { }; }; }; + evaluateSelection: { + parameters: { + query?: { + /** @description Space-separated selectors, as `sqb build -s` takes them; empty selects every node */ + select?: string; + /** @description Space-separated selectors, as `--exclude` takes them */ + exclude?: string; + /** @description Sorted keys to skip */ + offset?: number; + /** @description Page size; defaults to 1,000, clamped to 1..10,000 */ + limit?: number; + }; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description The selected keys, or why `sqb` refuses the selectors */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SelectionEvaluation"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + readSelectionVocabulary: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SelectionVocabulary"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; listPersonSnapshots: { parameters: { query?: { @@ -6409,6 +7309,62 @@ export interface operations { }; }; }; + report_changed_build: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ChangedBuildReport"]; + }; + }; + responses: { + /** @description Stored; it replaces the target's previous report */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Reader generation fenced */ + 409: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; subscribe: { parameters: { query?: never; @@ -6421,7 +7377,7 @@ export interface operations { }; requestBody?: never; responses: { - /** @description changed events: all, runs, run:, ticks, snapshot:, node_status:. Refetch REST on all/reconnect; no durable cursor. */ + /** @description changed events: all, runs, run:, ticks, snapshot:, node_status:. Refetch REST on all/reconnect; no durable cursor. node_status carries version, previous_version and the changed states (empty when none changed), or full: refetch node states when full is set or previous_version is not the version held. */ 200: { headers: { [name: string]: unknown; @@ -6556,6 +7512,42 @@ export interface operations { }; }; }; + read_metadata_retention: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WarehouseRetention"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; statistics_due: { parameters: { query?: never; diff --git a/crates/sqlbuild-analysis/src/graph/_helpers/selectors.rs b/crates/sqlbuild-analysis/src/graph/_helpers/selectors.rs index 5500c932ef..15580b2211 100644 --- a/crates/sqlbuild-analysis/src/graph/_helpers/selectors.rs +++ b/crates/sqlbuild-analysis/src/graph/_helpers/selectors.rs @@ -5,7 +5,7 @@ use std::convert::Infallible; use crate::assembly::project::types::ObjectKey; use crate::graph::constants::{ - EMPTY_SELECTOR_CODE, EMPTY_VALUE_CODE, EXPANSION_MARKER, FOLDER_SEPARATOR, + EMPTY_SELECTOR_CODE, EMPTY_VALUE_CODE, EXPANSION_MARKER, FOLDER_SEPARATOR, FUNCTION_KIND_ALIAS, INTERSECTION_SEPARATOR, JOB_KIND, KIND_SEPARATOR, MISPLACED_MARKER_CODE, MISSING_NAME_CODE, MODEL_RESOURCE, MODEL_ROOT, MODEL_ROOT_PREFIX, NAME_KIND, NODE_KIND_KIND, PATH_KIND, PATH_ROOT_CODE, PATH_ROOT_ERROR, PATH_SELECTOR_CODE, PATH_SEPARATOR, PATTERN_CHARACTERS, @@ -400,7 +400,7 @@ fn of_kind(graph: &ProjectGraph, value: &str) -> Resolved { let keys: Keys = graph .names .iter() - .filter(|(_, key)| key.0 == value) + .filter(|(_, key)| key.0 == value || (value == FUNCTION_KIND_ALIAS && is_function(key))) .map(|(_, key)| key.clone()) .collect(); if keys.is_empty() { diff --git a/crates/sqlbuild-analysis/src/graph/constants.rs b/crates/sqlbuild-analysis/src/graph/constants.rs index f3896c96ae..1540090e3f 100644 --- a/crates/sqlbuild-analysis/src/graph/constants.rs +++ b/crates/sqlbuild-analysis/src/graph/constants.rs @@ -21,9 +21,11 @@ pub(crate) const TEST_KIND: &str = "test"; pub(crate) const JOB_KIND: &str = "job"; pub(crate) const NODE_KIND_KIND: &str = "kind"; /// The node kinds `kind:` selects, as the catalog names them: SQL resources, then Python nodes. -pub(crate) const SELECTABLE_NODE_KINDS: [&str; 9] = [ - "model", "source", "seed", "udf", "table_fn", "task", "asset", "loader", "check", +pub(crate) const SELECTABLE_NODE_KINDS: [&str; 10] = [ + "model", "source", "seed", "udf", "table_fn", "function", "task", "asset", "loader", "check", ]; +/// `kind:function` selects both function kinds. +pub(crate) const FUNCTION_KIND_ALIAS: &str = "function"; /// Selector kinds Python parses but `sqb compile` cannot map to a compiled resource. pub(crate) const UNMAPPED_KINDS: [&str; 4] = ["task", "asset", "loader", "check"]; pub(crate) const SUGGESTION_CUTOFF: f64 = 0.8; diff --git a/crates/sqlbuild-analysis/src/graph/tests/helpers.rs b/crates/sqlbuild-analysis/src/graph/tests/helpers.rs index 3174ac179a..9b9d9e0f2a 100644 --- a/crates/sqlbuild-analysis/src/graph/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/graph/tests/helpers.rs @@ -140,6 +140,19 @@ pub(super) const TABLE_FUNCTION_PROJECT: &[ResourceSpec] = &[ ), ]; +/// A model reading a scalar function and a table function. +pub(super) const FUNCTIONS_PROJECT: &[ResourceSpec] = &[ + ("udf", "tax", &[], &[], None), + ("table_fn", "customer_orders", &[], &[], None), + ( + "model", + "orders", + &[("udf", "tax"), ("table_fn", "customer_orders")], + &[], + None, + ), +]; + pub(super) fn orders_graph() -> ProjectGraph { project_graph(ORDERS_PROJECT) } diff --git a/crates/sqlbuild-analysis/src/graph/tests/test_selector_scenarios.rs b/crates/sqlbuild-analysis/src/graph/tests/test_selector_scenarios.rs index e89b991aea..f618813937 100644 --- a/crates/sqlbuild-analysis/src/graph/tests/test_selector_scenarios.rs +++ b/crates/sqlbuild-analysis/src/graph/tests/test_selector_scenarios.rs @@ -1,7 +1,7 @@ use crate::graph::main::parse_selector::parse_selector; use crate::graph::tests::helpers::{ - DIAMOND_PROJECT, FOLDER_PROJECT, TABLE_FUNCTION_PROJECT, failure, kind, names, path, - resolved_names, + DIAMOND_PROJECT, FOLDER_PROJECT, FUNCTIONS_PROJECT, TABLE_FUNCTION_PROJECT, failure, kind, + names, path, resolved_names, }; use crate::graph::tests::test_types::{ ParseSelectorErrorTestCase, ParseSelectorTestCase, ProjectSelectorErrorTestCase, @@ -213,6 +213,20 @@ fn given_selectors_when_resolving_then_selects_expected_names() { exclude: &[], expected_names: &["raw_customers", "raw_orders"], }, + SelectedNamesTestCase { + description: "kind:function selects scalar and table functions alike", + project: FUNCTIONS_PROJECT, + select: &["kind:function"], + exclude: &[], + expected_names: &["customer_orders", "tax"], + }, + SelectedNamesTestCase { + description: "the precise function kinds still select one kind each", + project: FUNCTIONS_PROJECT, + select: &["kind:udf"], + exclude: &[], + expected_names: &["tax"], + }, SelectedNamesTestCase { description: "+ expands its own comma part before the kind narrows it", project: DIAMOND_PROJECT, diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs b/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs index 873f53de24..07f41b49ac 100644 --- a/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs @@ -1,7 +1,6 @@ -use std::collections::HashMap; - use crate::python_plan::models::LifecycleNode; use crate::python_plan::tests::test_types::LifecycleNodeSpec; +use std::collections::HashMap; pub(crate) fn edges(rows: &[(&str, &[&str])]) -> HashMap> { rows.iter() diff --git a/crates/sqlbuild-cli/src/adhoc/_helpers/evaluation.rs b/crates/sqlbuild-cli/src/adhoc/_helpers/evaluation.rs new file mode 100644 index 0000000000..c79c54c128 --- /dev/null +++ b/crates/sqlbuild-cli/src/adhoc/_helpers/evaluation.rs @@ -0,0 +1,294 @@ +//! A selection evaluated as `sqb build` resolves it, with each refusal traced to the token that causes it. + +use std::collections::{BTreeMap, BTreeSet}; + +use sqlbuild_analysis::graph::main::parse_selector::parse_selector; +use sqlbuild_analysis::graph::models::ParsedSelector; +use sqlbuild_core::text::main::close_matches::close_matches; +use sqlbuild_orchestration::runs::models::{ + SelectionField, SelectionLocation, SelectionProblem, SelectionRefusal, SelectionVocabulary, + VocabularyEntry, +}; + +use crate::adhoc::_helpers::planner::resolve_with_tests; +use crate::adhoc::_helpers::publication::{IndexedPublication, is_sql_test, name_of}; +use crate::adhoc::constants::{ + BUILD_VERB, FUNCTION_KIND_ALIAS, FUNCTION_KINDS, GLOB_CHARACTERS, GLOB_SELECTOR, NAME_SELECTOR, + PROJECT_FOLDER, SUGGESTION_CUTOFF, SUGGESTION_LIMIT, +}; +use crate::adhoc::models::Refusal; + +/// The sorted keys `sqb build --select --exclude ` selects, or why it refuses them; each text holds space-separated selectors. + fn evaluate(&self, select: &str, exclude: &str) -> Result, Box>; + + /// Every value each selector type takes here, with its node count; `snapshot_id` is left for the caller. + fn vocabulary(&self) -> SelectionVocabulary; } /// Which runs of one location and target to list. diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs index 455bda2395..73bca8cef0 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs @@ -1,9 +1,9 @@ //! Deterministic tick insertion gates all run facts; retries cannot queue a second run. -use crate::runs::main::queue_run::queue_run; +use crate::runs::main::requests::queue_run::queue_run; use crate::runs::models::{RunCommand, RunKind, RunOrigin, RunRequest}; use crate::scheduler::models::{DueTrigger, SchedulerLease, TickDecision, TickPlan}; -use crate::snapshots::main::job_write_set::job_write_set; +use crate::snapshots::main::write_sets::job_write_set::job_write_set; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{Change, ControlTransaction, Parameter}; @@ -67,9 +67,10 @@ pub(crate) async fn persist_ticks( }, settings: trigger.job_json.settings.clone(), window: None, - concurrency_groups: trigger.job_json.concurrency_groups.clone(), + pools: trigger.job_json.pools.clone(), narrowing: None, retry: None, + member_attempt: None, }; transaction = queue_run(transaction, &request).await?; } diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs index f27642c52d..e36219f183 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs @@ -5,7 +5,7 @@ use std::time::{Duration, Instant}; use chrono::Utc; use crate::runs::constants::MONITOR_BATCH_SIZE; -use crate::runs::main::monitor_runs::monitor_runs; +use crate::runs::main::execution::monitor_runs::monitor_runs; use crate::scheduler::_helpers::drain_due_triggers::{drain_due_triggers, wait_for_next_due}; use crate::scheduler::_helpers::tick_retention::prune_ticks_when_due; use crate::scheduler::constants::RENEW_INTERVAL_SECONDS; diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/custom.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/custom.rs index 6918d453ff..be4b12a9dd 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/custom.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/custom.rs @@ -5,7 +5,7 @@ use std::collections::BTreeSet; use chrono::TimeDelta; use serde_json::json; -use crate::platform::main::read_location_work::read_location_work; +use crate::platform::main::work::read_location_work::read_location_work; use crate::platform::models::{LocationWork, LocationWorkOutcome, LocationWorkRequest}; use crate::sensors::_helpers::requests::queue_sensor_run; use crate::sensors::models::SensorRequests; diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs index 87e6ea7618..5fee175431 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs @@ -14,7 +14,7 @@ use crate::sensors::models::{ NodeEventKind, NodeSensorEvent, SensorCursor, SensorEvaluationRequest, }; use crate::sensors::types::SourcePage; -use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::snapshots::main::reads::read_snapshot_nodes::read_snapshot_nodes; use crate::store::errors::ControlError; use crate::store::main::decode_row::decode_row; use crate::store::main::query::query; diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs index 277a88763b..20f5b74207 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs @@ -1,6 +1,6 @@ //! One bounded platform page yields matched run requests and a storage-order cursor. -use crate::platform::main::read_platform_events::read_platform_events; +use crate::platform::main::installation::read_platform_events::read_platform_events; use crate::platform::models::PlatformEventKind; use crate::sensors::_helpers::requests::queue_sensor_run; use crate::sensors::models::SensorRequests; diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs index 77ea2cab17..641bdeca14 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs @@ -5,8 +5,8 @@ use uuid::Uuid; use crate::credentials::models::AuditContext; use crate::declarations::main::published_job::published_job; use crate::declarations::models::Overlap; -use crate::runs::main::frozen_job_request::frozen_job_request; -use crate::runs::main::queue_run::queue_run; +use crate::runs::main::requests::frozen_job_request::frozen_job_request; +use crate::runs::main::requests::queue_run::queue_run; use crate::runs::models::{JobRunRequest, RunOrigin}; use crate::scheduler::main::active_job_run::active_job_run; use crate::sensors::models::{SensorEvaluationRequest, SensorRequests, SkippedRequest}; diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs index 791ed0093d..947f7eacd6 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs @@ -1,9 +1,10 @@ -//! Terminal projections carry the durable finish cursor; late observations cannot move it. +//! Terminal projections carry the durable finish cursor; a retried member fires on its final attempt. use std::collections::BTreeSet; use uuid::Uuid; +use crate::runs::constants::SUPERSEDED_CHILD; use crate::sensors::_helpers::requests::queue_sensor_run; use crate::sensors::constants::EXECUTION_NODE_BATCH_SIZE; use crate::sensors::models::SensorRequests; @@ -23,10 +24,10 @@ pub(crate) async fn consume_run_events( ) -> Result<(ControlTransaction, (SensorCursor, SensorRequests)), ControlError> { let (mut transaction, rows) = query( transaction, - "SELECT run_id,status,last_event_storage_id FROM sqb_runs \ - WHERE location_id=$1 AND target=$2 AND job_key=$3 AND last_event_storage_id>$4 \ - AND status IN ('success','warn','partial','failed','cancelled') \ - ORDER BY last_event_storage_id LIMIT 100", + &format!("SELECT r.run_id,r.status,r.last_event_storage_id FROM sqb_runs r \ + WHERE r.location_id=$1 AND r.target=$2 AND r.job_key=$3 AND r.last_event_storage_id>$4 \ + AND r.status IN ('success','warn','partial','failed','cancelled','skipped') AND NOT {SUPERSEDED_CHILD} \ + ORDER BY r.last_event_storage_id LIMIT 100"), vec![ Parameter::Id(evaluation.location_id), Parameter::Text(evaluation.target.clone()), @@ -54,7 +55,7 @@ pub(crate) async fn consume_run_events( Ok((transaction, (cursor, requests))) } -/// Each chunk intersects the previous matches: every selected node must qualify. +/// Each chunk narrows the previous matches; a member's node qualifies from any attempt in its group run. async fn matching_executions( mut transaction: ControlTransaction, mut eligible: BTreeSet, @@ -75,15 +76,19 @@ async fn matching_executions( let rows; (transaction, rows) = query(transaction, &format!( "SELECT r.run_id,r.status,r.last_event_storage_id FROM sqb_runs r \ - WHERE r.run_id IN ({run_parameters}) AND \ + WHERE r.run_id IN ({run_parameters}) AND (\ (SELECT COUNT(*) FROM sqb_run_nodes n WHERE n.run_id=r.run_id AND n.kind='node' \ AND n.node_key IN ({node_parameters}) AND n.status='success' AND n.ended_at IS NOT NULL \ AND ((r.kind<>'group' AND n.started_at IS NOT NULL) OR \ (r.kind='group' AND EXISTS(SELECT 1 FROM sqb_runs c \ JOIN sqb_run_nodes cn ON cn.run_id=c.run_id AND cn.node_key=n.node_key AND cn.kind='node' \ WHERE c.parent_run_id=r.run_id AND c.location_id=r.location_id AND c.target=r.target \ - AND cn.status='success' AND cn.started_at IS NOT NULL AND cn.ended_at IS NOT NULL))))={}", - chunk.len()), parameters).await?; + AND cn.status='success' AND cn.started_at IS NOT NULL AND cn.ended_at IS NOT NULL))))={count} \ + OR (r.parent_run_id IS NOT NULL AND \ + (SELECT COUNT(DISTINCT an.node_key) FROM sqb_runs a JOIN sqb_run_nodes an ON an.run_id=a.run_id AND an.kind='node' \ + WHERE a.parent_run_id=r.parent_run_id AND a.job_key=r.job_key AND an.node_key IN ({node_parameters}) \ + AND an.status='success' AND an.started_at IS NOT NULL AND an.ended_at IS NOT NULL)={count}))", + count = chunk.len()), parameters).await?; let matches: Vec = rows.into_iter().map(decode_row).collect::>()?; eligible = matches.into_iter().map(|run| run.run_id).collect(); diff --git a/crates/sqlbuild-orchestration/src/sensors/models.rs b/crates/sqlbuild-orchestration/src/sensors/models.rs index 936bddc237..bad48dc2b2 100644 --- a/crates/sqlbuild-orchestration/src/sensors/models.rs +++ b/crates/sqlbuild-orchestration/src/sensors/models.rs @@ -62,6 +62,8 @@ pub enum RunEventKind { Partial, Failed, Cancelled, + /// A group member the group did not run because a job it depends on failed. + Skipped, } impl RunEventKind { @@ -75,6 +77,7 @@ impl RunEventKind { | (Self::Partial, RunOutcome::Partial) | (Self::Failed, RunOutcome::Failed) | (Self::Cancelled, RunOutcome::Cancelled) + | (Self::Skipped, RunOutcome::Skipped) ) } } diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/connection_settings.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/connection_settings.rs new file mode 100644 index 0000000000..f357ff1551 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/connection_settings.rs @@ -0,0 +1,39 @@ +//! Named, non-secret connection settings read from a target's settings. + +use serde_json::{Map, Value}; + +use crate::snapshots::constants::{AUTHENTICATOR_METHODS, CREDENTIAL_KEY_METHODS}; + +/// A setting's trimmed text, when it is set and not blank. +fn text<'a>(settings: &'a Map, key: &str) -> Option<&'a str> { + settings + .get(key) + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) +} + +/// The first of `keys` that is set, as text. +pub(crate) fn first(settings: &Map, keys: &[&str]) -> Option { + keys.iter() + .find_map(|key| text(settings, key)) + .map(str::to_owned) +} + +/// The sign-in method by name: Snowflake's authenticator, then which credential is set, or `none` for a file database. +pub(crate) fn auth_method(settings: &Map, file_database: bool) -> Option { + if let Some(authenticator) = text(settings, "authenticator") { + let authenticator = authenticator.to_lowercase(); + return Some( + AUTHENTICATOR_METHODS + .iter() + .find(|(name, _)| *name == authenticator) + .map_or(authenticator.clone(), |(_, method)| (*method).to_owned()), + ); + } + CREDENTIAL_KEY_METHODS + .iter() + .find(|(key, _)| settings.get(*key).is_some_and(|value| !value.is_null())) + .map(|(_, method)| (*method).to_owned()) + .or_else(|| file_database.then(|| "none".to_owned())) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs index d7f4615435..d176a6b06e 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs @@ -1,5 +1,7 @@ +pub(crate) mod connection_settings; pub(crate) mod content; pub(crate) mod preparation; pub(crate) mod publication; +pub(crate) mod published_listing; pub(crate) mod reading; pub(crate) mod write_set; diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/published_listing.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/published_listing.rs new file mode 100644 index 0000000000..549c4f184c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/published_listing.rs @@ -0,0 +1,81 @@ +//! A target's publications as a person browses them, from their stored rows. + +use std::collections::BTreeMap; + +use chrono::{DateTime, Utc}; +use uuid::Uuid; + +use crate::snapshots::models::{ConnectionDetails, PublishedChange, PublishedSnapshot}; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, decompress::decompress}; +use crate::store::types::Row; + +/// Each publication's changes; role names stay hidden from people who cannot assign roles. +pub(crate) fn published_changes( + rows: Vec, + names_roles: bool, +) -> Result>, ControlError> { + let mut changes: BTreeMap> = BTreeMap::new(); + for row in rows { + let snapshot_id = id_of(&row)?; + let mut change: PublishedChange = decode_row(row)?; + if !names_roles && change.subject_key.starts_with("role:") { + change.subject_key = "role:".into(); + } + changes.entry(snapshot_id).or_default().push(change); + } + Ok(changes) +} + +/// Each publication's connection settings, from its stored connection document. +pub(crate) fn published_connections( + rows: Vec, +) -> Result, ControlError> { + rows.into_iter() + .map(|row| { + let details = serde_json::from_slice(&decompress(&row["body"])?) + .map_err(|error| ControlError::Invalid(format!("connection document: {error}")))?; + Ok((id_of(&row)?, details)) + }) + .collect() +} + +/// The publications, newest first, each with its changes and connection settings. +pub(crate) fn published_snapshots( + rows: Vec, + mut changes: BTreeMap>, + mut connections: BTreeMap, +) -> Result, ControlError> { + rows.into_iter() + .map(|row| { + let mut snapshot = snapshot_of(row)?; + snapshot.changes = changes.remove(&snapshot.snapshot_id).unwrap_or_default(); + snapshot.connection = connections.remove(&snapshot.snapshot_id); + Ok(snapshot) + }) + .collect() +} + +fn snapshot_of(row: Row) -> Result { + let snapshot_id = id_of(&row)?; + let text = |key: &str| row[key].as_str().map(String::from); + let published_at: DateTime = serde_json::from_value(row["published_at"].clone()) + .map_err(|error| ControlError::Invalid(format!("stored publication time: {error}")))?; + Ok(PublishedSnapshot { + snapshot_id, + generation: row["generation"].as_i64().unwrap_or_default(), + git_revision: text("git_revision"), + git_dirty: row["git_dirty"].as_i64() == Some(1), + git_author: text("git_author"), + environment_hash: text("environment_hash").unwrap_or_default(), + sqb_version: text("sqb_version").unwrap_or_default(), + published_at, + changes: Vec::new(), + connection: None, + }) +} + +fn id_of(row: &Row) -> Result { + serde_json::from_value(row["snapshot_id"].clone()) + .map_err(|error| ControlError::Invalid(format!("stored snapshot id: {error}"))) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs index 710686ba6b..abdcf27449 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs @@ -6,7 +6,7 @@ use crate::snapshots::models::SnapshotNode; use crate::store::errors::ControlError; use crate::store::main::decompress::decompress; use crate::store::main::query::query; -use crate::store::models::{ControlTransaction, Parameter}; +use crate::store::models::{Connection, ControlTransaction, Parameter}; use crate::store::types::Row; use serde::de::DeserializeOwned; use serde_json::Value; @@ -43,6 +43,15 @@ pub(crate) fn read_nodes(rows: Vec) -> Result String { + if matches!(transaction.connection, Some(Connection::Postgres(_))) { + format!("encode({column}, 'hex')") + } else { + format!("hex({column})") + } +} + fn read_bodies( rows: Vec, key: &str, diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/write_set.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/write_set.rs index 955f075b2b..5bbfffe685 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/write_set.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/write_set.rs @@ -5,14 +5,35 @@ use std::collections::{BTreeMap, BTreeSet}; use serde_json::Value; use crate::snapshots::constants::{ - LOADER_POINTER, LOADS_FIELD, MODEL_NODE_KIND, SOURCE_KEY_PREFIX, SOURCE_NODE_KIND, - WRITER_NODE_KINDS, + LOAD_SOURCES_SETTING, LOADER_POINTER, LOADS_FIELD, MODEL_NODE_KIND, SOURCE_KEY_PREFIX, + SOURCE_NODE_KIND, WRITER_NODE_KINDS, }; -use crate::snapshots::models::SnapshotNode; +use crate::snapshots::models::{SnapshotNode, WriterNode}; + +/// The parts of a published node a write set reads. +pub(crate) fn writer_node(node: &SnapshotNode) -> WriterNode { + let definition = &node.definition; + WriterNode { + kind: node.kind.clone(), + source_deps: keys(definition, "deps") + .filter(|key| key.starts_with(SOURCE_KEY_PREFIX)) + .collect(), + loads: keys(definition, LOADS_FIELD).collect(), + managed: node.kind == SOURCE_NODE_KIND + && definition + .pointer(LOADER_POINTER) + .is_some_and(|loader| !loader.is_null()), + } +} + +/// Whether a build with these settings loads the managed sources its selected models read. +pub(crate) fn loads_model_sources(settings: &BTreeMap) -> bool { + settings.get(LOAD_SOURCES_SETTING) != Some(&Value::Bool(false)) +} /// Selected writer keys, candidate source keys, and the sources selected models read when loading is on. -pub(crate) fn selected_writers( - selected: &BTreeMap, +pub(crate) fn selected_writers<'a>( + selected: impl IntoIterator, loads_model_sources: bool, ) -> (BTreeSet, BTreeSet, BTreeSet) { let mut writers: BTreeSet = BTreeSet::new(); @@ -25,31 +46,27 @@ pub(crate) fn selected_writers( if node.kind == SOURCE_NODE_KIND { sources.insert(key.clone()); } else if node.kind == MODEL_NODE_KIND && loads_model_sources { - model_reads.extend(source_deps(&node.definition)); - model_reads.extend(loads(&node.definition)); + model_reads.extend(node.source_deps.iter().cloned()); + model_reads.extend(node.loads.iter().cloned()); } } sources.extend(model_reads.iter().cloned()); (writers, sources, model_reads) } -/// A source with a non-null loader, which a loading build writes. -pub(crate) fn managed(node: &SnapshotNode) -> bool { - node.kind == SOURCE_NODE_KIND - && node - .definition - .pointer(LOADER_POINTER) - .is_some_and(|loader| !loader.is_null()) -} - -/// The source keys among a node's compiled `deps`. -pub(crate) fn source_deps(definition: &Value) -> impl Iterator + '_ { - keys(definition, "deps").filter(|key| key.starts_with(SOURCE_KEY_PREFIX)) -} - -/// The keys a node's definition lists in `loads`; snapshots published before it have none. -pub(crate) fn loads(definition: &Value) -> impl Iterator + '_ { - keys(definition, LOADS_FIELD) +/// The managed candidate sources, and the loader chains of those selected models read. +pub(crate) fn managed_sources<'a>( + candidates: impl IntoIterator, + model_reads: &BTreeSet, +) -> BTreeSet { + let mut written: BTreeSet = BTreeSet::new(); + for (key, node) in candidates.into_iter().filter(|(_, node)| node.managed) { + if model_reads.contains(key) { + written.extend(node.loads.iter().cloned()); + } + written.insert(key.clone()); + } + written } /// The string keys a node's definition lists under `field`. diff --git a/crates/sqlbuild-orchestration/src/snapshots/constants.rs b/crates/sqlbuild-orchestration/src/snapshots/constants.rs index 0072f17f66..4c29142d1c 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/constants.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/constants.rs @@ -6,8 +6,12 @@ pub(crate) const CONTENT_HASH_CHUNK: usize = 500; /// `{node_key: {column: {"expression_range": [start, end]}}}`; readers skip unknown column keys. pub(crate) const COLUMNS_DOCUMENT: &str = "columns"; +/// How long a publication waits for the server, which stores a large snapshot well past 30 s. +pub(crate) const PUBLISH_TIMEOUT_SECONDS: u64 = 300; +/// The target's non-secret connection settings, a `ConnectionDetails`. +pub const CONNECTION_DOCUMENT: &str = "connection"; /// Snapshot document kinds: per-snapshot metadata that never changes a node hash or node change. -pub(crate) const SNAPSHOT_DOCUMENT_KINDS: [&str; 7] = [ +pub(crate) const SNAPSHOT_DOCUMENT_KINDS: [&str; 8] = [ "jobs", "groups", "triggers", @@ -15,6 +19,36 @@ pub(crate) const SNAPSHOT_DOCUMENT_KINDS: [&str; 7] = [ "lineage", COLUMNS_DOCUMENT, "providers", + CONNECTION_DOCUMENT, +]; +/// Connection keys that may name the signed-in user, in preference order. +pub(crate) const USER_KEYS: [&str; 3] = ["user", "username", "login"]; +/// Connection keys that may name the server, in preference order. +pub(crate) const HOST_KEYS: [&str; 3] = ["host", "server_hostname", "server"]; +/// Connection keys that may name the default database, in preference order. +pub(crate) const DATABASE_KEYS: [&str; 4] = ["database", "dbname", "catalog", "project"]; +/// Adapters whose connection names a local database file, and the key that names it. +pub(crate) const FILE_DATABASE_KEYS: [(&str, &str); 2] = + [("duckdb", "database"), ("sqlite", "path")]; +/// Snowflake `authenticator` values by the sign-in method they name; others are shown as they are. +pub(crate) const AUTHENTICATOR_METHODS: [(&str, &str); 7] = [ + ("snowflake", "password"), + ("snowflake_jwt", "key_pair"), + ("externalbrowser", "external_browser"), + ("oauth", "oauth"), + ("oauth_authorization_code", "oauth"), + ("oauth_client_credentials", "oauth"), + ("programmatic_access_token", "token"), +]; +/// Keys whose presence names the sign-in method when no authenticator does, in preference order. +pub(crate) const CREDENTIAL_KEY_METHODS: [(&str, &str); 7] = [ + ("private_key", "key_pair"), + ("private_key_path", "key_pair"), + ("private_key_file", "key_pair"), + ("token", "token"), + ("access_token", "token"), + ("credentials_path", "service_account"), + ("password", "password"), ]; /// Beyond the newest generations, any retained reference keeps a snapshot. diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/list_person_snapshots.rs b/crates/sqlbuild-orchestration/src/snapshots/main/list_person_snapshots.rs deleted file mode 100644 index 33605212c7..0000000000 --- a/crates/sqlbuild-orchestration/src/snapshots/main/list_person_snapshots.rs +++ /dev/null @@ -1,87 +0,0 @@ -//! A person browses a target's publications only with view on every current node, like its jobs. - -use std::collections::BTreeMap; - -use chrono::{DateTime, Utc}; -use uuid::Uuid; - -use crate::authorization::main::authorize_listing::authorize_listing; -use crate::authorization::main::person_assigns_roles::person_assigns_roles; -use crate::credentials::models::{AuditContext, DefaultRole}; -use crate::snapshots::models::{PublishedChange, PublishedSnapshot}; -use crate::store::errors::ControlError; -use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; -use crate::store::types::Row; - -/// Newest first, at most 100 publications; changed roles are named only for people who assign roles. -pub async fn list_person_snapshots( - store: &ControlStore, - (context, credentials): (&AuditContext, Uuid), - ((location_id, target), limit): ((Uuid, &str), u32), - default_role: DefaultRole, -) -> Result, ControlError> { - let (transaction, _) = authorize_listing( - begin(store, TransactionMode::Write).await?, - (context, credentials), - (location_id, target), - default_role, - ) - .await?; - let scope = vec![ - Parameter::Id(location_id), - Parameter::Text(target.into()), - Parameter::Integer(i64::from(limit.min(100))), - ]; - let (transaction, published) = query(transaction, - "SELECT snapshot_id, generation, git_revision, CASE WHEN git_dirty THEN 1 ELSE 0 END AS git_dirty, git_author, environment_hash, sqb_version, published_at \ - FROM sqb_snapshots WHERE location_id=$1 AND target=$2 ORDER BY generation DESC LIMIT $3", - scope.clone()).await?; - let (transaction, changed) = query(transaction, - "SELECT c.snapshot_id, c.change_kind, c.subject_key FROM sqb_snapshot_changes c \ - WHERE c.snapshot_id IN (SELECT s.snapshot_id FROM sqb_snapshots s WHERE s.location_id=$1 AND s.target=$2 \ - ORDER BY s.generation DESC LIMIT $3) ORDER BY c.snapshot_id, c.change_seq", - scope).await?; - let (transaction, names_roles) = person_assigns_roles(transaction, credentials).await?; - commit(transaction).await?; - let mut changes: BTreeMap> = BTreeMap::new(); - for row in changed { - let snapshot_id = id_of(&row)?; - let mut change: PublishedChange = decode_row(row)?; - if !names_roles && change.subject_key.starts_with("role:") { - change.subject_key = "role:".into(); - } - changes.entry(snapshot_id).or_default().push(change); - } - published - .into_iter() - .map(|row| { - let mut snapshot = snapshot_of(row)?; - snapshot.changes = changes.remove(&snapshot.snapshot_id).unwrap_or_default(); - Ok(snapshot) - }) - .collect() -} - -fn snapshot_of(row: Row) -> Result { - let snapshot_id = id_of(&row)?; - let text = |key: &str| row[key].as_str().map(String::from); - let published_at: DateTime = serde_json::from_value(row["published_at"].clone()) - .map_err(|error| ControlError::Invalid(format!("stored publication time: {error}")))?; - Ok(PublishedSnapshot { - snapshot_id, - generation: row["generation"].as_i64().unwrap_or_default(), - git_revision: text("git_revision"), - git_dirty: row["git_dirty"].as_i64() == Some(1), - git_author: text("git_author"), - environment_hash: text("environment_hash").unwrap_or_default(), - sqb_version: text("sqb_version").unwrap_or_default(), - published_at, - changes: Vec::new(), - }) -} - -fn id_of(row: &Row) -> Result { - serde_json::from_value(row["snapshot_id"].clone()) - .map_err(|error| ControlError::Invalid(format!("stored snapshot id: {error}"))) -} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs index 43ec2c5452..7df0f24505 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs @@ -1,18 +1,3 @@ -pub(super) mod _publish_snapshot; -pub mod current_snapshot; -pub mod current_snapshot_record; -pub(crate) mod holds_on_current_nodes; -pub(crate) mod job_write_set; -pub mod list_person_snapshots; -pub mod next_remote_generation; -pub mod node_hashes; -pub(crate) mod prune_snapshots; -pub(crate) mod publish_development_snapshot; -pub mod publish_remote_snapshot; -pub mod publish_snapshot; -pub(crate) mod read_current_record; -pub mod read_person_node_sql; -pub(crate) mod read_snapshot_documents; -pub(crate) mod read_snapshot_keys; -pub(crate) mod read_snapshot_nodes; -pub mod verify_execution; +pub mod publication; +pub mod reads; +pub mod write_sets; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/_publish_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/publication/_publish_snapshot.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/snapshots/main/_publish_snapshot.rs rename to crates/sqlbuild-orchestration/src/snapshots/main/publication/_publish_snapshot.rs diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/publication/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/publication/mod.rs new file mode 100644 index 0000000000..72ab6c9726 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/publication/mod.rs @@ -0,0 +1,7 @@ +pub(in crate::snapshots) mod _publish_snapshot; +pub mod next_remote_generation; +pub(crate) mod prune_snapshots; +pub(crate) mod publish_development_snapshot; +pub mod publish_remote_snapshot; +pub mod publish_snapshot; +pub mod verify_execution; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs b/crates/sqlbuild-orchestration/src/snapshots/main/publication/next_remote_generation.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs rename to crates/sqlbuild-orchestration/src/snapshots/main/publication/next_remote_generation.rs diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/prune_snapshots.rs b/crates/sqlbuild-orchestration/src/snapshots/main/publication/prune_snapshots.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/snapshots/main/prune_snapshots.rs rename to crates/sqlbuild-orchestration/src/snapshots/main/publication/prune_snapshots.rs diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/publish_development_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/publication/publish_development_snapshot.rs similarity index 82% rename from crates/sqlbuild-orchestration/src/snapshots/main/publish_development_snapshot.rs rename to crates/sqlbuild-orchestration/src/snapshots/main/publication/publish_development_snapshot.rs index 9d3065c812..0316e7236e 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/publish_development_snapshot.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/publication/publish_development_snapshot.rs @@ -1,6 +1,6 @@ //! Publication keeps development safety defaults inside the snapshot transaction. -use crate::snapshots::main::_publish_snapshot::publish_snapshot_with_defaults; +use crate::snapshots::main::publication::_publish_snapshot::publish_snapshot_with_defaults; use crate::snapshots::models::{SnapshotInput, SnapshotRecord}; use crate::store::errors::ControlError; use crate::store::models::ControlStore; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/publish_remote_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/publication/publish_remote_snapshot.rs similarity index 92% rename from crates/sqlbuild-orchestration/src/snapshots/main/publish_remote_snapshot.rs rename to crates/sqlbuild-orchestration/src/snapshots/main/publication/publish_remote_snapshot.rs index f278709c98..a250a35c25 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/publish_remote_snapshot.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/publication/publish_remote_snapshot.rs @@ -1,5 +1,7 @@ //! A code location sends retained compiled data over HTTP and never opens the control store. +use std::time::Duration; + use reqwest::StatusCode; use reqwest::header::{CONTENT_ENCODING, CONTENT_TYPE}; @@ -7,6 +9,7 @@ use crate::api::constants::{BODY_BYTES, GZIP_ENCODING, JSON_MEDIA_TYPE}; use crate::api::errors::ClientError; use crate::api::main::client_response::client_response; use crate::api::main::connect_client::connect_client; +use crate::snapshots::constants::PUBLISH_TIMEOUT_SECONDS; use crate::snapshots::models::{SnapshotInput, SnapshotRecord}; use crate::store::main::compress::compress; @@ -33,6 +36,7 @@ pub async fn publish_remote_snapshot( connection .client .post(url.clone()) + .timeout(Duration::from_secs(PUBLISH_TIMEOUT_SECONDS)) .header(CONTENT_TYPE, JSON_MEDIA_TYPE) .body(body) }; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/publication/publish_snapshot.rs similarity index 83% rename from crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs rename to crates/sqlbuild-orchestration/src/snapshots/main/publication/publish_snapshot.rs index 471a2bebe6..8172a26845 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/publication/publish_snapshot.rs @@ -1,6 +1,6 @@ //! Publish a generation using ordinary production trigger defaults. -use crate::snapshots::main::_publish_snapshot::publish_snapshot_with_defaults; +use crate::snapshots::main::publication::_publish_snapshot::publish_snapshot_with_defaults; use crate::snapshots::models::{SnapshotInput, SnapshotRecord}; use crate::store::errors::ControlError; use crate::store::models::ControlStore; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/verify_execution.rs b/crates/sqlbuild-orchestration/src/snapshots/main/publication/verify_execution.rs similarity index 96% rename from crates/sqlbuild-orchestration/src/snapshots/main/verify_execution.rs rename to crates/sqlbuild-orchestration/src/snapshots/main/publication/verify_execution.rs index f4a305b5ba..a827222cb7 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/verify_execution.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/publication/verify_execution.rs @@ -1,7 +1,7 @@ //! Refuse execution when a worker's local compilation differs from the pinned publication. use crate::runs::models::RunExecution; -use crate::snapshots::main::node_hashes::node_hashes; +use crate::snapshots::main::reads::node_hashes::node_hashes; use crate::snapshots::models::SnapshotInput; use crate::store::errors::ControlError; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/reads/connection_details.rs b/crates/sqlbuild-orchestration/src/snapshots/main/reads/connection_details.rs new file mode 100644 index 0000000000..9c76b8c8af --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/reads/connection_details.rs @@ -0,0 +1,32 @@ +//! A target's publishable connection settings: named fields only, so no secret value can leak. + +use serde_json::{Map, Value}; + +use crate::snapshots::_helpers::connection_settings::{auth_method, first}; +use crate::snapshots::constants::{DATABASE_KEYS, FILE_DATABASE_KEYS, HOST_KEYS, USER_KEYS}; +use crate::snapshots::models::ConnectionDetails; + +/// Only these named, non-secret settings are read; every other key, including every credential value, is ignored. +#[must_use] +pub fn connection_details(adapter: &str, settings: &Map) -> ConnectionDetails { + let file_key = FILE_DATABASE_KEYS + .iter() + .find(|(name, _)| *name == adapter) + .map(|(_, key)| *key); + ConnectionDetails { + adapter: adapter.to_owned(), + account: first(settings, &["account"]), + host: match file_key { + Some(key) => first(settings, &[key]), + None => first(settings, &HOST_KEYS), + }, + user: first(settings, &USER_KEYS), + role: first(settings, &["role"]), + warehouse: first(settings, &["warehouse"]), + database: file_key + .is_none() + .then(|| first(settings, &DATABASE_KEYS)) + .flatten(), + auth_method: auth_method(settings, file_key.is_some()), + } +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/reads/current_snapshot.rs similarity index 89% rename from crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs rename to crates/sqlbuild-orchestration/src/snapshots/main/reads/current_snapshot.rs index ddffa096f8..00df2fa292 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/reads/current_snapshot.rs @@ -3,9 +3,9 @@ use uuid::Uuid; use crate::snapshots::_helpers::reading::attach_sources; -use crate::snapshots::main::read_current_record::read_current_record; -use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents; -use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::snapshots::main::reads::read_current_record::read_current_record; +use crate::snapshots::main::reads::read_snapshot_documents::read_snapshot_documents; +use crate::snapshots::main::reads::read_snapshot_nodes::read_snapshot_nodes; use crate::snapshots::models::{CurrentSnapshot, SnapshotChange}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs b/crates/sqlbuild-orchestration/src/snapshots/main/reads/current_snapshot_record.rs similarity index 89% rename from crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs rename to crates/sqlbuild-orchestration/src/snapshots/main/reads/current_snapshot_record.rs index 940c0a3804..2659bd9ab1 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/reads/current_snapshot_record.rs @@ -2,7 +2,7 @@ use uuid::Uuid; -use crate::snapshots::main::read_current_record::read_current_record; +use crate::snapshots::main::reads::read_current_record::read_current_record; use crate::snapshots::models::SnapshotRecord; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit}; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/reads/list_person_snapshots.rs b/crates/sqlbuild-orchestration/src/snapshots/main/reads/list_person_snapshots.rs new file mode 100644 index 0000000000..566f70a4d8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/reads/list_person_snapshots.rs @@ -0,0 +1,58 @@ +//! A person browses a target's publications, with their connection settings, only with view on every current node, like its jobs. + +use uuid::Uuid; + +use crate::authorization::main::authorize_listing::authorize_listing; +use crate::authorization::main::person_assigns_roles::person_assigns_roles; +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::snapshots::_helpers::published_listing::{ + published_changes, published_connections, published_snapshots, +}; +use crate::snapshots::constants::CONNECTION_DOCUMENT; +use crate::snapshots::models::PublishedSnapshot; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Newest first, at most 100 publications; changed roles are named only for people who assign roles. +pub async fn list_person_snapshots( + store: &ControlStore, + (context, credentials): (&AuditContext, Uuid), + ((location_id, target), limit): ((Uuid, &str), u32), + default_role: DefaultRole, +) -> Result, ControlError> { + let (transaction, _) = authorize_listing( + begin(store, TransactionMode::Write).await?, + (context, credentials), + (location_id, target), + default_role, + ) + .await?; + let scope = vec![ + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::Integer(i64::from(limit.min(100))), + ]; + let (transaction, published) = query(transaction, + "SELECT snapshot_id, generation, git_revision, CASE WHEN git_dirty THEN 1 ELSE 0 END AS git_dirty, git_author, environment_hash, sqb_version, published_at \ + FROM sqb_snapshots WHERE location_id=$1 AND target=$2 ORDER BY generation DESC LIMIT $3", + scope.clone()).await?; + let (transaction, changed) = query(transaction, + "SELECT c.snapshot_id, c.change_kind, c.subject_key FROM sqb_snapshot_changes c \ + WHERE c.snapshot_id IN (SELECT s.snapshot_id FROM sqb_snapshots s WHERE s.location_id=$1 AND s.target=$2 \ + ORDER BY s.generation DESC LIMIT $3) ORDER BY c.snapshot_id, c.change_seq", + scope.clone()).await?; + let (transaction, connected) = query(transaction, + &format!("SELECT d.snapshot_id, b.body FROM sqb_snapshot_documents d JOIN sqb_documents b \ + ON b.document_hash=d.document_hash WHERE d.kind='{CONNECTION_DOCUMENT}' AND d.snapshot_id IN \ + (SELECT s.snapshot_id FROM sqb_snapshots s WHERE s.location_id=$1 AND s.target=$2 \ + ORDER BY s.generation DESC LIMIT $3)"), + scope).await?; + let (transaction, names_roles) = person_assigns_roles(transaction, credentials).await?; + commit(transaction).await?; + published_snapshots( + published, + published_changes(changed, names_roles)?, + published_connections(connected)?, + ) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/reads/listing_read.rs b/crates/sqlbuild-orchestration/src/snapshots/main/reads/listing_read.rs new file mode 100644 index 0000000000..d04bcc8406 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/reads/listing_read.rs @@ -0,0 +1,33 @@ +//! One read of a target in a transaction where the person may list its current publication. + +use std::future::Future; + +use uuid::Uuid; + +use crate::authorization::main::authorize_listing::authorize_listing; +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, ControlTransaction, TransactionMode}; + +/// Authorize the person to list `(location_id, target)`, run `read`, then commit. +pub(crate) async fn listing_read( + store: &ControlStore, + (credentials, default_role): ((&AuditContext, Uuid), DefaultRole), + (location_id, target): (Uuid, &str), + read: impl FnOnce(ControlTransaction) -> Read, +) -> Result +where + Read: Future>, +{ + let (transaction, _) = authorize_listing( + begin(store, TransactionMode::Write).await?, + credentials, + (location_id, target), + default_role, + ) + .await?; + let (transaction, value) = read(transaction).await?; + commit(transaction).await?; + Ok(value) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/reads/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/reads/mod.rs new file mode 100644 index 0000000000..02bc5d9c03 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/reads/mod.rs @@ -0,0 +1,12 @@ +pub mod connection_details; +pub mod current_snapshot; +pub mod current_snapshot_record; +pub mod list_person_snapshots; +pub(crate) mod listing_read; +pub mod node_hashes; +pub(crate) mod read_current_record; +pub(crate) mod read_kind_nodes; +pub mod read_person_node_sql; +pub(crate) mod read_snapshot_documents; +pub(crate) mod read_snapshot_keys; +pub(crate) mod read_snapshot_nodes; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/node_hashes.rs b/crates/sqlbuild-orchestration/src/snapshots/main/reads/node_hashes.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/snapshots/main/node_hashes.rs rename to crates/sqlbuild-orchestration/src/snapshots/main/reads/node_hashes.rs diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/read_current_record.rs b/crates/sqlbuild-orchestration/src/snapshots/main/reads/read_current_record.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/snapshots/main/read_current_record.rs rename to crates/sqlbuild-orchestration/src/snapshots/main/reads/read_current_record.rs diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/reads/read_kind_nodes.rs b/crates/sqlbuild-orchestration/src/snapshots/main/reads/read_kind_nodes.rs new file mode 100644 index 0000000000..758a8a08a0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/reads/read_kind_nodes.rs @@ -0,0 +1,31 @@ +//! Read one snapshot's nodes of the given kinds, so callers skip the definitions they never look at. + +use std::collections::BTreeMap; + +use uuid::Uuid; + +use crate::snapshots::_helpers::reading::{hex_column, query_bodies, read_nodes}; +use crate::snapshots::models::SnapshotNode; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +/// The snapshot's nodes whose kind is listed; an empty list reads none. +pub(crate) async fn read_kind_nodes( + transaction: ControlTransaction, + snapshot_id: Uuid, + kinds: &[&str], +) -> Result<(ControlTransaction, BTreeMap), ControlError> { + let select = format!( + "SELECT n.node_key, {} AS body FROM sqb_snapshot_nodes n JOIN sqb_node_definitions d \ + ON d.node_hash=n.node_hash WHERE n.snapshot_id=$1", + hex_column(&transaction, "d.body"), + ); + let (transaction, rows) = query_bodies( + transaction, + (&select, "n.node_kind"), + snapshot_id, + Some(kinds), + ) + .await?; + Ok((transaction, read_nodes(rows)?)) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs b/crates/sqlbuild-orchestration/src/snapshots/main/reads/read_person_node_sql.rs similarity index 94% rename from crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs rename to crates/sqlbuild-orchestration/src/snapshots/main/reads/read_person_node_sql.rs index b73be257f8..7b8d1ed5d2 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/reads/read_person_node_sql.rs @@ -10,9 +10,9 @@ use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; use crate::snapshots::_helpers::reading::attach_sources; use crate::snapshots::constants::COLUMNS_DOCUMENT; -use crate::snapshots::main::read_current_record::read_current_record; -use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents; -use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::snapshots::main::reads::read_current_record::read_current_record; +use crate::snapshots::main::reads::read_snapshot_documents::read_snapshot_documents; +use crate::snapshots::main::reads::read_snapshot_nodes::read_snapshot_nodes; use crate::snapshots::models::NodeSql; use crate::store::errors::ControlError; use crate::store::main::{commit::commit, query::query}; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_documents.rs b/crates/sqlbuild-orchestration/src/snapshots/main/reads/read_snapshot_documents.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_documents.rs rename to crates/sqlbuild-orchestration/src/snapshots/main/reads/read_snapshot_documents.rs diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_keys.rs b/crates/sqlbuild-orchestration/src/snapshots/main/reads/read_snapshot_keys.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_keys.rs rename to crates/sqlbuild-orchestration/src/snapshots/main/reads/read_snapshot_keys.rs diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_nodes.rs b/crates/sqlbuild-orchestration/src/snapshots/main/reads/read_snapshot_nodes.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_nodes.rs rename to crates/sqlbuild-orchestration/src/snapshots/main/reads/read_snapshot_nodes.rs diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/write_sets/held_write_set.rs b/crates/sqlbuild-orchestration/src/snapshots/main/write_sets/held_write_set.rs new file mode 100644 index 0000000000..ce7d1f45c2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/write_sets/held_write_set.rs @@ -0,0 +1,26 @@ +//! Resolve a selection's physical writers from write-set facts already held in memory. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value; + +use crate::snapshots::_helpers::write_set::{ + loads_model_sources, managed_sources, selected_writers, +}; +use crate::snapshots::models::WriterNode; + +/// [`crate::snapshots::main::write_sets::job_write_set::job_write_set`] over every node's facts, read once. +pub(crate) fn held_write_set( + nodes: &BTreeMap, + (selection, settings): (&BTreeSet, &BTreeMap), +) -> BTreeSet { + let present = |keys: &BTreeSet| -> Vec<(&String, &WriterNode)> { + keys.iter() + .filter_map(|key| nodes.get_key_value(key)) + .collect() + }; + let (mut written, candidates, model_reads) = + selected_writers(present(selection), loads_model_sources(settings)); + written.extend(managed_sources(present(&candidates), &model_reads)); + written +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/holds_on_current_nodes.rs b/crates/sqlbuild-orchestration/src/snapshots/main/write_sets/holds_on_current_nodes.rs similarity index 60% rename from crates/sqlbuild-orchestration/src/snapshots/main/holds_on_current_nodes.rs rename to crates/sqlbuild-orchestration/src/snapshots/main/write_sets/holds_on_current_nodes.rs index 0110ab2067..40c190566e 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/holds_on_current_nodes.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/write_sets/holds_on_current_nodes.rs @@ -4,8 +4,8 @@ use std::collections::BTreeSet; use uuid::Uuid; -use crate::authorization::main::decide_person_scopes::decide_person_scopes; -use crate::credentials::models::{Action, PermissionScope}; +use crate::credentials::models::Action; +use crate::snapshots::main::write_sets::holds_on_nodes::holds_on_nodes; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; @@ -20,16 +20,15 @@ pub(crate) async fn holds_on_current_nodes( "SELECT n.node_key FROM sqb_snapshot_nodes n JOIN sqb_current_snapshots c ON c.snapshot_id=n.snapshot_id \ WHERE c.location_id=$1 AND c.target=$2", vec![Parameter::Id(location_id), Parameter::Text(target.into())]).await?; - let scope = PermissionScope { - action, - location_id: Some(location_id), - target: Some(target.into()), - nodes: current - .iter() - .filter_map(|row| row["node_key"].as_str().map(str::to_owned)) - .collect::>(), - }; - let (transaction, refusals) = - decide_person_scopes(transaction, principal_id, (target, &[scope])).await?; - Ok((transaction, refusals.as_deref() == Some(&[None]))) + let nodes = current + .iter() + .filter_map(|row| row["node_key"].as_str().map(str::to_owned)) + .collect::>(); + holds_on_nodes( + transaction, + principal_id, + (action, location_id, target), + nodes, + ) + .await } diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/write_sets/holds_on_nodes.rs b/crates/sqlbuild-orchestration/src/snapshots/main/write_sets/holds_on_nodes.rs new file mode 100644 index 0000000000..c149045542 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/write_sets/holds_on_nodes.rs @@ -0,0 +1,28 @@ +//! Decide whether a person holds one action on every listed node of a target, for checks that must not audit. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::authorization::main::decide_person_scopes::decide_person_scopes; +use crate::credentials::models::{Action, PermissionScope}; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +/// Whether the person holds `action` on every one of `nodes`, decided without recording a denial. +pub(crate) async fn holds_on_nodes( + transaction: ControlTransaction, + principal_id: Uuid, + (action, location_id, target): (Action, Uuid, &str), + nodes: BTreeSet, +) -> Result<(ControlTransaction, bool), ControlError> { + let scope = PermissionScope { + action, + location_id: Some(location_id), + target: Some(target.into()), + nodes, + }; + let (transaction, refusals) = + decide_person_scopes(transaction, principal_id, (target, &[scope])).await?; + Ok((transaction, refusals.as_deref() == Some(&[None]))) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs b/crates/sqlbuild-orchestration/src/snapshots/main/write_sets/job_write_set.rs similarity index 56% rename from crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs rename to crates/sqlbuild-orchestration/src/snapshots/main/write_sets/job_write_set.rs index 488e6cf351..3b775b4c2e 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/write_sets/job_write_set.rs @@ -5,9 +5,11 @@ use std::collections::{BTreeMap, BTreeSet}; use serde_json::Value; use uuid::Uuid; -use crate::snapshots::_helpers::write_set::{loads, managed, selected_writers}; -use crate::snapshots::constants::LOAD_SOURCES_SETTING; -use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::snapshots::_helpers::write_set::{ + loads_model_sources, managed_sources, selected_writers, +}; +use crate::snapshots::main::reads::read_snapshot_nodes::read_snapshot_nodes; +use crate::snapshots::main::write_sets::writer_nodes::writer_nodes; use crate::store::errors::ControlError; use crate::store::models::ControlTransaction; @@ -19,15 +21,10 @@ pub(crate) async fn job_write_set( ) -> Result<(ControlTransaction, BTreeSet), ControlError> { let (transaction, selected) = read_snapshot_nodes(transaction, snapshot_id, Some(selection)).await?; - let loads_model_sources = settings.get(LOAD_SOURCES_SETTING) != Some(&Value::Bool(false)); - let (mut writers, candidates, model_reads) = selected_writers(&selected, loads_model_sources); + let (mut written, candidates, model_reads) = + selected_writers(&writer_nodes(&selected), loads_model_sources(settings)); let (transaction, sources) = read_snapshot_nodes(transaction, snapshot_id, Some(&candidates)).await?; - for (key, node) in sources.into_iter().filter(|(_, node)| managed(node)) { - if model_reads.contains(&key) { - writers.extend(loads(&node.definition)); - } - writers.insert(key); - } - Ok((transaction, writers)) + written.extend(managed_sources(&writer_nodes(&sources), &model_reads)); + Ok((transaction, written)) } diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/write_sets/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/write_sets/mod.rs new file mode 100644 index 0000000000..90001b1954 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/write_sets/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod held_write_set; +pub(crate) mod holds_on_current_nodes; +pub(crate) mod holds_on_nodes; +pub(crate) mod job_write_set; +pub(crate) mod writer_nodes; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/write_sets/writer_nodes.rs b/crates/sqlbuild-orchestration/src/snapshots/main/write_sets/writer_nodes.rs new file mode 100644 index 0000000000..2028ae7b49 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/write_sets/writer_nodes.rs @@ -0,0 +1,14 @@ +//! Keep only what each published node contributes to a write set. + +use std::collections::BTreeMap; + +use crate::snapshots::_helpers::write_set::writer_node; +use crate::snapshots::models::{SnapshotNode, WriterNode}; + +/// Each node's write-set facts, by key. +pub(crate) fn writer_nodes(nodes: &BTreeMap) -> BTreeMap { + nodes + .iter() + .map(|(key, node)| (key.clone(), writer_node(node))) + .collect() +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/mod.rs index e833f67102..ca8a58aa7c 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/mod.rs @@ -1,5 +1,5 @@ pub(crate) mod _helpers; -pub(crate) mod constants; +pub mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-orchestration/src/snapshots/models.rs b/crates/sqlbuild-orchestration/src/snapshots/models.rs index 3e035cc258..2e8e121b1d 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/models.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/models.rs @@ -81,6 +81,18 @@ pub(crate) struct NodeContent<'a> { pub(crate) sources: BTreeMap<&'static str, (String, &'a str)>, } +/// What a published node contributes to a write set, without the rest of its definition. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct WriterNode { + pub(crate) kind: String, + /// The `source:` keys among the node's compiled `deps`. + pub(crate) source_deps: Vec, + /// The keys the definition lists in `loads`; snapshots published before it have none. + pub(crate) loads: Vec, + /// A source with a non-null loader, which a loading build writes. + pub(crate) managed: bool, +} + #[derive(Debug)] pub(crate) struct PreparedNode { pub(crate) kind: String, @@ -144,6 +156,25 @@ pub struct PublishedSnapshot { pub published_at: DateTime, /// Added, changed and removed nodes and roles, in publication order. pub changes: Vec, + /// The target's non-secret connection settings; null when the location published none. + pub connection: Option, +} + +/// A target's connection settings as its location publishes them: names only, never a secret. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ConnectionDetails { + /// The adapter, such as `snowflake`, `postgres` or `duckdb`. + pub adapter: String, + pub account: Option, + /// The server host, or for a file database its path. + pub host: Option, + pub user: Option, + pub role: Option, + pub warehouse: Option, + /// The database the connection opens by default. + pub database: Option, + /// How the location signs in, by name only, such as `key_pair`, `password` or `oauth`. + pub auth_method: Option, } /// A changed subject, such as `model:orders`, or `role:analyst` (`role:` for those who can't assign roles). diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs index 420775f72e..c22b0fb5a5 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs @@ -7,3 +7,4 @@ pub(crate) mod recall_schema; pub(crate) mod release_version; #[cfg(test)] pub(crate) mod statement_count; +pub(crate) mod writer_gate; diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs b/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs index 4257371c1f..162f6f9899 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs @@ -47,7 +47,7 @@ pub(crate) fn open_sqlite(path: PathBuf) -> Result { | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX, )?; connection.busy_timeout(Duration::from_secs(30))?; - connection.execute_batch("PRAGMA journal_mode=WAL")?; + connection.execute_batch("PRAGMA auto_vacuum=INCREMENTAL; PRAGMA journal_mode=WAL")?; connection.query_row("SELECT count(*) FROM sqlite_schema", [], |_| Ok(()))?; Ok(ControlStore::Sqlite(SqliteStore { path, diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/writer_gate.rs b/crates/sqlbuild-orchestration/src/store/_helpers/writer_gate.rs new file mode 100644 index 0000000000..6ae5cef58c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/writer_gate.rs @@ -0,0 +1,33 @@ +//! One process's writers to a SQLite store take turns in arrival order, not SQLite's busy backoff. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, OnceLock}; + +use tokio::sync::OwnedMutexGuard; + +use crate::store::errors::ControlError; + +/// How long a writer waits for its turn; the same bound SQLite's busy handler gives it. +const WRITER_WAIT: std::time::Duration = std::time::Duration::from_secs(30); + +/// Wait for this process's turn to write `path`; the guard ends the turn when it drops. +pub(crate) async fn writer_turn(path: &Path) -> Result, ControlError> { + static GATES: OnceLock>>>> = OnceLock::new(); + let gate = Arc::clone( + GATES + .get_or_init(Mutex::default) + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .entry(path.canonicalize()?) + .or_default(), + ); + tokio::time::timeout(WRITER_WAIT, gate.lock_owned()) + .await + .map_err(|_| { + ControlError::Sqlite(rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_BUSY), + Some("another writer in this process held the control store".into()), + )) + }) +} diff --git a/crates/sqlbuild-orchestration/src/store/indexes.sql b/crates/sqlbuild-orchestration/src/store/indexes.sql index 4f141c93ee..811e4e24f6 100644 --- a/crates/sqlbuild-orchestration/src/store/indexes.sql +++ b/crates/sqlbuild-orchestration/src/store/indexes.sql @@ -8,3 +8,7 @@ CREATE INDEX IF NOT EXISTS sqb_location_work_snapshot ON sqb_location_work(snaps CREATE INDEX IF NOT EXISTS sqb_run_requests_run ON sqb_run_requests(run_id); CREATE INDEX IF NOT EXISTS sqb_warehouse_fingerprints_text ON sqb_warehouse_fingerprints(definition_text_hash); CREATE INDEX IF NOT EXISTS sqb_role_assignments_location ON sqb_role_assignments(location_id); +CREATE INDEX IF NOT EXISTS sqb_warehouse_catalog_received ON sqb_warehouse_catalog(location_id, target, received_at); +CREATE INDEX IF NOT EXISTS sqb_warehouse_fingerprints_received ON sqb_warehouse_fingerprints(location_id, target, received_at); +CREATE INDEX IF NOT EXISTS sqb_warehouse_consumption_received ON sqb_warehouse_consumption(location_id, target, received_at); +CREATE INDEX IF NOT EXISTS sqb_warehouse_observations_received ON sqb_warehouse_observations(location_id, target, received_at); diff --git a/crates/sqlbuild-orchestration/src/store/main/begin.rs b/crates/sqlbuild-orchestration/src/store/main/begin.rs index 7deced8adc..36c07dca74 100644 --- a/crates/sqlbuild-orchestration/src/store/main/begin.rs +++ b/crates/sqlbuild-orchestration/src/store/main/begin.rs @@ -7,6 +7,7 @@ use rusqlite::OpenFlags; use rusqlite::config::DbConfig; use crate::store::_helpers::changes::sqlite_changes; +use crate::store::_helpers::writer_gate::writer_turn; use crate::store::errors::ControlError; use crate::store::main::refresh_clock::refresh_clock; use crate::store::models::{Connection, ControlStore, ControlTransaction, TransactionMode}; @@ -15,8 +16,12 @@ pub(crate) async fn begin( store: &ControlStore, mode: TransactionMode, ) -> Result { + let mut writer = None; let (connection, notifications) = match store { ControlStore::Sqlite(path) => { + if mode == TransactionMode::Write { + writer = Some(writer_turn(path).await?); + } let path = path.clone(); tokio::task::spawn_blocking(move || { let connection = rusqlite::Connection::open_with_flags( @@ -62,6 +67,7 @@ pub(crate) async fn begin( now: Utc::now(), changes: Default::default(), notifications, + _writer: writer, }; refresh_clock(transaction).await } diff --git a/crates/sqlbuild-orchestration/src/store/main/decompress.rs b/crates/sqlbuild-orchestration/src/store/main/decompress.rs index 520cd4e7d7..f62b09dc48 100644 --- a/crates/sqlbuild-orchestration/src/store/main/decompress.rs +++ b/crates/sqlbuild-orchestration/src/store/main/decompress.rs @@ -7,9 +7,35 @@ use serde_json::Value; use crate::store::errors::ControlError; +/// A compressed body read as a blob, or as hexadecimal text, which a store returns far faster for large reads. pub(crate) fn decompress(value: &Value) -> Result, ControlError> { - let bytes: Vec = serde_json::from_value(value.clone()) - .map_err(|error| ControlError::Invalid(error.to_string()))?; + let invalid = + || ControlError::Invalid("stored body is neither bytes nor hexadecimal text".into()); + let bytes: Vec = match value { + Value::Array(items) => items + .iter() + .map(|item| { + item.as_u64() + .ok_or_else(invalid) + .and_then(|byte| u8::try_from(byte).map_err(|_| invalid())) + }) + .collect::, ControlError>>()?, + Value::String(text) => { + let (pairs, rest) = text.as_bytes().as_chunks::<2>(); + if !rest.is_empty() { + return Err(invalid()); + } + let digit = |byte: u8| char::from(byte).to_digit(16).ok_or_else(invalid); + pairs + .iter() + .map(|[high, low]| { + u8::try_from(digit(*high)? * 16 + digit(*low)?).map_err(|_| invalid()) + }) + .collect::, ControlError>>()? + } + other => serde_json::from_value(other.clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + }; let mut output: Vec = Vec::new(); GzDecoder::new(bytes.as_slice()).read_to_end(&mut output)?; Ok(output) diff --git a/crates/sqlbuild-orchestration/src/store/models.rs b/crates/sqlbuild-orchestration/src/store/models.rs index fc9a7371b2..cb4af29923 100644 --- a/crates/sqlbuild-orchestration/src/store/models.rs +++ b/crates/sqlbuild-orchestration/src/store/models.rs @@ -79,6 +79,8 @@ pub(crate) struct ControlTransaction { pub(crate) now: DateTime, pub(crate) changes: BTreeSet, pub(crate) notifications: Option>, + /// This process's turn to write a SQLite store, held until the transaction ends. + pub(crate) _writer: Option>, } #[derive(Debug)] diff --git a/crates/sqlbuild-orchestration/src/store/schema.sql b/crates/sqlbuild-orchestration/src/store/schema.sql index cd25e62407..24adc0bcf8 100644 --- a/crates/sqlbuild-orchestration/src/store/schema.sql +++ b/crates/sqlbuild-orchestration/src/store/schema.sql @@ -56,7 +56,7 @@ CREATE TABLE sqb_snapshot_sources ( ); CREATE TABLE sqb_snapshot_documents ( snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), - kind TEXT NOT NULL CHECK (kind IN ('jobs', 'groups', 'triggers', 'roles', 'lineage', 'columns', 'providers')), + kind TEXT NOT NULL CHECK (kind IN ('jobs', 'groups', 'triggers', 'roles', 'lineage', 'columns', 'providers', 'connection')), document_hash TEXT NOT NULL REFERENCES sqb_documents(document_hash), PRIMARY KEY (snapshot_id, kind) ); @@ -68,6 +68,14 @@ CREATE TABLE sqb_current_snapshots ( FOREIGN KEY (location_id, target, snapshot_id) REFERENCES sqb_snapshots(location_id, target, snapshot_id) ); +CREATE TABLE sqb_changed_builds ( + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + generation BIGINT NOT NULL, + reported_at {time} NOT NULL, + report_json {json} NOT NULL, + PRIMARY KEY (location_id, target) +); CREATE TABLE sqb_snapshot_changes ( snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), change_seq BIGINT NOT NULL, @@ -148,7 +156,7 @@ CREATE TABLE sqb_runs ( authored_json {json}, write_set_digest TEXT NOT NULL, rewrite_history BOOLEAN NOT NULL DEFAULT FALSE, - status TEXT NOT NULL CHECK (status IN ('queued', 'launched', 'running', 'unresponsive', 'success', 'warn', 'partial', 'failed', 'cancelled')), + status TEXT NOT NULL CHECK (status IN ('queued', 'launched', 'running', 'unresponsive', 'success', 'warn', 'partial', 'failed', 'cancelled', 'skipped')), status_reason TEXT, queued_at {time} NOT NULL, launched_at {time}, @@ -156,6 +164,7 @@ CREATE TABLE sqb_runs ( ended_at {time}, cancel_requested_at {time}, node_counts_json {json} NOT NULL, + ignored_audit_failures BIGINT NOT NULL DEFAULT 0, last_event_storage_id BIGINT NOT NULL, worker_instance {uuid}, container_id TEXT, @@ -196,8 +205,7 @@ CREATE TABLE sqb_concurrency_slots ( group_key TEXT NOT NULL, slot BIGINT NOT NULL CHECK (slot > 0), run_id {uuid} NOT NULL REFERENCES sqb_runs(run_id), - PRIMARY KEY (group_key, slot), - UNIQUE (group_key, run_id) + PRIMARY KEY (group_key, slot) ); CREATE INDEX sqb_concurrency_slots_run ON sqb_concurrency_slots(run_id); CREATE TABLE sqb_principals ( @@ -263,6 +271,12 @@ CREATE TABLE sqb_location_work ( completed_at {time} ); CREATE INDEX sqb_location_work_pending ON sqb_location_work(location_id, target, requested_at); +CREATE TABLE sqb_plan_diffs ( + request_id {uuid} NOT NULL REFERENCES sqb_location_work(request_id), + model TEXT NOT NULL, + diff_json {json} NOT NULL, + PRIMARY KEY (request_id, model) +); CREATE TABLE sqb_platform_sink_deliveries ( sink_id TEXT NOT NULL, storage_id BIGINT NOT NULL, @@ -289,7 +303,7 @@ CREATE TABLE sqb_run_requests ( PRIMARY KEY (trigger_id, run_key) ); CREATE INDEX sqb_runs_terminal_cursor ON sqb_runs(location_id, target, job_key, last_event_storage_id) -WHERE status IN ('success', 'warn', 'partial', 'failed', 'cancelled'); +WHERE status IN ('success', 'warn', 'partial', 'failed', 'cancelled', 'skipped'); CREATE INDEX sqb_runs_parent ON sqb_runs(parent_run_id) WHERE parent_run_id IS NOT NULL; CREATE TABLE sqb_warehouse_freshness ( event_id {uuid} PRIMARY KEY, @@ -399,7 +413,8 @@ CREATE TABLE sqb_warehouse_audit_results ( record_json {json} NOT NULL, received_at {time} NOT NULL, scan_id {uuid}, - binding_key TEXT + binding_key TEXT, + ignored BOOLEAN NOT NULL DEFAULT FALSE ); CREATE INDEX sqb_audit_results_namespace ON sqb_warehouse_audit_results (location_id,target,state_database,state_schema,observed_at,event_id); CREATE INDEX sqb_audit_results_binding ON sqb_warehouse_audit_results (location_id,target,binding_key,observed_at,event_id); diff --git a/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/checks.rs b/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/checks.rs index d4a78fbad3..92065c1a7d 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/checks.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/checks.rs @@ -19,6 +19,8 @@ use crate::warehouse_metadata::models::WarehouseAuditRecord; pub(crate) struct AuditRow { pub(crate) event_id: Uuid, pub(crate) record: WarehouseAuditRecord, + /// An error-severity failure its build let through. + pub(crate) ignored: bool, } /// The newest result of every check in the target. @@ -27,7 +29,7 @@ pub(crate) async fn latest_results( (location_id, target): (Uuid, &str), ) -> Result<(ControlTransaction, Vec), ControlError> { query_records(transaction, - "SELECT a.event_id,a.record_json AS record FROM sqb_warehouse_audit_results a \ + "SELECT a.event_id,a.record_json AS record,a.ignored FROM sqb_warehouse_audit_results a \ WHERE a.location_id=$1 AND a.target=$2 AND NOT EXISTS (SELECT 1 FROM sqb_warehouse_audit_results b \ WHERE b.location_id=a.location_id AND b.target=a.target AND b.binding_key=a.binding_key \ AND (b.observed_at,b.event_id)>(a.observed_at,a.event_id)) \ @@ -43,7 +45,7 @@ pub(crate) async fn check_result( ) -> Result<(ControlTransaction, Option), ControlError> { let (transaction, mut rows) = query_records( transaction, - "SELECT event_id,record_json AS record FROM sqb_warehouse_audit_results \ + "SELECT event_id,record_json AS record,ignored FROM sqb_warehouse_audit_results \ WHERE location_id=$1 AND target=$2 AND binding_key=$3 AND event_id=$4", vec![ Parameter::Id(location_id), @@ -98,7 +100,7 @@ pub(crate) async fn check_results( limit: u32, ) -> Result<(ControlTransaction, Vec), ControlError> { query_records(transaction, - "SELECT event_id,record_json AS record FROM sqb_warehouse_audit_results \ + "SELECT event_id,record_json AS record,ignored FROM sqb_warehouse_audit_results \ WHERE location_id=$1 AND target=$2 AND binding_key=$3 ORDER BY observed_at DESC,event_id DESC LIMIT $4", vec![Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Text(binding_key.into()), Parameter::Integer(i64::from(limit.clamp(1, 1000)))]).await diff --git a/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/run_recall.rs b/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/run_recall.rs index e8fbfca2c9..840050a339 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/run_recall.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/run_recall.rs @@ -9,15 +9,16 @@ use crate::authorization::main::authorize_listing::authorize_listing; use crate::authorization::main::authorize_person::authorize_person; use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::runs::constants::{MEMBER_RECOVERED_EVENT, MEMBER_RETRYING_EVENT}; use crate::runs::models::{ RunErrorText, RunEventRecord, RunOutput, RunOutputLine, RunStatementRecord, RunStatementSql, }; use crate::store::errors::ControlError; use crate::store::main::decode_row::decode_row; use crate::store::main::query::query; -use crate::store::main::query_records::query_records; use crate::store::main::{begin::begin, commit::commit}; use crate::store::models::{ControlStore, ControlTransaction, Parameter, TransactionMode}; +use crate::store::types::Row; /// The Recall invocations of one run and the nodes it selected. pub(crate) struct RunRecallScope { @@ -156,45 +157,125 @@ pub(crate) async fn run_events_in( scope: &RunRecallScope, (after, limit): (i64, u32), ) -> Result<(ControlTransaction, Vec), ControlError> { - let (filter, mut parameters) = invocation_filter(scope, 1); - let next = parameters.len() + 1; - parameters.extend([ - Parameter::Integer(after), - Parameter::Integer(i64::from(limit.min(5000))), - ]); - query_records(transaction, - &format!("SELECT storage_id, event_id, invocation_id, occurred_at, event_type, severity, producer, resource_id, \ - resource_kind, resource_name, resource_attempt_id, statement_id, error_type, error_code, duration_ms, outcome, \ - row_count, affected_rows FROM events WHERE {filter} AND storage_id>${next} ORDER BY storage_id LIMIT ${}", next + 1), - parameters).await + recall_records( + transaction, + scope, + ( + |filter: &str, next: usize| { + format!("SELECT storage_id, event_id, invocation_id, occurred_at, event_type, severity, producer, resource_id, \ + resource_kind, resource_name, resource_attempt_id, statement_id, error_type, error_code, duration_ms, outcome, \ + row_count, affected_rows, CASE WHEN event_type IN ('{MEMBER_RETRYING_EVENT}','{MEMBER_RECOVERED_EVENT}') \ + THEN payload END AS member_payload FROM events WHERE {filter} AND storage_id>${next} ORDER BY storage_id LIMIT ${}", next + 1) + }, + vec![ + Parameter::Integer(after), + Parameter::Integer(i64::from(limit.min(5000))), + ], + ), + |row: &mut Row| { + let payload = row.remove("member_payload").unwrap_or(Value::Null); + if let Some(message) = member_message(&payload)? { + row.insert("message".into(), Value::String(message)); + } + Ok(()) + }, + ) + .await } -pub(crate) async fn run_statements_in( +/// One recall read: `sql(filter, first extra parameter)` over the run's invocations, each row adjusted then decoded. +async fn recall_records( transaction: ControlTransaction, scope: &RunRecallScope, - limit: u32, -) -> Result<(ControlTransaction, Vec), ControlError> { + (sql, extra): (impl FnOnce(&str, usize) -> String, Vec), + mut adjust: impl FnMut(&mut Row) -> Result<(), ControlError>, +) -> Result<(ControlTransaction, Vec), ControlError> { let (filter, mut parameters) = invocation_filter(scope, 1); let next = parameters.len() + 1; - parameters.push(Parameter::Integer(i64::from(limit.min(5000)))); - let (transaction, rows) = query(transaction, - &format!("SELECT invocation_id, statement_id, resource_id, resource_name, resource_kind, intent, statement_kind, \ - purpose, query_id, started_at, ended_at, outcome, error_type, error_code, duration_ms, row_count, affected_rows, \ - sql_available, sql_unavailable_reason FROM statements WHERE {filter} ORDER BY started_at, statement_id LIMIT ${next}"), - parameters).await?; + parameters.extend(extra); + let (transaction, rows) = query(transaction, &sql(&filter, next), parameters).await?; let records = rows .into_iter() .map(|mut row| { - let available = row - .get("sql_available") - .is_some_and(|value| value.as_bool().unwrap_or_else(|| value.as_i64() == Some(1))); - row.insert("sql_available".into(), Value::Bool(available)); + adjust(&mut row)?; decode_row(row) }) .collect::>()?; Ok((transaction, records)) } +/// The scheduler's summary of a member retry or recovery on its group run, from its own payload. +fn member_message(payload: &Value) -> Result, ControlError> { + let payload: Value = match payload { + Value::String(text) => serde_json::from_str(text) + .map_err(|error| ControlError::Invalid(format!("member event payload: {error}")))?, + Value::Null => return Ok(None), + other => other.clone(), + }; + Ok(member_summary(&payload)) +} + +/// The summary a member retry or recovery payload names, when it names a member. +fn member_summary(payload: &Value) -> Option { + let member = payload.get("member")?; + let job_key = member.get("job_key")?.as_str()?; + let attempt = member.get("attempt")?.as_u64()?; + let Some(attempts) = member.get("attempts").and_then(Value::as_u64) else { + return Some(format!("{job_key} succeeded on attempt {attempt}")); + }; + let delay = member + .get("delay_seconds") + .and_then(Value::as_u64) + .unwrap_or_default(); + let when = if delay == 0 { + "now".to_owned() + } else { + format!("in {}", delay_text(delay)) + }; + Some(format!( + "{job_key} failed, retrying {when} (attempt {attempt} of {attempts})" + )) +} + +/// A retry delay as people read it, such as `45 s`, `15 min` or `1 h 30 min`. +fn delay_text(seconds: u64) -> String { + let (hours, minutes, rest) = (seconds / 3600, seconds % 3600 / 60, seconds % 60); + match (hours, minutes, rest) { + (0, 0, rest) => format!("{rest} s"), + (0, minutes, 0) => format!("{minutes} min"), + (0, minutes, rest) => format!("{minutes} min {rest} s"), + (hours, 0, _) => format!("{hours} h"), + (hours, minutes, _) => format!("{hours} h {minutes} min"), + } +} + +pub(crate) async fn run_statements_in( + transaction: ControlTransaction, + scope: &RunRecallScope, + limit: u32, +) -> Result<(ControlTransaction, Vec), ControlError> { + recall_records( + transaction, + scope, + ( + |filter: &str, next: usize| { + format!("SELECT invocation_id, statement_id, resource_id, resource_name, resource_kind, intent, statement_kind, \ + purpose, query_id, started_at, ended_at, outcome, error_type, error_code, duration_ms, row_count, affected_rows, \ + sql_available, sql_unavailable_reason FROM statements WHERE {filter} ORDER BY started_at, statement_id LIMIT ${next}") + }, + vec![Parameter::Integer(i64::from(limit.min(5000)))], + ), + |row: &mut Row| { + let available = row + .get("sql_available") + .is_some_and(|value| value.as_bool().unwrap_or_else(|| value.as_i64() == Some(1))); + row.insert("sql_available".into(), Value::Bool(available)); + Ok(()) + }, + ) + .await +} + /// One artifact reference names each digest per invocation, so a statement finds its SQL by digest. pub(crate) async fn statement_sql_in( transaction: ControlTransaction, diff --git a/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_check.rs b/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_check.rs index e48000c366..43ffbf10b4 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_check.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_check.rs @@ -28,6 +28,6 @@ pub async fn read_person_check( commit(transaction).await?; Ok(rows .into_iter() - .map(|row| CheckResult::new(row.event_id, row.record)) + .map(|row| CheckResult::new(row.event_id, row.record, row.ignored)) .collect()) } diff --git a/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_checks.rs b/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_checks.rs index 4a31d398bd..1f6ad577d9 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_checks.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_checks.rs @@ -2,31 +2,25 @@ use uuid::Uuid; -use crate::authorization::main::authorize_listing::authorize_listing; use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::snapshots::main::reads::listing_read::listing_read; use crate::store::errors::ControlError; -use crate::store::main::{begin::begin, commit::commit}; -use crate::store::models::{ControlStore, TransactionMode}; +use crate::store::models::ControlStore; use crate::warehouse_history::_helpers::checks::latest_results; use crate::warehouse_history::models::CheckResult; pub async fn read_person_checks( store: &ControlStore, credentials: (&AuditContext, Uuid), - (location_id, target): (Uuid, &str), + scope: (Uuid, &str), default_role: DefaultRole, ) -> Result, ControlError> { - let (transaction, _) = authorize_listing( - begin(store, TransactionMode::Write).await?, - credentials, - (location_id, target), - default_role, - ) + let rows = listing_read(store, (credentials, default_role), scope, |transaction| { + latest_results(transaction, scope) + }) .await?; - let (transaction, rows) = latest_results(transaction, (location_id, target)).await?; - commit(transaction).await?; Ok(rows .into_iter() - .map(|row| CheckResult::new(row.event_id, row.record)) + .map(|row| CheckResult::new(row.event_id, row.record, row.ignored)) .collect()) } diff --git a/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_node_error.rs b/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_node_error.rs index c976559688..8b0db84818 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_node_error.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_node_error.rs @@ -7,7 +7,7 @@ use crate::authorization::main::begin_authorized::begin_authorized; use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; use crate::runs::models::{RunChange, RunChangeRequest, RunNodeFailure}; -use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::snapshots::main::reads::read_snapshot_nodes::read_snapshot_nodes; use crate::store::errors::ControlError; use crate::store::main::{commit::commit, query::query}; use crate::store::models::{ControlStore, Parameter}; diff --git a/crates/sqlbuild-orchestration/src/warehouse_history/models.rs b/crates/sqlbuild-orchestration/src/warehouse_history/models.rs index bc8ba4e55a..e00488310a 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_history/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_history/models.rs @@ -37,10 +37,13 @@ pub struct CheckResult { pub evidence_count: Option, pub evidence_truncated: Option, pub reused: String, + /// An error-severity failure its build let through instead of holding anything back. + #[serde(default)] + pub ignored: bool, } impl CheckResult { - pub(crate) fn new(event_id: Uuid, record: WarehouseAuditRecord) -> Self { + pub(crate) fn new(event_id: Uuid, record: WarehouseAuditRecord, ignored: bool) -> Self { Self { event_id, binding_key: record.binding_key, @@ -67,6 +70,7 @@ impl CheckResult { evidence_count: record.evidence_count, evidence_truncated: record.evidence_truncated, reused: record.reused, + ignored, } } } diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs index 922be872a5..78dfb2346c 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs @@ -8,7 +8,7 @@ use serde_json::Value; use uuid::Uuid; use crate::sensors::constants::SOURCE_NODE_KIND; -use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::snapshots::main::reads::read_snapshot_nodes::read_snapshot_nodes; use crate::snapshots::models::SnapshotNode; use crate::store::errors::ControlError; use crate::store::main::query_records::query_records; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs index 9ae184aaee..0ef8ba206d 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs @@ -6,8 +6,10 @@ use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter}; use crate::warehouse_metadata::_helpers::ingestion::{ - begin_ingestion, commit_ingestion, outside_scan, scoped_event_id, validate_fields, + begin_ingestion, commit_ingestion, next_ingestion_chunk, outside_scan, scoped_event_id, + validate_fields, }; +use crate::warehouse_metadata::constants::INGESTION_CHUNK_RECORDS; use crate::warehouse_metadata::models::{MetadataBatch, StateEventImport, WarehouseSyncResponse}; /// Mirror a state-history page; a source id keeps its earliest row (its latest for kinds whose `mirror_keeps_latest` holds), then its smallest record text in byte order. SQLite and PostgreSQL format that text differently, so on an exact tie each store is deterministic but they may keep different rows; a store migration rebuilds the mirror. @@ -52,7 +54,12 @@ pub(crate) async fn ingest_events( .await?; let mut applied = false; let mut keys: Vec = Vec::new(); - for record in records { + for (index, record) in records.into_iter().enumerate() { + if index > 0 && index % INGESTION_CHUNK_RECORDS == 0 { + transaction = + next_ingestion_chunk(transaction, (store, location_id, target), batch, applied) + .await?; + } let event_id = scoped_event_id( (location_id, target), ( diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprints.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprints.rs index 887cd6fa21..3a4c8d26af 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprints.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprints.rs @@ -1,10 +1,16 @@ //! Fixed legacy fingerprint keys and validation of bounded source history pages. use serde_json::json; +use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; use crate::store::errors::ControlError; -use crate::warehouse_metadata::_helpers::ingestion::{outside_scan, validate_fields}; +use crate::store::main::{compress::compress, query::query, store_source_text::store_source_text}; +use crate::store::models::{ControlTransaction, Parameter}; +use crate::warehouse_metadata::_helpers::ingestion::{ + outside_scan, scoped_event_id, validate_fields, +}; use crate::warehouse_metadata::models::{MetadataBatch, WarehouseFingerprintRecord}; pub(crate) fn validate_batch( @@ -63,3 +69,55 @@ pub(crate) fn source_event_id(record: &WarehouseFingerprintRecord) -> String { .as_bytes(), ) } + +/// Store one fingerprint and its definition text; its event ID, and whether it was new. +pub(crate) async fn insert_fingerprint( + mut transaction: ControlTransaction, + (location_id, target): (Uuid, &str), + record: &WarehouseFingerprintRecord, +) -> Result<(ControlTransaction, Uuid, bool), ControlError> { + let source_event_id = source_event_id(record); + let event_id = scoped_event_id( + (location_id, target), + ( + record.state_database.as_deref(), + &record.state_schema, + FINGERPRINT_TABLE_NAME, + ), + &source_event_id, + )?; + let existing; + (transaction, existing) = query( + transaction, + "SELECT event_id FROM sqb_warehouse_fingerprints WHERE event_id=$1", + vec![Parameter::Id(event_id)], + ) + .await?; + if !existing.is_empty() { + return Ok((transaction, event_id, false)); + } + let definition_text_hash = sha256_hex(record.definition.as_bytes()); + let mut record_json = json!(record); + if let serde_json::Value::Object(object) = &mut record_json { + object.remove("definition"); + } + let received_at = transaction.now; + transaction = store_source_text( + transaction, + ( + definition_text_hash.clone(), + compress(record.definition.as_bytes())?, + ), + ) + .await?; + let inserted; + (transaction, inserted) = query(transaction, + "INSERT INTO sqb_warehouse_fingerprints (event_id,location_id,target,source_event_id,observed_at,definition_text_hash,record_json,received_at,state_database,state_schema,node_type,node_name,target_database,target_schema,target_name) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", + vec![Parameter::Id(event_id), Parameter::Id(location_id), Parameter::Text(target.into()), + Parameter::Text(source_event_id), Parameter::Time(record.ts), Parameter::Text(definition_text_hash), + Parameter::Json(record_json), Parameter::Time(received_at), Parameter::OptionalText(record.state_database.clone()), Parameter::Text(record.state_schema.clone()), + Parameter::Text(record.node_type.clone()), Parameter::Text(record.node_name.clone()), Parameter::OptionalText(record.target_database.clone()), + Parameter::OptionalText(record.target_schema.clone()), Parameter::OptionalText(record.target_name.clone())]).await?; + Ok((transaction, event_id, !inserted.is_empty())) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs index 5ba9033f3f..d8fde4e13f 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs @@ -3,12 +3,16 @@ use serde::de::DeserializeOwned; use uuid::Uuid; -use crate::platform::main::require_instance::require_instance; +use crate::platform::main::installation::require_instance::require_instance; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; use crate::store::models::{Change, ControlStore, ControlTransaction, Parameter, TransactionMode}; -use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; -use crate::warehouse_metadata::models::{MetadataBatch, MetadataScan, WarehouseSyncResponse}; +use crate::warehouse_metadata::constants::{ + IGNORED_KEY, MAX_HISTORY_PAGE_RECORDS, MAX_HISTORY_RECORDS, +}; +use crate::warehouse_metadata::models::{ + MetadataBatch, MetadataScan, WarehouseHistoryKind, WarehouseSyncResponse, +}; pub(crate) async fn begin_ingestion( store: &ControlStore, @@ -55,6 +59,27 @@ pub(crate) fn scoped_event_id( Ok(Uuid::new_v5(&location_id, &identity)) } +/// Commit the records a page applied so far and open its next transaction behind the same fence. +pub(crate) async fn next_ingestion_chunk( + mut transaction: ControlTransaction, + (store, location_id, target): (&ControlStore, Uuid, &str), + batch: &MetadataBatch, + applied: bool, +) -> Result { + if applied { + transaction + .changes + .insert(Change::NodeStatus(location_id, target.into())); + } + commit(transaction).await?; + begin_ingestion( + store, + (location_id, target), + (batch.session_id, batch.generation), + ) + .await +} + /// Delivered rows take the scan's id; a complete scan deletes every row it did not deliver. pub(crate) async fn commit_ingestion( mut transaction: ControlTransaction, @@ -128,9 +153,14 @@ pub(crate) fn validate_fields( invalid_scope: bool, ) -> Result<(), ControlError> { let kind = batch.kind.as_str().replace('_', " "); - if batch.generation < 1 || batch.records.len() > MAX_HISTORY_RECORDS || invalid_identity { + let most = if batch.kind == WarehouseHistoryKind::Statistics { + MAX_HISTORY_RECORDS + } else { + MAX_HISTORY_PAGE_RECORDS + }; + if batch.generation < 1 || batch.records.len() > most || invalid_identity { return Err(ControlError::Invalid(format!( - "{kind} sync needs a positive generation, at most {MAX_HISTORY_RECORDS} records and nonempty state identities" + "{kind} sync needs a positive generation, at most {most} records and nonempty state identities" ))); } if invalid_scope @@ -145,3 +175,18 @@ pub(crate) fn validate_fields( } Ok(()) } + +/// Whether an audit result's metadata marks it as a failure its build ignored. +pub(crate) fn ignored_failure(metadata_json: Option<&str>) -> bool { + let Some(text) = metadata_json else { + return false; + }; + match serde_json::from_str::(text) { + Ok(metadata) => metadata + .get(IGNORED_KEY) + .and_then(serde_json::Value::as_bool) + .unwrap_or(false), + Err(error) if error.is_syntax() || error.is_eof() => false, + Err(_) => false, + } +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs index e23899c68d..986abcd818 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs @@ -5,26 +5,29 @@ use std::collections::{BTreeMap, BTreeSet}; use chrono::{DateTime, Utc}; use serde::de::DeserializeOwned; use serde_json::Value; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::authorization::models::CurrentSnapshotId; use crate::platform::constants::LOCATION_RETENTION_DAYS; -use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::snapshots::main::reads::read_snapshot_nodes::read_snapshot_nodes; use crate::store::errors::ControlError; use crate::store::main::decode_row::decode_row; use crate::store::main::decompress::decompress; use crate::store::main::query::query; -use crate::store::models::{ControlTransaction, Parameter}; +use crate::store::models::{Connection, ControlTransaction, Parameter}; use crate::store::types::Row; use crate::warehouse_metadata::constants::PREVIEW_REQUIRED_TABLES; use crate::warehouse_metadata::models::{ - CleanSync, PreviewLatestRun, WarehouseCatalogRecord, WarehouseConsumptionRecord, - WarehouseFingerprintRecord, WarehouseObservationRecord, + CleanSync, PreviewLatestRun, WarehouseCatalogColumn, WarehouseCatalogRecord, + WarehouseConsumptionRecord, WarehouseFingerprintRecord, WarehouseObservationRecord, }; +use sqlbuild_state::warehouse_state::models::{StateColumn, StateLocation, StateRelation}; const RELATIONS_SQL: &str = "SELECT record_json FROM sqb_warehouse_catalog \ WHERE location_id=$1 AND target=$2 ORDER BY state_database, state_schema, event_id"; -const FINGERPRINTS_SQL: &str = "SELECT f.record_json, t.body FROM (SELECT record_json, definition_text_hash, \ +/// `{body}` is the definition body as hexadecimal text in the transaction's dialect. +const FINGERPRINTS_SQL: &str = "SELECT f.record_json, {body} AS body FROM (SELECT record_json, definition_text_hash, \ ROW_NUMBER() OVER (PARTITION BY state_database, state_schema, node_type, node_name, target_database, \ target_schema, target_name ORDER BY observed_at DESC, event_id DESC) AS newest \ FROM sqb_warehouse_fingerprints WHERE location_id=$1 AND target=$2) f \ @@ -44,6 +47,21 @@ const OBSERVATIONS_SQL: &str = "SELECT record_json FROM (SELECT record_json, kin FROM sqb_warehouse_observations WHERE location_id=$1 AND target=$2) observations WHERE newest=1 \ ORDER BY kind, node"; +/// The mirrored tables a preview copy reads; any write to them changes the mirror version. +const MIRROR_TABLES: [&str; 4] = [ + "sqb_warehouse_catalog", + "sqb_warehouse_fingerprints", + "sqb_warehouse_consumption", + "sqb_warehouse_observations", +]; +const CURRENT_SNAPSHOT_SQL: &str = + "SELECT snapshot_id FROM sqb_current_snapshots WHERE location_id=$1 AND target=$2"; +/// The newest clean sync of today's location invocation, which moves the copy's `synced_at`. +const NEWEST_CLEAN_SYNC_SQL: &str = "SELECT MAX(occurred_at) AS synced_at FROM recall_events \ + WHERE invocation_id=$1 AND event_type='location_health' AND payload->'report'->>'target'=$2 \ + AND payload->'report'->'health'->>'kind'='warehouse_sync' \ + AND payload->'report'->'health'->>'failure' IS NULL"; + /// Adds scope, cursor and page size to the parameters, with the SQL after the caller's conditions. pub(crate) fn page_tail( mut parameters: Vec, @@ -74,6 +92,36 @@ pub(crate) fn page_tail( (parameters, tail) } +/// The definition body as hexadecimal text, which a store returns far faster than bytes. +fn body_hex(transaction: &ControlTransaction) -> String { + if matches!(transaction.connection, Some(Connection::Postgres(_))) { + "encode(t.body, 'hex')".to_owned() + } else { + "hex(t.body)".to_owned() + } +} + +pub(crate) fn location(database: Option, schema: String) -> StateLocation { + StateLocation { database, schema } +} + +fn column(column: WarehouseCatalogColumn) -> StateColumn { + StateColumn { + name: column.name, + data_type: column.data_type, + } +} + +pub(crate) fn relation(record: WarehouseCatalogRecord) -> StateRelation { + StateRelation { + database: record.state_database, + schema: record.state_schema, + name: record.name, + relation_type: record.relation_type, + columns: record.columns.into_iter().map(column).collect(), + } +} + fn scope_parameters((location_id, target): (Uuid, &str)) -> Vec { vec![Parameter::Id(location_id), Parameter::Text(target.into())] } @@ -109,7 +157,8 @@ pub(crate) async fn preview_fingerprints( transaction: ControlTransaction, scope: (Uuid, &str), ) -> Result<(ControlTransaction, Vec), ControlError> { - let (transaction, rows) = query(transaction, FINGERPRINTS_SQL, scope_parameters(scope)).await?; + let sql: String = FINGERPRINTS_SQL.replace("{body}", &body_hex(&transaction)); + let (transaction, rows) = query(transaction, &sql, scope_parameters(scope)).await?; let mut fingerprints: Vec = Vec::with_capacity(rows.len()); for mut row in rows { let body: Vec = decompress(&row.remove("body").unwrap_or(Value::Null))?; @@ -126,6 +175,52 @@ pub(crate) async fn preview_fingerprints( Ok((transaction, fingerprints)) } +/// The mirror version: a digest of the mirrored tables' row counts and newest receipts, the snapshot and today's newest clean sync. +pub(crate) async fn preview_version( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), +) -> Result<(ControlTransaction, String), ControlError> { + let tables: Vec = MIRROR_TABLES + .iter() + .map(|table| { + format!( + "SELECT '{table}' AS mirror, COUNT(*) AS records, MAX(received_at) AS received_at \ + FROM {table} WHERE location_id=$1 AND target=$2" + ) + }) + .collect(); + let (transaction, mirrored) = query( + transaction, + &tables.join(" UNION ALL "), + scope_parameters((location_id, target)), + ) + .await?; + let (transaction, snapshot) = query( + transaction, + CURRENT_SNAPSHOT_SQL, + scope_parameters((location_id, target)), + ) + .await?; + let today = transaction.now.date_naive(); + let (transaction, synced) = query( + transaction, + NEWEST_CLEAN_SYNC_SQL, + vec![ + Parameter::Text(format!("location/{location_id}/{today}")), + Parameter::Text(target.into()), + ], + ) + .await?; + let mut mirrored: Vec = mirrored; + mirrored.sort_by(|left, right| { + left.get("mirror") + .map(Value::to_string) + .cmp(&right.get("mirror").map(Value::to_string)) + }); + let identity: Value = serde_json::json!([mirrored, snapshot, synced]); + Ok((transaction, sha256_hex(identity.to_string().as_bytes()))) +} + /// Each node's newest succeeded build that names its node and run. pub(crate) async fn preview_latest_runs( transaction: ControlTransaction, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/node_status.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/node_status.rs index 43f3352405..008345ff11 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/node_status.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/node_status.rs @@ -30,7 +30,7 @@ pub(crate) fn destination(node: &SnapshotNode) -> Option { } /// A node's newest build: its record, recorded metadata JSON and recorded definition text hash. -type RecordedBuild = (FingerprintObservation, String, String); +pub(crate) type RecordedBuild = (FingerprintObservation, String, String); /// Each node's definition evidence, reading the recorded texts its comparison re-fingerprints. pub(crate) async fn definition_evidences( diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs index a879c1ede9..b1eb03479b 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs @@ -2,9 +2,39 @@ use crate::warehouse_metadata::models::WarehouseHistoryKind; +/// Statistics relations sampled per page, and other bounded statistics work; each sample is a query. pub const MAX_HISTORY_RECORDS: usize = 100; +/// Rows of one state-table or catalog page; the reply's byte budget usually ends a page first. +pub const MAX_HISTORY_PAGE_RECORDS: usize = 2000; +/// Encoded records one page carries before the reader ends it, far below the API body bound. +pub const HISTORY_PAGE_BYTES: usize = 8 * 1024 * 1024; pub(crate) const CATALOG_MIRROR: &str = "sqb_warehouse_catalog"; pub(crate) const DEFAULT_STATISTICS_RETENTION_DAYS: u16 = 90; +/// Builds and check results a SQLite store keeps per node or check unless configured. +pub const DEFAULT_SQLITE_STATE_HISTORY_VERSIONS: u16 = 5; +/// The PostgreSQL default matches the janitor's `direct_state_history_versions` default. +pub const DEFAULT_POSTGRES_STATE_HISTORY_VERSIONS: u16 = 20; +/// Mirrored state-history rows one pruning transaction deletes, so writers never wait long. +pub(crate) const STATE_HISTORY_PRUNE_BATCH: usize = 500; +/// Builds ranked past the limit within their node, in one state namespace; the newest always stays. +pub(crate) const PRUNE_FINGERPRINTS_SQL: &str = "DELETE FROM sqb_warehouse_fingerprints WHERE event_id IN (SELECT event_id FROM \ + (SELECT event_id, ROW_NUMBER() OVER (PARTITION BY location_id, target, state_database, state_schema, \ + node_type, node_name, target_database, target_schema, target_name ORDER BY observed_at DESC, event_id DESC) \ + AS version FROM sqb_warehouse_fingerprints) ranked WHERE version > $1 LIMIT $2) RETURNING definition_text_hash"; + +/// Check results ranked past the limit within their check binding, in one state namespace. +pub(crate) const PRUNE_AUDIT_RESULTS_SQL: &str = "DELETE FROM sqb_warehouse_audit_results WHERE event_id IN (SELECT event_id FROM \ + (SELECT event_id, ROW_NUMBER() OVER (PARTITION BY location_id, target, state_database, state_schema, \ + COALESCE(binding_key, audit_name) ORDER BY observed_at DESC, event_id DESC) AS version \ + FROM sqb_warehouse_audit_results) ranked WHERE version > $1 LIMIT $2) RETURNING event_id"; +/// Free pages, as a share of the SQLite file, above which pruning returns the space to the disk. +pub(crate) const SQLITE_RECLAIM_FREE_SHARE: f64 = 0.25; +/// Free pages below this are not worth reclaiming, whatever their share. +pub(crate) const SQLITE_RECLAIM_MIN_FREE_PAGES: i64 = 1024; +/// Free pages one incremental vacuum step returns to the disk. +pub(crate) const SQLITE_RECLAIM_STEP_PAGES: i64 = 4096; +/// SQLite's `auto_vacuum` value for incremental mode. +pub(crate) const SQLITE_AUTO_VACUUM_INCREMENTAL: i64 = 2; /// Column-lineage node definitions read per query, well under every backend's bound parameter limit. pub(crate) const LINEAGE_DEFINITION_BATCH: usize = 1000; /// Mirrored tables a plan preview needs synced cleanly; observations only refine it into knowns. @@ -18,3 +48,8 @@ pub(crate) const PREVIEW_REQUIRED_TABLES: [WarehouseHistoryKind; 3] = [ pub(crate) const OUTCOME_RECENT_RUNS: i64 = 25; /// How far back, in runs, the newest failed or warned runs are looked for. pub(crate) const OUTCOME_FAILURE_WINDOW: i64 = 500; +/// Records one ingestion transaction applies before committing, so a page never holds the write lock long. +pub(crate) const INGESTION_CHUNK_RECORDS: usize = 200; + +/// The audit result metadata field marking a failure its build let through. +pub(crate) const IGNORED_KEY: &str = "ignored"; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/begin_ingestion.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/begin_ingestion.rs new file mode 100644 index 0000000000..12347fc2ec --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/begin_ingestion.rs @@ -0,0 +1,17 @@ +//! Open a transaction that ingests one location session's metadata page. + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::{ControlStore, ControlTransaction}; +use crate::warehouse_metadata::_helpers::ingestion; + +/// Open a transaction that ingests one page from a location session of a target. +#[inline] +pub(crate) async fn begin_ingestion( + store: &ControlStore, + scope: (Uuid, &str), + session: (Uuid, i64), +) -> Result { + ingestion::begin_ingestion(store, scope, session).await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_audit_results.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_audit_results.rs index b3e5fd568f..e60d396cd8 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_audit_results.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_audit_results.rs @@ -6,7 +6,7 @@ use uuid::Uuid; use crate::store::errors::ControlError; use crate::store::models::{ControlStore, Parameter}; use crate::warehouse_metadata::_helpers::events::ingest_events; -use crate::warehouse_metadata::_helpers::ingestion::batch_records; +use crate::warehouse_metadata::_helpers::ingestion::{batch_records, ignored_failure}; use crate::warehouse_metadata::models::{ MetadataBatch, StateEventImport, WarehouseAuditRecord, WarehouseStateLocation, WarehouseSyncResponse, @@ -36,6 +36,10 @@ pub async fn ingest_audit_results( ("audit_name", Parameter::Text(record.audit_name.clone())), ("outcome", Parameter::Text(record.outcome.clone())), ("binding_key", Parameter::Text(record.binding_key.clone())), + ( + "ignored", + Parameter::Boolean(ignored_failure(record.metadata_json.as_deref())), + ), ], record_json: json!(record), }) diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_catalog.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_catalog.rs index 998ff9dc9a..90a1876e53 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_catalog.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_catalog.rs @@ -8,9 +8,10 @@ use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter}; use crate::warehouse_metadata::_helpers::catalog::{project_catalog_record, validate_catalog}; use crate::warehouse_metadata::_helpers::ingestion::{ - batch_records, begin_ingestion, commit_ingestion, scoped_event_id, + batch_records, begin_ingestion, commit_ingestion, next_ingestion_chunk, scoped_event_id, }; use crate::warehouse_metadata::constants::CATALOG_MIRROR; +use crate::warehouse_metadata::constants::INGESTION_CHUNK_RECORDS; use crate::warehouse_metadata::models::{ MetadataBatch, WarehouseCatalogRecord, WarehouseSyncResponse, }; @@ -26,7 +27,12 @@ pub async fn ingest_catalog( begin_ingestion(store, scope, (batch.session_id, batch.generation)).await?; let mut keys: Vec = Vec::new(); let mut applied = false; - for record in &records { + for (index, record) in records.iter().enumerate() { + if index > 0 && index % INGESTION_CHUNK_RECORDS == 0 { + transaction = + next_ingestion_chunk(transaction, (store, scope.0, scope.1), batch, applied) + .await?; + } let relation_id = scoped_event_id( scope, ( diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_fingerprints.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_fingerprints.rs index 24d5ab0d55..22483f3b53 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_fingerprints.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_fingerprints.rs @@ -1,17 +1,14 @@ //! Import bounded fingerprint history without conflating semantic and exact-content hashes. -use serde_json::json; -use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; -use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::store::errors::ControlError; -use crate::store::main::{compress::compress, query::query, store_source_text::store_source_text}; -use crate::store::models::{ControlStore, Parameter}; -use crate::warehouse_metadata::_helpers::fingerprints::{source_event_id, validate_batch}; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::_helpers::fingerprints::{insert_fingerprint, validate_batch}; use crate::warehouse_metadata::_helpers::ingestion::{ - batch_records, begin_ingestion, commit_ingestion, scoped_event_id, + batch_records, begin_ingestion, commit_ingestion, next_ingestion_chunk, }; +use crate::warehouse_metadata::constants::INGESTION_CHUNK_RECORDS; use crate::warehouse_metadata::models::{ MetadataBatch, WarehouseFingerprintRecord, WarehouseSyncResponse, }; @@ -31,52 +28,17 @@ pub async fn ingest_fingerprints( .await?; let mut applied = false; let mut keys: Vec = Vec::new(); - for record in &records { - let source_event_id = source_event_id(record); - let event_id = scoped_event_id( - (location_id, target), - ( - record.state_database.as_deref(), - &record.state_schema, - FINGERPRINT_TABLE_NAME, - ), - &source_event_id, - )?; - keys.push(event_id); - let existing; - (transaction, existing) = query( - transaction, - "SELECT event_id FROM sqb_warehouse_fingerprints WHERE event_id=$1", - vec![Parameter::Id(event_id)], - ) - .await?; - if !existing.is_empty() { - continue; - } - let definition_text_hash = sha256_hex(record.definition.as_bytes()); - let mut record_json = json!(record); - if let serde_json::Value::Object(object) = &mut record_json { - object.remove("definition"); + for (index, record) in records.iter().enumerate() { + if index > 0 && index % INGESTION_CHUNK_RECORDS == 0 { + transaction = + next_ingestion_chunk(transaction, (store, location_id, target), batch, applied) + .await?; } - let received_at = transaction.now; - transaction = store_source_text( - transaction, - ( - definition_text_hash.clone(), - compress(record.definition.as_bytes())?, - ), - ) - .await?; - let inserted; - (transaction, inserted) = query(transaction, - "INSERT INTO sqb_warehouse_fingerprints (event_id,location_id,target,source_event_id,observed_at,definition_text_hash,record_json,received_at,state_database,state_schema,node_type,node_name,target_database,target_schema,target_name) \ - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", - vec![Parameter::Id(event_id), Parameter::Id(location_id), Parameter::Text(target.into()), - Parameter::Text(source_event_id), Parameter::Time(record.ts), Parameter::Text(definition_text_hash), - Parameter::Json(record_json), Parameter::Time(received_at), Parameter::OptionalText(record.state_database.clone()), Parameter::Text(record.state_schema.clone()), - Parameter::Text(record.node_type.clone()), Parameter::Text(record.node_name.clone()), Parameter::OptionalText(record.target_database.clone()), - Parameter::OptionalText(record.target_schema.clone()), Parameter::OptionalText(record.target_name.clone())]).await?; - applied |= !inserted.is_empty(); + let (next, event_id, inserted) = + insert_fingerprint(transaction, (location_id, target), record).await?; + transaction = next; + keys.push(event_id); + applied |= inserted; } commit_ingestion( transaction, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_source_freshness.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_source_freshness.rs index 1d7f925f04..e60a131609 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_source_freshness.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_source_freshness.rs @@ -11,8 +11,9 @@ use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter}; use crate::warehouse_metadata::_helpers::freshness::validate_batch; use crate::warehouse_metadata::_helpers::ingestion::{ - batch_records, begin_ingestion, commit_ingestion, scoped_event_id, + batch_records, begin_ingestion, commit_ingestion, next_ingestion_chunk, scoped_event_id, }; +use crate::warehouse_metadata::constants::INGESTION_CHUNK_RECORDS; use crate::warehouse_metadata::models::{ MetadataBatch, WarehouseFreshnessRecord, WarehouseSyncResponse, }; @@ -32,7 +33,12 @@ pub async fn ingest_source_freshness( .await?; let mut applied = false; let mut keys: Vec = Vec::new(); - for record in &records { + for (index, record) in records.iter().enumerate() { + if index > 0 && index % INGESTION_CHUNK_RECORDS == 0 { + transaction = + next_ingestion_chunk(transaction, (store, location_id, target), batch, applied) + .await?; + } let source_event_id = record.event_id.clone().unwrap_or_else(|| { legacy_event_id(&FreshnessEventIdentity { source_name: &record.source_name, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/mod.rs index c51faaae90..a21454bcde 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod begin_ingestion; pub mod ingest_audit_results; pub mod ingest_catalog; pub mod ingest_column_migrations; @@ -12,4 +13,5 @@ pub mod ingest_old_name_views; pub mod ingest_resource_costs; pub mod ingest_source_freshness; pub mod ingest_statistics; +pub(crate) mod prune_state_history; pub(crate) mod prune_statistics; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/prune_state_history.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/prune_state_history.rs new file mode 100644 index 0000000000..4b22f80c78 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/prune_state_history.rs @@ -0,0 +1,64 @@ +//! Keep the newest builds of each node and results of each check in bounded pages. + +use std::collections::BTreeSet; + +use crate::store::errors::ControlError; +use crate::store::main::{query::query, write::write}; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::constants::{ + PRUNE_AUDIT_RESULTS_SQL, PRUNE_FINGERPRINTS_SQL, STATE_HISTORY_PRUNE_BATCH, +}; +use crate::warehouse_metadata::models::StateHistoryRetention; + +/// Remove one page of builds and of check results past the limit; returns rows deleted and whether more remain. +pub(crate) async fn prune_state_history( + store: &ControlStore, + retention: StateHistoryRetention, +) -> Result<(usize, bool), ControlError> { + let versions = i64::from(retention.kept_versions(store)); + let limit = STATE_HISTORY_PRUNE_BATCH as i64; + let fingerprints = write(store, |transaction| async move { + let (mut transaction, rows) = query( + transaction, + PRUNE_FINGERPRINTS_SQL, + vec![Parameter::Integer(versions), Parameter::Integer(limit)], + ) + .await?; + let texts: Vec = rows + .iter() + .filter_map(|row| row["definition_text_hash"].as_str()) + .collect::>() + .into_iter() + .map(|hash| Parameter::Text(hash.to_owned())) + .collect(); + if !texts.is_empty() { + (transaction, _) = query( + transaction, + &format!( + "DELETE FROM sqb_source_texts WHERE text_hash IN ({}) \ + AND NOT EXISTS (SELECT 1 FROM sqb_warehouse_fingerprints f WHERE f.definition_text_hash=sqb_source_texts.text_hash) \ + AND NOT EXISTS (SELECT 1 FROM sqb_snapshot_sources s WHERE s.text_hash=sqb_source_texts.text_hash)", + Parameter::placeholders(1..=texts.len()) + ), + texts, + ) + .await?; + } + Ok((transaction, rows.len())) + }) + .await?; + let results = write(store, |transaction| async move { + let (transaction, rows) = query( + transaction, + PRUNE_AUDIT_RESULTS_SQL, + vec![Parameter::Integer(versions), Parameter::Integer(limit)], + ) + .await?; + Ok((transaction, rows.len())) + }) + .await?; + Ok(( + fingerprints + results, + fingerprints == STATE_HISTORY_PRUNE_BATCH || results == STATE_HISTORY_PRUNE_BATCH, + )) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs index d54f5f9745..e38b47279c 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs @@ -1,3 +1,4 @@ +pub mod preview_state_parts; pub mod read_preview_copy; pub mod read_statistics_due; pub mod read_sync_due; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/preview_state_parts.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/preview_state_parts.rs new file mode 100644 index 0000000000..32990874fb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/preview_state_parts.rs @@ -0,0 +1,84 @@ +//! A location plans from the warehouse state its synced copy describes. + +use sqlbuild_state::warehouse_state::models::{ + StateConsumption, StateFingerprint, StateLatestRun, StateObservation, SyncedProvenance, + WarehouseStateParts, +}; + +use crate::warehouse_metadata::_helpers::mirror_pages::{location, relation}; +use crate::warehouse_metadata::models::PreviewCopy; + +/// The state facts of `copy`, in the copy's order; `has_consumption` when it was read with history. +pub fn preview_state_parts(copy: PreviewCopy, has_consumption: bool) -> WarehouseStateParts { + WarehouseStateParts { + provenance: SyncedProvenance { + synced_at: copy.synced_at, + version: copy.version, + }, + relations: copy.relations.into_iter().map(relation).collect(), + fingerprints: copy + .fingerprints + .into_iter() + .map(|record| StateFingerprint { + location: location(record.state_database, record.state_schema), + node_type: record.node_type, + node_name: record.node_name, + target_database: record.target_database, + target_schema: record.target_schema, + target_name: record.target_name, + run_id: record.run_id, + definition_hash: record.definition_hash, + version_hash: record.version_hash, + schema_fingerprint: record.schema_fingerprint, + definition: record.definition, + metadata_json: record.metadata_json, + ts: record.ts, + }) + .collect(), + latest_runs: copy + .latest_runs + .into_iter() + .map(|run| StateLatestRun { + location: location(run.state_database, run.state_schema), + node_type: run.node_type, + node_name: run.node_name, + run_id: run.run_id, + }) + .collect(), + observations: copy + .observations + .into_iter() + .map(|record| StateObservation { + location: location(record.state_database, record.state_schema), + kind: record.kind, + node: record.node, + cursor_type: record.cursor_type, + value: record.value, + run_id: record.run_id, + observed_at: record.observed_at, + }) + .collect(), + consumption: copy + .consumption + .into_iter() + .map(|record| StateConsumption { + location: location(record.state_database, record.state_schema), + execution_id: record.execution_id, + phase: record.phase, + run_id: record.run_id, + node_type: record.node_type, + node_name: record.node_name, + target_database: record.target_database, + target_schema: record.target_schema, + target_name: record.target_name, + version_hash: record.version_hash, + input_count: record.input_count, + input_kind: record.input_kind, + input_name: record.input_name, + input_version: record.input_version, + recorded_at: record.recorded_at, + }) + .collect(), + has_consumption, + } +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs index 46c9305908..b0ac7457cb 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs @@ -1,4 +1,4 @@ -//! A plan preview plans from the mirror, so the copy carries only what its facade may answer. +//! A plan preview plans from the mirror; a location holding the current mirror version reads no records. use uuid::Uuid; @@ -8,7 +8,7 @@ use crate::store::models::ControlStore; use crate::warehouse_metadata::_helpers::ingestion::begin_ingestion; use crate::warehouse_metadata::_helpers::mirror_pages::{ preview_consumption, preview_fingerprints, preview_latest_runs, preview_observations, - preview_relations, preview_synced_at, + preview_relations, preview_synced_at, preview_version, }; use crate::warehouse_metadata::models::{PreviewCopy, PreviewCopyRead}; @@ -19,6 +19,25 @@ pub async fn read_preview_copy( ) -> Result { let transaction = begin_ingestion(store, scope, (request.session_id, request.generation)).await?; + let (transaction, version) = if request.versioned { + let (transaction, version) = preview_version(transaction, scope).await?; + (transaction, Some(version)) + } else { + (transaction, None) + }; + if version.is_some() && version == request.known_version { + commit(transaction).await?; + return Ok(PreviewCopy { + synced_at: None, + relations: Vec::new(), + fingerprints: Vec::new(), + latest_runs: Vec::new(), + observations: Vec::new(), + consumption: Vec::new(), + version, + unchanged: true, + }); + } let (transaction, relations) = preview_relations(transaction, scope).await?; let (transaction, fingerprints) = preview_fingerprints(transaction, scope).await?; let (transaction, latest_runs) = preview_latest_runs(transaction, scope).await?; @@ -37,5 +56,7 @@ pub async fn read_preview_copy( latest_runs, observations, consumption, + version, + unchanged: false, }) } diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs index d3bb217ec4..91a0c97ff7 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -1,3 +1,4 @@ pub mod ingest; pub mod location_reads; +pub(crate) mod node_reads; pub mod person_reads; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/catalog_node.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/catalog_node.rs new file mode 100644 index 0000000000..cb8e80cdef --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/catalog_node.rs @@ -0,0 +1,20 @@ +//! A node's catalog record from its snapshot definition, last build and checks. + +use uuid::Uuid; + +use crate::snapshots::models::SnapshotNode; +use crate::warehouse_metadata::_helpers::catalog_nodes; +use crate::warehouse_metadata::models::{ + CatalogNodeBuild, CatalogNodeChecks, CatalogNodeOutcome, CatalogNodeRecord, +}; + +/// A node's catalog record from its snapshot definition, last build and checks. +#[inline] +pub(crate) fn catalog_node( + node: (&str, &SnapshotNode), + snapshot_id: Uuid, + builds: (Option, Option, bool), + checks: CatalogNodeChecks, +) -> CatalogNodeRecord { + catalog_nodes::catalog_node(node, snapshot_id, builds, checks) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/definition_evidences.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/definition_evidences.rs new file mode 100644 index 0000000000..b04f671423 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/definition_evidences.rs @@ -0,0 +1,16 @@ +//! Whether each node's definition changed since its recorded build, and why. + +use crate::snapshots::models::SnapshotNode; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; +use crate::warehouse_metadata::_helpers::node_status::{self, RecordedBuild}; +use crate::warehouse_metadata::models::DefinitionEvidence; + +/// Whether each node's definition changed since its recorded build, and why. +#[inline] +pub(crate) async fn definition_evidences( + transaction: ControlTransaction, + builds: &[(&SnapshotNode, Option<&RecordedBuild>)], +) -> Result<(ControlTransaction, Vec), ControlError> { + node_status::definition_evidences(transaction, builds).await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/destination.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/destination.rs new file mode 100644 index 0000000000..64a6590b81 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/destination.rs @@ -0,0 +1,11 @@ +//! Where a node's builds are recorded, when it builds a relation. + +use crate::snapshots::models::SnapshotNode; +use crate::warehouse_metadata::_helpers::node_status; +use crate::warehouse_metadata::models::FingerprintQuery; + +/// Where a node's builds are recorded, when it builds a relation. +#[inline] +pub(crate) fn destination(node: &SnapshotNode) -> Option { + node_status::destination(node) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/last_outcome.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/last_outcome.rs new file mode 100644 index 0000000000..8131fd12de --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/last_outcome.rs @@ -0,0 +1,13 @@ +//! A node's last outcome: its newest attempt, or its last build when no attempt is newer. + +use crate::warehouse_metadata::_helpers::catalog_nodes; +use crate::warehouse_metadata::models::{CatalogNodeBuild, CatalogNodeOutcome}; + +/// A node's last outcome: its newest attempt, or its last build when no attempt is newer. +#[inline] +pub(crate) fn last_outcome( + attempt: Option, + build: Option<&CatalogNodeBuild>, +) -> Option { + catalog_nodes::last_outcome(attempt, build) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/latest_attempts.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/latest_attempts.rs new file mode 100644 index 0000000000..3170556f57 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/latest_attempts.rs @@ -0,0 +1,19 @@ +//! Each node's newest run attempt in a target. + +use std::collections::HashMap; + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; +use crate::warehouse_metadata::_helpers::catalog_nodes; +use crate::warehouse_metadata::models::CatalogNodeOutcome; + +/// Each node's newest run attempt in a target. +#[inline] +pub(crate) async fn latest_attempts( + transaction: ControlTransaction, + scope: (Uuid, &str), +) -> Result<(ControlTransaction, HashMap), ControlError> { + catalog_nodes::latest_attempts(transaction, scope).await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/latest_fingerprints.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/latest_fingerprints.rs new file mode 100644 index 0000000000..a9aba281af --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/latest_fingerprints.rs @@ -0,0 +1,23 @@ +//! The newest mirrored build of each queried destination. + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; +use crate::warehouse_metadata::_helpers::fingerprint_reads; +use crate::warehouse_metadata::models::{FingerprintObservation, FingerprintQuery}; + +/// The newest mirrored build of each queried destination, with its definition texts. +#[inline] +pub(crate) async fn latest_fingerprints( + transaction: ControlTransaction, + scope: (Uuid, &str, &[FingerprintQuery]), +) -> Result< + ( + ControlTransaction, + Vec>, + ), + ControlError, +> { + fingerprint_reads::latest_fingerprints(transaction, scope).await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/mod.rs new file mode 100644 index 0000000000..d54ea37654 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/node_reads/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod catalog_node; +pub(crate) mod definition_evidences; +pub(crate) mod destination; +pub(crate) mod last_outcome; +pub(crate) mod latest_attempts; +pub(crate) mod latest_fingerprints; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/list_person_catalog_nodes.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/list_person_catalog_nodes.rs deleted file mode 100644 index 90b60ff129..0000000000 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/list_person_catalog_nodes.rs +++ /dev/null @@ -1,93 +0,0 @@ -//! A person lists the current nodes with view on all of them; builds are batched indexed seeks, attempts and check results one read each. - -use uuid::Uuid; - -use crate::authorization::main::authorize_listing::authorize_listing; -use crate::credentials::models::{AuditContext, DefaultRole}; -use crate::snapshots::main::read_current_record::read_current_record; -use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; -use crate::store::errors::ControlError; -use crate::store::main::{begin::begin, commit::commit}; -use crate::store::models::{ControlStore, TransactionMode}; -use crate::warehouse_history::main::read_node_check_counts::read_node_check_counts; -use crate::warehouse_metadata::_helpers::catalog_nodes::{ - catalog_node, last_outcome, latest_attempts, -}; -use crate::warehouse_metadata::_helpers::fingerprint_reads::latest_fingerprints; -use crate::warehouse_metadata::_helpers::node_status::{definition_evidences, destination}; -use crate::warehouse_metadata::models::{ - CatalogNodeBuild, CatalogNodeRecord, DefinitionEvidence, FingerprintQuery, -}; - -pub async fn list_person_catalog_nodes( - store: &ControlStore, - credentials: (&AuditContext, Uuid), - (location_id, target): (Uuid, &str), - default_role: DefaultRole, -) -> Result, ControlError> { - let (transaction, _) = authorize_listing( - begin(store, TransactionMode::Write).await?, - credentials, - (location_id, target), - default_role, - ) - .await?; - let (transaction, snapshot) = read_current_record(transaction, (location_id, target)).await?; - let Some(snapshot) = snapshot else { - commit(transaction).await?; - return Ok(Vec::new()); - }; - let (transaction, nodes) = read_snapshot_nodes(transaction, snapshot.snapshot_id, None).await?; - let nodes: Vec<_> = nodes.iter().collect(); - let mut positions: Vec = Vec::new(); - let mut queries: Vec = Vec::new(); - for (position, (_, node)) in nodes.iter().enumerate() { - if let Some(query) = destination(node) { - positions.push(position); - queries.push(query); - } - } - let (transaction, found) = - latest_fingerprints(transaction, (location_id, target, &queries)).await?; - let (transaction, mut attempts) = latest_attempts(transaction, (location_id, target)).await?; - let (transaction, mut checks) = - read_node_check_counts(transaction, (location_id, target)).await?; - let (transaction, evidences) = definition_evidences( - transaction, - &positions - .iter() - .zip(&found) - .map(|(position, build)| (nodes[*position].1, build.as_ref())) - .collect::>(), - ) - .await?; - let mut builds: Vec> = vec![None; nodes.len()]; - let mut changed: Vec = vec![false; nodes.len()]; - for ((position, build), evidence) in positions.iter().zip(found).zip(evidences) { - changed[*position] = matches!(evidence, DefinitionEvidence::Changed { .. }); - builds[*position] = build.map(|(record, _, _)| CatalogNodeBuild { - observed_at: record.observed_at, - run_id: record.run_id, - version_hash: record.version_hash, - }); - } - let records: Vec = nodes - .iter() - .zip(builds.into_iter().zip(changed)) - .map(|((key, node), (build, changed))| { - let outcome = last_outcome(attempts.remove(key.as_str()), build.as_ref()); - let node_checks = checks - .remove(key.as_str()) - .or_else(|| checks.remove(&format!("{}:{key}", node.kind))) - .unwrap_or_default(); - catalog_node( - (key, node), - snapshot.snapshot_id, - (build, outcome, changed), - node_checks, - ) - }) - .collect(); - commit(transaction).await?; - Ok(records) -} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/mod.rs index eb7fc45d0f..0183857cf1 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/mod.rs @@ -1,4 +1,3 @@ -pub mod list_person_catalog_nodes; pub mod read_person_catalog; pub mod read_person_column_lineage; pub mod read_person_fingerprint; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_column_lineage.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_column_lineage.rs index d119d5d222..2c7a9a1b86 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_column_lineage.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_column_lineage.rs @@ -4,9 +4,9 @@ use uuid::Uuid; use crate::authorization::main::authorize_listing::authorize_listing; use crate::credentials::models::{AuditContext, DefaultRole}; -use crate::snapshots::main::read_current_record::read_current_record; -use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents; -use crate::snapshots::main::read_snapshot_keys::read_snapshot_keys; +use crate::snapshots::main::reads::read_current_record::read_current_record; +use crate::snapshots::main::reads::read_snapshot_documents::read_snapshot_documents; +use crate::snapshots::main::reads::read_snapshot_keys::read_snapshot_keys; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit}; use crate::store::models::{ControlStore, TransactionMode}; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_node_status.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_node_status.rs index 737f00bb45..07ff3e48c6 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_node_status.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_node_status.rs @@ -4,8 +4,8 @@ use uuid::Uuid; use crate::authorization::main::begin_authorized::begin_authorized; use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; -use crate::snapshots::main::read_current_record::read_current_record; -use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::snapshots::main::reads::read_current_record::read_current_record; +use crate::snapshots::main::reads::read_snapshot_nodes::read_snapshot_nodes; use crate::store::errors::ControlError; use crate::store::main::commit::commit; use crate::store::models::ControlStore; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/mod.rs index de8b740018..da47634a00 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/mod.rs @@ -4,3 +4,7 @@ pub(crate) mod _helpers; pub mod constants; pub mod main; pub mod models; + +#[cfg(test)] +#[path = "../../tests/warehouse_metadata/test_contracts.rs"] +mod tests; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index cc663321d7..631fd9fcd2 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -12,7 +12,11 @@ use sqlbuild_core::constants::{ }; use crate::store::errors::ControlError; -use crate::warehouse_metadata::constants::DEFAULT_STATISTICS_RETENTION_DAYS; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::constants::{ + DEFAULT_POSTGRES_STATE_HISTORY_VERSIONS, DEFAULT_SQLITE_STATE_HISTORY_VERSIONS, + DEFAULT_STATISTICS_RETENTION_DAYS, +}; /// Preserve observations, not an inferred latest version, age status or run outcome. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] @@ -482,6 +486,12 @@ pub struct PreviewCopyRead { /// Whether the copy carries consumption history; sent only when set, as older servers reject it. #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub consumption: bool, + /// Whether the location holds copies by mirror version; sent only when set, as older servers reject it. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub versioned: bool, + /// The mirror version of the copy the location holds; an unchanged mirror returns no records. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub known_version: Option, } /// The newest successful build of a node, which a `cursor_max` observation must come from. @@ -516,6 +526,12 @@ pub struct PreviewCopy { /// Every mirrored consumption row when asked for; omitted otherwise, as older locations reject it. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub consumption: Vec, + /// The mirror version these records were read at; sent only to a versioned read. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub version: Option, + /// Set when the mirror is still at the read's known version: the records are left out. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub unchanged: bool, } /// Private interpreter input; cursor values are data and receive adapter literal rendering. @@ -597,6 +613,9 @@ pub struct FingerprintRead { /// The table's change token after the last complete read; an equal token skips this read. #[serde(default)] pub unchanged_token: Option, + /// A full read delivers only each node's newest builds; incremental reads ignore it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub newest_versions: Option, } /// Node results carry no event identity, so the cursor holds the row's identity columns. @@ -636,6 +655,9 @@ pub struct StateEventsRead { /// The table's change token after the last complete read; an equal token skips this read. #[serde(default)] pub unchanged_token: Option, + /// A full read of check results delivers only each check's newest; other tables ignore it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub newest_versions: Option, } /// The private interpreter accepts only known state tables, never arbitrary SQL. @@ -688,6 +710,9 @@ pub struct WarehouseHistoryPage { /// The token still equals the request's: nothing was read and nothing is delivered. #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub unchanged: bool, + /// The adapter's history reads wake billed compute, so an idle location defers interval syncs. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub wakes_compute: bool, } /// A location runtime's failed history read: its redacted error, and whether a smaller page could succeed. @@ -921,6 +946,41 @@ impl Default for StatisticsRetention { } } +/// Builds and check results the store mirrors per node or check; unset follows the store's default. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct StateHistoryRetention { + versions: Option, +} + +impl StateHistoryRetention { + pub fn versions(versions: u16) -> Result { + if versions == 0 { + return Err(ControlError::Invalid( + "state history retention must keep at least one version".into(), + )); + } + Ok(Self { + versions: Some(versions), + }) + } + + /// The configured limit, or the store's default; never zero, so a node's latest version stays. + #[must_use] + pub fn kept_versions(self, store: &ControlStore) -> u16 { + self.versions.unwrap_or(match store { + ControlStore::Sqlite(_) => DEFAULT_SQLITE_STATE_HISTORY_VERSIONS, + ControlStore::Postgres(_) => DEFAULT_POSTGRES_STATE_HISTORY_VERSIONS, + }) + } +} + +/// The newest versions per node or check a full read of state history needs to deliver. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseRetention { + pub state_history_versions: u16, +} + /// A current snapshot node's declared metadata and latest build, without SQL, source or config. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct CatalogNodeRecord { diff --git a/crates/sqlbuild-orchestration/src/webhooks/_helpers/events.rs b/crates/sqlbuild-orchestration/src/webhooks/_helpers/events.rs index 2893510c0c..e7374e0512 100644 --- a/crates/sqlbuild-orchestration/src/webhooks/_helpers/events.rs +++ b/crates/sqlbuild-orchestration/src/webhooks/_helpers/events.rs @@ -1,7 +1,7 @@ //! Source-specific readers retain authority; delivery owns only acknowledgements. use crate::credentials::main::audit::read_audit_events::read_audit_events; -use crate::platform::main::read_platform_events::read_platform_events; +use crate::platform::main::installation::read_platform_events::read_platform_events; use crate::store::errors::ControlError; use crate::store::models::ControlTransaction; use crate::webhooks::models::{WebhookEvent, WebhookSource}; diff --git a/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs b/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs index 72569448be..d764600a26 100644 --- a/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs +++ b/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs @@ -3,7 +3,7 @@ use uuid::Uuid; use crate::runs::models::RunChange; -use crate::snapshots::main::verify_execution::verify_execution; +use crate::snapshots::main::publication::verify_execution::verify_execution; use crate::snapshots::models::SnapshotInput; use crate::worker::_helpers::heartbeat::start_session; use crate::worker::errors::WorkerError; diff --git a/crates/sqlbuild-orchestration/tests/api/changes/helpers.rs b/crates/sqlbuild-orchestration/tests/api/changes/helpers.rs index e52bfcc7e9..1271f24122 100644 --- a/crates/sqlbuild-orchestration/tests/api/changes/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/changes/helpers.rs @@ -65,6 +65,13 @@ pub(crate) async fn opened(response: reqwest::Response) -> SseConnection { } pub(crate) async fn next_topic(stream: &mut SseConnection) -> Option { + next_notice(stream) + .await + .map(|notice| notice["topic"].as_str().expect("topic").to_owned()) +} + +/// The next change event's data, such as a `node_status` event's version and states. +pub(crate) async fn next_notice(stream: &mut SseConnection) -> Option { tokio::time::timeout(Duration::from_secs(30), async { for _ in 0..4096 { for _ in 0..4096 { @@ -77,9 +84,7 @@ pub(crate) async fn next_topic(stream: &mut SseConnection) -> Option { continue; }; assert!(frame.lines().any(|line| line == "event: changed")); - let body: serde_json::Value = - serde_json::from_str(data).expect("typed change notice"); - return Some(body["topic"].as_str().expect("topic").to_owned()); + return Some(serde_json::from_str(data).expect("typed change notice")); } let chunk = stream .response diff --git a/crates/sqlbuild-orchestration/tests/api/fingerprint_reads/helpers.rs b/crates/sqlbuild-orchestration/tests/api/fingerprint_reads/helpers.rs index 45eef6062a..37a68f556e 100644 --- a/crates/sqlbuild-orchestration/tests/api/fingerprint_reads/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/fingerprint_reads/helpers.rs @@ -4,7 +4,7 @@ use serde_json::{Value, json}; use uuid::Uuid; use crate::credentials::models::{AuditContext, DefaultRole}; -use crate::platform::main::report_location_health::report_location_health; +use crate::platform::main::health::report_location_health::report_location_health; use crate::platform::models::{LocationHealth, LocationHealthReport}; use crate::store::errors::ControlError; use crate::store::models::ControlStore; diff --git a/crates/sqlbuild-orchestration/tests/api/helpers.rs b/crates/sqlbuild-orchestration/tests/api/helpers.rs index f63874ad36..2e3ce11dab 100644 --- a/crates/sqlbuild-orchestration/tests/api/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/helpers.rs @@ -156,3 +156,22 @@ pub(crate) fn large_snapshot(location_id: uuid::Uuid) -> crate::snapshots::model .collect(); input } + +/// Each `pool_capped` observation's warnings, with the generation and event ID they carry. +pub(crate) fn pool_capped( + observations: &[crate::platform::models::LocationHealthObservation], +) -> Vec<(i64, uuid::Uuid, crate::platform::models::PoolCapWarning)> { + let mut warnings = Vec::new(); + for observation in observations { + let health = serde_json::to_value(&observation.report.health).expect("health as JSON"); + let jobs: Vec = + serde_json::from_value(health.get("jobs").cloned().unwrap_or_default()) + .unwrap_or_default(); + let generation = health["generation"].as_i64().unwrap_or_default(); + warnings.extend( + jobs.into_iter() + .map(|warning| (generation, observation.report.event_id, warning)), + ); + } + warnings +} diff --git a/crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs b/crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs index d34c00e80e..1066cf562f 100644 --- a/crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs @@ -6,7 +6,7 @@ use uuid::Uuid; use crate::api::tests::person_sessions_helpers::request; use crate::api::tests::person_sessions_test_types::BrowserSession; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::snapshots::models::SnapshotInput; use crate::store::main::{begin::begin, commit::commit, query::query}; use crate::store::models::{ControlStore, TransactionMode}; diff --git a/crates/sqlbuild-orchestration/tests/api/node_states/helpers.rs b/crates/sqlbuild-orchestration/tests/api/node_states/helpers.rs new file mode 100644 index 0000000000..74248a0033 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/node_states/helpers.rs @@ -0,0 +1,88 @@ +//! Snapshots, reader sessions, build pages and change notices for node state API tests. + +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::api::tests::changes_helpers::next_notice; +use crate::credentials::tests::role_helpers::{SUPPORT_ROLE, orders_grant}; +use crate::platform::main::health::report_location_health::report_location_health; +use crate::platform::models::{LocationHealth, LocationHealthReport}; +use crate::runs::tests::helpers::snapshot; +use crate::snapshots::models::{SnapshotInput, SnapshotNode}; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::models::MetadataBatch; + +/// The orders location's snapshot with the support role and one built model per name. +pub(crate) fn orders_snapshot(location: Uuid, names: &[&str]) -> SnapshotInput { + let mut input = snapshot(location); + input.documents.insert( + "roles".into(), + json!({SUPPORT_ROLE: {"description": "Answer order questions", + "grants": [orders_grant(json!(["view"]), &["orders"])]}}), + ); + for name in names { + input.nodes.insert( + format!("model:{name}"), + SnapshotNode { + kind: "model".into(), + definition: json!({"name": name, "destination": {"database": null, "schema": "main", "name": name}}), + raw_sql: None, + compiled_sql: None, + python_source: None, + }, + ); + } + input +} + +/// A registered location reader session, which metadata pages must name. +pub(crate) async fn reader(store: &ControlStore, location: Uuid) -> Uuid { + let session_id = Uuid::now_v7(); + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id, + target: "development".into(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("reader"); + session_id +} + +/// One recorded build of each named model in this snapshot generation. +pub(crate) fn builds( + session_id: Uuid, + (generation, names): (i64, &[&str]), + at: &str, +) -> MetadataBatch { + let records: Vec = names + .iter() + .map(|name| { + json!({"event_id": format!("build-{name}-{at}"), "state_database": null, "state_schema": "build_history", + "node_type": "model", "node_name": name, "target_database": null, "target_schema": "main", + "target_name": name, "run_id": "orders-run", "definition_hash": format!("definition-{name}"), + "version_hash": format!("orders-version-{name}"), "schema_fingerprint": "orders-schema", + "definition": "SELECT order_id FROM orders", "metadata_json": "{}", "ts": at}) + }) + .collect(); + serde_json::from_value(json!({"kind": "fingerprints", "session_id": session_id, + "generation": generation, "records": records})) + .expect("build page") +} + +/// The next `count` events' data by topic, because one commit's events arrive in either order. +pub(crate) async fn notices( + feed: &mut crate::api::tests::test_types::SseConnection, + count: usize, +) -> Vec { + let mut notices = Vec::new(); + for _ in 0..count { + notices.push(next_notice(feed).await.expect("change event")); + } + notices.sort_by_key(|notice| notice["topic"].to_string()); + notices +} diff --git a/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs b/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs index 00b594f9d2..30af0488af 100644 --- a/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs @@ -8,7 +8,7 @@ use sqlbuild_core::fingerprint_identity::main::recorded_model_local_hash::record use crate::credentials::models::{AuditContext, DefaultRole}; use crate::runs::tests::helpers::{request, request_run, snapshot}; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::snapshots::models::SnapshotInput; use crate::snapshots::models::SnapshotNode; use crate::store::_helpers::statement_count::STATEMENTS; @@ -18,13 +18,38 @@ use crate::store::models::{ControlStore, Parameter, TransactionMode}; use crate::warehouse_metadata::main::{ ingest::ingest_catalog::ingest_catalog, ingest::ingest_fingerprints::ingest_fingerprints, ingest::ingest_source_freshness::ingest_source_freshness, - person_reads::list_person_catalog_nodes::list_person_catalog_nodes, person_reads::read_person_node_status::read_person_node_status, }; -use crate::warehouse_metadata::models::{MetadataBatch, NodeStatusQuery}; +use crate::warehouse_metadata::models::{CatalogNodeRecord, MetadataBatch, NodeStatusQuery}; -/// Statements a catalog listing issues besides one build read per 100 nodes: 8 fixed, 1 per extra namespace shape. -const WIDE_LISTING_FIXED_STATEMENTS: usize = 9; +/// A cold listing's authorization statements: its clock read, the authorized node keys, the principal, its roles and the current snapshot. +const LISTING_AUTHORIZATION_STATEMENTS: usize = 5; +/// A cold listing's view statements: its clock read, the current snapshot again, node definitions, latest attempts, check counts and the changed build. +const LISTING_VIEW_STATEMENTS: usize = 6; +/// Statements a cold node-list cache issues besides one build read per 100 nodes; the last is the second namespace shape's build read. +const WIDE_LISTING_FIXED_STATEMENTS: usize = + LISTING_AUTHORIZATION_STATEMENTS + LISTING_VIEW_STATEMENTS + 1; + +/// A new server's node list, so every listing reads the store. +async fn list_person_catalog_nodes( + store: &ControlStore, + credentials: (&AuditContext, Uuid), + scope: (Uuid, &str), + default_role: DefaultRole, +) -> Result, ControlError> { + let (cache, _) = crate::node_states::main::start_node_states::start_node_states( + store, + &tokio::sync::broadcast::channel(1).0, + ); + let listing = crate::node_states::main::list_person_catalog_nodes::list_person_catalog_nodes( + &cache, + credentials, + scope, + default_role, + ) + .await?; + Ok(serde_json::from_slice(&listing.body).expect("node list JSON")) +} pub(crate) async fn node_status_reads( store: &ControlStore, diff --git a/crates/sqlbuild-orchestration/tests/api/plans/helpers.rs b/crates/sqlbuild-orchestration/tests/api/plans/helpers.rs index adc0e0fa7c..a933e0eb71 100644 --- a/crates/sqlbuild-orchestration/tests/api/plans/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/plans/helpers.rs @@ -8,11 +8,11 @@ use crate::credentials::main::sessions::set_password::set_password; use crate::credentials::models::{ AuditContext, BuiltinRole, PasswordAuthority, RoleAssignmentRequest, }; -use crate::platform::main::report_location_health::report_location_health; +use crate::platform::main::health::report_location_health::report_location_health; use crate::platform::models::{ LocationHealth, LocationHealthReport, LocationWorkPoll, PlanRequest, }; -use crate::runs::main::request_person_run::request_person_run; +use crate::runs::main::requests::request_person_run::request_person_run; use crate::runs::models::PersonRunRequest; use crate::runs::tests::test_types::ExactKeyPlanner; use crate::store::errors::ControlError; diff --git a/crates/sqlbuild-orchestration/tests/api/state_history/helpers.rs b/crates/sqlbuild-orchestration/tests/api/state_history/helpers.rs new file mode 100644 index 0000000000..9e8b4c7b41 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/state_history/helpers.rs @@ -0,0 +1,116 @@ +//! Builds, check results and store page counts for state history retention tests. + +use chrono::{Duration, TimeZone, Utc}; +use serde_json::{Value, json}; + +use crate::api::tests::warehouse_history_helpers::fixture; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// `builds` builds of `node`, an hour apart. +pub(crate) fn fingerprints(node: &str, builds: i64) -> Vec { + let base = fixture("fingerprint_lookup.json")["record"].clone(); + (0..builds) + .map(|build| { + let mut record = base.clone(); + record["event_id"] = json!(format!("{node}-build-{build:03}")); + record["state_schema"] = json!("main"); + record["node_name"] = json!(node); + record["target_name"] = json!(node); + record["run_id"] = json!(format!("{node}-run-{build:03}")); + record["definition"] = json!(format!("SELECT {build} AS {node}_version")); + record["ts"] = json!(at(build)); + record + }) + .collect() +} + +/// `results` results of check `binding`, an hour apart. +pub(crate) fn audit_results(binding: &str, results: i64) -> Vec { + let base = fixture("audit_result_history.json")["record"].clone(); + (0..results) + .map(|result| { + let mut record = base.clone(); + record["result_id"] = json!(format!("{binding}-{result:03}")); + record["binding_key"] = json!(binding); + record["occurred_at"] = json!(at(result)); + record + }) + .collect() +} + +/// The time of the `index`th version. +pub(crate) fn at(index: i64) -> chrono::DateTime { + Utc.with_ymd_and_hms(2026, 3, 1, 0, 0, 0) + .single() + .expect("time") + + Duration::hours(index) +} + +/// The newest of `count` versions, as the store renders it. +pub(crate) fn newest(count: i64) -> chrono::DateTime { + at(count - 1) +} + +/// Rows kept per name and the newest time, by a `name, kept, newest` query. +pub(crate) async fn counts( + store: &ControlStore, + sql: &str, +) -> Vec<(String, usize, chrono::DateTime)> { + let transaction = begin(store, TransactionMode::Read).await.expect("read"); + let (transaction, rows) = query(transaction, sql, Vec::::new()) + .await + .expect("count"); + commit(transaction).await.expect("commit"); + rows.into_iter() + .map(|row| { + ( + row["name"].as_str().expect("name").to_owned(), + usize::try_from(row["kept"].as_i64().expect("count")).expect("count"), + serde_json::from_value(row["newest"].clone()).expect("newest"), + ) + }) + .collect() +} + +/// Page count, free pages and auto-vacuum mode of a SQLite store. +pub(crate) async fn pages(store: &ControlStore) -> (i64, i64, i64) { + let ControlStore::Sqlite(path) = store else { + panic!("SQLite store"); + }; + let connection = rusqlite::Connection::open(&**path).expect("open"); + let pragma = |name: &str| -> i64 { + connection + .query_row(&format!("PRAGMA {name}"), [], |row| row.get(0)) + .expect("pragma") + }; + ( + pragma("page_count"), + pragma("freelist_count"), + pragma("auto_vacuum"), + ) +} + +/// Builds of `node` whose metadata is about 128 KiB each, so pruning frees most of a store. +pub(crate) fn large_fingerprints(node: &str, builds: i64) -> Vec { + let large = "x".repeat(128 * 1024); + fingerprints(node, builds) + .into_iter() + .map(|mut record| { + record["metadata_json"] = + json!(format!("{{\"note\":\"{large}{}\"}}", record["run_id"])); + record + }) + .collect() +} + +/// Rebuild a SQLite store in `mode` (`NONE` or `INCREMENTAL`) auto-vacuum. +pub(crate) fn set_auto_vacuum(store: &ControlStore, mode: &str) { + let ControlStore::Sqlite(path) = store else { + panic!("SQLite store"); + }; + rusqlite::Connection::open(&**path) + .expect("open") + .execute_batch(&format!("PRAGMA auto_vacuum={mode}; VACUUM")) + .expect("auto-vacuum mode"); +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_acknowledgements.rs b/crates/sqlbuild-orchestration/tests/api/test_acknowledgements.rs index cfb5cbf894..fdda4e366a 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_acknowledgements.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_acknowledgements.rs @@ -9,10 +9,10 @@ use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; use crate::api::tests::test_types::AcknowledgementTestCase; use crate::credentials::tests::location_helpers::create_location; use crate::credentials::tests::role_helpers::role_fixture; -use crate::runs::main::request_job_run::request_job_run; +use crate::runs::main::requests::request_job_run::request_job_run; use crate::runs::models::JobRunRequest; use crate::runs::tests::helpers::snapshot; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs index 96bfd0d3d9..03bb9e16fd 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs @@ -15,10 +15,10 @@ use crate::credentials::models::{ AuditContext, BuiltinRole, DefaultRole, PasswordAuthority, PasswordLogin, RoleAssignmentRequest, }; use crate::credentials::tests::location_helpers::create_location; -use crate::platform::main::report_location_health::report_location_health; +use crate::platform::main::health::report_location_health::report_location_health; use crate::platform::models::{LocationHealth, LocationHealthReport}; use crate::runs::tests::helpers::snapshot; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; diff --git a/crates/sqlbuild-orchestration/tests/api/test_changes.rs b/crates/sqlbuild-orchestration/tests/api/test_changes.rs index 5262110726..ad82b6354d 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_changes.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_changes.rs @@ -18,12 +18,12 @@ use crate::credentials::models::{AuditContext, CredentialScope}; use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::{change_trigger::change_trigger, list_triggers::list_triggers}; use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; -use crate::runs::main::request_job_run::request_job_run; +use crate::runs::main::requests::request_job_run::request_job_run; use crate::runs::models::JobRunRequest; use crate::runs::tests::helpers::snapshot; use crate::scheduler::models::TickDecision; use crate::scheduler::tests::contracts::helpers::{acquire_lease, evaluate_trigger}; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use crate::store::models::ControlStore; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; diff --git a/crates/sqlbuild-orchestration/tests/api/test_checks.rs b/crates/sqlbuild-orchestration/tests/api/test_checks.rs index 3fd541528c..c52691ef12 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_checks.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_checks.rs @@ -80,7 +80,7 @@ fn given_checks_sharing_an_audit_name_when_people_read_them_then_binding_keys_an let (status, list) = get(&client, (&api.url, &viewer), &base_path).await; assert_eq!(status, StatusCode::OK, "{}", test_case.description); - let keys: Vec<(&str, &str, &str)> = list + let keys: Vec<(&str, &str, &str, bool)> = list .as_array() .expect("checks") .iter() @@ -89,17 +89,18 @@ fn given_checks_sharing_an_audit_name_when_people_read_them_then_binding_keys_an check["binding_key"].as_str().expect("key"), check["outcome"].as_str().expect("outcome"), check["audit_name"].as_str().expect("name"), + check["ignored"].as_bool().expect("ignored"), ) }) .collect(); assert_eq!( keys, [ - (customers_key, "error", "not_null"), - (orders_key, "pass", "not_null"), - (archived_key, "pass", "not_null") + (customers_key, "error", "not_null", true), + (orders_key, "pass", "not_null", false), + (archived_key, "pass", "not_null", false) ], - "one entry per binding key, each its newest result" + "one entry per binding key, each its newest result, marked when its build ignored it" ); let (status, nodes) = get( &client, diff --git a/crates/sqlbuild-orchestration/tests/api/test_column_lineage.rs b/crates/sqlbuild-orchestration/tests/api/test_column_lineage.rs index eb255b6e40..0a1d6ae2c9 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_column_lineage.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_column_lineage.rs @@ -9,7 +9,7 @@ use crate::credentials::main::identities::create_principal::create_principal; use crate::credentials::models::{AuditContext, DefaultRole}; use crate::credentials::tests::location_helpers::create_location; use crate::runs::tests::helpers::snapshot; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::snapshots::models::SnapshotNode; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs index 0eecbe0f8a..178fa78ecd 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -5,15 +5,15 @@ use crate::api::errors::ClientError; use crate::credentials::main::tokens::issue_token::issue_token; use crate::credentials::main::tokens::revoke_token::revoke_token; use crate::credentials::models::{AuditContext, CredentialScope}; -use crate::runs::main::change_run::change_run; -use crate::runs::main::claim_next_run::claim_next_run; -use crate::runs::main::list_runs::list_runs; +use crate::runs::main::execution::change_run::change_run; +use crate::runs::main::execution::claim_next_run::claim_next_run; +use crate::runs::main::listings::list_runs::list_runs; use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunExecution, RunStatus}; use crate::runs::tests::helpers::request_run; use crate::scheduler::tests::contracts::helpers::acquire_lease; -use crate::snapshots::main::next_remote_generation::next_remote_generation; -use crate::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; -use crate::snapshots::main::verify_execution::verify_execution; +use crate::snapshots::main::publication::next_remote_generation::next_remote_generation; +use crate::snapshots::main::publication::publish_remote_snapshot::publish_remote_snapshot; +use crate::snapshots::main::publication::verify_execution::verify_execution; use crate::snapshots::models::{SnapshotInput, SnapshotRecord}; use crate::store::main::migrate_store::migrate_store; use chrono::Utc; diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index d7e3e17264..606270678b 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -10,6 +10,8 @@ mod fingerprint_reads_helpers; pub(crate) mod helpers; #[path = "node_sql/helpers.rs"] mod node_sql_helpers; +#[path = "node_states/helpers.rs"] +mod node_states_helpers; #[path = "node_status/helpers.rs"] mod node_status_helpers; #[path = "oidc/helpers.rs"] @@ -26,6 +28,8 @@ mod person_sessions_test_types; mod plans_helpers; #[path = "recall/helpers.rs"] mod recall_helpers; +#[path = "state_history/helpers.rs"] +mod state_history_helpers; #[path = "statistics/eligibility/helpers.rs"] mod statistics_eligibility; #[path = "statistics/helpers.rs"] @@ -56,6 +60,8 @@ mod test_denials; mod test_development; #[path = "test_external_sessions.rs"] mod test_external_sessions; +#[path = "test_ingestion_turns.rs"] +mod test_ingestion_turns; #[path = "test_location_health.rs"] mod test_location_health; #[path = "test_location_work.rs"] @@ -64,6 +70,8 @@ mod test_location_work; mod test_maintenance; #[path = "test_microbatch_coverage.rs"] mod test_microbatch_coverage; +#[path = "test_node_states.rs"] +mod test_node_states; #[path = "test_oidc.rs"] mod test_oidc; #[path = "test_oidc_groups.rs"] @@ -92,6 +100,8 @@ mod test_resource_costs; mod test_run_ids; #[path = "test_server_errors.rs"] mod test_server_errors; +#[path = "test_state_history_retention.rs"] +mod test_state_history_retention; #[path = "test_statistics_recovery.rs"] mod test_statistics_recovery; #[path = "test_trigger_upcoming.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_denials.rs b/crates/sqlbuild-orchestration/tests/api/test_denials.rs index 3f16d3c708..e7689586a2 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_denials.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_denials.rs @@ -10,11 +10,11 @@ use crate::api::tests::test_types::{DenialRequest, DenialTestCase}; use crate::credentials::tests::location_helpers::create_location; use crate::credentials::tests::role_helpers::role_fixture; use crate::declarations::main::list_triggers::list_triggers; -use crate::runs::main::request_job_run::request_job_run; +use crate::runs::main::requests::request_job_run::request_job_run; use crate::runs::models::JobRunRequest; use crate::runs::tests::helpers::snapshot; use crate::scheduler::tests::contracts::ticks::helpers::calendar_text; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/api/test_development.rs b/crates/sqlbuild-orchestration/tests/api/test_development.rs index b41baed731..f873c54819 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_development.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_development.rs @@ -5,6 +5,7 @@ use uuid::Uuid; use crate::api::models::{ApiAuthentication, DevelopmentScope, HumanLogin}; use crate::credentials::main::identities::development_person::development_person; use crate::credentials::main::locations::development_location::development_location; +use crate::credentials::main::locations::location_named::location_named; use crate::credentials::models::AuditContext; use crate::declarations::main::{ change_trigger::change_trigger, list_triggers::list_triggers, @@ -12,7 +13,7 @@ use crate::declarations::main::{ }; use crate::declarations::models::{TriggerChange, TriggerChangeRequest, TriggerStatus}; use crate::snapshots::main::{ - current_snapshot::current_snapshot, publish_snapshot::publish_snapshot, + publication::publish_snapshot::publish_snapshot, reads::current_snapshot::current_snapshot, }; use crate::store::main::migrate_store::migrate_store; @@ -33,11 +34,13 @@ fn given_development_when_defaults_publish_then_identity_audit_and_trigger_pause description: "SQLite atomic development defaults", backend: Backend::Sqlite, expected_status: TriggerStatus::Paused, + expected_names: ["orders", "orders (2)"], }, DevelopmentTestCase { description: "PostgreSQL atomic development defaults", backend: Backend::Postgres, expected_status: TriggerStatus::Paused, + expected_names: ["orders", "orders (2)"], }, ]; for test_case in test_cases { @@ -63,26 +66,49 @@ fn given_development_when_defaults_publish_then_identity_audit_and_trigger_pause }; install_fault(store, &fault).await; assert!( - development_location(store, "/synthetic/orders", "development", &context) - .await - .is_err() + development_location( + store, + ("/synthetic/orders", "orders"), + "development", + &context + ) + .await + .is_err() ); assert_eq!(location_state(store).await, (0, 0)); remove_fault(store, &fault).await; - let location = - development_location(store, "/synthetic/orders", "development", &context) - .await - .expect("create identity"); + let location = development_location( + store, + ("/synthetic/orders", "orders"), + "development", + &context, + ) + .await + .expect("create identity"); assert_eq!( - development_location(store, "/synthetic/orders", "development", &context) - .await - .expect("retained identity"), + development_location( + store, + ("/synthetic/orders", "orders"), + "development", + &context + ) + .await + .expect("retained identity"), location ); - let other = development_location(store, "/synthetic/orders", "preview", &context) - .await - .expect("target-specific identity"); + let other = + development_location(store, ("/synthetic/orders", "orders"), "preview", &context) + .await + .expect("target-specific identity"); assert_ne!(location, other); + for (name, expected) in test_case.expected_names.into_iter().zip([location, other]) { + assert_eq!( + location_named(store, name).await.expect("named location"), + Some(expected), + "{}: development locations are named after the project", + test_case.description + ); + } let person = development_person(store, &context) .await .expect("development person"); diff --git a/crates/sqlbuild-orchestration/tests/api/test_ingestion_turns.rs b/crates/sqlbuild-orchestration/tests/api/test_ingestion_turns.rs new file mode 100644 index 0000000000..564e203c30 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_ingestion_turns.rs @@ -0,0 +1,71 @@ +use serde_json::{Value, json}; +use tokio::time::Instant; +use uuid::Uuid; + +use crate::api::tests::test_types::IngestionTurnTestCase; +use crate::api::tests::warehouse_history_helpers::seed_target; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::TransactionMode; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::warehouse_metadata::main::ingest::ingest_catalog::ingest_catalog; +use crate::warehouse_metadata::models::MetadataBatch; + +#[test] +fn given_a_sync_ingesting_pages_when_another_writer_waits_then_it_writes_before_the_sync_ends() { + let test_cases = [IngestionTurnTestCase { + description: "a scheduler lease during three 2,000-relation catalog pages", + pages: 3, + relations_per_page: 2000, + expected_writer_first: true, + }]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(Backend::Sqlite).await else { + return; + }; + let store = &fixture.store; + let (location, session_id) = seed_target(store).await; + let columns: Vec = (0..12) + .map(|index| json!({"name": format!("column_{index}"), "data_type": "INTEGER"})) + .collect(); + let batches: Vec = (0..test_case.pages) + .map(|page| { + let records: Vec = (0..test_case.relations_per_page) + .map(|index| { + json!({"state_database": null, "state_schema": "main", + "name": format!("orders_{page}_{index:05}"), + "relation_type": "BASE TABLE", "columns": columns}) + }) + .collect(); + serde_json::from_value(json!({"kind": "catalog", "session_id": session_id, + "generation": 1, "records": records, "scan": {"scan_id": Uuid::now_v7(), + "location": {"database": null, "schema": "main"}, "complete": false}})) + .expect("batch") + }) + .collect(); + let sync = async { + for batch in &batches { + ingest_catalog(store, (location, "development"), batch) + .await + .expect("ingest"); + } + Instant::now() + }; + let lease = async { + tokio::task::yield_now().await; + let transaction = begin(store, TransactionMode::Write).await.expect("begin"); + commit(transaction).await.expect("commit"); + Instant::now() + }; + let (synced, leased) = tokio::join!(sync, lease); + assert_eq!( + leased < synced, + test_case.expected_writer_first, + "{}: the waiting writer takes its turn between the sync's short transactions", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_health.rs b/crates/sqlbuild-orchestration/tests/api/test_location_health.rs index 14a932e7cc..8166d424f7 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_health.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_health.rs @@ -1,10 +1,12 @@ use crate::authorization::main::bootstrap_user::bootstrap_user; use crate::credentials::main::tokens::issue_token::issue_token; use crate::credentials::models::{AuditContext, CredentialScope, DefaultRole}; -use crate::platform::main::read_person_location_health::read_person_location_health; -use crate::platform::main::report_remote_location_health::report_remote_location_health; -use crate::platform::models::{LocationHealth, LocationHealthObservation, LocationHealthReport}; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::platform::main::health::read_person_location_health::read_person_location_health; +use crate::platform::main::health::report_remote_location_health::report_remote_location_health; +use crate::platform::models::{ + ConnectionCheckTrigger, LocationHealth, LocationHealthObservation, LocationHealthReport, +}; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use crate::store::main::{begin::begin, commit::commit, query::query}; use crate::store::models::{Parameter, TransactionMode}; @@ -14,9 +16,11 @@ use reqwest::StatusCode; use serde_json::json; use uuid::Uuid; -use crate::api::tests::helpers::{start_api, stop_api}; -use crate::api::tests::test_types::LocationHealthTestCase; +use crate::api::tests::helpers::{pool_capped, start_api, stop_api}; +use crate::api::tests::test_types::{LocationHealthTestCase, PoolCapHealthTestCase}; use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::models::ServerPoolCaps; +use crate::platform::main::health::report_location_health::report_location_health; use crate::runs::tests::helpers::snapshot; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; @@ -137,6 +141,8 @@ fn given_scoped_location_observations_when_retried_or_read_then_history_and_auth health: LocationHealth::Connection { generation: 1, healthy: false, + trigger: Some(ConnectionCheckTrigger::Startup), + elapsed_millis: Some(410), }, ..heartbeat.clone() }; @@ -184,12 +190,13 @@ fn given_scoped_location_observations_when_retried_or_read_then_history_and_auth let administrator = bootstrap_user(store, &audit, "orders-admin") .await .expect("administrator"); + let no_caps = ServerPoolCaps::default(); let person_read = |target: &'static str| { read_person_location_health( store, (&audit, administrator), (location_id, target), - DefaultRole::None, + (DefaultRole::None, &no_caps), ) }; assert_eq!( @@ -266,6 +273,8 @@ fn given_scoped_location_observations_when_retried_or_read_then_history_and_auth health: LocationHealth::Connection { generation: 1, healthy: true, + trigger: None, + elapsed_millis: None, }, ..busy_heartbeat.clone() }; @@ -289,3 +298,166 @@ fn given_scoped_location_observations_when_retried_or_read_then_history_and_auth }); } } + +#[test] +fn given_pool_amounts_above_this_servers_caps_when_reading_health_then_a_pool_capped_warning_lists_them() + { + let test_cases = [ + PoolCapHealthTestCase { + description: "a cap below a job's amount warns about that job only", + caps: &[("warehouse_heavy", 1)], + expected_warnings: &[( + "orders", + "warehouse_heavy", + 3, + 1, + "job orders takes 3 of pool warehouse_heavy; this server caps it at 1, so it runs \ + alone in that pool", + )], + }, + PoolCapHealthTestCase { + description: "a cap below a group's amount names it a job group", + caps: &[("supplier_api", 1)], + expected_warnings: &[( + "nightly_group", + "supplier_api", + 2, + 1, + "job group nightly_group takes 2 of pool supplier_api; this server caps it at 1, \ + so it runs alone in that pool", + )], + }, + PoolCapHealthTestCase { + description: "a cap at or above every amount warns about nothing", + caps: &[("warehouse_heavy", 3), ("supplier_api", 2)], + expected_warnings: &[], + }, + PoolCapHealthTestCase { + description: "an uncapped server warns about nothing", + caps: &[], + expected_warnings: &[], + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(Backend::Sqlite).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location_id); + input.documents.insert( + "jobs".into(), + json!({ + "orders": {"description": "Build orders", "selection": ["orders"], + "pools": {"warehouse_heavy": {"amount": 3, "limit": 4}}}, + "customers": {"description": "Build customers", "selection": ["customers"], + "pools": {"warehouse_heavy": {"amount": 1, "limit": 4}}}, + }), + ); + input.documents.insert( + "groups".into(), + json!({"nightly_group": {"description": "Everything nightly", + "selection": ["customers", "orders"], + "pools": {"supplier_api": {"amount": 2, "limit": 2}}, + "jobs": {"customers": [], "orders": []}}}), + ); + let published = publish_snapshot(store, &input).await.expect("publish"); + let audit = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let administrator = bootstrap_user(store, &audit, "orders-admin") + .await + .expect("administrator"); + let caps = ServerPoolCaps { + launchers: [( + (location_id, "development".to_owned()), + test_case + .caps + .iter() + .map(|(pool, cap)| { + ( + (*pool).to_owned(), + std::num::NonZeroU32::new(*cap).expect("cap"), + ) + }) + .collect(), + )] + .into(), + }; + + let observations = read_person_location_health( + store, + (&audit, administrator), + (location_id, "development"), + (DefaultRole::None, &caps), + ) + .await + .expect("health"); + + let capped = pool_capped(&observations); + assert!( + capped + .iter() + .all(|(generation, event_id, _)| (*generation, *event_id) + == (1, published.snapshot_id)), + "{}: the warning belongs to the current snapshot", + test_case.description + ); + let warnings: Vec<(String, String, u32, u32, String)> = capped + .into_iter() + .map(|(_, _, warning)| { + ( + warning.job_key, + warning.pool, + warning.amount, + warning.cap, + warning.message, + ) + }) + .collect(); + assert_eq!( + warnings, + test_case + .expected_warnings + .iter() + .map(|(job, pool, amount, cap, message)| ( + (*job).to_owned(), + (*pool).to_owned(), + *amount, + *cap, + (*message).to_owned() + )) + .collect::>(), + "{}", + test_case.description + ); + assert!( + report_location_health( + store, + location_id, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: Uuid::now_v7(), + target: "development".into(), + health: LocationHealth::PoolCapped { + generation: 1, + jobs: Vec::new(), + }, + }, + ) + .await + .is_err(), + "{}: a location cannot report the server's own warning", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index 5752d74e7f..8ccd1afb05 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -1,15 +1,15 @@ use crate::api::errors::ClientError; use crate::credentials::main::tokens::issue_token::issue_token; use crate::credentials::models::{AuditContext, CredentialScope}; -use crate::platform::main::connect_location_work::connect_location_work; -use crate::platform::main::location_work::location_work; -use crate::platform::main::report_location_health::report_location_health; -use crate::platform::main::request_location_work::request_location_work; +use crate::platform::main::health::report_location_health::report_location_health; +use crate::platform::main::work::connect_location_work::connect_location_work; +use crate::platform::main::work::location_work::location_work; +use crate::platform::main::work::request_location_work::request_location_work; use crate::platform::models::{ LocationHealth, LocationHealthReport, LocationWork, LocationWorkOutcome, LocationWorkPoll, LocationWorkRecord, LocationWorkRequest, LocationWorkResult, }; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use chrono::{Duration, Utc}; use reqwest::StatusCode; diff --git a/crates/sqlbuild-orchestration/tests/api/test_maintenance.rs b/crates/sqlbuild-orchestration/tests/api/test_maintenance.rs index 9fe6f671fa..efb9785c55 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_maintenance.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_maintenance.rs @@ -9,10 +9,10 @@ use crate::api::tests::test_types::WarehouseMetadataTestCase; use crate::api::tests::warehouse_history_helpers::{ batch, fixture, get, ids, login, records, seed_target, start_password_api, }; -use crate::platform::main::report_location_health::report_location_health; +use crate::platform::main::health::report_location_health::report_location_health; use crate::platform::models::{LocationHealth, LocationHealthReport}; use crate::runs::tests::helpers::snapshot; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; use crate::warehouse_metadata::main::ingest::{ @@ -181,6 +181,46 @@ fn given_moves_views_and_cleanups_when_people_read_maintenance_then_each_model_s .0, StatusCode::FORBIDDEN ); + let totals_path = + format!("/api/control/locations/{location}/targets/development/mirrored-totals"); + let (status, totals) = get(&client, (&api.url, &viewer), &totals_path).await; + assert_eq!(status, StatusCode::OK, "{}", test_case.description); + let counted: Vec<(&str, u64)> = totals + .as_array() + .expect("totals") + .iter() + .filter(|total| total["records"] != 0) + .map(|total| { + ( + total["table"].as_str().expect("table"), + total["records"].as_u64().expect("records"), + ) + }) + .collect(); + assert_eq!( + (totals.as_array().expect("totals").len(), counted), + ( + 14, + vec![ + ("janitor_events", 2), + ("migrations", 3), + ("old_name_views", 4) + ] + ), + "{}: every state table, counting only this target's mirrored records", + test_case.description + ); + assert_eq!( + get( + &client, + (&api.url, &viewer), + &totals_path.replace("development", "preview") + ) + .await + .0, + StatusCode::FORBIDDEN, + "mirrored totals need the same view as the target's health" + ); stop_api(api).await; close_test_store(fixture_store).await; }); diff --git a/crates/sqlbuild-orchestration/tests/api/test_node_states.rs b/crates/sqlbuild-orchestration/tests/api/test_node_states.rs new file mode 100644 index 0000000000..78bcbcc971 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_node_states.rs @@ -0,0 +1,529 @@ +//! Held node lists and states: ETag revalidation, pushed state changes, recovery reads and snapshot resets on both control stores. + +use std::collections::BTreeSet; + +use reqwest::{StatusCode, header}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::api::models::{ApiAuthentication, HumanLogin}; +use crate::api::tests::changes_helpers::{next_notice, opened, request_person_feed}; +use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; +use crate::api::tests::node_states_helpers::{builds, notices, orders_snapshot, reader}; +use crate::api::tests::test_types::{ + ChangedBuildTestCase, NodeStatesTestCase, PushedStatesTestCase, +}; +use crate::credentials::tests::role_helpers::role_fixture; +use crate::node_states::_helpers::views::{current_view, refresh_view, target_cache}; +use crate::node_states::main::report_changed_build::report_changed_build; +use crate::node_states::main::start_node_states::start_node_states; +use crate::node_states::models::ChangedBuildReport; +use crate::runs::main::requests::request_job_run::request_job_run; +use crate::runs::models::JobRunRequest; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::warehouse_metadata::main::ingest::ingest_fingerprints::ingest_fingerprints; + +#[test] +fn given_held_node_states_when_people_revalidate_and_builds_land_then_only_changes_are_pushed_and_snapshots_reset() + { + let test_cases = [ + NodeStatesTestCase { + description: "SQLite held node states", + backend: Backend::Sqlite, + expected_listed: 5, + }, + NodeStatesTestCase { + description: "PostgreSQL held node states", + backend: Backend::Postgres, + expected_listed: 5, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let roles = role_fixture(store).await; + let location = roles.location_id; + let mut input = orders_snapshot(location, &["orders"]); + input.generation = 2; + let published = publish_snapshot(store, &input).await.expect("orders"); + let session_id = reader(store, location).await; + let api = start_api_with_authentication( + store, + ApiAuthentication { + login: HumanLogin::Password("https://control.example.com".into()), + ..ApiAuthentication::default() + }, + ) + .await; + let client = reqwest::Client::new(); + let read = |path: &str, bearer: &str, tag: Option<&str>| { + let request = client + .get(format!( + "{}/api/control/locations/{location}/targets/development/{path}", + api.url + )) + .header(header::HOST, "control.example.com") + .header(header::COOKIE, format!("__Host-sqb-session={bearer}")); + tag.into_iter() + .fold(request, |request, tag| { + request.header(header::IF_NONE_MATCH, tag) + }) + .send() + }; + let admin = roles.administrator.bearer.clone(); + let etag = |response: &reqwest::Response| { + response.headers()[header::ETAG] + .to_str() + .expect("ASCII ETag") + .to_owned() + }; + + let listed = read("nodes", &admin, None).await.expect("nodes"); + let first_tag = etag(&listed); + let nodes: Vec = listed.json().await.expect("node list"); + let unchanged = read("nodes", &admin, Some(&first_tag)) + .await + .expect("revalidate"); + let other_tag = read("nodes", &admin, Some("\"elsewhere:1\"")) + .await + .expect("other tag") + .status(); + let states: Value = read("node-states", &admin, None) + .await + .expect("states") + .json() + .await + .expect("states body"); + assert_eq!( + ( + first_tag.clone(), + nodes.len(), + unchanged.status(), + etag(&unchanged), + other_tag + ), + ( + format!("\"{}:1\"", published.snapshot_id), + test_case.expected_listed, + StatusCode::NOT_MODIFIED, + first_tag.clone(), + StatusCode::OK + ), + "{}: the list is tagged with its snapshot and version and revalidates to 304", + test_case.description + ); + assert_eq!( + ( + &states["snapshot_id"], + &states["version"], + states["states"] + .as_object() + .expect("states") + .keys() + .cloned() + .collect::>(), + &states["states"]["model:orders"]["last_build"], + ), + ( + &json!(published.snapshot_id), + &json!(1), + nodes + .iter() + .map(|node| node["node_key"].as_str().expect("key").to_owned()) + .collect(), + &Value::Null, + ), + "{}: node states cover every listed node at the list's version", + test_case.description + ); + + let member = roles.member.bearer.clone(); + let denied = [ + read("nodes", &member, None) + .await + .expect("member nodes") + .status(), + read("nodes", &member, Some(&first_tag)) + .await + .expect("member revalidation") + .status(), + read("node-states", &member, None) + .await + .expect("member states") + .status(), + ]; + assert_eq!( + denied, + [StatusCode::FORBIDDEN; 3], + "{}: a person without view of every node is refused before any held body or 304", + test_case.description + ); + + let mut feed = + opened(request_person_feed(&client, &api, (location, "development", &admin)).await) + .await; + ingest_fingerprints( + store, + (location, "development"), + &builds(session_id, (2, &["orders"]), "2026-01-03T10:00:00Z"), + ) + .await + .expect("orders build"); + let pushed = next_notice(&mut feed).await.expect("node status"); + let recovered: Value = read("node-states", &admin, None) + .await + .expect("recovery read") + .json() + .await + .expect("recovered states"); + assert_eq!( + ( + &pushed["topic"], + &pushed["version"], + &pushed["previous_version"], + pushed["states"] + .as_object() + .expect("pushed states") + .keys() + .cloned() + .collect::>(), + &pushed["states"]["model:orders"]["last_build"]["version_hash"], + &pushed.get("full"), + ), + ( + &json!("node_status:development"), + &json!(2), + &json!(1), + vec!["model:orders".to_owned()], + &json!("orders-version-orders"), + &None, + ), + "{}: a build pushes only the node whose state changed", + test_case.description + ); + assert_eq!( + (&recovered["version"], &recovered["states"]["model:orders"]), + (&json!(2), &pushed["states"]["model:orders"]), + "{}: a client with a version gap recovers the pushed state from node-states", + test_case.description + ); + let relisted = read("nodes", &admin, Some(&first_tag)) + .await + .expect("stale revalidation"); + let second_tag = etag(&relisted); + let relisted: Vec = relisted.json().await.expect("relisted"); + let orders = relisted + .iter() + .find(|node| node["node_key"] == "model:orders") + .expect("orders listed"); + assert_eq!( + (second_tag, &orders["last_build"]), + ( + format!("\"{}:2\"", published.snapshot_id), + &pushed["states"]["model:orders"]["last_build"] + ), + "{}: a changed state invalidates the old ETag", + test_case.description + ); + + let mut jobs = input.clone(); + jobs.generation = 3; + jobs.documents.insert( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), + ); + let republished = publish_snapshot(store, &jobs).await.expect("new snapshot"); + let reset = notices(&mut feed, 2).await; + let reset_tag = etag( + &read("nodes", &admin, Some(&first_tag)) + .await + .expect("new snapshot list"), + ); + assert_eq!( + (reset, reset_tag), + ( + vec![ + json!({"topic": "node_status:development", "version": 3, "previous_version": 2, "full": true}), + json!({"topic": format!("snapshot:{location}")}), + ], + format!("\"{}:3\"", republished.snapshot_id), + ), + "{}: a new snapshot resets the list and tells clients to refetch the states", + test_case.description + ); + let request = JobRunRequest { + run_id: Uuid::now_v7(), + location_id: location, + target: "development".into(), + job_key: "orders".into(), + reason: "manual verification".into(), + priority: 0, + window: None, + full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), + }; + assert!( + request_job_run(store, (&roles.context, None), &request) + .await + .expect("queue job") + ); + assert_eq!( + notices(&mut feed, 2).await, + vec![ + json!({"topic": "node_status:development", "version": 3, "previous_version": 3, "states": {}}), + json!({"topic": "runs"}), + ], + "{}: a signal that changes no state keeps the version and pushes no states", + test_case.description + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_changed_build_reports_when_people_read_node_states_then_only_the_current_generations_plan_is_served() + { + let test_cases = [ + ChangedBuildTestCase { + description: "SQLite changed build", + backend: Backend::Sqlite, + expected_before_plan: Value::Null, + }, + ChangedBuildTestCase { + description: "PostgreSQL changed build", + backend: Backend::Postgres, + expected_before_plan: Value::Null, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let roles = role_fixture(store).await; + let location = roles.location_id; + let mut input = orders_snapshot(location, &["orders", "customers"]); + input.generation = 2; + publish_snapshot(store, &input).await.expect("orders"); + let session_id = reader(store, location).await; + let api = start_api_with_authentication( + store, + ApiAuthentication { + login: HumanLogin::Password("https://control.example.com".into()), + ..ApiAuthentication::default() + }, + ) + .await; + let client = reqwest::Client::new(); + let admin = roles.administrator.bearer.clone(); + let changed_build = || async { + let states: Value = client + .get(format!( + "{}/api/control/locations/{location}/targets/development/node-states", + api.url + )) + .header(header::HOST, "control.example.com") + .header(header::COOKIE, format!("__Host-sqb-session={admin}")) + .send() + .await + .expect("states") + .json() + .await + .expect("states body"); + let mut build = states["changed_build"].clone(); + build.as_object_mut().into_iter().for_each(|build| { + assert!(build.remove("reported_at").is_some_and(|at| at.is_string())); + }); + build + }; + assert_eq!( + changed_build().await, + test_case.expected_before_plan, + "{}: unknown before the location plans it", + test_case.description + ); + let mut feed = + opened(request_person_feed(&client, &api, (location, "development", &admin)).await) + .await; + let report = |generation: i64, changed: &[&str]| ChangedBuildReport { + session_id, + generation, + changed: changed.iter().map(|key| (*key).to_owned()).collect(), + downstream: vec!["model:customers".into()], + failing: None, + }; + let failing = |reason: String| ChangedBuildReport { + session_id, + generation: 2, + changed: Vec::new(), + downstream: Vec::new(), + failing: Some(reason), + }; + report_changed_build( + store, + (location, "development"), + &report(2, &["model:orders"]), + ) + .await + .expect("changed build"); + assert_eq!( + next_notice(&mut feed).await.expect("node status"), + json!({"topic": "node_status:development", "version": 2, "previous_version": 1, "full": true}), + "{}: a new changed build tells clients to read the states again", + test_case.description + ); + assert_eq!( + changed_build().await, + json!({"generation": 2, "changed": ["model:orders"], "downstream": ["model:customers"]}), + "{}: served with the node states", + test_case.description + ); + report_changed_build( + store, + (location, "development"), + &failing("Model orders cannot plan a full rebuild".into()), + ) + .await + .expect("failing changed build"); + next_notice(&mut feed).await.expect("node status"); + assert_eq!( + changed_build().await, + json!({"generation": 2, "changed": [], "downstream": [], "failing": "Model orders cannot plan a full rebuild"}), + "{}: a failed plan replaces the count with its reason", + test_case.description + ); + assert!( + matches!( + report_changed_build( + store, + (location, "development"), + &failing("x".repeat(2_001)) + ) + .await, + Err(ControlError::Invalid(_)) + ), + "{}: an unbounded reason is refused", + test_case.description + ); + let refused = [ + report_changed_build( + store, + (location, "development"), + &report(1, &["model:orders"]), + ) + .await, + report_changed_build(store, (location, "development"), &report(2, &["orders"])) + .await, + ]; + assert!( + matches!( + refused, + [Err(ControlError::Fenced), Err(ControlError::Invalid(_))] + ), + "{}: an older generation is fenced and nodes must be keys: {refused:?}", + test_case.description + ); + let mut republished = input.clone(); + republished.generation = 3; + publish_snapshot(store, &republished) + .await + .expect("new snapshot"); + assert_eq!( + changed_build().await, + Value::Null, + "{}: a new snapshot's changed build is unknown until the location plans it", + test_case.description + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_builds_of_many_nodes_when_states_refresh_then_large_changes_ask_clients_to_refetch() { + let test_cases = [ + PushedStatesTestCase { + description: "a few changed nodes are pushed", + built: 3, + expected_pushed: Some(3), + }, + PushedStatesTestCase { + description: "more than 2,000 changed nodes are announced in full", + built: 2_001, + expected_pushed: None, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(Backend::Sqlite).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = role_fixture(store).await.location_id; + let names: Vec = (0..test_case.built) + .map(|index| format!("orders_{index:04}")) + .collect(); + let names: Vec<&str> = names.iter().map(String::as_str).collect(); + let mut input = orders_snapshot(location, &names); + input.generation = 2; + publish_snapshot(store, &input) + .await + .expect("wide snapshot"); + let session_id = reader(store, location).await; + let (cache, _) = start_node_states(store, &tokio::sync::broadcast::channel(1).0); + let mut updates = cache.updates.subscribe(); + let held = current_view(&cache, (location, "development"), None) + .await + .expect("first view"); + for chunk in names.chunks(100) { + ingest_fingerprints( + store, + (location, "development"), + &builds(session_id, (2, chunk), "2026-01-03T10:00:00Z"), + ) + .await + .expect("builds"); + } + let entry = target_cache(&cache, (location, "development")); + let view = refresh_view(&cache, (location, "development"), (&entry, true)) + .await + .expect("refresh"); + let update: Value = + serde_json::from_str(&updates.recv().await.expect("update").data).expect("JSON"); + assert_eq!( + ( + held.version, + view.version, + &update["previous_version"], + update["states"].as_object().map(serde_json::Map::len), + update["full"].as_bool(), + ), + ( + 1, + 2, + &json!(1), + test_case.expected_pushed, + test_case.expected_pushed.is_none().then_some(true), + ), + "{}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_live.rs b/crates/sqlbuild-orchestration/tests/api/test_person_live.rs index 09f414f988..c324901e5e 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_live.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_live.rs @@ -12,10 +12,10 @@ use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; use crate::api::tests::test_types::PersonLiveTestCase; use crate::credentials::tests::location_helpers::create_location; use crate::credentials::tests::role_helpers::{SUPPORT_ROLE, orders_grant, role_fixture}; -use crate::runs::main::request_job_run::request_job_run; +use crate::runs::main::requests::request_job_run::request_job_run; use crate::runs::models::JobRunRequest; use crate::runs::tests::helpers::snapshot; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs index 5e6e61e5a1..0d72ba8d3d 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs @@ -15,13 +15,13 @@ use crate::credentials::models::{ }; use crate::credentials::tests::location_helpers::create_location; use crate::platform::main::{ - read_location_work::read_location_work, - read_person_connection_test::read_person_connection_test, - request_location_work::request_location_work, + health::read_person_connection_test::read_person_connection_test, + work::read_location_work::read_location_work, + work::request_location_work::request_location_work, }; use crate::platform::models::{LocationWork, LocationWorkRequest}; use crate::runs::tests::helpers::snapshot; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs b/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs index d8ee50dc0e..4b45899618 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs @@ -1,22 +1,31 @@ //! A person's plan is frozen like an ad-hoc build, queued as location work and read only by them. +use std::collections::BTreeMap; + use serde_json::json; +use sqlbuild_core::plan_display::main::detach_model_diffs::detach_model_diffs; use uuid::Uuid; use crate::api::tests::plans_helpers::{ask, person, plan_request, register_instance, run_rows}; -use crate::api::tests::test_types::PlanTestCase; +use crate::api::tests::test_types::{PlanDiffTestCase, PlanTestCase}; +use crate::api::tests::work_helpers::execute; use crate::authorization::main::bootstrap_user::bootstrap_user; use crate::credentials::models::{AuditContext, BuiltinRole}; use crate::credentials::tests::location_helpers::create_location; -use crate::platform::main::claim_location_work::claim_location_work; -use crate::platform::main::complete_location_work::complete_location_work; -use crate::platform::main::read_person_plan::read_person_plan; -use crate::platform::main::request_location_work::request_location_work; +use crate::platform::_helpers::maintenance::retention::retain; +use crate::platform::main::person_reads::read_person_plan::read_person_plan; +use crate::platform::main::person_reads::read_person_plan_diff::read_person_plan_diff; +use crate::platform::main::person_reads::read_person_plan_rows::read_person_plan_rows; +use crate::platform::main::person_reads::read_person_plan_summary::read_person_plan_summary; +use crate::platform::main::work::claim_location_work::claim_location_work; +use crate::platform::main::work::complete_location_work::complete_location_work; +use crate::platform::main::work::request_location_work::request_location_work; use crate::platform::models::{ - LocationWork, LocationWorkOutcome, LocationWorkRequest, LocationWorkResult, PlanRequest, + LocationWork, LocationWorkOutcome, LocationWorkRequest, LocationWorkResult, PlanModelDiff, + PlanRequest, PlanRowSection, PlanRowsQuery, PlanTimingStep, PlanTimings, }; use crate::runs::tests::helpers::snapshot; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; @@ -310,7 +319,10 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo let complete = |document: serde_json::Value| LocationWorkResult { claim_id: claimed.claim_id, session_id: poll.session_id, - outcome: LocationWorkOutcome::Plan { document }, + outcome: LocationWorkOutcome::Plan { + document, + diffs: BTreeMap::new(), + }, }; for invalid in [ json!(["models"]), @@ -326,9 +338,11 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo ); } let document = json!({ - "models": [{"name": "orders", "action": "create_table", "reason": "first_run", "query_diff": ["-SELECT 1", "+SELECT 2"], "config_diff": ["-a", "+b"]}], + "models": [{"name": "orders", "action": "create_table", "reason": "first_run", "query_diff": ["-SELECT 1", "+SELECT 2"], "config_diff": ["-a", "+b"]}, + {"name": "customers", "action": "create_view", "reason": "no_change", "config_diff": ["-a", "+b"]}], "warnings": [{"severity": "warning", "message": "connect failed: db_password=orders-pass"}], "python_nodes": [{"name": "orders_export", "identity_diff": {"source_diff": ["-a", "+b"]}}], + "timings": {"total_seconds": 1.5, "steps": [{"name": "Reading warehouse state", "seconds": 1.0}, {"name": "Other", "seconds": 0.5}], "planned_from": "synced_state"}, }); assert!( complete_location_work( @@ -357,7 +371,7 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo ) }; let record = read(viewer).await.expect("read").expect("own plan"); - let Some(LocationWorkOutcome::Plan { document }) = + let Some(LocationWorkOutcome::Plan { document, .. }) = record.result_json.map(|result| result.outcome) else { panic!( @@ -384,8 +398,97 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo "{description}: viewers lack view.sql", description = test_case.description ); + let plan = (location, "development", first.request_id); + let summary = read_person_plan_summary(store, (&context, viewer), plan) + .await + .expect("summary") + .expect("own plan"); + assert_eq!( + summary.timings, + Some(PlanTimings { + total_seconds: 1.5, + steps: vec![ + PlanTimingStep { + name: "Reading warehouse state".to_owned(), + seconds: 1.0, + }, + PlanTimingStep { + name: "Other".to_owned(), + seconds: 0.5, + }, + ], + planned_from: "synced_state".to_owned(), + }), + "{description}: typed plan timings", + description = test_case.description + ); + let Some(LocationWorkOutcome::Plan { + document: summary, .. + }) = summary.outcome + else { + panic!( + "{description}: plan summary", + description = test_case.description + ) + }; + let rows = |reader: Uuid, offset: usize| { + read_person_plan_rows( + store, + (&context, reader), + plan, + PlanRowsQuery { + section: PlanRowSection::RoutineModels, + offset: Some(offset), + limit: Some(1), + name: None, + }, + ) + }; + let page = rows(viewer, 0).await.expect("rows").expect("planned"); + assert_eq!( + ( + summary["models"].as_array().map(Vec::len), + summary["models"][0].get("query_diff").is_none(), + summary["built_model_names"].clone(), + summary["pages"].clone(), + page.total, + page.rows.first().map(|row| row["name"].clone()), + page.rows + .first() + .map(|row| row.get("config_diff").is_none()), + rows(viewer, 1) + .await + .expect("rows") + .map(|page| page.rows.len()), + ), + ( + Some(0), + true, + serde_json::Value::Null, + json!({"first_run": 1, "routine_models": 1, "python_nodes": 1, "warnings": 1}), + 1, + Some(json!("customers")), + Some(true), + Some(0), + ), + "{description}: the summary counts each section and carries no rows; pages leave diffs out", + description = test_case.description + ); + assert!( + read_person_plan_diff(store, (&context, viewer), (plan, "orders")) + .await + .expect("diff") + .is_none(), + "{description}: no diff without view.sql", + description = test_case.description + ); assert!( - read(developer).await.expect("read").is_none(), + read(developer).await.expect("read").is_none() + && read_person_plan_summary(store, (&context, developer), plan) + .await + .expect("summary") + .is_none() + && rows(developer, 0).await.expect("rows").is_none(), "{description}: only the requester reads a plan", description = test_case.description ); @@ -398,3 +501,205 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo }); } } + +#[test] +fn given_a_plan_whose_document_left_diffs_out_when_read_then_pages_count_them_and_each_diff_is_served_on_demand() + { + let test_cases = [ + PlanDiffTestCase { + description: "SQLite", + backend: Backend::Sqlite, + expected_detached_diff: &["-select id", "+select id"], + }, + PlanDiffTestCase { + description: "PostgreSQL", + backend: Backend::Postgres, + expected_detached_diff: &["-select id", "+select id"], + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let admin = bootstrap_user(store, &context, "orders-admin") + .await + .expect("bootstrap"); + let location = create_location(store, "orders", "test-owner") + .await + .expect("location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publish"); + let poll = register_instance(store, location).await; + let request = plan_request(&["build", "orders", "customers"]); + ask(store, (&context, admin), location, &request) + .await + .expect("plan"); + let claimed = claim_location_work(store, location, &poll) + .await + .expect("claim") + .expect("plan work"); + let models: Vec = (0..450) + .map(|index| { + json!({"name": format!("orders_{index:03}"), "action": "create_table", + "reason": "query_changed", "query_diff": ["-select id", "+select id"]}) + }) + .chain([ + json!({"name": "customers", "action": "create_view", "reason": "config_changed", + "config_diff": ["-a", "+b"]}), + ]) + .collect(); + let (detached_plan, detached) = + detach_model_diffs(json!({"models": models[..450].to_vec(), "warnings": []})); + let document = json!({ + "models": detached_plan["models"] + .as_array() + .into_iter() + .flatten() + .chain([&models[450]]) + .collect::>(), + "warnings": [], + }); + let diffs: BTreeMap = detached + .into_iter() + .map(|(model, diff)| { + ( + model, + PlanModelDiff { + query_diff: diff.query_diff, + config_diff: diff.config_diff, + }, + ) + }) + .collect(); + let result = LocationWorkResult { + claim_id: claimed.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::Plan { document, diffs }, + }; + assert!( + complete_location_work(store, (location, request.request_id), &result) + .await + .expect("complete"), + "{}: the plan and its detached diffs are stored", + test_case.description + ); + assert!( + !complete_location_work(store, (location, request.request_id), &result) + .await + .expect("retry"), + "{}: a retried completion stores nothing twice", + test_case.description + ); + let plan = (location, "development", request.request_id); + let reader = (&context, admin); + let page = |section: PlanRowSection, name: Option<&str>| { + read_person_plan_rows( + store, + reader, + plan, + PlanRowsQuery { + section, + offset: Some(0), + limit: Some(50), + name: name.map(str::to_owned), + }, + ) + }; + + let changed = page(PlanRowSection::QueryChanged, None) + .await + .expect("rows") + .expect("planned"); + let found = page(PlanRowSection::QueryChanged, Some("ORDERS_44")) + .await + .expect("rows") + .expect("planned"); + let detached_diff = read_person_plan_diff(store, reader, (plan, "orders_123")) + .await + .expect("diff"); + let inline_diff = read_person_plan_diff(store, reader, (plan, "customers")) + .await + .expect("diff"); + let missing = read_person_plan_diff(store, reader, (plan, "nowhere")) + .await + .expect("diff"); + let stored = read_person_plan(store, reader, plan) + .await + .expect("plan") + .expect("own plan"); + + assert_eq!( + ( + ( + changed.total, + changed.rows.len(), + changed.rows[0]["diff_stats"].clone() + ), + (found.total, found.section_total), + detached_diff.map(|read| read.diff.query_diff), + inline_diff.map(|read| read.diff.config_diff), + missing, + stored.result_json.map(|result| matches!( + result.outcome, + LocationWorkOutcome::Plan { diffs, .. } if diffs.is_empty() + )), + ), + ( + ( + 450, + 50, + json!({"query": {"added": 1, "removed": 1, "whitespace_only": true}}) + ), + (10, 450), + Some(Some( + test_case + .expected_detached_diff + .iter() + .map(|line| (*line).to_owned()) + .collect() + )), + Some(Some(vec!["-a".to_owned(), "+b".to_owned()])), + None, + Some(true), + ), + "{}: pages count the left-out diffs, each diff reads from the document or apart from it, \ + and the stored plan never carries them", + test_case.description + ); + execute( + store, + "UPDATE sqb_location_work SET expires_at='2000-01-01T00:00:00Z'", + ) + .await; + retain(store, Default::default(), &[]) + .await + .expect("retention pass"); + assert_eq!( + ( + read_person_plan(store, reader, plan) + .await + .expect("plan") + .is_none(), + read_person_plan_diff(store, reader, (plan, "orders_123")) + .await + .expect("diff") + .is_none(), + ), + (true, true), + "{}: an expired plan goes with the diffs it stored apart", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs index fb05efb12f..f5e9a3a37c 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs @@ -23,13 +23,13 @@ use crate::credentials::models::{ }; use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::list_triggers::list_triggers; -use crate::platform::main::read_location_work::read_location_work; +use crate::platform::main::work::read_location_work::read_location_work; use crate::platform::models::LocationWork; use crate::runs::tests::helpers::{request as run_request, request_run}; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; use crate::sensors::tests::helpers::snapshot; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::snapshots::models::SnapshotNode; use crate::store::main::{begin::begin, commit::commit, migrate_store::migrate_store}; use crate::store::models::TransactionMode; @@ -93,8 +93,8 @@ fn given_browser_session_roles_when_work_is_requested_then_scope_revocation_and_ let mut input = snapshot(location); input.documents.get_mut("triggers").expect("triggers")["health_orders"]["when"] = json!({"kind":"custom", "function":"orders_ready", "code_hash":"orders_v1"}); - input.documents.get_mut("jobs").expect("jobs")["orders"]["concurrency_groups"] = - json!(["warehouse_heavy"]); + input.documents.get_mut("jobs").expect("jobs")["orders"]["pools"] = + json!({"warehouse_heavy": {"amount": 2, "limit": 3}}); input.documents.insert( "groups".into(), json!({"nightly": {"description": "Nightly orders", "selection": ["orders"], "jobs": {"orders": []}}}), @@ -418,10 +418,10 @@ fn given_browser_session_roles_when_work_is_requested_then_scope_revocation_and_ jobs.as_array().into_iter().flatten().cloned().collect(); let expected_jobs = [ json!({"job_key": "nightly", "kind": "group", "description": "Nightly orders", - "selection": ["orders"], "concurrency_groups": [], "snapshot_id": published.snapshot_id, + "selection": ["orders"], "pools": [], "concurrency": null, "resolved_concurrency": null, "snapshot_id": published.snapshot_id, "select": null, "declared_at": null, "owner": null, "overlap": "queue", "on_failure": null}), json!({"job_key": "orders", "kind": "job", "description": "Refresh orders", - "selection": ["orders"], "concurrency_groups": ["warehouse_heavy"], "snapshot_id": published.snapshot_id, + "selection": ["orders"], "pools": [{"pool": "warehouse_heavy", "amount": 2, "limit": 3, "cap": null, "server_limit": 3, "clamped": false, "shared_with": []}], "concurrency": null, "resolved_concurrency": null, "snapshot_id": published.snapshot_id, "select": null, "declared_at": null, "owner": null, "overlap": "queue", "on_failure": null}), ]; assert_eq!( diff --git a/crates/sqlbuild-orchestration/tests/api/test_providers.rs b/crates/sqlbuild-orchestration/tests/api/test_providers.rs index 72a747f786..21de021109 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_providers.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_providers.rs @@ -10,7 +10,7 @@ use crate::credentials::models::{AuditContext, DefaultRole}; use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::read_person_providers::read_person_providers; use crate::runs::tests::helpers::snapshot; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; @@ -123,6 +123,10 @@ fn given_published_providers_when_people_read_them_then_jobs_using_their_nodes_a ], "nodes": ["check:stock_levels", "model:orders"], "jobs": test_case.expected_jobs, + "job_nodes": { + "order_exports": ["model:orders"], + "stock_alerts": ["check:stock_levels"], + }, }]), "{}: jobs selecting a node that runs the provider, including a model through its hook, and none the document names", test_case.description diff --git a/crates/sqlbuild-orchestration/tests/api/test_publish_encoding.rs b/crates/sqlbuild-orchestration/tests/api/test_publish_encoding.rs index 9b2aefbcfc..643c901844 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_publish_encoding.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_publish_encoding.rs @@ -19,7 +19,7 @@ use crate::credentials::main::tokens::issue_token::issue_token; use crate::credentials::models::{AuditContext, CredentialScope}; use crate::credentials::tests::location_helpers::create_location; use crate::runs::tests::helpers::snapshot; -use crate::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; +use crate::snapshots::main::publication::publish_remote_snapshot::publish_remote_snapshot; use crate::snapshots::models::{SnapshotInput, SnapshotRecord}; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::store_helpers::{open_test_store, run}; diff --git a/crates/sqlbuild-orchestration/tests/api/test_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_reads.rs index d23c33b8ed..6de532a2a5 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_reads.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_reads.rs @@ -5,7 +5,7 @@ use crate::authorization::main::bootstrap_user::bootstrap_user; use crate::credentials::main::identities::create_principal::create_principal; use crate::declarations::main::list_person_triggers::list_person_triggers; use crate::scheduler::tests::contracts::ticks::helpers::{calendar_text, calendar_time}; -use crate::snapshots::main::list_person_snapshots::list_person_snapshots; +use crate::snapshots::main::reads::list_person_snapshots::list_person_snapshots; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; use crate::store::models::{Parameter, TransactionMode}; @@ -24,12 +24,12 @@ use crate::credentials::models::{ use crate::declarations::main::change_trigger::change_trigger; use crate::declarations::main::list_triggers::list_triggers; use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; -use crate::runs::main::change_run::change_run; -use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::execution::change_run::change_run; +use crate::runs::main::execution::claim_next_run::claim_next_run; use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunOutcome}; use crate::runs::tests::helpers::request_run; use crate::scheduler::tests::contracts::helpers::{acquire_lease, evaluate_trigger}; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::helpers::publish_recall; use crate::worker::main::connect_worker::connect_worker; @@ -382,6 +382,12 @@ fn given_published_locations_and_scheduled_work_when_read_through_http_then_reso json!({"order_support": {"description": "Answer order questions", "grants": [{"nodes": ["tag:orders"], "permissions": ["view"], "resolved_nodes": {"tag:orders": ["orders"]}}]}}), ); + let connection = json!({"adapter": "postgres", "account": null, "host": "db.example.com", + "user": "orders", "role": null, "warehouse": null, "database": "orders", + "auth_method": "password"}); + extra + .documents + .insert("connection".into(), connection.clone()); publish_snapshot(store, &extra) .await .expect("second publication"); @@ -405,6 +411,11 @@ fn given_published_locations_and_scheduled_work_when_read_through_http_then_reso {"change_kind": "added", "subject_key": "role:order_support"}]), "only the subjects that changed; a role assigner sees the role's name" ); + assert_eq!( + [json!(history[0].connection), json!(history[1].connection)], + [connection.clone(), json!(null)], + "each publication carries the connection settings it was published with" + ); let viewer = create_principal(store, &audit, "orders-viewer") .await .expect("viewer"); @@ -437,6 +448,21 @@ fn given_published_locations_and_scheduled_work_when_read_through_http_then_reso {"change_kind": "added", "subject_key": "role:"}]), "a viewer sees that a role changed, not which" ); + assert_eq!( + json!( + list_person_snapshots( + store, + (&audit, viewer), + ((location_id, "development"), 1), + DefaultRole::None, + ) + .await + .expect("viewer snapshots")[0] + .connection + ), + connection, + "everyone who can view the location sees its connection settings" + ); let run = (location_id, "development", claim.request.run_id); let execution = format!("worker/{}/execution", claim.claim_id); let run_events = read_person_run_events( diff --git a/crates/sqlbuild-orchestration/tests/api/test_run_ids.rs b/crates/sqlbuild-orchestration/tests/api/test_run_ids.rs index 87aa51f573..318be89a3e 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_run_ids.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_run_ids.rs @@ -8,10 +8,10 @@ use crate::api::tests::denials_helpers::{job, person_send}; use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; use crate::api::tests::test_types::RunIdRefusalTestCase; use crate::credentials::tests::role_helpers::role_fixture; -use crate::runs::main::request_job_run::request_job_run; +use crate::runs::main::requests::request_job_run::request_job_run; use crate::runs::models::JobRunRequest; use crate::runs::tests::helpers::snapshot; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/api/test_state_history_retention.rs b/crates/sqlbuild-orchestration/tests/api/test_state_history_retention.rs new file mode 100644 index 0000000000..bef6081273 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_state_history_retention.rs @@ -0,0 +1,176 @@ +//! The control store keeps a bounded number of builds per node and results per check. + +use crate::api::tests::state_history_helpers::{ + audit_results, counts, fingerprints, large_fingerprints, newest, pages, set_auto_vacuum, +}; +use crate::api::tests::test_types::{SqliteReclaimTestCase, StateHistoryRetentionTestCase}; +use crate::api::tests::warehouse_history_helpers::{batch, seed_target}; +use crate::platform::_helpers::maintenance::retention::reclaim_sqlite_space; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::warehouse_metadata::main::ingest::prune_state_history::prune_state_history; +use crate::warehouse_metadata::main::ingest::{ + ingest_audit_results::ingest_audit_results, ingest_fingerprints::ingest_fingerprints, +}; +use crate::warehouse_metadata::models::StateHistoryRetention; + +#[test] +fn given_many_builds_and_check_results_when_pruning_then_each_node_and_check_keeps_its_newest() { + let test_cases = [ + StateHistoryRetentionTestCase { + description: "SQLite keeps five by default", + backend: Backend::Sqlite, + configured: None, + expected_versions: 5, + }, + StateHistoryRetentionTestCase { + description: "PostgreSQL keeps twenty by default", + backend: Backend::Postgres, + configured: None, + expected_versions: 20, + }, + StateHistoryRetentionTestCase { + description: "a configured limit applies on SQLite", + backend: Backend::Sqlite, + configured: Some(2), + expected_versions: 2, + }, + StateHistoryRetentionTestCase { + description: "a configured limit applies on PostgreSQL", + backend: Backend::Postgres, + configured: Some(1), + expected_versions: 1, + }, + ]; + for test_case in test_cases { + run(async { + let fixture_store = open_test_store(test_case.backend).await.expect("store"); + let store = &fixture_store.store; + let (location, session_id) = seed_target(store).await; + let scope = (location, "development"); + let retention = test_case + .configured + .map_or_else(StateHistoryRetention::default, |versions| { + StateHistoryRetention::versions(versions).expect("valid limit") + }); + assert_eq!( + retention.kept_versions(store), + test_case.expected_versions, + "{}", + test_case.description + ); + let builds = 25; + for node in ["orders", "customers"] { + for records in fingerprints(node, builds).chunks(50) { + ingest_fingerprints( + store, + scope, + &batch(session_id, "fingerprints", records.to_vec()), + ) + .await + .expect("fingerprint ingestion"); + } + } + ingest_fingerprints( + store, + scope, + &batch(session_id, "fingerprints", fingerprints("inventory", 1)), + ) + .await + .expect("single build"); + for binding in ["orders.valid_orders", "customers.unique_email"] { + ingest_audit_results( + store, + scope, + &batch(session_id, "audit_results", audit_results(binding, builds)), + ) + .await + .expect("check result ingestion"); + } + prune_state_history(store, retention).await.expect("prune"); + let kept = usize::from(test_case.expected_versions); + assert_eq!( + counts(store, "SELECT node_name AS name, COUNT(*) AS kept, MAX(observed_at) AS newest FROM sqb_warehouse_fingerprints GROUP BY node_name ORDER BY node_name").await, + vec![ + ("customers".to_owned(), kept, newest(builds)), + ("inventory".to_owned(), 1, newest(1)), + ("orders".to_owned(), kept, newest(builds)), + ], + "{}: builds kept per node, the newest always among them", + test_case.description + ); + assert_eq!( + counts(store, "SELECT binding_key AS name, COUNT(*) AS kept, MAX(observed_at) AS newest FROM sqb_warehouse_audit_results GROUP BY binding_key ORDER BY binding_key").await, + vec![ + ("customers.unique_email".to_owned(), kept, newest(builds)), + ("orders.valid_orders".to_owned(), kept, newest(builds)), + ], + "{}: results kept per check", + test_case.description + ); + assert_eq!( + prune_state_history(store, retention).await.expect("repeat"), + (0, false), + "{}: a repeated pass finds nothing", + test_case.description + ); + close_test_store(fixture_store).await; + }); + } +} + +#[test] +fn given_a_sqlite_store_when_pruning_frees_most_of_it_then_the_file_shrinks() { + let test_cases = [ + SqliteReclaimTestCase { + description: "a store in incremental mode frees pages step by step", + auto_vacuum: "INCREMENTAL", + expected_mode: 2, + }, + SqliteReclaimTestCase { + description: "a store without incremental mode is vacuumed into it once", + auto_vacuum: "NONE", + expected_mode: 2, + }, + ]; + for test_case in test_cases { + run(async { + let fixture_store = open_test_store(Backend::Sqlite).await.expect("store"); + let store = &fixture_store.store; + let (location, session_id) = seed_target(store).await; + set_auto_vacuum(store, test_case.auto_vacuum); + for records in large_fingerprints("orders", 48).chunks(10) { + ingest_fingerprints( + store, + (location, "development"), + &batch(session_id, "fingerprints", records.to_vec()), + ) + .await + .expect("large builds"); + } + let before = pages(store).await; + prune_state_history(store, StateHistoryRetention::versions(1).expect("one")) + .await + .expect("prune"); + assert!( + reclaim_sqlite_space(store).await.expect("reclaim"), + "{}: one pass reclaims space", + test_case.description + ); + let after = pages(store).await; + assert!( + after.0 * 2 < before.0, + "{}: the file shrinks from {} to {} pages", + test_case.description, + before.0, + after.0 + ); + assert_eq!( + after.2, test_case.expected_mode, + "{}: the store reclaims space incrementally from now on", + test_case.description + ); + close_test_store(fixture_store).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_statistics_recovery.rs b/crates/sqlbuild-orchestration/tests/api/test_statistics_recovery.rs index a39529267c..64dd637f2c 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_statistics_recovery.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_statistics_recovery.rs @@ -7,7 +7,7 @@ use serde_json::Value; use crate::api::tests::changes_helpers::reset_proxy; use crate::api::tests::helpers::{start_api, stop_api}; use crate::api::tests::test_types::WorkerRedirectTestCase; -use crate::platform::main::run_maintenance::run_maintenance; +use crate::platform::main::installation::run_maintenance::run_maintenance; use crate::store::main::migrate_store::migrate_store; use crate::store::models::ControlStore; use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; diff --git a/crates/sqlbuild-orchestration/tests/api/test_trigger_upcoming.rs b/crates/sqlbuild-orchestration/tests/api/test_trigger_upcoming.rs index 5897709eb4..f7e49509b8 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_trigger_upcoming.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_trigger_upcoming.rs @@ -10,7 +10,7 @@ use crate::credentials::models::{AuditContext, DefaultRole}; use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::list_person_triggers::list_person_triggers; use crate::scheduler::tests::contracts::ticks::helpers::snapshot; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 21b1c025ab..a16e610540 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -33,6 +33,8 @@ pub(crate) struct DevelopmentTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, pub(crate) expected_status: crate::declarations::models::TriggerStatus, + /// The names of the first target's location and of a second target's location. + pub(crate) expected_names: [&'static str; 2], } pub(crate) struct OlderServerReportTestCase { @@ -299,3 +301,56 @@ pub(crate) struct ProvidersTestCase { pub(crate) backend: Backend, pub(crate) expected_jobs: Vec<&'static str>, } + +pub(crate) struct NodeStatesTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_listed: usize, +} + +pub(crate) struct ChangedBuildTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_before_plan: serde_json::Value, +} + +pub(crate) struct PushedStatesTestCase { + pub(crate) description: &'static str, + pub(crate) built: usize, + pub(crate) expected_pushed: Option, +} + +pub(crate) struct PoolCapHealthTestCase { + pub(crate) description: &'static str, + pub(crate) caps: &'static [(&'static str, u32)], + /// Each warning as `(job_key, pool, amount, cap, message)`; empty when no pool_capped observation is expected. + pub(crate) expected_warnings: &'static [(&'static str, &'static str, u32, u32, &'static str)], +} + +pub(crate) struct StateHistoryRetentionTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) configured: Option, + pub(crate) expected_versions: u16, +} + +/// A SQLite store in one auto-vacuum mode whose pruning frees most of its pages. +pub(crate) struct SqliteReclaimTestCase { + pub(crate) description: &'static str, + pub(crate) auto_vacuum: &'static str, + pub(crate) expected_mode: i64, +} + +pub(crate) struct PlanDiffTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + /// The diff of a model the document left out, as it reads back. + pub(crate) expected_detached_diff: &'static [&'static str], +} + +pub(crate) struct IngestionTurnTestCase { + pub(crate) description: &'static str, + pub(crate) pages: usize, + pub(crate) relations_per_page: usize, + pub(crate) expected_writer_first: bool, +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs index 649c1272ff..3aea4d97c4 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -18,11 +18,11 @@ use crate::api::tests::warehouse_history_helpers::{ use crate::credentials::main::tokens::issue_token::issue_token; use crate::credentials::models::{AuditContext, CredentialScope}; use crate::credentials::tests::location_helpers::create_location; -use crate::platform::main::connect_location_work::connect_location_work; -use crate::platform::main::report_location_health::report_location_health; +use crate::platform::main::health::report_location_health::report_location_health; +use crate::platform::main::work::connect_location_work::connect_location_work; use crate::platform::models::{LocationHealth, LocationHealthReport}; use crate::runs::tests::helpers::snapshot; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::{ begin::begin, commit::commit, migrate_store::migrate_store, query::query, }; @@ -31,6 +31,7 @@ use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::fault_types::TransactionFault; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; +use crate::warehouse_metadata::constants::MAX_HISTORY_PAGE_RECORDS; use crate::warehouse_metadata::models::{MetadataBatch, WarehouseHistoryKind}; #[test] @@ -199,7 +200,7 @@ fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicit let mut invalid_id = batch.clone(); invalid_id["records"][0]["event_id"] = json!(" "); let mut oversized = batch.clone(); - oversized["records"] = json!(vec![record.clone(); 101]); + oversized["records"] = json!(vec![record.clone(); MAX_HISTORY_PAGE_RECORDS + 1]); let mut unknown = batch.clone(); unknown["records"][0]["status"] = json!("success"); let foreign = url.replace(&location.to_string(), &other.to_string()); diff --git a/crates/sqlbuild-orchestration/tests/api/test_worker.rs b/crates/sqlbuild-orchestration/tests/api/test_worker.rs index 4c4f7f78a2..d43d66c30c 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_worker.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_worker.rs @@ -2,14 +2,14 @@ use std::collections::BTreeMap; use crate::credentials::main::tokens::issue_token::issue_token; use crate::credentials::models::{AuditContext, CredentialScope}; -use crate::runs::main::change_run::change_run; -use crate::runs::main::claim_next_run::claim_next_run; -use crate::runs::main::list_runs::list_runs; +use crate::runs::main::execution::change_run::change_run; +use crate::runs::main::execution::claim_next_run::claim_next_run; +use crate::runs::main::listings::list_runs::list_runs; use crate::runs::models::{ NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, }; use crate::scheduler::tests::contracts::helpers::{acquire_lease, hold_lease}; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::worker::errors::WorkerError; diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_history/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_history/helpers.rs index f9236e0040..67319966ff 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_history/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_history/helpers.rs @@ -20,10 +20,10 @@ use crate::credentials::models::{ AuditContext, BuiltinRole, PasswordAuthority, PasswordLogin, RoleAssignmentRequest, }; use crate::credentials::tests::location_helpers::create_location; -use crate::platform::main::report_location_health::report_location_health; +use crate::platform::main::health::report_location_health::report_location_health; use crate::platform::models::{LocationHealth, LocationHealthReport}; use crate::runs::tests::helpers::snapshot; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; use crate::store::main::{ begin::begin, commit::commit, migrate_store::migrate_store, query::query, diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs index 6925eac3df..915651c62a 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs @@ -11,6 +11,7 @@ use crate::store::main::{ use crate::store::models::{ControlStore, TransactionMode}; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::fault_types::TransactionFault; +use crate::warehouse_metadata::constants::MAX_HISTORY_PAGE_RECORDS; use crate::warehouse_metadata::models::{ WarehouseHistoryKind, WarehouseSyncResponse, WarehouseWatermarkRead, }; @@ -60,7 +61,7 @@ pub(crate) async fn event_history( ), ( "records", - json!(vec![record.clone(); 101]), + json!(vec![record.clone(); MAX_HISTORY_PAGE_RECORDS + 1]), StatusCode::BAD_REQUEST, ), ] { diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs index 103e9001b4..eeb54e9281 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs @@ -12,6 +12,7 @@ use crate::store::main::{ use crate::store::models::{ControlStore, Parameter, TransactionMode}; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::fault_types::TransactionFault; +use crate::warehouse_metadata::constants::MAX_HISTORY_PAGE_RECORDS; use crate::warehouse_metadata::models::{MetadataBatch, WarehouseWatermarkRead}; pub(crate) async fn fingerprint_history( @@ -114,7 +115,7 @@ pub(crate) async fn fingerprint_history( let mut stale = batch.clone(); stale["generation"] = json!(2); let mut oversized = batch.clone(); - oversized["records"] = json!(vec![record.clone(); 101]); + oversized["records"] = json!(vec![record.clone(); MAX_HISTORY_PAGE_RECORDS + 1]); let mut blank = batch.clone(); blank["records"][0]["event_id"] = json!(" "); let mut foreign = batch.clone(); diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs index aa112308cb..e936923cc5 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs @@ -159,6 +159,8 @@ pub(crate) async fn preview_copy_scope( session_id: preview_session, generation: 1, consumption: false, + versioned: false, + known_version: None, }; let copy = || async { outbound @@ -202,9 +204,67 @@ pub(crate) async fn preview_copy_scope( .await .expect("copy body"); assert!( - unsynced.consumption.is_empty() && older_location.get("consumption").is_none(), - "a read that does not ask for consumption never carries the field, which older locations reject" + unsynced.consumption.is_empty() + && ["consumption", "version", "unchanged"] + .iter() + .all(|field| older_location.get(field).is_none()), + "a read that asks for neither consumption nor versions never carries the fields, which older locations reject" + ); + let versioned = |known_version: Option| PreviewCopyRead { + versioned: true, + known_version, + ..read.clone() + }; + let current = outbound + .preview_copy("preview", &versioned(None)) + .await + .expect("versioned copy"); + let held = outbound + .preview_copy("preview", &versioned(current.version.clone())) + .await + .expect("copy at the held version"); + assert_eq!( + ( + current.version.is_some() && !current.unchanged, + ours(¤t) == ours(&unsynced), + held.unchanged, + held.version == current.version, + ours(&held), + ), + (true, true, true, true, (vec![], vec![], vec![], vec![])), + "a versioned read names its version; the same version again reads no records" + ); + post( + client, + (&metadata, bearer), + batch( + "fingerprints", + vec![fingerprint( + "copy-run-3", + "SELECT 3 AS id", + "2026-03-01T12:00:00Z", + )], + ), + ) + .await; + let changed = outbound + .preview_copy("preview", &versioned(current.version.clone())) + .await + .expect("copy after a mirrored write"); + assert_eq!( + ( + changed.unchanged, + changed.version != current.version, + ours(&changed).0, + ), + ( + false, + true, + vec![("copy-run-3".into(), "SELECT 3 AS id".into())] + ), + "a mirrored write moves the version, so the held copy is read again" ); + let before_sync: Option = changed.version.clone(); let with_consumption = outbound .preview_copy( "preview", @@ -265,6 +325,15 @@ pub(crate) async fn preview_copy_scope( None, "consumption never synced cleanly" ); + assert_ne!( + outbound + .preview_copy("preview", &versioned(before_sync.clone())) + .await + .expect("copy after a clean sync") + .version, + before_sync, + "a clean sync can move `synced_at`, so it moves the version" + ); for report in [ sync_report(preview_session, "consumption", None), sync_report(preview_session, "fingerprints", Some("read")), @@ -325,6 +394,8 @@ pub(crate) async fn preview_copy_scope( session_id, generation: 1, consumption: false, + versioned: false, + known_version: None, }, ) .await diff --git a/crates/sqlbuild-orchestration/tests/credentials/roles/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/roles/helpers.rs index aeb4358e1c..f7baf18137 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/roles/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/roles/helpers.rs @@ -21,11 +21,11 @@ use crate::declarations::main::change_person_trigger::change_person_trigger; use crate::declarations::main::list_person_trigger_changes::list_person_trigger_changes; use crate::declarations::main::list_triggers::list_triggers; use crate::declarations::models::{PersonTriggerChange, TriggerChange, TriggerChangeRequest}; -use crate::runs::main::list_person_runs::list_person_runs; +use crate::runs::main::listings::list_person_runs::list_person_runs; use crate::runs::tests::helpers::snapshot; use crate::runs::types::RunsListing; use crate::scheduler::tests::contracts::ticks::helpers::calendar_text; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; use crate::store::models::{ControlStore, TransactionMode}; diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_audit_retention.rs b/crates/sqlbuild-orchestration/tests/credentials/test_audit_retention.rs index db6d2c6c57..425079bba8 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_audit_retention.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_audit_retention.rs @@ -9,10 +9,10 @@ use crate::credentials::models::{AuditContext, AuditRetention}; use crate::credentials::tests::helpers::{acknowledge_audit, age_audit, audit_rows}; use crate::credentials::tests::location_helpers::create_location; use crate::credentials::tests::test_types::AuditRetentionTestCase; -use crate::runs::main::request_job_run::request_job_run; +use crate::runs::main::requests::request_job_run::request_job_run; use crate::runs::models::JobRunRequest; use crate::runs::tests::helpers::snapshot; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::{begin::begin, commit::commit, migrate_store::migrate_store}; use crate::store::models::TransactionMode; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; @@ -52,8 +52,7 @@ fn given_expired_audit_rows_when_pruned_then_undelivered_rows_survive_and_retrie input.documents.insert( "jobs".into(), json!({"daily_orders": { - "description": "Build orders", "selection": ["orders"], "settings": {}, - "concurrency_groups": [] + "description": "Build orders", "selection": ["orders"], "settings": {} }}), ); publish_snapshot(store, &input).await.expect("publish job"); diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs index 3e346698de..6b0b9169c8 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs @@ -7,9 +7,9 @@ use crate::credentials::main::tokens::issue_token::issue_token; use crate::credentials::main::tokens::revoke_token::revoke_token; use crate::credentials::main::tokens::rotate_token::rotate_token; use crate::credentials::models::{AuditContext, CredentialScope}; -use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::execution::claim_next_run::claim_next_run; use crate::scheduler::tests::contracts::helpers::acquire_lease; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use chrono::{Duration, Utc}; use uuid::Uuid; diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_custom_roles.rs b/crates/sqlbuild-orchestration/tests/credentials/test_custom_roles.rs index 48679facee..25cb02d621 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_custom_roles.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_custom_roles.rs @@ -8,7 +8,7 @@ use crate::credentials::tests::role_helpers::{ }; use crate::credentials::tests::test_types::CustomRoleTestCase; use crate::runs::tests::helpers::snapshot; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_project_roles.rs b/crates/sqlbuild-orchestration/tests/credentials/test_project_roles.rs index 3179c22693..e0e96d094e 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_project_roles.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_project_roles.rs @@ -19,7 +19,7 @@ use crate::credentials::tests::role_helpers::{ use crate::credentials::tests::test_types::{ProjectRoleTestCase, RolesFileTestCase}; use crate::declarations::models::TriggerChange; use crate::runs::tests::helpers::snapshot; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::store::main::{begin::begin, commit::commit, query::query}; diff --git a/crates/sqlbuild-orchestration/tests/declarations/helpers.rs b/crates/sqlbuild-orchestration/tests/declarations/helpers.rs index 10b270cdad..1ea5b31302 100644 --- a/crates/sqlbuild-orchestration/tests/declarations/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/declarations/helpers.rs @@ -2,7 +2,14 @@ use std::collections::{BTreeMap, BTreeSet}; +use serde_json::json; +use uuid::Uuid; + +use crate::declarations::main::group_dependencies::group_dependencies; +use crate::declarations::models::JobDefinition; use crate::declarations::tests::test_types::Pairs; +use crate::snapshots::models::{SnapshotInput, SnapshotNode}; +use crate::store::errors::ControlError; pub(crate) fn map(pairs: Pairs) -> BTreeMap> { pairs @@ -45,3 +52,59 @@ pub(crate) fn chained_group( deps.insert("model:tail".to_owned(), BTreeSet::from([node(last)])); (selections, deps) } + +/// A snapshot whose models read the nodes `lineage` lists. +pub(crate) fn input(lineage: &[(&str, Vec<&str>)]) -> SnapshotInput { + SnapshotInput { + location_id: Uuid::nil(), + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: None, + environment_hash: "orders-environment".into(), + sqb_version: "1.0.0".into(), + nodes: lineage + .iter() + .map(|(key, deps)| { + ( + (*key).to_owned(), + SnapshotNode { + kind: "model".into(), + definition: json!({"deps": deps}), + raw_sql: None, + compiled_sql: None, + python_source: None, + }, + ) + }) + .collect(), + documents: BTreeMap::new(), + } +} + +/// Member jobs selecting the nodes `selections` lists. +pub(crate) fn jobs(selections: &[(&str, Vec<&str>)]) -> BTreeMap { + selections + .iter() + .map(|(name, selection)| { + let definition = + json!({"description": format!("Build {name}"), "selection": selection}); + ( + (*name).to_owned(), + serde_json::from_value(definition).expect("job definition"), + ) + }) + .collect() +} + +/// The group waits of members `selections` over `lineage`. +pub(crate) fn waits( + lineage: &[(&str, Vec<&str>)], + selections: &[(&str, Vec<&str>)], +) -> Result>, ControlError> { + let jobs = jobs(selections); + let members: BTreeSet = jobs.keys().cloned().collect(); + group_dependencies(("orders_job_group", &input(lineage)), &jobs, &members) +} diff --git a/crates/sqlbuild-orchestration/tests/declarations/test_contracts.rs b/crates/sqlbuild-orchestration/tests/declarations/test_contracts.rs index 89785f7cea..49d631ddd5 100644 --- a/crates/sqlbuild-orchestration/tests/declarations/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/declarations/test_contracts.rs @@ -2,7 +2,12 @@ #[path = "helpers.rs"] pub(crate) mod helpers; + #[path = "test_group_wait_graph.rs"] mod test_group_wait_graph; + #[path = "test_types.rs"] pub(crate) mod test_types; + +#[path = "test_group_dependencies.rs"] +mod test_group_dependencies; diff --git a/crates/sqlbuild-orchestration/tests/declarations/test_group_dependencies.rs b/crates/sqlbuild-orchestration/tests/declarations/test_group_dependencies.rs new file mode 100644 index 0000000000..33a20b8d44 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/declarations/test_group_dependencies.rs @@ -0,0 +1,190 @@ +//! A group member waits for the members building the first nodes it reaches upstream. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::declarations::tests::helpers::waits; +use crate::declarations::tests::test_types::{GroupCycleTestCase, GroupWaitsTestCase}; + +#[test] +fn given_member_lineage_when_deriving_group_waits_then_each_member_waits_for_the_first_owners_upstream() + { + let test_cases = [ + GroupWaitsTestCase { + description: "a chain through a member's nodes waits only for that member", + lineage: vec![ + ("model:raw_orders", vec![]), + ("model:stg_orders", vec!["model:raw_orders"]), + ("model:orders", vec!["model:stg_orders"]), + ], + jobs: vec![ + ("load_job", vec!["model:raw_orders"]), + ("staging_job", vec!["model:stg_orders"]), + ("marts_job", vec!["model:orders"]), + ], + expected_waits: vec![ + ("load_job", vec![]), + ("staging_job", vec!["load_job"]), + ("marts_job", vec!["staging_job"]), + ], + }, + GroupWaitsTestCase { + description: "a shared function behind another member's node adds none of its builders", + lineage: vec![ + ("function:normalise_sku", vec![]), + ("model:stg_inventory", vec!["function:normalise_sku"]), + ("model:stg_orders", vec!["function:normalise_sku"]), + ("model:stg_customers", vec!["function:normalise_sku"]), + ("model:inventory", vec!["model:stg_inventory"]), + ], + jobs: vec![ + ( + "inventory_staging_job", + vec!["function:normalise_sku", "model:stg_inventory"], + ), + ( + "orders_staging_job", + vec!["function:normalise_sku", "model:stg_orders"], + ), + ( + "customers_staging_job", + vec!["function:normalise_sku", "model:stg_customers"], + ), + ("inventory_job", vec!["model:inventory"]), + ], + expected_waits: vec![ + ("inventory_staging_job", vec![]), + ("orders_staging_job", vec![]), + ("customers_staging_job", vec![]), + ("inventory_job", vec!["inventory_staging_job"]), + ], + }, + GroupWaitsTestCase { + description: "a path through nodes no member builds still waits for the owner beyond", + lineage: vec![ + ("model:raw_customers", vec![]), + ("model:customer_view", vec!["model:raw_customers"]), + ("model:customer_lookup", vec!["model:customer_view"]), + ("model:customers", vec!["model:customer_lookup"]), + ], + jobs: vec![ + ("load_job", vec!["model:raw_customers"]), + ("customers_job", vec!["model:customers"]), + ], + expected_waits: vec![("load_job", vec![]), ("customers_job", vec!["load_job"])], + }, + GroupWaitsTestCase { + description: "a shared node reached first waits for every member building it", + lineage: vec![ + ("function:normalise_sku", vec![]), + ("model:inventory", vec!["function:normalise_sku"]), + ], + jobs: vec![ + ("orders_staging_job", vec!["function:normalise_sku"]), + ("inventory_staging_job", vec!["function:normalise_sku"]), + ("inventory_job", vec!["model:inventory"]), + ], + expected_waits: vec![ + ("orders_staging_job", vec![]), + ("inventory_staging_job", vec![]), + ( + "inventory_job", + vec!["inventory_staging_job", "orders_staging_job"], + ), + ], + }, + GroupWaitsTestCase { + description: "a member keeps walking past a shared node it builds itself", + lineage: vec![ + ("model:raw_orders", vec![]), + ("function:normalise_sku", vec!["model:raw_orders"]), + ("model:stg_inventory", vec!["function:normalise_sku"]), + ], + jobs: vec![ + ("load_job", vec!["model:raw_orders"]), + ("orders_staging_job", vec!["function:normalise_sku"]), + ( + "inventory_staging_job", + vec!["function:normalise_sku", "model:stg_inventory"], + ), + ], + expected_waits: vec![ + ("load_job", vec![]), + ("orders_staging_job", vec!["load_job"]), + ("inventory_staging_job", vec!["load_job"]), + ], + }, + GroupWaitsTestCase { + description: "a member reading another's node past a widely shared function waits once", + lineage: vec![ + ("function:normalise_sku", vec![]), + ("model:orders_a", vec!["function:normalise_sku"]), + ("model:orders_b", vec!["function:normalise_sku"]), + ("model:orders_c", vec!["function:normalise_sku"]), + ("model:stg_orders", vec!["model:orders_a"]), + ("model:orders", vec!["model:stg_orders"]), + ], + jobs: vec![ + ("a_job", vec!["function:normalise_sku", "model:orders_a"]), + ("b_job", vec!["function:normalise_sku", "model:orders_b"]), + ("c_job", vec!["function:normalise_sku", "model:orders_c"]), + ("staging_job", vec!["model:stg_orders"]), + ("marts_job", vec!["model:orders"]), + ("functions_job", vec!["function:normalise_sku"]), + ], + expected_waits: vec![ + ("a_job", vec![]), + ("b_job", vec![]), + ("c_job", vec![]), + ("functions_job", vec![]), + ("staging_job", vec!["a_job"]), + ("marts_job", vec!["staging_job"]), + ], + }, + ]; + for test_case in test_cases { + let actual = waits(&test_case.lineage, &test_case.jobs).expect(test_case.description); + assert_eq!( + actual, + test_case + .expected_waits + .iter() + .map(|(name, deps)| { + ( + (*name).to_owned(), + deps.iter().map(|dep| (*dep).to_owned()).collect(), + ) + }) + .collect::>>(), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_members_reading_each_other_when_deriving_group_waits_then_the_cycle_names_both_waits() { + let test_cases = [GroupCycleTestCase { + description: "two members reading each other's nodes", + lineage: vec![ + ("model:stg_orders", vec!["model:customers"]), + ("model:customers", vec!["model:stg_orders"]), + ], + jobs: vec![ + ("orders_job", vec!["model:stg_orders"]), + ("customers_job", vec!["model:customers"]), + ], + expected_error: "invalid control-plane data: group dependencies contain a cycle in job group \ + orders_job_group: customers_job waits for orders_job (model:customers reads \ + model:stg_orders); orders_job waits for customers_job (model:stg_orders reads \ + model:customers)", + }]; + for test_case in test_cases { + let error = waits(&test_case.lineage, &test_case.jobs).expect_err("a cycle is rejected"); + assert_eq!( + error.to_string(), + test_case.expected_error, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/declarations/test_group_wait_graph.rs b/crates/sqlbuild-orchestration/tests/declarations/test_group_wait_graph.rs index f686a87a53..e686eb189f 100644 --- a/crates/sqlbuild-orchestration/tests/declarations/test_group_wait_graph.rs +++ b/crates/sqlbuild-orchestration/tests/declarations/test_group_wait_graph.rs @@ -41,7 +41,7 @@ fn given_group_members_and_node_lineage_when_deriving_waits_then_every_cycle_is_ ], expected_edges: &[ ("customers_job", &["orders_job"]), - ("finance_job", &["customers_job", "orders_job"]), + ("finance_job", &["orders_job"]), ("inventory_job", &["shipments_job"]), ("orders_job", &["customers_job"]), ("shipments_job", &["inventory_job"]), @@ -109,7 +109,7 @@ fn given_a_large_group_on_a_long_chain_when_deriving_waits_then_one_cycle_is_rep members: 50, nodes_per_member: 200, expected_cycles: 1, - expected_waits: 49 + (1..50).sum::(), + expected_waits: 50, expected_limit: std::time::Duration::from_secs(2), }]; for test_case in test_cases { diff --git a/crates/sqlbuild-orchestration/tests/declarations/test_types.rs b/crates/sqlbuild-orchestration/tests/declarations/test_types.rs index d0063a71ab..d550229d5a 100644 --- a/crates/sqlbuild-orchestration/tests/declarations/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/declarations/test_types.rs @@ -21,3 +21,21 @@ pub(crate) struct LargeGroupTestCase { pub(crate) expected_waits: usize, pub(crate) expected_limit: std::time::Duration, } + +pub(crate) struct GroupWaitsTestCase { + pub(crate) description: &'static str, + /// Each node and the nodes it reads. + pub(crate) lineage: Vec<(&'static str, Vec<&'static str>)>, + /// Each member job and the nodes it selects. + pub(crate) jobs: Vec<(&'static str, Vec<&'static str>)>, + /// Each member and the members it waits for. + pub(crate) expected_waits: Vec<(&'static str, Vec<&'static str>)>, +} + +/// Members whose nodes read each other, and the cycle error that names both waits. +pub(crate) struct GroupCycleTestCase { + pub(crate) description: &'static str, + pub(crate) lineage: Vec<(&'static str, Vec<&'static str>)>, + pub(crate) jobs: Vec<(&'static str, Vec<&'static str>)>, + pub(crate) expected_error: &'static str, +} diff --git a/crates/sqlbuild-orchestration/tests/platform/helpers.rs b/crates/sqlbuild-orchestration/tests/platform/helpers.rs index d658c71fcc..21000b3d54 100644 --- a/crates/sqlbuild-orchestration/tests/platform/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/platform/helpers.rs @@ -8,8 +8,8 @@ use axum::http::{HeaderMap, StatusCode}; use axum::response::IntoResponse; use crate::platform::models::{PlatformEvent, PlatformEventEnvelope}; -use crate::runs::main::claim_next_run::claim_next_run; -use crate::runs::main::publish_run_recall::publish_run_recall; +use crate::runs::main::execution::claim_next_run::claim_next_run; +use crate::runs::main::execution::publish_run_recall::publish_run_recall; use crate::runs::models::{RecallBatch, RecallRecord}; use crate::snapshots::models::{SnapshotInput, SnapshotNode}; use crate::store::errors::ControlError; diff --git a/crates/sqlbuild-orchestration/tests/platform/test_compile_failure.rs b/crates/sqlbuild-orchestration/tests/platform/test_compile_failure.rs new file mode 100644 index 0000000000..6b32d8ba2b --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/test_compile_failure.rs @@ -0,0 +1,99 @@ +use uuid::Uuid; + +use crate::credentials::main::locations::create_location::create_location; +use crate::credentials::models::AuditContext; +use crate::platform::main::health::report_location_health::report_location_health; +use crate::platform::models::{ + LocationHealth, LocationHealthReport, LocationRuleFailure, LocationRuleFinding, +}; +use crate::platform::tests::test_types::CompileFailureTestCase; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_rule_findings_when_a_location_reports_a_failed_load_then_only_bounded_lists_are_accepted() +{ + let test_cases = [ + CompileFailureTestCase { + description: "one finding", + finding_count: 1, + listed: 1, + message_chars: 60, + expected_accepted: true, + }, + CompileFailureTestCase { + description: "the first 50 of many findings", + finding_count: 222, + listed: 50, + message_chars: 500, + expected_accepted: true, + }, + CompileFailureTestCase { + description: "more than 50 listed findings", + finding_count: 51, + listed: 51, + message_chars: 60, + expected_accepted: false, + }, + CompileFailureTestCase { + description: "a count below the listed findings", + finding_count: 1, + listed: 2, + message_chars: 60, + expected_accepted: false, + }, + CompileFailureTestCase { + description: "an over-long message", + finding_count: 1, + listed: 1, + message_chars: 501, + expected_accepted: false, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(Backend::Sqlite).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let location_id = create_location(store, "orders", &context) + .await + .expect("location"); + let finding = LocationRuleFinding { + code: "SQBRMODEL102".into(), + message: "m".repeat(test_case.message_chars), + path: Some("models/orders.sql".into()), + line: Some(2), + }; + let report = LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: Uuid::now_v7(), + target: "development".into(), + health: LocationHealth::CompileFailed { + generation: 1, + rules: Some(LocationRuleFailure { + finding_count: test_case.finding_count, + findings: vec![finding; test_case.listed], + }), + }, + }; + assert_eq!( + report_location_health(store, location_id, &report) + .await + .is_ok(), + test_case.expected_accepted, + "{}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs b/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs index 894a2240a1..5c72e85e00 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs @@ -2,6 +2,8 @@ #[path = "helpers.rs"] mod helpers; +#[path = "test_compile_failure.rs"] +mod test_compile_failure; #[path = "test_installation.rs"] mod test_installation; #[path = "test_problems.rs"] diff --git a/crates/sqlbuild-orchestration/tests/platform/test_installation.rs b/crates/sqlbuild-orchestration/tests/platform/test_installation.rs index 63ac1fed38..dec6d6cc89 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_installation.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_installation.rs @@ -1,7 +1,7 @@ use crate::authorization::main::bootstrap_user::bootstrap_user; use crate::credentials::main::identities::create_principal::create_principal; use crate::credentials::models::AuditContext; -use crate::platform::main::read_installation::read_installation; +use crate::platform::main::installation::read_installation::read_installation; use crate::platform::models::StorageKind; use crate::platform::tests::test_types::InstallationTestCase; use crate::store::errors::ControlError; diff --git a/crates/sqlbuild-orchestration/tests/platform/test_problems.rs b/crates/sqlbuild-orchestration/tests/platform/test_problems.rs index 25afe12c83..7fdf46f88a 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_problems.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_problems.rs @@ -3,16 +3,16 @@ use crate::authorization::main::{ }; use crate::credentials::main::identities::create_principal::create_principal; use crate::credentials::models::{AuditContext, BuiltinRole, DefaultRole, RoleAssignmentRequest}; -use crate::platform::_helpers::observations::observe; -use crate::platform::main::list_person_platform_events::list_person_platform_events; -use crate::platform::main::read_person_problems::read_person_problems; -use crate::platform::main::report_location_health::report_location_health; +use crate::platform::_helpers::health::observations::observe; +use crate::platform::main::health::report_location_health::report_location_health; +use crate::platform::main::person_reads::list_person_platform_events::list_person_platform_events; +use crate::platform::main::person_reads::read_person_problems::read_person_problems; use crate::platform::models::{ LocationHealth, LocationHealthReport, PersonPlatformEvent, PersonPlatformEventPage, PlatformEvent, PlatformEventKind, }; use crate::scheduler::tests::contracts::helpers::acquire_lease; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::begin::begin; use crate::store::main::commit::commit; use crate::store::main::migrate_store::migrate_store; diff --git a/crates/sqlbuild-orchestration/tests/platform/test_types.rs b/crates/sqlbuild-orchestration/tests/platform/test_types.rs index 9b0dcf7bb7..850c358382 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_types.rs @@ -64,3 +64,11 @@ pub(crate) struct InstallationTestCase { pub(crate) backend: Backend, pub(crate) expected_storage: StorageKind, } + +pub(crate) struct CompileFailureTestCase { + pub(crate) description: &'static str, + pub(crate) finding_count: u64, + pub(crate) listed: usize, + pub(crate) message_chars: usize, + pub(crate) expected_accepted: bool, +} diff --git a/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs index 2c86bf4d88..ab20eec76c 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs @@ -1,13 +1,13 @@ use std::collections::BTreeSet; use std::time::Duration; -use crate::platform::_helpers::retention::retain; -use crate::platform::_helpers::watchdog::scan_health; -use crate::platform::main::report_location_health::report_location_health; -use crate::platform::main::run_maintenance::run_maintenance; +use crate::platform::_helpers::maintenance::retention::retain; +use crate::platform::_helpers::maintenance::watchdog::scan_health; +use crate::platform::main::health::report_location_health::report_location_health; +use crate::platform::main::installation::run_maintenance::run_maintenance; use crate::platform::models::{LocationHealth, LocationHealthReport, PlatformEvent}; use crate::scheduler::tests::contracts::helpers::acquire_lease; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::begin::begin; use crate::store::main::commit::commit; use crate::store::main::migrate_store::migrate_store; diff --git a/crates/sqlbuild-orchestration/tests/platform/test_webhook.rs b/crates/sqlbuild-orchestration/tests/platform/test_webhook.rs index 6cb93e77ae..e57b7d7a5c 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_webhook.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_webhook.rs @@ -6,8 +6,8 @@ use axum::Router; use axum::routing::post; use uuid::Uuid; -use crate::platform::main::run_maintenance::run_maintenance; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::platform::main::installation::run_maintenance::run_maintenance; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use crate::webhooks::_helpers::delivery::deliver_webhook; use crate::webhooks::main::{connect_webhook::connect_webhook, run_webhook::run_webhook}; diff --git a/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs index a9b30517ba..ca5aed6411 100644 --- a/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs @@ -1,11 +1,14 @@ //! Real slot accounting and successful completion through the public worker lifecycle. use std::collections::BTreeMap; +use std::num::NonZeroU32; -use crate::runs::main::change_run::change_run; +use crate::runs::main::execution::change_run::change_run; +use crate::runs::main::execution::claim_next_run::claim_next_run; use crate::runs::models::{ NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome, }; +use crate::scheduler::models::SchedulerLease; use crate::store::models::ControlStore; use uuid::Uuid; @@ -21,6 +24,7 @@ pub(crate) async fn slot_counts(store: &ControlStore) -> BTreeMap { .expect("slot query") .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) .expect("SQLite slot counts") + .map(|row| row.map(by_group)) .collect::>() .expect("decode slot counts"); }; @@ -29,18 +33,54 @@ pub(crate) async fn slot_counts(store: &ControlStore) -> BTreeMap { .await .expect("PostgreSQL slot counts") .into_iter() + .map(by_group) .collect() } +/// Slot keys carry their location and target; tests count by group name. +fn by_group((key, count): (String, i64)) -> (String, i64) { + ( + key.rsplit_once('/') + .map_or(key.as_str(), |(_, group)| group) + .to_owned(), + count, + ) +} + pub(crate) async fn succeed(store: &ControlStore, claim: &RunClaim) { finish(store, claim, (NodeOutcome::Success, RunOutcome::Success)).await; } +/// Every selected node and the run succeed, the run having let `ignored` audit failures through. +pub(crate) async fn succeed_ignoring_audit_failures( + store: &ControlStore, + claim: &RunClaim, + ignored: u32, +) { + finish_reporting( + store, + claim, + (NodeOutcome::Success, RunOutcome::Success), + vec![RunChange::AuditFailuresIgnored { count: ignored }], + ) + .await; +} + /// Every selected node and the run end with the given outcomes. pub(crate) async fn finish( + store: &ControlStore, + claim: &RunClaim, + outcomes: (NodeOutcome, RunOutcome), +) { + finish_reporting(store, claim, outcomes, Vec::new()).await; +} + +/// `finish`, publishing `reported` after the nodes and before the run finishes. +async fn finish_reporting( store: &ControlStore, claim: &RunClaim, (node_outcome, run_outcome): (NodeOutcome, RunOutcome), + reported: Vec, ) { let mut changes = vec![RunChange::Started]; changes.extend( @@ -57,6 +97,7 @@ pub(crate) async fn finish( error_message: None, }), ); + changes.extend(reported); changes.push(RunChange::Finished { outcome: run_outcome, reason: "synthetic child completed".into(), @@ -80,3 +121,202 @@ pub(crate) async fn finish( .expect("worker fact"); } } + +/// Claim `count` runs with `caps`, each of which must be claimable, then refuse one more. +pub(crate) async fn claim_exactly( + store: &ControlStore, + lease: &SchedulerLease, + (caps, count): (&BTreeMap, usize), +) -> Vec { + let mut claims: Vec = Vec::new(); + for _ in 0..count { + claims.push( + claim_next_run(store, lease, caps, None) + .await + .expect("claim") + .expect("claimable within the limit"), + ); + } + assert!( + claim_next_run(store, lease, caps, None) + .await + .expect("claim") + .is_none(), + "no run past the limit is claimable" + ); + claims +} + +/// Publish the orders location's nightly group: staging, then marts and inventory reading customers. +pub(crate) async fn publish_dependency_group( + store: &ControlStore, + location: uuid::Uuid, + dependency_failure: &serde_json::Value, +) { + let mut input = crate::runs::tests::helpers::snapshot(location); + for node in ["orders", "inventory"] { + input.nodes.get_mut(node).expect("node").definition["deps"] = + serde_json::json!(["customers"]); + } + input.documents.insert( + "jobs".into(), + serde_json::json!({ + "staging_job": {"description": "Stage customers", "selection": ["customers"]}, + "marts_job": {"description": "Build orders", "selection": ["orders"]}, + "inventory_job": {"description": "Build inventory", "selection": ["inventory"]} + }), + ); + input.documents.insert("groups".into(), serde_json::json!({"nightly_job_group": { + "description": "Stage, then build orders and inventory", + "selection": ["customers", "inventory", "orders"], + "jobs": {"staging_job": [], "marts_job": ["staging_job"], "inventory_job": ["staging_job"]}, + "dependency_failure": dependency_failure + }})); + crate::snapshots::main::publication::publish_snapshot::publish_snapshot(store, &input) + .await + .expect("publish"); +} + +/// Run each dependent the failed staging job left runnable, then read the ones the group ended. +pub(crate) async fn member_reactions( + store: &ControlStore, + lease: &crate::scheduler::models::SchedulerLease, + (limits, scope): ( + &std::collections::BTreeMap, + (uuid::Uuid, &str), + ), +) -> std::collections::BTreeMap { + use crate::runs::tests::test_types::MemberReaction; + let mut reactions = std::collections::BTreeMap::new(); + for _ in 0..2 { + let Some(claim) = crate::runs::main::execution::claim_next_run::claim_next_run( + store, + lease, + limits, + Some(scope), + ) + .await + .expect("claim dependent") else { + break; + }; + let execution = crate::runs::main::execution::run_execution::run_execution( + store, + (claim.request.run_id, claim.claim_id), + ) + .await + .expect("execution") + .expect("claimed"); + let mut skipped: Vec<(&'static str, &'static str)> = execution + .skipped_nodes + .iter() + .map(|(node, cause)| { + ( + ["orders", "inventory"] + .into_iter() + .find(|known| known == node) + .expect("known node"), + ["customers"] + .into_iter() + .find(|known| known == cause) + .unwrap_or("other"), + ) + }) + .collect(); + skipped.sort_unstable(); + let outcome = [ + (NodeOutcome::Skipped, RunOutcome::Failed), + (NodeOutcome::Success, RunOutcome::Success), + ][usize::from(skipped.is_empty())]; + finish(store, &claim, outcome).await; + reactions.insert( + claim.request.job_key.clone().expect("member"), + MemberReaction::Runs(skipped), + ); + } + let runs = crate::runs::main::listings::list_runs::list_runs(store, scope, 100) + .await + .expect("runs"); + let unclaimed: Vec<&str> = ["marts_job", "inventory_job"] + .into_iter() + .filter(|member| !reactions.contains_key(*member)) + .collect(); + for member in unclaimed { + let record = runs + .iter() + .find(|record| record.job_key.as_deref() == Some(member)) + .expect("member run"); + assert_eq!( + (record.status, record.launched_at), + (crate::runs::models::RunStatus::Skipped, None), + "{member} ends skipped without launching" + ); + let nodes = crate::runs::tests::helpers::rows( + store, + "SELECT status, skipped_because_of FROM sqb_run_nodes WHERE run_id=$1 AND kind='node'", + vec![crate::store::models::Parameter::Id(record.run_id)], + ) + .await; + assert!( + nodes.iter().all( + |node| node["status"] == "skipped" && node["skipped_because_of"] == "customers" + ), + "{member}: {nodes:?}" + ); + reactions.insert(member.into(), MemberReaction::Skipped); + } + reactions +} + +/// The lease, pool limits and `(location, target)` a test claims runs with. +type ClaimScope<'a> = ( + &'a SchedulerLease, + (&'a BTreeMap, (Uuid, &'a str)), +); + +/// With a retry delay, nothing starts until the test moves the retry's `not_before` into the past. +pub(crate) async fn wait_out_retry_delay( + store: &ControlStore, + (lease, (limits, scope)): ClaimScope<'_>, + (run_id, delay_seconds): (Uuid, u32), + description: &str, +) { + let 1.. = delay_seconds else { + return; + }; + assert!( + claim_next_run(store, lease, limits, Some(scope)) + .await + .expect("claim during the delay") + .is_none(), + "{description}: nothing starts while the member waits out its delay" + ); + let stored = crate::runs::tests::helpers::rows( + store, + "SELECT request_json FROM sqb_runs WHERE run_id=$1", + vec![crate::store::models::Parameter::Id(run_id)], + ) + .await + .remove(0)["request_json"] + .clone(); + let mut request = stored.as_str().map_or(stored.clone(), |text| { + serde_json::from_str(text).expect("request") + }); + request["retry"]["not_before"] = serde_json::json!("2000-01-01T00:00:00Z"); + let transaction = + crate::store::main::begin::begin(store, crate::store::models::TransactionMode::Write) + .await + .expect("begin"); + let (transaction, _) = crate::store::main::query::query( + transaction, + "UPDATE sqb_runs SET request_json=$1 WHERE run_id=$2", + vec![ + crate::store::models::Parameter::Json(request), + crate::store::models::Parameter::Id(run_id), + ], + ) + .await + .expect("end the delay"); + crate::store::main::commit::commit(transaction) + .await + .expect("commit"); +} diff --git a/crates/sqlbuild-orchestration/tests/runs/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/helpers.rs index 525baad6a1..4e0186c9b6 100644 --- a/crates/sqlbuild-orchestration/tests/runs/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/helpers.rs @@ -8,12 +8,13 @@ use crate::authorization::main::bootstrap_user::bootstrap_user; use crate::credentials::main::identities::create_principal::create_principal; use crate::credentials::models::{AuditContext, BuiltinRole, RoleAssignmentRequest}; use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::models::PoolSlots; use crate::runs::_helpers::requests::publication::publish_request; use crate::runs::models::{ AdhocCheck, AdhocCheckRequest, AdhocRunRequest, RunCommand, RunKind, RunOrigin, RunRequest, }; use crate::runs::tests::test_types::{AdhocCommandTestCase, AdhocPeople}; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::snapshots::models::{SnapshotInput, SnapshotNode}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -31,7 +32,7 @@ use sqlbuild_recall::records::models::{ }; use uuid::Uuid; -use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::execution::claim_next_run::claim_next_run; use crate::runs::models::RunClaim; use crate::runs::tests::claim_helpers::succeed; use crate::scheduler::models::SchedulerLease; @@ -107,9 +108,10 @@ pub(crate) fn request( selection, settings: BTreeMap::new(), window: None, - concurrency_groups: BTreeSet::new(), + pools: BTreeMap::new(), narrowing: None, retry: None, + member_attempt: None, } } @@ -432,3 +434,86 @@ pub(crate) async fn seed_logged_runs( claim_logged_run(store, scope, &lease, position as u128 + 1).await; } } + +/// A published snapshot node of `kind` with only its `definition`. +pub(crate) fn snapshot_node(kind: &str, definition: serde_json::Value) -> SnapshotNode { + SnapshotNode { + kind: kind.into(), + definition, + raw_sql: None, + compiled_sql: None, + python_source: None, + } +} + +/// One slot of each pool, with a project limit of 100 above every test's server cap, so the caps govern. +pub(crate) fn one_slot_each(pools: &[&str]) -> serde_json::Value { + let mut slots = serde_json::Map::new(); + for pool in pools { + slots.insert( + (*pool).to_owned(), + serde_json::json!({"amount": 1, "limit": 100}), + ); + } + serde_json::Value::Object(slots) +} + +/// `amount` slots of a pool whose project limit is `limit`. +pub(crate) fn pool_slots(amount: u32, limit: u32) -> PoolSlots { + PoolSlots { + amount: std::num::NonZeroU32::new(amount).expect("positive amount"), + limit: std::num::NonZeroU32::new(limit).expect("positive limit"), + } +} + +/// IDs that differ by a case's small numbers, so cases read as `target 1, snapshot 2`. +pub(crate) fn cache_id(number: u8) -> Uuid { + Uuid::from_u128(u128::from(number) + 1) +} + +/// The cached value of `snapshot` in target `scope`, built as the snapshot's number, and whether this lookup built it. +pub(crate) async fn cache_lookup( + slots: &std::sync::Mutex>, + (scope, snapshot): (u8, u8), +) -> (u8, bool) { + let built = std::cell::Cell::new(false); + let ((), value) = crate::runs::_helpers::lifecycle::publications::cached( + slots, + ((cache_id(scope), "development".into()), cache_id(snapshot)), + (), + |()| async { + built.set(true); + Ok(((), snapshot)) + }, + ) + .await + .expect("cached value"); + (*value, built.get()) +} + +/// A selection evaluation as `(count, page, next offset, refusal message)`. +pub(crate) fn selection_outcome( + evaluation: &crate::runs::models::SelectionEvaluation, +) -> (u64, Vec, Option, Option) { + ( + evaluation.count, + evaluation.keys.clone(), + evaluation.next_offset, + evaluation + .error + .as_ref() + .map(|problem| problem.message.clone()), + ) +} + +/// An expected selection in [`selection_outcome`]'s owned form. +pub(crate) fn owned_selection( + (count, keys, next, error): crate::runs::tests::test_types::ExpectedSelection, +) -> (u64, Vec, Option, Option) { + ( + count, + keys.iter().map(|key| (*key).to_owned()).collect(), + next, + error.map(str::to_owned), + ) +} diff --git a/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs index 4f99fce097..96c48670d7 100644 --- a/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs @@ -2,8 +2,8 @@ use std::collections::BTreeMap; -use crate::runs::main::change_run::change_run; -use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::execution::change_run::change_run; +use crate::runs::main::execution::claim_next_run::claim_next_run; use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunClaim, RunRequest}; use crate::scheduler::models::SchedulerLease; use crate::store::models::ControlStore; diff --git a/crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs index 5b31b83c2f..8151177fe3 100644 --- a/crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs @@ -1,6 +1,6 @@ //! Terminal facts exercise claim release through the public run lifecycle. -use crate::runs::main::change_run::change_run; +use crate::runs::main::execution::change_run::change_run; use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome}; use crate::scheduler::models::SchedulerLease; use crate::store::models::ControlStore; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs b/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs index f8da807f9d..b214462916 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs @@ -4,8 +4,8 @@ use serde_json::{Value, json}; use uuid::Uuid; use crate::credentials::models::AuditContext; -use crate::runs::main::check_adhoc_run::check_adhoc_run; -use crate::runs::main::request_person_run::request_person_run; +use crate::runs::main::requests::check_adhoc_run::check_adhoc_run; +use crate::runs::main::requests::request_person_run::request_person_run; use crate::runs::models::PersonRunRequest; use crate::runs::tests::helpers::{ adhoc_check, adhoc_fixture, adhoc_launch, check_projection, json_column, launch_outcome, rows, diff --git a/crates/sqlbuild-orchestration/tests/runs/test_checks.rs b/crates/sqlbuild-orchestration/tests/runs/test_checks.rs index 08b78b9dc2..3a4d570e86 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_checks.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_checks.rs @@ -7,8 +7,8 @@ use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; use crate::runs::main::{ - change_run::change_run, claim_next_run::claim_next_run, list_run_nodes::list_run_nodes, - list_runs::list_runs, + execution::change_run::change_run, execution::claim_next_run::claim_next_run, + listings::list_run_nodes::list_run_nodes, listings::list_runs::list_runs, }; use crate::runs::models::{ NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, @@ -16,7 +16,7 @@ use crate::runs::models::{ use crate::runs::tests::helpers::{queue, request, snapshot}; use crate::runs::tests::test_types::RunLifecycleTestCase; use crate::scheduler::tests::contracts::helpers::acquire_lease; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::snapshots::models::SnapshotNode; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::helpers::read_recall_events; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs index 270ea19c73..db4c844343 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs @@ -1,12 +1,12 @@ use std::collections::BTreeMap; use std::num::NonZeroU32; -use crate::runs::main::claim_next_run::claim_next_run; -use crate::runs::main::list_run_claims::list_run_claims; -use crate::runs::main::list_runs::list_runs; +use crate::runs::main::execution::claim_next_run::claim_next_run; +use crate::runs::main::listings::list_run_claims::list_run_claims; +use crate::runs::main::listings::list_runs::list_runs; use crate::runs::models::RunStatus; use crate::scheduler::tests::contracts::helpers::acquire_lease; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::helpers::read_recall_events; use uuid::Uuid; @@ -15,7 +15,7 @@ use crate::credentials::tests::location_helpers::create_location; use crate::runs::_helpers::lifecycle::reads::list_runs_in; use crate::runs::models::{QueueWait, RunQueueState}; use crate::runs::tests::claim_helpers::succeed; -use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::helpers::{pool_slots, queue, request, snapshot}; use crate::runs::tests::terminal_helpers::finish; use crate::runs::tests::test_types::{ BacklogClaimTestCase, GroupLimitTestCase, PriorityClaimsTestCase, QueueStateTestCase, @@ -325,15 +325,9 @@ fn given_named_group_limits_when_capacity_is_lowered_then_active_high_slots_stil for id in 1..=3 { let mut input = request((location, published.snapshot_id), id, 0, &["orders"]); input.write_set.clear(); - input.concurrency_groups.insert("warehouse".into()); + input.pools.insert("warehouse".into(), pool_slots(1, 100)); queue(store, &input).await; } - assert!( - claim_next_run(store, &lease, &BTreeMap::new(), None) - .await - .is_err(), - "unknown groups cannot silently become unlimited" - ); let first = claim_next_run(store, &lease, &limits, None) .await .expect("first slot") @@ -483,7 +477,10 @@ fn given_a_blocked_multi_limit_waiter_when_younger_work_arrives_then_shared_limi (4, 0, &["products"][..], &[][..]), ] { let mut input = request(scope, id, priority, nodes); - input.concurrency_groups = groups.iter().map(|key| (*key).into()).collect(); + input.pools = groups + .iter() + .map(|key| ((*key).into(), pool_slots(1, 100))) + .collect(); queue(store, &input).await; } let lease = acquire_lease(store, Uuid::now_v7()) @@ -575,9 +572,11 @@ fn given_claimed_work_here_and_elsewhere_when_listing_the_queue_then_each_run_sh .expect("elect") .expect("leader"); let mut running = request((location, here.snapshot_id), 1, 0, &["orders"]); - running.concurrency_groups.insert("warehouse".into()); + running.pools.insert("warehouse".into(), pool_slots(2, 100)); let mut elsewhere = request((other, there.snapshot_id), 2, 0, &["products"]); - elsewhere.concurrency_groups.insert("warehouse".into()); + elsewhere + .pools + .insert("warehouse".into(), pool_slots(1, 100)); for input in [&running, &elsewhere] { queue(store, input).await; claim_next_run(store, &lease, &limits, None) @@ -586,9 +585,9 @@ fn given_claimed_work_here_and_elsewhere_when_listing_the_queue_then_each_run_sh .expect("claimable"); } let mut blocked = request((location, here.snapshot_id), 3, 0, &["customers", "orders"]); - blocked.concurrency_groups.insert("warehouse".into()); + blocked.pools.insert("warehouse".into(), pool_slots(1, 100)); let mut urgent = request((location, here.snapshot_id), 4, 5, &["inventory"]); - urgent.concurrency_groups.insert("warehouse".into()); + urgent.pools.insert("warehouse".into(), pool_slots(1, 100)); let free = request((location, here.snapshot_id), 5, 0, &["products"]); for input in [&blocked, &urgent, &free] { queue(store, input).await; @@ -602,10 +601,9 @@ fn given_claimed_work_here_and_elsewhere_when_listing_the_queue_then_each_run_sh .await .expect("queue"); commit(transaction).await.expect("commit"); - let slot = QueueWait::ConcurrencyLimit { - limit: "warehouse".into(), + let slot = QueueWait::PoolFull { + pool: "warehouse".into(), holders: vec![running.run_id], - elsewhere: 1, }; let expected = [ ( @@ -628,7 +626,7 @@ fn given_claimed_work_here_and_elsewhere_when_listing_the_queue_then_each_run_sh slot, QueueWait::EarlierRun { run_id: Some(urgent.run_id), - limit: "warehouse".into(), + pool: "warehouse".into(), }, ], }, diff --git a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs index 64b25eae06..6d811497dc 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs @@ -1,14 +1,14 @@ use std::collections::BTreeMap; use crate::declarations::main::list_triggers::list_triggers; -use crate::runs::main::claim_next_run::claim_next_run; -use crate::runs::main::list_runs::list_runs; +use crate::runs::main::execution::claim_next_run::claim_next_run; +use crate::runs::main::listings::list_runs::list_runs; use crate::runs::tests::claim_helpers::succeed; use crate::scheduler::main::list_ticks::list_ticks; use crate::scheduler::models::{TickDecision, TickOutcome}; use crate::scheduler::tests::contracts::helpers::{acquire_lease, evaluate_trigger}; use crate::scheduler::tests::contracts::ticks::helpers::calendar_text; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use serde_json::json; use uuid::Uuid; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs index a5153ebe6d..fe5e12c686 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs @@ -1,5 +1,5 @@ #[path = "claims/helpers.rs"] -mod claim_helpers; +pub(crate) mod claim_helpers; #[path = "helpers.rs"] pub(crate) mod helpers; #[path = "monitor/helpers.rs"] @@ -18,6 +18,14 @@ mod test_coalesce; mod test_faults; #[path = "test_group_admission.rs"] mod test_group_admission; +#[path = "test_group_declarations.rs"] +mod test_group_declarations; +#[path = "test_group_dependencies.rs"] +mod test_group_dependencies; +#[path = "test_group_retries.rs"] +mod test_group_retries; +#[path = "test_group_round_trips.rs"] +mod test_group_round_trips; #[path = "test_groups.rs"] mod test_groups; #[path = "test_job_detail.rs"] @@ -32,6 +40,10 @@ mod test_monitor; mod test_node_errors; #[path = "test_permissions.rs"] mod test_permissions; +#[path = "test_project_limits.rs"] +mod test_project_limits; +#[path = "test_publication_cache.rs"] +mod test_publication_cache; #[path = "test_requests.rs"] mod test_requests; #[path = "test_reruns.rs"] @@ -40,5 +52,7 @@ mod test_reruns; mod test_retries; #[path = "test_run_logs.rs"] mod test_run_logs; +#[path = "test_selection.rs"] +mod test_selection; #[path = "test_types.rs"] pub(crate) mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs index 8b4107f241..5cf5a3f632 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs @@ -1,20 +1,20 @@ use std::collections::BTreeMap; use std::num::NonZeroU32; -use crate::runs::main::change_run::change_run; -use crate::runs::main::claim_next_run::claim_next_run; -use crate::runs::main::heartbeat_run::heartbeat_run; -use crate::runs::main::list_runs::list_runs; +use crate::runs::main::execution::change_run::change_run; +use crate::runs::main::execution::claim_next_run::claim_next_run; +use crate::runs::main::execution::heartbeat_run::heartbeat_run; +use crate::runs::main::listings::list_runs::list_runs; use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest}; use crate::scheduler::tests::contracts::helpers::acquire_lease; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::helpers::read_recall_events; use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; -use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::helpers::{pool_slots, queue, request, snapshot}; use crate::runs::tests::test_types::{ClaimFailureTestCase, TransactionFault}; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::lease_helpers::expire_lease; @@ -55,7 +55,7 @@ fn given_failures_during_claim_publication_when_retried_then_claims_slots_livene 1, &["customers", "orders"], ); - input.concurrency_groups.insert("warehouse".into()); + input.pools.insert("warehouse".into(), pool_slots(1, 100)); queue(store, &input).await; let lease = acquire_lease(store, Uuid::now_v7()) .await diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs index 643b151e5c..207fe334a2 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs @@ -5,19 +5,19 @@ use std::num::NonZeroU32; use crate::credentials::models::AuditContext; use crate::runs::main::{ - advance_groups::advance_groups, claim_next_run::claim_next_run, list_runs::list_runs, - request_job_run::request_job_run, + execution::advance_groups::advance_groups, execution::claim_next_run::claim_next_run, + listings::list_runs::list_runs, requests::request_job_run::request_job_run, }; use crate::runs::models::{JobRunRequest, RunDispatch, RunStatus}; use crate::scheduler::tests::contracts::helpers::acquire_lease; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use serde_json::json; use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; use crate::runs::tests::claim_helpers::{slot_counts, succeed}; -use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::helpers::{one_slot_each, pool_slots, queue, request, snapshot}; use crate::runs::tests::test_types::{ AdmissionParent, GroupAdmissionTestCase, GroupChildLimitTestCase, GroupQueueTestCase, MixedAdmissionTestCase, RunLifecycleTestCase, @@ -71,7 +71,7 @@ fn given_mixed_physical_and_parent_waiters_when_admitting_then_priority_and_queu "jobs".into(), json!({"stage": {"description": "Build orders", "selection": ["orders"]}}), ); - input.documents.insert("groups".into(), json!({"daily": {"description": "Group orders", "selection": ["orders"], "concurrency_groups": ["warehouse"], "jobs": {"stage": []}}})); + input.documents.insert("groups".into(), json!({"daily": {"description": "Group orders", "selection": ["orders"], "pools": {"warehouse": {"amount": 1, "limit": 100}}, "jobs": {"stage": []}}})); let published = publish_snapshot(store, &input).await.expect("publish"); let mut physical = request( (location, published.snapshot_id), @@ -79,7 +79,9 @@ fn given_mixed_physical_and_parent_waiters_when_admitting_then_priority_and_queu test_case.physical_priority, &["inventory"], ); - physical.concurrency_groups.insert("warehouse".into()); + physical + .pools + .insert("warehouse".into(), pool_slots(1, 100)); queue(store, &physical).await; request_job_run( store, @@ -284,7 +286,7 @@ fn given_an_unserved_queued_group_when_dispatching_an_eligible_scope_then_unknow .expect("served location"); for location in [unserved, served] { let mut input = snapshot(location); - input.documents.insert("jobs".into(), json!({"stage": {"description": "Build orders", "selection": ["orders"], "concurrency_groups": if location == unserved {vec!["inventory_limit"]} else {vec![]}}})); + input.documents.insert("jobs".into(), json!({"stage": {"description": "Build orders", "selection": ["orders"], "pools": if location == unserved {one_slot_each(&["inventory_limit"])} else {json!({})}}})); input.documents.insert("groups".into(), json!({"daily": {"description": "Group orders", "selection": ["orders"], "jobs": {"stage": []}}})); publish_snapshot(store, &input).await.expect("publish"); request_job_run( @@ -502,13 +504,13 @@ fn given_capacity_and_multistage_cycles_when_replicas_admit_and_resume_groups_th input.nodes.get_mut(node).expect("stage node").definition["deps"] = json!(earlier_nodes); earlier_nodes.push(node); - jobs.insert(name.clone(), json!({"description": "Build a synthetic stage", "selection": [node], "concurrency_groups": needs})); + jobs.insert(name.clone(), json!({"description": "Build a synthetic stage", "selection": [node], "pools": one_slot_each(needs)})); let after: Vec = (0..stage) .map(|earlier| format!("orders_job_{index}_{earlier}")) .collect(); members.insert(name, json!(after)); } - groups.insert(format!("orders_group_{index}"), json!({"description": "Complete a synthetic group", "selection": selection, "concurrency_groups": parent.holds, "jobs": members})); + groups.insert(format!("orders_group_{index}"), json!({"description": "Complete a synthetic group", "selection": selection, "pools": one_slot_each(parent.holds), "jobs": members})); } input.documents.insert("jobs".into(), json!(jobs)); input.documents.insert("groups".into(), json!(groups)); @@ -659,7 +661,7 @@ fn given_an_older_blocked_allow_group_when_an_independent_generation_is_queued_t "jobs".into(), json!({"stage": {"description": "Build a pinned stage", "selection": ["orders"]}}), ); - input.documents.insert("groups".into(), json!({"daily": {"description": "Allow independent generations", "selection": ["orders"], "concurrency_groups": ["orders_limit"], "jobs": {"stage": []}}})); + input.documents.insert("groups".into(), json!({"daily": {"description": "Allow independent generations", "selection": ["orders"], "pools": {"orders_limit": {"amount": 1, "limit": 100}}, "jobs": {"stage": []}}})); let first = publish_snapshot(store, &input) .await .expect("first snapshot"); @@ -683,8 +685,8 @@ fn given_an_older_blocked_allow_group_when_an_independent_generation_is_queued_t json!(["inventory"]); input.documents.get_mut("groups").expect("groups")["daily"]["selection"] = json!(["inventory"]); - input.documents.get_mut("groups").expect("groups")["daily"]["concurrency_groups"] = - json!(["inventory_limit"]); + input.documents.get_mut("groups").expect("groups")["daily"]["pools"] = + one_slot_each(&["inventory_limit"]); publish_snapshot(store, &input) .await .expect("second snapshot"); @@ -697,7 +699,9 @@ fn given_an_older_blocked_allow_group_when_an_independent_generation_is_queued_t .await .expect("newer request"); let mut physical = request((location, first.snapshot_id), 3000, 10, &["products"]); - physical.concurrency_groups.insert("orders_limit".into()); + physical + .pools + .insert("orders_limit".into(), pool_slots(1, 100)); queue(store, &physical).await; let lease = acquire_lease(store, Uuid::now_v7()) .await @@ -812,7 +816,7 @@ fn given_a_large_healthy_project_with_blocked_groups_when_claiming_then_unrelate json!(format!("Orders {index} {}", "details ".repeat(128))); input.nodes.insert(format!("orders_{index}"), node); } - let jobs = (0..test_case.group_count).map(|index| (format!("job_{index}"), json!({"description": "Build orders", "selection": ["orders"], "concurrency_groups": ["warehouse"]}))).collect::>(); + let jobs = (0..test_case.group_count).map(|index| (format!("job_{index}"), json!({"description": "Build orders", "selection": ["orders"], "pools": {"warehouse": {"amount": 1, "limit": 100}}}))).collect::>(); let groups = (0..test_case.group_count).map(|index| (format!("group_{index}"), json!({"description": "Independent order group", "selection": ["orders"], "jobs": {format!("job_{index}"): []}}))).collect::>(); input.documents.insert("jobs".into(), json!(jobs)); input.documents.insert("groups".into(), json!(groups)); @@ -831,7 +835,9 @@ fn given_a_large_healthy_project_with_blocked_groups_when_claiming_then_unrelate let limits = BTreeMap::from([("warehouse".into(), NonZeroU32::new(1).expect("one"))]); let scope = (location, "development"); let mut physical = request((location, published.snapshot_id), 1, 10, &["products"]); - physical.concurrency_groups.insert("warehouse".into()); + physical + .pools + .insert("warehouse".into(), pool_slots(1, 100)); queue(store, &physical).await; let blocker = claim_next_run(store, &lease, &limits, Some(scope)) .await diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_declarations.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_declarations.rs new file mode 100644 index 0000000000..56be9559c7 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_declarations.rs @@ -0,0 +1,102 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::json; +use uuid::Uuid; + +use crate::declarations::main::group_wait_reasons::group_wait_reasons; +use crate::declarations::main::snapshot_definitions::snapshot_definitions; +use crate::declarations::models::JobDefinition; +use crate::runs::tests::helpers::snapshot; +use crate::runs::tests::test_types::{DeclarationCheckTestCase, WaitReasonTestCase}; +use crate::store::errors::ControlError; + +#[test] +fn given_group_members_when_publishing_then_each_wait_has_its_node_level_reasons() { + let test_cases = [WaitReasonTestCase { + description: "through a node no member selects, and never through another member's node", + deps: &[ + ("products", &["orders"]), + ("customers", &["products"]), + ("inventory", &["customers"]), + ], + jobs: json!({ + "load_orders": {"description": "Orders", "selection": ["orders"]}, + "build_customers": {"description": "Customers", "selection": ["customers"]}, + "build_inventory": {"description": "Inventory", "selection": ["inventory"]}, + }), + waits: json!({ + "load_orders": [], + "build_customers": ["load_orders"], + "build_inventory": ["build_customers", "load_orders"], + }), + expected_reasons: json!({ + "build_customers": [ + {"upstream_job": "load_orders", "from_node": "orders", "to_node": "customers"}], + "build_inventory": [ + {"upstream_job": "build_customers", "from_node": "customers", "to_node": "inventory"}], + }), + }]; + for test_case in test_cases { + let mut input = snapshot(Uuid::nil()); + for (node, upstream) in test_case.deps { + input.nodes.get_mut(*node).expect("node").definition["deps"] = json!(upstream); + } + let jobs: BTreeMap = + serde_json::from_value(test_case.jobs).expect("jobs"); + let waits: BTreeMap> = + serde_json::from_value(test_case.waits).expect("waits"); + assert_eq!( + json!(group_wait_reasons(&input, &jobs, &waits)), + test_case.expected_reasons, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_invalid_limits_or_reasons_when_publishing_then_the_snapshot_is_refused() { + let test_cases = [ + DeclarationCheckTestCase { + description: "an amount above its pool's limit", + jobs: json!({"orders": {"description": "Orders", "selection": ["orders"], + "pools": {"warehouse": {"amount": 2, "limit": 1}}}}), + groups: json!({}), + expected_error: "orders takes more slots of pool warehouse than its limit", + }, + DeclarationCheckTestCase { + description: "one group with two limits", + jobs: json!({ + "orders": {"description": "Orders", "selection": ["orders"], + "pools": {"warehouse": {"amount": 1, "limit": 1}}}, + "customers": {"description": "Customers", "selection": ["customers"], + "pools": {"warehouse": {"amount": 1, "limit": 2}}}, + }), + groups: json!({}), + expected_error: "pool warehouse has more than one limit", + }, + DeclarationCheckTestCase { + description: "a wait reason naming a node outside the snapshot", + jobs: json!({ + "orders": {"description": "Orders", "selection": ["orders"]}, + "customers": {"description": "Customers", "selection": ["customers"]}, + }), + groups: json!({"daily": {"description": "Daily", "selection": ["customers", "orders"], + "jobs": {"orders": [], "customers": []}, + "wait_reasons": {"customers": [ + {"upstream_job": "orders", "from_node": "returns", "to_node": "customers"}]}}}), + expected_error: "group daily has wait reasons outside its members' waits or the snapshot", + }, + ]; + for test_case in test_cases { + let mut input = snapshot(Uuid::nil()); + input.documents.insert("jobs".into(), test_case.jobs); + input.documents.insert("groups".into(), test_case.groups); + let refused = snapshot_definitions(&input).map(|_| ()); + assert!( + matches!(&refused, Err(ControlError::Invalid(message)) if message == test_case.expected_error), + "{}: {refused:?}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_dependencies.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_dependencies.rs new file mode 100644 index 0000000000..a62c083840 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_dependencies.rs @@ -0,0 +1,177 @@ +//! A group's `[dependencies] on_failure` decides what each member does after a job it depends on fails. + +use std::collections::BTreeMap; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::main::{ + execution::advance_groups::advance_groups, execution::claim_next_run::claim_next_run, + listings::list_runs::list_runs, requests::request_job_run::request_job_run, +}; +use crate::runs::models::{JobRunRequest, NodeOutcome, RunOutcome, RunStatus}; +use crate::runs::tests::claim_helpers::{finish, member_reactions, publish_dependency_group}; +use crate::runs::tests::test_types::{GroupDependencyTestCase, MemberReaction}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_failed_member_when_its_dependents_react_then_each_dependency_policy_and_override_applies() + { + let test_cases = [ + GroupDependencyTestCase { + description: "SQLite: partial by default skips only downstream nodes", + backend: Backend::Sqlite, + dependency_failure: json!({}), + expected_marts: MemberReaction::Runs(vec![("orders", "customers")]), + expected_inventory: MemberReaction::Runs(vec![("inventory", "customers")]), + }, + GroupDependencyTestCase { + description: "SQLite: skip ends dependents without running them", + backend: Backend::Sqlite, + dependency_failure: json!({"marts_job": "skip", "inventory_job": "skip"}), + expected_marts: MemberReaction::Skipped, + expected_inventory: MemberReaction::Skipped, + }, + GroupDependencyTestCase { + description: "SQLite: run builds every node regardless", + backend: Backend::Sqlite, + dependency_failure: json!({"marts_job": "run", "inventory_job": "run"}), + expected_marts: MemberReaction::Runs(Vec::new()), + expected_inventory: MemberReaction::Runs(Vec::new()), + }, + GroupDependencyTestCase { + description: "SQLite: a member override differs from the group default", + backend: Backend::Sqlite, + dependency_failure: json!({"inventory_job": "skip"}), + expected_marts: MemberReaction::Runs(vec![("orders", "customers")]), + expected_inventory: MemberReaction::Skipped, + }, + GroupDependencyTestCase { + description: "PostgreSQL: partial by default skips only downstream nodes", + backend: Backend::Postgres, + dependency_failure: json!({}), + expected_marts: MemberReaction::Runs(vec![("orders", "customers")]), + expected_inventory: MemberReaction::Runs(vec![("inventory", "customers")]), + }, + GroupDependencyTestCase { + description: "PostgreSQL: skip ends dependents without running them", + backend: Backend::Postgres, + dependency_failure: json!({"marts_job": "skip", "inventory_job": "skip"}), + expected_marts: MemberReaction::Skipped, + expected_inventory: MemberReaction::Skipped, + }, + GroupDependencyTestCase { + description: "PostgreSQL: run builds every node regardless", + backend: Backend::Postgres, + dependency_failure: json!({"marts_job": "run", "inventory_job": "run"}), + expected_marts: MemberReaction::Runs(Vec::new()), + expected_inventory: MemberReaction::Runs(Vec::new()), + }, + GroupDependencyTestCase { + description: "PostgreSQL: a member override differs from the group default", + backend: Backend::Postgres, + dependency_failure: json!({"inventory_job": "skip"}), + expected_marts: MemberReaction::Runs(vec![("orders", "customers")]), + expected_inventory: MemberReaction::Skipped, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + publish_dependency_group(store, location, &test_case.dependency_failure).await; + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let group = Uuid::now_v7(); + request_job_run( + store, + (&context, None), + &JobRunRequest { + run_id: group, + location_id: location, + target: "development".into(), + job_key: "nightly_job_group".into(), + reason: "Build the nightly group".into(), + priority: 0, + window: None, + full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), + }, + ) + .await + .expect("request group"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::new(); + let scope = (location, "development"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("start group"); + let staging = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim staging") + .expect("staging ready"); + assert_eq!( + staging.request.job_key.as_deref(), + Some("staging_job"), + "{}", + test_case.description + ); + assert!( + claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("dependents wait") + .is_none(), + "{}: dependents wait for the job they read from", + test_case.description + ); + finish(store, &staging, (NodeOutcome::Failed, RunOutcome::Failed)).await; + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("react to the failure"); + let reactions: BTreeMap = + member_reactions(store, &lease, (&limits, scope)).await; + assert_eq!( + (&reactions["marts_job"], &reactions["inventory_job"]), + (&test_case.expected_marts, &test_case.expected_inventory), + "{}", + test_case.description + ); + for _ in 0..2 { + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish group"); + } + let runs = list_runs(store, scope, 100).await.expect("runs"); + let parent = runs + .iter() + .find(|record| record.run_id == group) + .expect("group"); + assert!( + matches!(parent.status, RunStatus::Failed | RunStatus::Partial), + "{}: {:?}", + test_case.description, + parent.status + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_retries.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_retries.rs new file mode 100644 index 0000000000..a572d3db96 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_retries.rs @@ -0,0 +1,257 @@ +//! A member job's `[retry]` reruns it inside its group run while the members that depend on it wait. + +use std::collections::BTreeMap; + +use serde_json::json; +use uuid::Uuid; + +use crate::authorization::main::bootstrap_user::bootstrap_user; +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::_helpers::lifecycle::reads::list_runs_in; +use crate::runs::main::{ + execution::advance_groups::advance_groups, execution::claim_next_run::claim_next_run, + execution::run_execution::run_execution, listings::list_runs::list_runs, + requests::request_job_run::request_job_run, +}; +use crate::runs::models::{JobRunRequest, NodeOutcome, QueueWait, RunOutcome, RunStatus}; +use crate::runs::tests::claim_helpers::{finish, wait_out_retry_delay}; +use crate::runs::tests::helpers::{rows, snapshot}; +use crate::runs::tests::test_types::GroupMemberRetryTestCase; +use crate::runs::types::RunsListing; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; +use crate::store::main::{begin::begin, commit::commit, migrate_store::migrate_store}; +use crate::store::models::TransactionMode; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::warehouse_history::main::read_person_run_events::read_person_run_events; + +#[test] +fn given_a_member_with_retry_when_it_fails_in_a_group_then_it_retries_inside_the_group_while_dependents_wait() + { + let test_cases = [ + GroupMemberRetryTestCase { + description: "SQLite member retry", + backend: Backend::Sqlite, + delay_seconds: 900, + expected_messages: &[ + "staging_job failed, retrying in 15 min (attempt 2 of 3)", + "staging_job succeeded on attempt 2", + ], + }, + GroupMemberRetryTestCase { + description: "PostgreSQL member retry", + backend: Backend::Postgres, + delay_seconds: 0, + expected_messages: &[ + "staging_job failed, retrying now (attempt 2 of 3)", + "staging_job succeeded on attempt 2", + ], + }, + ]; + let mut completed = 0; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.nodes.get_mut("orders").expect("orders").definition["deps"] = + json!(["customers"]); + input.documents.insert("jobs".into(), json!({ + "staging_job": {"description": "Stage customers", "selection": ["customers"], + "on_failure": {"retries": 2, "retry_delay_seconds": test_case.delay_seconds}}, + "marts_job": {"description": "Build orders", "selection": ["orders"]} + })); + input.documents.insert( + "groups".into(), + json!({"nightly_job_group": { + "description": "Stage then build orders", "selection": ["customers", "orders"], + "jobs": {"staging_job": [], "marts_job": ["staging_job"]} + }}), + ); + publish_snapshot(store, &input).await.expect("publish"); + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let group = Uuid::now_v7(); + request_job_run( + store, + (&context, None), + &JobRunRequest { + run_id: group, + location_id: location, + target: "development".into(), + job_key: "nightly_job_group".into(), + reason: "Build the nightly group".into(), + priority: 0, + window: None, + full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), + }, + ) + .await + .expect("request group"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::new(); + let scope = (location, "development"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("start group"); + let first = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim staging") + .expect("staging ready"); + finish(store, &first, (NodeOutcome::Failed, RunOutcome::Failed)).await; + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("advance after the failure"); + + let alerts = rows( + store, + "SELECT event_id FROM recall_events WHERE event_type='platform_run_failed'", + vec![], + ) + .await; + assert!( + alerts.is_empty(), + "{}: a member that will retry raises no failure alert: {alerts:?}", + test_case.description + ); + let transaction = begin(store, TransactionMode::Read).await.expect("begin"); + let (transaction, queued) = list_runs_in(transaction, scope, RunsListing::Queue(20)) + .await + .expect("queue"); + commit(transaction).await.expect("commit"); + let retry = queued + .iter() + .find(|record| record.job_key.as_deref() == Some("staging_job")) + .expect("queued retry"); + assert_eq!( + ( + retry.parent_run_id, + retry.rerun_of_run_id, + retry.member_attempt + ), + (Some(group), Some(first.request.run_id), Some(2)), + "{}: the retry is another child of the same group run", + test_case.description + ); + let marts = queued + .iter() + .find(|record| record.job_key.as_deref() == Some("marts_job")) + .expect("waiting dependent"); + assert_eq!( + marts.queue.as_ref().expect("queue state").waits, + vec![QueueWait::UpstreamJob { + run_id: retry.run_id, + job_key: "staging_job".into(), + attempt: Some(2), + }], + "{}: the dependent waits for the retrying member instead of skipping", + test_case.description + ); + let runs = list_runs(store, scope, 100).await.expect("runs"); + assert_eq!( + runs.iter() + .find(|record| record.run_id == group) + .expect("group") + .status, + RunStatus::Running, + "{}: the group keeps running", + test_case.description + ); + + wait_out_retry_delay( + store, + (&lease, (&limits, scope)), + (retry.run_id, test_case.delay_seconds), + test_case.description, + ) + .await; + let second = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim retry") + .expect("retry ready"); + assert_eq!(second.request.run_id, retry.run_id); + assert_eq!(second.request.selection, ["customers".to_owned()].into()); + finish(store, &second, (NodeOutcome::Success, RunOutcome::Success)).await; + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("advance after the retry"); + let dependent = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim dependent") + .expect("dependent ready"); + assert_eq!(dependent.request.job_key.as_deref(), Some("marts_job")); + assert!( + run_execution(store, (dependent.request.run_id, dependent.claim_id)) + .await + .expect("execution") + .expect("claimed") + .skipped_nodes + .is_empty(), + "{}: the dependent builds everything once the retry succeeded", + test_case.description + ); + finish( + store, + &dependent, + (NodeOutcome::Success, RunOutcome::Success), + ) + .await; + for _ in 0..2 { + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish group"); + } + let runs = list_runs(store, scope, 100).await.expect("runs"); + assert_eq!( + runs.iter() + .find(|record| record.run_id == group) + .expect("group") + .status, + RunStatus::Success, + "{}: the group succeeds on the member's final outcome", + test_case.description + ); + let admin = bootstrap_user(store, &context, "orders-admin") + .await + .expect("administrator"); + let events = read_person_run_events( + store, + (&context, admin), + ((location, "development", group), (0, 1000)), + DefaultRole::None, + ) + .await + .expect("group events") + .expect("group run"); + let messages: Vec<&str> = events + .iter() + .filter_map(|event| event.message.as_deref()) + .collect(); + assert_eq!( + messages, test_case.expected_messages, + "{}", + test_case.description + ); + completed += 1; + close_test_store(fixture).await; + }); + } + assert_eq!(completed, 2, "both backends run"); +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_round_trips.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_round_trips.rs new file mode 100644 index 0000000000..01e9b0009a --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_round_trips.rs @@ -0,0 +1,172 @@ +//! A group orders a warehouse round trip by its nearest lineage, SQL to task to loader to models, on both stores. + +use std::collections::BTreeMap; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::main::{ + execution::advance_groups::advance_groups, execution::claim_next_run::claim_next_run, + requests::request_job_run::request_job_run, +}; +use crate::runs::models::{JobRunRequest, NodeOutcome, RunOutcome}; +use crate::runs::tests::claim_helpers::finish; +use crate::runs::tests::helpers::{snapshot, snapshot_node}; +use crate::runs::tests::test_types::RoundTripGroupTestCase; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_round_trip_split_across_jobs_when_the_group_runs_then_members_follow_the_loader_edge() { + let test_cases = [ + RoundTripGroupTestCase { + description: "SQLite: the task waits for SQL, the loader for the task, the model for the loader", + backend: Backend::Sqlite, + expected_waits: json!({ + "orders_job": [], + "score_job": ["orders_job"], + "ingest_job": ["score_job"], + "report_job": ["ingest_job", "orders_job"], + }), + expected_order: &["orders_job", "score_job", "ingest_job", "report_job"], + }, + RoundTripGroupTestCase { + description: "PostgreSQL: the task waits for SQL, the loader for the task, the model for the loader", + backend: Backend::Postgres, + expected_waits: json!({ + "orders_job": [], + "score_job": ["orders_job"], + "ingest_job": ["score_job"], + "report_job": ["ingest_job", "orders_job"], + }), + expected_order: &["orders_job", "score_job", "ingest_job", "report_job"], + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.nodes = BTreeMap::from([ + ( + "model:orders".into(), + snapshot_node("model", json!({"deps": []})), + ), + ( + "task:score_orders".into(), + snapshot_node("task", json!({"deps": ["model:orders"]})), + ), + ( + "loader:partner_scores".into(), + snapshot_node("loader", json!({"deps": ["task:score_orders"]})), + ), + ( + "source:partner_scores".into(), + snapshot_node( + "source", + json!({"name": "partner_scores", "entry": {"loader": "partner_scores"}}), + ), + ), + ( + "model:scored_orders".into(), + snapshot_node( + "model", + json!({"deps": ["model:orders", "source:partner_scores"]}), + ), + ), + ]); + input.documents.insert( + "jobs".into(), + json!({ + "orders_job": {"description": "Build orders", "selection": ["model:orders"]}, + "score_job": {"description": "Score orders", "selection": ["task:score_orders"]}, + "ingest_job": { + "description": "Load partner scores", + "selection": ["loader:partner_scores", "source:partner_scores"] + }, + "report_job": { + "description": "Build scored orders", + "selection": ["model:scored_orders"] + } + }), + ); + input.documents.insert( + "groups".into(), + json!({"scoring_job_group": { + "description": "Score orders through the partner and report on them", + "selection": [ + "loader:partner_scores", "model:orders", "model:scored_orders", + "source:partner_scores", "task:score_orders" + ], + "jobs": test_case.expected_waits, + }}), + ); + publish_snapshot(store, &input) + .await + .unwrap_or_else(|error| panic!("{}: publish: {error:?}", test_case.description)); + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + request_job_run( + store, + (&context, None), + &JobRunRequest { + run_id: Uuid::now_v7(), + location_id: location, + target: "development".into(), + job_key: "scoring_job_group".into(), + reason: "Score the orders".into(), + priority: 0, + window: None, + full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), + }, + ) + .await + .expect("request group"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::new(); + let scope = (location, "development"); + let mut order: Vec = Vec::new(); + for _ in test_case.expected_order { + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("advance group"); + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim member") + .unwrap_or_else(|| panic!("{}: a member is ready", test_case.description)); + assert!( + claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim another") + .is_none(), + "{}: one member at a time along the chain", + test_case.description + ); + order.push(claim.request.job_key.clone().expect("member")); + finish(store, &claim, (NodeOutcome::Success, RunOutcome::Success)).await; + } + assert_eq!(order, test_case.expected_order, "{}", test_case.description); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs index 82d54108a1..6309917fbc 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs @@ -5,23 +5,26 @@ use std::num::NonZeroU32; use crate::credentials::models::AuditContext; use crate::runs::main::{ - advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, - list_runs::list_runs, request_job_run::request_job_run, run_execution::run_execution, + execution::advance_groups::advance_groups, execution::change_run::change_run, + execution::claim_next_run::claim_next_run, execution::run_execution::run_execution, + listings::list_runs::list_runs, requests::request_job_run::request_job_run, }; use crate::runs::models::{ JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, }; use crate::scheduler::tests::contracts::helpers::acquire_lease; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::helpers::read_recall_events; use serde_json::json; use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; -use crate::runs::tests::claim_helpers::{slot_counts, succeed}; +use crate::runs::tests::claim_helpers::{slot_counts, succeed, succeed_ignoring_audit_failures}; use crate::runs::tests::helpers::snapshot; -use crate::runs::tests::test_types::{RunLifecycleTestCase, TransactionFault}; +use crate::runs::tests::test_types::{ + GroupMemberSettingsTestCase, RunLifecycleTestCase, TransactionFault, +}; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; @@ -61,12 +64,12 @@ fn given_a_pinned_group_when_redeployed_and_cancelled_then_children_keep_scope_a input.nodes.get_mut("orders").expect("orders").definition["deps"] = json!(["customers"]); input.documents.insert("jobs".into(), json!({ - "staging": {"description": "Stage customers", "selection": ["customers"], "concurrency_groups": ["warehouse"]}, - "marts": {"description": "Build orders", "selection": ["orders"], "concurrency_groups": ["warehouse"]} + "staging": {"description": "Stage customers", "selection": ["customers"], "pools": {"warehouse": {"amount": 1, "limit": 100}}}, + "marts": {"description": "Build orders", "selection": ["orders"], "pools": {"warehouse": {"amount": 1, "limit": 100}}} })); input.documents.insert("groups".into(), json!({"daily_orders": { "description": "Stage then build orders", "selection": ["customers", "orders"], - "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["staging"]} + "pools": {"warehouse": {"amount": 1, "limit": 100}}, "jobs": {"staging": [], "marts": ["staging"]} }})); let published = publish_snapshot(store, &input) .await @@ -340,12 +343,12 @@ fn given_opposing_parent_and_child_limits_when_advancing_groups_then_both_groups }; let mut input = snapshot(location); input.documents.insert("jobs".into(), json!({ - "orders_job": {"description": "Build orders", "selection": ["orders"], "concurrency_groups": ["inventory_limit"]}, - "inventory_job": {"description": "Build inventory", "selection": ["customers"], "concurrency_groups": ["orders_limit"]} + "orders_job": {"description": "Build orders", "selection": ["orders"], "pools": {"inventory_limit": {"amount": 1, "limit": 100}}}, + "inventory_job": {"description": "Build inventory", "selection": ["customers"], "pools": {"orders_limit": {"amount": 1, "limit": 100}}} })); input.documents.insert("groups".into(), json!({ - "orders_group": {"description": "Orders group", "selection": ["orders"], "concurrency_groups": ["orders_limit"], "jobs": {"orders_job": []}}, - "inventory_group": {"description": "Inventory group", "selection": ["customers"], "concurrency_groups": ["inventory_limit"], "jobs": {"inventory_job": []}} + "orders_group": {"description": "Orders group", "selection": ["orders"], "pools": {"orders_limit": {"amount": 1, "limit": 100}}, "jobs": {"orders_job": []}}, + "inventory_group": {"description": "Inventory group", "selection": ["customers"], "pools": {"inventory_limit": {"amount": 1, "limit": 100}}, "jobs": {"inventory_job": []}} })); publish_snapshot(store, &input) .await @@ -434,3 +437,153 @@ fn given_opposing_parent_and_child_limits_when_advancing_groups_then_both_groups } assert_eq!(completed.len(), 2, "both backends must run"); } + +#[test] +fn given_group_members_with_different_audit_settings_when_run_then_each_keeps_its_own_and_the_group_sums_ignored_failures() + { + let test_cases = [ + GroupMemberSettingsTestCase { + description: "SQLite members keep their own settings", + backend: Backend::Sqlite, + members: [("staging", true, 2), ("marts", false, 0)], + expected_group_status: RunStatus::Success, + expected_group_ignored: 2, + }, + GroupMemberSettingsTestCase { + description: "PostgreSQL members keep their own settings", + backend: Backend::Postgres, + members: [("staging", false, 0), ("marts", true, 3)], + expected_group_status: RunStatus::Success, + expected_group_ignored: 3, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let [ + (first_job, first_ignores, _), + (second_job, second_ignores, _), + ] = test_case.members; + let mut input = snapshot(location); + input.documents.insert( + "jobs".into(), + json!({ + first_job: {"description": "Stage customers", "selection": ["customers"], + "settings": {"ignore_audit_failures": first_ignores, "concurrency": 2}}, + second_job: {"description": "Build orders", "selection": ["orders"], + "settings": {"ignore_audit_failures": second_ignores}} + }), + ); + input.documents.insert( + "groups".into(), + json!({"daily_orders": { + "description": "Both members at once", "selection": ["customers", "orders"], + "jobs": {first_job: [], second_job: []} + }}), + ); + publish_snapshot(store, &input) + .await + .expect("publish group"); + let request = JobRunRequest { + run_id: Uuid::now_v7(), + location_id: location, + target: "development".into(), + job_key: "daily_orders".into(), + reason: "Build the group".into(), + priority: 1, + window: None, + full_refresh: true, + select: Vec::new(), + exclude: Vec::new(), + }; + request_job_run(store, (&context, None), &request) + .await + .expect("request group"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::new(); + let scope = (location, "development"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("start virtual parent"); + let mut settings = BTreeMap::new(); + for _ in test_case.members { + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim member") + .expect("member ready"); + let job = claim.request.job_key.clone().expect("member job"); + let (_, _, ignored) = test_case + .members + .into_iter() + .find(|(name, _, _)| *name == job) + .expect("declared member"); + settings.insert(job, claim.request.settings.clone()); + succeed_ignoring_audit_failures(store, &claim, ignored).await; + } + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish parent"); + + assert_eq!( + settings, + BTreeMap::from([ + ( + first_job.to_owned(), + BTreeMap::from([ + ("concurrency".to_owned(), json!(2)), + ("full_refresh".to_owned(), json!(true)), + ("ignore_audit_failures".to_owned(), json!(first_ignores)), + ]), + ), + ( + second_job.to_owned(), + BTreeMap::from([ + ("full_refresh".to_owned(), json!(true)), + ("ignore_audit_failures".to_owned(), json!(second_ignores)), + ]), + ), + ]), + "{}: each member runs with its own settings and the group's full refresh", + test_case.description + ); + let runs = list_runs(store, scope, 100).await.expect("runs"); + let parent = runs + .iter() + .find(|run| run.run_id == request.run_id) + .expect("parent"); + assert_eq!( + ( + parent.status, + parent.ignored_audit_failures, + parent + .node_counts_json + .contains_key("audit_failures_ignored"), + ), + ( + test_case.expected_group_status, + test_case.expected_group_ignored, + false, + ), + "{}: the group succeeds with its members' ignored failures summed", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs b/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs index f0cbec653a..ebc26a0c8a 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs @@ -1,6 +1,7 @@ //! One job's declaration reads like the job list; a full-refresh launch needs run.rewrite_history down to group children and records it. use std::collections::BTreeMap; +use std::num::NonZeroU32; use serde_json::json; use uuid::Uuid; @@ -12,15 +13,17 @@ use crate::credentials::tests::role_helpers::{ }; use crate::declarations::main::list_person_jobs::list_person_jobs; use crate::declarations::main::read_person_job::read_person_job; +use crate::declarations::models::ServerPoolCaps; use crate::runs::_helpers::requests::job::frozen_group_children; use crate::runs::main::{ - advance_groups::advance_groups, list_runs::list_runs, request_person_run::request_person_run, + execution::advance_groups::advance_groups, listings::list_runs::list_runs, + requests::request_person_run::request_person_run, }; use crate::runs::models::{JobRunRequest, PersonRunRequest, RunRecord, RunRequest, RunWindow}; use crate::runs::tests::helpers::{json_column, rows, snapshot}; use crate::runs::tests::test_types::{ExactKeyPlanner, JobDetailTestCase}; use crate::scheduler::tests::contracts::helpers::acquire_lease; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::store::main::{begin::begin, commit::commit}; @@ -68,6 +71,8 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r "select": "orders", "targets": ["development"], "declared_at": {"path": "orchestration/jobs.py", "line": 7}, "owner": "Orders team", "overlap": "skip", + "pools": {"warehouse": {"amount": 3, "limit": 4}}, + "resolved_concurrency": {"limit": 4, "source": "job"}, "on_failure": {"retries": 2, "retry_delay_seconds": 300}}, "customers": {"description": "Build customers", "selection": ["customers"]}, })); @@ -75,7 +80,10 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r "groups".into(), json!({"orders_group": { "description": "Orders, then customers", "selection": ["customers", "orders"], - "jobs": {"customers": ["daily_orders"], "daily_orders": []}}}), + "pools": {"warehouse": {"amount": 1, "limit": 4}}, + "jobs": {"customers": ["daily_orders"], "daily_orders": []}, + "wait_reasons": {"customers": [ + {"upstream_job": "daily_orders", "from_node": "orders", "to_node": "customers"}]}}}), ); input .nodes @@ -91,12 +99,18 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r publish_snapshot(store, &input).await.expect("publish jobs"); let administrator = (&roles.context, credentials(&roles.administrator)); let location = roles.location_id; + let caps = ServerPoolCaps { + launchers: BTreeMap::from([( + (location, "development".to_owned()), + BTreeMap::from([("warehouse".to_owned(), NonZeroU32::new(2).expect("cap"))]), + )]), + }; let read = |who, job: &'static str| { read_person_job( store, who, (location, "development", job), - DefaultRole::None, + (DefaultRole::None, &caps), ) }; @@ -144,7 +158,7 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r store, administrator, (location, "development"), - DefaultRole::None, + (DefaultRole::None, &caps), ) .await .expect("administrator lists jobs"); @@ -203,6 +217,31 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r "{}: members in run order, not name order, with their waits", test_case.description ); + assert_eq!( + ( + json!(group.members[1].reasons), + json!(job.pools), + (job.concurrency, json!(job.resolved_concurrency)), + json!(listed[1].pools), + (listed[1].concurrency, json!(listed[1].resolved_concurrency)), + json!(group.pools), + json!(group.resolved_concurrency) + ), + ( + json!([{"upstream_job": "daily_orders", "from_node": "orders", "to_node": "customers"}]), + json!([{"pool": "warehouse", "amount": 3, "limit": 4, "cap": 2, "server_limit": 2, + "clamped": true, "shared_with": ["orders_group"]}]), + (Some(4), json!({"limit": 4, "source": "job"})), + json!([{"pool": "warehouse", "amount": 3, "limit": 4, "cap": 2, "server_limit": 2, + "clamped": true, "shared_with": ["orders_group"]}]), + (Some(4), json!({"limit": 4, "source": "job"})), + json!([{"pool": "warehouse", "amount": 1, "limit": 4, "cap": 2, "server_limit": 2, + "clamped": false, "shared_with": ["daily_orders"]}]), + json!(null) + ), + "{}: a member's node-level reasons, each pool against this server's cap with what shares it, and the run's concurrency", + test_case.description + ); assert!( read(administrator, "nightly") .await diff --git a/crates/sqlbuild-orchestration/tests/runs/test_job_subsets.rs b/crates/sqlbuild-orchestration/tests/runs/test_job_subsets.rs index 26cae64df5..d3d17e8007 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_job_subsets.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_job_subsets.rs @@ -10,12 +10,12 @@ use crate::credentials::tests::location_helpers::create_location; use crate::credentials::tests::role_helpers::{ SUPPORT_ROLE, credentials, orders_grant, role_fixture, }; -use crate::runs::main::change_run::change_run; -use crate::runs::main::list_person_runs::list_person_runs; -use crate::runs::main::list_runs::list_runs; -use crate::runs::main::request_job_run::request_job_run; -use crate::runs::main::request_person_run::request_person_run; -use crate::runs::main::request_rerun::request_rerun; +use crate::runs::main::execution::change_run::change_run; +use crate::runs::main::listings::list_person_runs::list_person_runs; +use crate::runs::main::listings::list_runs::list_runs; +use crate::runs::main::requests::request_job_run::request_job_run; +use crate::runs::main::requests::request_person_run::request_person_run; +use crate::runs::main::requests::request_rerun::request_rerun; use crate::runs::models::{ JobRunRequest, PersonRunRequest, RerunRequest, RunAuthority, RunChange, RunChangeRequest, RunKind, RunNarrowing, RunRequest, @@ -25,7 +25,7 @@ use crate::runs::tests::test_types::{ ExactKeyPlanner, JobSubsetTestCase, NamedRerunTestCase, NarrowingDenialTestCase, }; use crate::runs::types::RunsListing; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::store::models::Parameter; @@ -110,7 +110,7 @@ fn given_a_published_job_when_a_run_narrows_it_then_only_the_jobs_own_selected_n .definition["deps"] = json!(["orders"]); input.documents.insert("jobs".into(), json!({ "daily_orders": {"description": "Orders and stock", "selection": ["customers", "inventory", "orders"], - "concurrency_groups": ["orders-load"]}, + "pools": {"orders-load": {"amount": 1, "limit": 100}}}, "customer_sync": {"description": "Customers", "selection": ["customers"]}, })); input.documents.insert( @@ -184,7 +184,7 @@ fn given_a_published_job_when_a_run_narrows_it_then_only_the_jobs_own_selected_n frozen.job_key, frozen.selection, frozen.write_set, - frozen.concurrency_groups, + frozen.pools.keys().cloned().collect::>(), )), test_case.expected_selection.map(|expected| ( RunKind::Job, diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs index fcc3bb4fca..ac859a9b89 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -2,17 +2,17 @@ use std::collections::BTreeMap; use crate::credentials::models::AuditContext; use crate::platform::models::PlatformEvent; -use crate::runs::main::change_run::change_run; -use crate::runs::main::claim_next_run::claim_next_run; -use crate::runs::main::heartbeat_run::heartbeat_run; -use crate::runs::main::list_run_nodes::list_run_nodes; -use crate::runs::main::list_runs::list_runs; +use crate::runs::main::execution::change_run::change_run; +use crate::runs::main::execution::claim_next_run::claim_next_run; +use crate::runs::main::execution::heartbeat_run::heartbeat_run; +use crate::runs::main::listings::list_run_nodes::list_run_nodes; +use crate::runs::main::listings::list_runs::list_runs; use crate::runs::models::{ NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunKind, RunNodeFailure, RunOutcome, RunStatus, }; use crate::scheduler::tests::contracts::helpers::acquire_lease; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::helpers::read_recall_events; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs b/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs index 1f874b411e..83d2f805f2 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs @@ -3,15 +3,15 @@ use std::collections::BTreeMap; use std::time::Duration; -use crate::runs::main::change_run::change_run; -use crate::runs::main::claim_next_run::claim_next_run; -use crate::runs::main::heartbeat_run::heartbeat_run; -use crate::runs::main::list_runs::list_runs; -use crate::runs::main::monitor_runs::monitor_runs; +use crate::runs::main::execution::change_run::change_run; +use crate::runs::main::execution::claim_next_run::claim_next_run; +use crate::runs::main::execution::heartbeat_run::heartbeat_run; +use crate::runs::main::execution::monitor_runs::monitor_runs; +use crate::runs::main::listings::list_runs::list_runs; use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus}; use crate::scheduler::models::SchedulerState; use crate::scheduler::tests::contracts::helpers::{acquire_lease, scheduler_cycle}; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::helpers::read_recall_events; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_node_errors.rs b/crates/sqlbuild-orchestration/tests/runs/test_node_errors.rs index e8572b9594..76bb999bc8 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_node_errors.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_node_errors.rs @@ -6,13 +6,13 @@ use serde_json::json; use uuid::Uuid; use crate::credentials::tests::role_helpers::{credentials, orders_grant, role_fixture}; -use crate::runs::main::change_run::change_run; -use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::execution::change_run::change_run; +use crate::runs::main::execution::claim_next_run::claim_next_run; use crate::runs::models::{RunAuthority, RunChangeRequest}; use crate::runs::tests::helpers::{queue, request, rows, snapshot}; use crate::runs::tests::test_types::RunNodeErrorTestCase; use crate::scheduler::tests::contracts::helpers::acquire_lease; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::store::models::Parameter; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs b/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs index 187b602e8a..f1845cd435 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs @@ -17,9 +17,10 @@ use crate::credentials::models::{ }; use crate::credentials::tests::location_helpers::create_location; use crate::runs::main::{ - advance_groups::advance_groups, change_run::change_run, list_person_runs::list_person_runs, - list_runs::list_runs, request_job_run::request_job_run, request_person_run::request_person_run, - request_rerun::request_rerun, + execution::advance_groups::advance_groups, execution::change_run::change_run, + listings::list_person_runs::list_person_runs, listings::list_runs::list_runs, + requests::request_job_run::request_job_run, requests::request_person_run::request_person_run, + requests::request_rerun::request_rerun, }; use crate::runs::models::{ CancelRunRequest, JobRunRequest, PersonRunRequest, RerunRequest, RunAuthority, RunChange, @@ -29,7 +30,7 @@ use crate::runs::tests::helpers::{request_run, snapshot}; use crate::runs::tests::test_types::{ExactKeyPlanner, PersonRunPermissionTestCase}; use crate::runs::types::RunsListing; use crate::scheduler::tests::contracts::helpers::acquire_lease; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_project_limits.rs b/crates/sqlbuild-orchestration/tests/runs/test_project_limits.rs new file mode 100644 index 0000000000..2921cf2c5a --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_project_limits.rs @@ -0,0 +1,204 @@ +use std::collections::BTreeMap; +use std::num::NonZeroU32; + +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::models::PoolSlots; +use crate::runs::tests::claim_helpers::claim_exactly; +use crate::runs::tests::claim_helpers::slot_counts; +use crate::runs::tests::helpers::pool_slots; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::test_types::{PoolAmountTestCase, ProjectLimitTestCase}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_project_limit_and_a_server_cap_when_claiming_then_each_location_runs_up_to_the_lower_of_them() + { + let test_cases = [ + ProjectLimitTestCase { + description: "the project's limit holds without a server cap", + backend: Backend::Sqlite, + project_limit: 2, + server_cap: None, + expected_running_per_location: 2, + }, + ProjectLimitTestCase { + description: "a lower server cap lowers the project's limit", + backend: Backend::Sqlite, + project_limit: 2, + server_cap: Some(1), + expected_running_per_location: 1, + }, + ProjectLimitTestCase { + description: "a higher server cap never raises the project's limit", + backend: Backend::Postgres, + project_limit: 1, + server_cap: Some(3), + expected_running_per_location: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate run store"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limit = NonZeroU32::new(test_case.project_limit).expect("positive limit"); + let mut locations: Vec = Vec::new(); + for (index, name) in ["orders", "returns"].into_iter().enumerate() { + let location = create_location(store, name, "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + for run_index in 0..3 { + let id = u128::try_from(index * 10 + run_index + 1).expect("run id"); + let mut input = request((location, published.snapshot_id), id, 0, &["orders"]); + input.write_set.clear(); + input.pools.insert( + "warehouse".into(), + PoolSlots { + amount: NonZeroU32::MIN, + limit, + }, + ); + queue(store, &input).await; + } + locations.push(location); + } + let caps: BTreeMap = test_case + .server_cap + .map(|cap| ("warehouse".to_owned(), NonZeroU32::new(cap).expect("cap"))) + .into_iter() + .collect(); + let mut running: BTreeMap = BTreeMap::new(); + let claims = claim_exactly( + store, + &lease, + (&caps, test_case.expected_running_per_location * 2), + ) + .await; + for claim in claims { + *running.entry(claim.request.location_id).or_default() += 1; + } + assert_eq!( + locations + .iter() + .map(|location| running.get(location).copied().unwrap_or_default()) + .collect::>(), + vec![test_case.expected_running_per_location; 2], + "{}: each location's group has its own slots", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_pool_amounts_when_claiming_then_a_run_takes_all_its_slots_at_once_or_none() { + let test_cases = [ + PoolAmountTestCase { + description: "a run holding two of three slots leaves no room for another taking two", + runs: &[ + (1, 0, &[("supplier_api", 2, 3)]), + (2, 0, &[("supplier_api", 2, 3)]), + ], + caps: &[], + expected_claimed: &[1], + expected_slots: &[("supplier_api", 2)], + }, + PoolAmountTestCase { + description: "a run using two pools waits when one is full, takes nothing of the other, and goes before later runs of it", + runs: &[ + (1, 5, &[("warehouse", 1, 1)]), + (2, 0, &[("supplier_api", 1, 1), ("warehouse", 1, 1)]), + (3, 0, &[("supplier_api", 1, 1)]), + ], + caps: &[], + expected_claimed: &[1], + expected_slots: &[("warehouse", 1)], + }, + PoolAmountTestCase { + description: "a server cap below a run's amount gives it the whole capped pool", + runs: &[(1, 0, &[("supplier_api", 3, 3)])], + caps: &[("supplier_api", 2)], + expected_claimed: &[1], + expected_slots: &[("supplier_api", 2)], + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(Backend::Sqlite).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate run store"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + for (id, priority, pools) in test_case.runs { + let mut input = request( + (location, published.snapshot_id), + *id, + *priority, + &["orders"], + ); + input.write_set.clear(); + for (pool, amount, limit) in *pools { + input + .pools + .insert((*pool).to_owned(), pool_slots(*amount, *limit)); + } + queue(store, &input).await; + } + let caps: BTreeMap = test_case + .caps + .iter() + .map(|(pool, cap)| ((*pool).to_owned(), NonZeroU32::new(*cap).expect("cap"))) + .collect(); + + let claims = + claim_exactly(store, &lease, (&caps, test_case.expected_claimed.len())).await; + + assert_eq!( + claims + .iter() + .map(|claim| claim.request.run_id.as_u128()) + .collect::>(), + test_case.expected_claimed, + "{}", + test_case.description + ); + assert_eq!( + slot_counts(store).await.into_iter().collect::>(), + test_case + .expected_slots + .iter() + .map(|(pool, count)| ((*pool).to_owned(), *count)) + .collect::>(), + "{}: a run holds as many slots as it takes", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_publication_cache.rs b/crates/sqlbuild-orchestration/tests/runs/test_publication_cache.rs new file mode 100644 index 0000000000..40c2637018 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_publication_cache.rs @@ -0,0 +1,232 @@ +//! A publication is indexed once per snapshot, shared by concurrent first requests and replaced by newer publications. + +use std::cell::Cell; +use std::sync::{Arc, Mutex}; + +use serde_json::json; +use tokio::sync::oneshot; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::runs::_helpers::lifecycle::publications::cached; +use crate::runs::main::requests::check_adhoc_run::check_adhoc_run; +use crate::runs::models::{AdhocCheckRequest, SnapshotSlots}; +use crate::runs::tests::helpers::{ + adhoc_fixture, cache_id, cache_lookup, check_projection, snapshot, +}; +use crate::runs::tests::test_types::{ + CacheLookupTestCase, ConcurrentLookupTestCase, ExactKeyPlanner, FailedBuildTestCase, + NewerPublicationTestCase, +}; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; +use crate::snapshots::models::SnapshotNode; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_snapshot_lookups_when_cached_then_each_is_built_once_until_two_newer_snapshots_of_its_target_replace_it() + { + let test_cases = [ + CacheLookupTestCase { + description: "a repeated lookup reuses the value its first lookup built", + lookups: &[(1, 1), (1, 1)], + expected_built: &[true, false], + }, + CacheLookupTestCase { + description: "the current snapshot and the one before it stay cached together", + lookups: &[(1, 1), (1, 2), (1, 1), (1, 2)], + expected_built: &[true, true, false, false], + }, + CacheLookupTestCase { + description: "a third snapshot of one target evicts its least recently used", + lookups: &[(1, 1), (1, 2), (1, 3), (1, 2), (1, 1)], + expected_built: &[true, true, true, false, true], + }, + CacheLookupTestCase { + description: "another target's publications never evict this target's", + lookups: &[(1, 1), (2, 2), (2, 3), (2, 4), (1, 1)], + expected_built: &[true, true, true, true, false], + }, + ]; + for test_case in test_cases { + run(async { + let slots = Mutex::new(SnapshotSlots::new()); + let mut values: Vec = Vec::new(); + let mut built: Vec = Vec::new(); + for lookup in test_case.lookups { + let (value, was_built) = cache_lookup(&slots, *lookup).await; + values.push(value); + built.push(was_built); + } + let snapshots: Vec = test_case + .lookups + .iter() + .map(|(_, snapshot)| *snapshot) + .collect(); + assert_eq!( + (values, built.as_slice()), + (snapshots, test_case.expected_built), + "{}: each lookup gets its snapshot's value", + test_case.description + ); + }); + } +} + +#[test] +fn given_concurrent_first_lookups_when_one_is_building_then_the_others_wait_for_its_value() { + let test_cases = [ConcurrentLookupTestCase { + description: "the second lookup waits for the first build and shares its value", + first_value: 7, + second_value: 8, + expected_outcome: (7, 7, 1, true), + }]; + for test_case in test_cases { + run(async { + let slots: Mutex> = Mutex::new(SnapshotSlots::new()); + let builds = Cell::new(0); + let (release, released) = oneshot::channel::<()>(); + let key = || ((cache_id(1), "development".to_owned()), cache_id(1)); + let first = cached(&slots, key(), (), |()| async { + builds.set(builds.get() + 1); + released.await.expect("release the build"); + Ok(((), test_case.first_value)) + }); + let second = cached(&slots, key(), (), |()| async { + builds.set(builds.get() + 1); + Ok(((), test_case.second_value)) + }); + let releasing = async { + tokio::task::yield_now().await; + release.send(()).expect("build still waiting"); + }; + let (first, second, ()) = tokio::join!(first, second, releasing); + let ((), first) = first.expect("first lookup"); + let ((), second) = second.expect("second lookup"); + assert_eq!( + (*first, *second, builds.get(), Arc::ptr_eq(&first, &second)), + test_case.expected_outcome, + "{}", + test_case.description + ); + }); + } +} + +#[test] +fn given_a_failed_build_when_looked_up_again_then_nothing_was_cached() { + let test_cases = [FailedBuildTestCase { + description: "a failed build leaves the next lookup to build again", + failure: "store unavailable", + retried_value: 3, + expected_outcome: ( + Some("invalid control-plane data: store unavailable"), + Some(3), + ), + }]; + for test_case in test_cases { + run(async { + let slots: Mutex> = Mutex::new(SnapshotSlots::new()); + let key = || ((cache_id(1), "development".to_owned()), cache_id(1)); + let failed = cached(&slots, key(), (), |()| async { + Err::<((), u8), _>(ControlError::Invalid(test_case.failure.into())) + }) + .await; + let retried = cached(&slots, key(), (), |()| async { + Ok(((), test_case.retried_value)) + }) + .await; + assert_eq!( + ( + failed.err().map(|error| error.to_string()), + retried.ok().map(|((), value)| *value) + ), + ( + test_case.expected_outcome.0.map(str::to_owned), + test_case.expected_outcome.1 + ), + "{}", + test_case.description + ); + }); + } +} + +#[test] +fn given_a_newer_publication_when_a_command_is_checked_again_then_it_resolves_over_the_new_snapshot() + { + let test_cases = [NewerPublicationTestCase { + description: "a node the newer publication adds is selectable once it is current", + command: &["build", "model:invoices"], + added_node: "model:invoices", + expected_before_error: "model:invoices is not available for ad-hoc runs", + expected_after_selection: &["model:invoices"], + }]; + for test_case in test_cases { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let (location_id, _, people) = adhoc_fixture(store, &context).await; + let request = AdhocCheckRequest { + location_id, + target: "development".into(), + command: test_case + .command + .iter() + .map(|token| (*token).to_owned()) + .collect(), + priority: 0, + }; + let check = || async { + check_adhoc_run(store, (&context, people.admin), &ExactKeyPlanner, &request) + .await + .expect("dry run") + }; + let before = check().await; + let mut input = snapshot(location_id); + input.generation = 2; + input.nodes.insert( + test_case.added_node.into(), + SnapshotNode { + kind: "model".into(), + definition: json!({"materialization": "table"}), + raw_sql: None, + compiled_sql: Some("select 1 as id".into()), + python_source: None, + }, + ); + let newer = publish_snapshot(store, &input) + .await + .expect("publish") + .snapshot_id; + let after = check().await; + assert_eq!( + ( + check_projection(&before)["error"].clone(), + after.snapshot_id, + check_projection(&after)["selection"].clone(), + ), + ( + json!(test_case.expected_before_error), + Some(newer), + json!(test_case.expected_after_selection), + ), + "{backend:?}: {}", + test_case.description + ); + }); + } + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs index eaec5f2d71..816485dbf3 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs @@ -1,14 +1,14 @@ //! Published-job requests freeze execution inputs and audit atomically on both databases. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use crate::credentials::models::AuditContext; -use crate::runs::main::claim_next_run::claim_next_run; -use crate::runs::main::list_runs::list_runs; -use crate::runs::main::request_job_run::request_job_run; +use crate::runs::main::execution::claim_next_run::claim_next_run; +use crate::runs::main::listings::list_runs::list_runs; +use crate::runs::main::requests::request_job_run::request_job_run; use crate::runs::models::{JobRunRequest, RunOrigin, RunStatus, RunWindow}; use crate::scheduler::tests::contracts::helpers::acquire_lease; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::helpers::read_recall_events; use serde_json::json; @@ -58,7 +58,7 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an input.nodes.insert("warehouse_orders".into(), source); input.documents.insert("jobs".into(), json!({"daily_orders": { "description": "Build orders from the warehouse source", "selection": ["orders", "warehouse_orders"], - "settings": {"full_refresh": true}, "concurrency_groups": ["orders-load"] + "settings": {"full_refresh": true}, "pools": {"orders-load": {"amount": 1, "limit": 100}} }})); let published = publish_snapshot(store, &input).await.expect("publish job"); let request = JobRunRequest { @@ -227,8 +227,13 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an BTreeMap::from([("full_refresh".into(), json!(true))]) ); assert_eq!( - claim.request.concurrency_groups, - ["orders-load".into()].into() + claim + .request + .pools + .keys() + .cloned() + .collect::>(), + BTreeSet::from(["orders-load".to_owned()]) ); assert_eq!(claim.request.priority, 7); assert_eq!(claim.request.requested_by, "test-operator"); diff --git a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs index 339285d199..214fdd53e1 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs @@ -4,15 +4,16 @@ use std::collections::{BTreeMap, BTreeSet}; use crate::credentials::models::AuditContext; use crate::runs::main::{ - advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, - list_runs::list_runs, request_job_run::request_job_run, request_rerun::request_rerun, + execution::advance_groups::advance_groups, execution::change_run::change_run, + execution::claim_next_run::claim_next_run, listings::list_runs::list_runs, + requests::request_job_run::request_job_run, requests::request_rerun::request_rerun, }; use crate::runs::models::{ JobRunRequest, NodeOutcome, RerunRequest, RunAuthority, RunChange, RunChangeRequest, RunOrigin, RunOutcome, RunStatus, RunWindow, }; use crate::scheduler::tests::contracts::helpers::acquire_lease; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use serde_json::json; use uuid::Uuid; @@ -79,13 +80,13 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ .expect("inventory model") .definition["deps"] = json!(["products"]); input.documents.insert("jobs".into(), json!({ - "staging": {"description": "Stage orders", "selection": ["orders", "customers"], "concurrency_groups": ["warehouse"], "settings": {"vars": {"batch": 23}}}, - "marts": {"description": "Build inventory", "selection": ["inventory"], "concurrency_groups": ["inventory_limit"]}, + "staging": {"description": "Stage orders", "selection": ["orders", "customers"], "pools": {"warehouse": {"amount": 1, "limit": 100}}, "settings": {"vars": {"batch": 23}}}, + "marts": {"description": "Build inventory", "selection": ["inventory"], "pools": {"inventory_limit": {"amount": 1, "limit": 100}}}, "completed": {"description": "Build products and shipments", "selection": ["products", "shipments"]} })); input.documents.insert("groups".into(), json!({"daily_orders": { "description": "Build daily orders", "selection": ["customers", "orders", "inventory", "products", "shipments"], - "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["completed"], "completed": ["staging"]} + "pools": {"warehouse": {"amount": 1, "limit": 100}}, "jobs": {"staging": [], "marts": ["completed"], "completed": ["staging"]} }})); let published = publish_snapshot(store, &input).await.expect("publish"); let source_id = Uuid::now_v7(); diff --git a/crates/sqlbuild-orchestration/tests/runs/test_retries.rs b/crates/sqlbuild-orchestration/tests/runs/test_retries.rs index d3cf93578f..2fe527cf05 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_retries.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_retries.rs @@ -9,8 +9,9 @@ use crate::credentials::models::AuditContext; use crate::credentials::tests::location_helpers::create_location; use crate::runs::_helpers::lifecycle::reads::list_runs_in; use crate::runs::main::{ - change_run::change_run, claim_next_run::claim_next_run, list_runs::list_runs, - request_job_run::request_job_run, request_rerun::request_rerun, + execution::change_run::change_run, execution::claim_next_run::claim_next_run, + listings::list_runs::list_runs, requests::request_job_run::request_job_run, + requests::request_rerun::request_rerun, }; use crate::runs::models::{ JobRunRequest, NodeOutcome, QueueWait, RerunRequest, RunAuthority, RunChange, RunChangeRequest, @@ -21,7 +22,7 @@ use crate::runs::tests::helpers::snapshot; use crate::runs::tests::test_types::RetryChainTestCase; use crate::runs::types::RunsListing; use crate::scheduler::tests::contracts::helpers::acquire_lease; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::{begin::begin, commit::commit, migrate_store::migrate_store}; use crate::store::models::TransactionMode; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_run_logs.rs b/crates/sqlbuild-orchestration/tests/runs/test_run_logs.rs index bc29b86ceb..0e76bd7df8 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_run_logs.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_run_logs.rs @@ -3,11 +3,11 @@ use serde_json::{Value, json}; use crate::credentials::tests::location_helpers::create_location; -use crate::platform::_helpers::retention::prune_run_logs; +use crate::platform::_helpers::maintenance::retention::prune_run_logs; use crate::runs::models::RunLogRetention; use crate::runs::tests::helpers::{logged_worker, rows, seed_logged_runs, snapshot}; use crate::runs::tests::test_types::RunLogRetentionTestCase; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_selection.rs b/crates/sqlbuild-orchestration/tests/runs/test_selection.rs new file mode 100644 index 0000000000..85e637d2bc --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_selection.rs @@ -0,0 +1,154 @@ +//! People who may view a target evaluate selections and read selector values against its current publication. + +use uuid::Uuid; + +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::main::requests::evaluate_selection::evaluate_selection; +use crate::runs::main::requests::read_selection_vocabulary::read_selection_vocabulary; +use crate::runs::models::{SelectionQuery, VocabularyEntry}; +use crate::runs::tests::helpers::{ + adhoc_fixture, launch_outcome, owned_selection, selection_outcome, +}; +use crate::runs::tests::test_types::{AdhocPerson, ExactKeyPlanner, SelectionRequestTestCase}; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_selection_requests_when_people_evaluate_them_then_viewers_get_pages_and_refusals() { + let test_cases = [ + SelectionRequestTestCase { + description: "a viewer pages through the selected keys in order", + person: AdhocPerson::Developer, + select: "model:orders model:customers", + exclude: "", + limit: Some(1), + expected_outcome: Ok((2, &["model:customers"], Some(1), None)), + }, + SelectionRequestTestCase { + description: "an unknown selector is refused without failing the request", + person: AdhocPerson::Developer, + select: "model:invoices", + exclude: "", + limit: None, + expected_outcome: Ok((0, &[], None, Some("unknown selector name 'model:invoices'"))), + }, + SelectionRequestTestCase { + description: "a person without view on the target is refused", + person: AdhocPerson::Stranger, + select: "model:orders", + exclude: "", + limit: None, + expected_outcome: Err(403), + }, + ]; + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let (location_id, _, people) = adhoc_fixture(store, &context).await; + for test_case in &test_cases { + let query = SelectionQuery { + select: test_case.select.into(), + exclude: test_case.exclude.into(), + offset: None, + limit: test_case.limit, + }; + let evaluated = evaluate_selection( + store, + (&ExactKeyPlanner, (&context, people.of(test_case.person))), + (location_id, "development", &query), + DefaultRole::None, + ) + .await; + assert_eq!( + evaluated + .map(|evaluation| selection_outcome(&evaluation)) + .map_err(|error| launch_outcome(Err(error)).err()), + test_case + .expected_outcome + .map(owned_selection) + .map_err(Some), + "{backend:?}: {}", + test_case.description + ); + } + }); + } +} + +#[test] +fn given_a_target_when_people_read_its_selector_values_then_they_describe_its_current_publication() +{ + let test_cases = [SelectionRequestTestCase { + description: "a viewer reads each kind's node count; a target without a publication has none", + person: AdhocPerson::Developer, + select: "", + exclude: "", + limit: None, + expected_outcome: Ok((3, &["model:2", "seed:1", "source:3"], None, None)), + }]; + for test_case in test_cases { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let (location_id, _, people) = adhoc_fixture(store, &context).await; + let unpublished = create_location(store, "customers", "test-owner") + .await + .expect("location"); + let credentials = (&context, people.of(test_case.person)); + let read = |location| { + read_selection_vocabulary( + store, + (&ExactKeyPlanner, credentials), + (location, "development"), + DefaultRole::None, + ) + }; + let published = read(location_id).await.expect("vocabulary"); + let empty = read(unpublished).await.expect("empty vocabulary"); + let kinds: Vec = published + .kinds + .iter() + .map(|VocabularyEntry { value, count }| format!("{value}:{count}")) + .collect(); + assert_eq!( + ( + Ok((published.kinds.len() as u64, kinds)), + empty.snapshot_id, + empty.kinds.len() + ), + ( + test_case + .expected_outcome + .map(|expected| (expected.0, owned_selection(expected).1)), + None, + 0 + ), + "{backend:?}: {}", + test_case.description + ); + }); + } + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index fd113a5bca..625a97e45b 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -165,15 +165,92 @@ pub(crate) struct AdhocCommandTestCase { /// The CLI's planner reduced to exact node keys, `--full-refresh`, `--no-load-sources` and `--warehouse NAME`. pub(crate) struct ExactKeyPlanner; +/// Every test's publications share one cache; snapshot IDs and scopes are unique per test. +static EXACT_KEY_PUBLICATIONS: crate::runs::models::PublicationCache = + crate::runs::models::PublicationCache::new(); + impl crate::runs::types::CommandPlanner for ExactKeyPlanner { - fn plan( + fn prepare( + &self, + resources: Vec, + jobs: crate::runs::types::PublishedJobs, + ) -> std::sync::Arc { + std::sync::Arc::new(ExactKeyPublication { resources, jobs }) + } + + fn publications(&self) -> &crate::runs::models::PublicationCache { + &EXACT_KEY_PUBLICATIONS + } +} + +/// One publication as [`ExactKeyPlanner`] prepared it. +#[derive(Debug)] +pub(crate) struct ExactKeyPublication { + resources: Vec, + jobs: crate::runs::types::PublishedJobs, +} + +impl crate::runs::types::PreparedPublication for ExactKeyPublication { + /// Exact keys only: each token names one published key, and an unknown one is refused with its position. + fn evaluate( &self, - command: &[String], - (resources, jobs): ( - &[crate::runs::models::PlannerResource], - &crate::runs::types::PublishedJobs, - ), - ) -> Result { + select: &str, + exclude: &str, + ) -> Result, Box> { + let known = |field: crate::runs::models::SelectionField, text: &str| { + let mut keys: std::collections::BTreeSet = Default::default(); + for token in text.split(' ').filter(|token| !token.is_empty()) { + if !self.resources.iter().any(|resource| resource.key == token) { + let start = text.find(token).unwrap_or_default(); + return Err(Box::new(crate::runs::models::SelectionProblem { + code: Some("S007".into()), + message: format!("unknown selector name '{token}'"), + help: None, + location: Some(crate::runs::models::SelectionLocation { + field, + token: token.to_owned(), + start: u32::try_from(start).unwrap_or_default(), + end: u32::try_from(start + token.len()).unwrap_or_default(), + }), + suggestions: Vec::new(), + refused: None, + })); + } + keys.insert(token.to_owned()); + } + Ok(keys) + }; + let selected = known(crate::runs::models::SelectionField::Select, select)?; + let excluded = known(crate::runs::models::SelectionField::Exclude, exclude)?; + let selected = if select.trim().is_empty() { + self.resources + .iter() + .map(|resource| resource.key.clone()) + .collect() + } else { + selected + }; + Ok(selected.difference(&excluded).cloned().collect()) + } + + /// One entry per published kind. + fn vocabulary(&self) -> crate::runs::models::SelectionVocabulary { + let mut kinds: std::collections::BTreeMap = Default::default(); + for resource in &self.resources { + let kind = resource.key.split_once(':').map_or("", |(kind, _)| kind); + *kinds.entry(kind.to_owned()).or_default() += 1; + } + crate::runs::models::SelectionVocabulary { + kinds: kinds + .into_iter() + .map(|(value, count)| crate::runs::models::VocabularyEntry { value, count }) + .collect(), + ..Default::default() + } + } + + fn plan(&self, command: &[String]) -> Result { + let (resources, jobs) = (&self.resources, &self.jobs); let mut plan = crate::runs::models::CommandPlan { verb: command.first().cloned().unwrap_or_default(), selection: Default::default(), @@ -308,3 +385,135 @@ pub(crate) struct RunLogRetentionTestCase { pub(crate) expected_artifacts: i64, pub(crate) expected_runs: i64, } + +/// What a member of a group does once the job it depends on has failed. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum MemberReaction { + /// It runs, and the worker skips these nodes, each because of the failed node named. + Runs(Vec<(&'static str, &'static str)>), + /// The group ends it without running: its nodes are skipped because of the failed node. + Skipped, +} + +pub(crate) struct GroupDependencyTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + /// The group's `dependency_failure` document: each member whose policy is not `partial`. + pub(crate) dependency_failure: serde_json::Value, + pub(crate) expected_marts: MemberReaction, + pub(crate) expected_inventory: MemberReaction, +} + +pub(crate) struct GroupMemberRetryTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + /// The failing member's `retry_delay_seconds`. + pub(crate) delay_seconds: u32, + /// The member events the group run's timeline shows, in order. + pub(crate) expected_messages: &'static [&'static str], +} + +pub(crate) struct RoundTripGroupTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + /// Each member's jobs it waits for, as publication freezes them. + pub(crate) expected_waits: serde_json::Value, + /// The order the group launches its members in. + pub(crate) expected_order: &'static [&'static str], +} + +pub(crate) struct ProjectLimitTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) project_limit: u32, + pub(crate) server_cap: Option, + pub(crate) expected_running_per_location: usize, +} + +pub(crate) struct WaitReasonTestCase { + pub(crate) description: &'static str, + /// Each node's upstream nodes. + pub(crate) deps: &'static [(&'static str, &'static [&'static str])], + pub(crate) jobs: serde_json::Value, + pub(crate) waits: serde_json::Value, + pub(crate) expected_reasons: serde_json::Value, +} + +pub(crate) struct DeclarationCheckTestCase { + pub(crate) description: &'static str, + pub(crate) jobs: serde_json::Value, + pub(crate) groups: serde_json::Value, + pub(crate) expected_error: &'static str, +} + +/// Lookups of `(location and target, snapshot)` by small numbers, and which ones built their value. +pub(crate) struct CacheLookupTestCase { + pub(crate) description: &'static str, + pub(crate) lookups: &'static [(u8, u8)], + pub(crate) expected_built: &'static [bool], +} + +/// Two first lookups of one snapshot, the first still building when the second arrives. +pub(crate) struct ConcurrentLookupTestCase { + pub(crate) description: &'static str, + pub(crate) first_value: u8, + pub(crate) second_value: u8, + /// The value each lookup got, how many builds ran and whether both share one value. + pub(crate) expected_outcome: (u8, u8, usize, bool), +} + +/// A lookup whose build fails, then another of the same snapshot. +pub(crate) struct FailedBuildTestCase { + pub(crate) description: &'static str, + pub(crate) failure: &'static str, + pub(crate) retried_value: u8, + /// The failed lookup's error and the retried lookup's value. + pub(crate) expected_outcome: (Option<&'static str>, Option), +} + +/// A command checked before and after a publication adds the node it names. +pub(crate) struct NewerPublicationTestCase { + pub(crate) description: &'static str, + pub(crate) command: &'static [&'static str], + pub(crate) added_node: &'static str, + pub(crate) expected_before_error: &'static str, + pub(crate) expected_after_selection: &'static [&'static str], +} + +/// A selection's count, page of keys, next offset and refusal message. +pub(crate) type ExpectedSelection = ( + u64, + &'static [&'static str], + Option, + Option<&'static str>, +); + +/// One person's selection request: the count, page, next offset and refusal message it returns, or the HTTP status refusing it. +pub(crate) struct SelectionRequestTestCase { + pub(crate) description: &'static str, + pub(crate) person: AdhocPerson, + pub(crate) select: &'static str, + pub(crate) exclude: &'static str, + pub(crate) limit: Option, + pub(crate) expected_outcome: Result, +} + +/// A queued run: its ID, priority and each pool it takes as `(pool, amount, limit)`. +pub(crate) type PooledRun = (u128, i64, &'static [(&'static str, u32, u32)]); + +pub(crate) struct PoolAmountTestCase { + pub(crate) description: &'static str, + pub(crate) runs: &'static [PooledRun], + pub(crate) caps: &'static [(&'static str, u32)], + pub(crate) expected_claimed: &'static [u128], + pub(crate) expected_slots: &'static [(&'static str, i64)], +} + +pub(crate) struct GroupMemberSettingsTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + /// Each member job's `ignore_audit_failures` setting and the audit failures its run ignores. + pub(crate) members: [(&'static str, bool, u32); 2], + pub(crate) expected_group_status: RunStatus, + pub(crate) expected_group_ignored: u32, +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_sensor_fairness.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_sensor_fairness.rs index 5259d9ee0b..b91669c637 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/test_sensor_fairness.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_sensor_fairness.rs @@ -10,7 +10,7 @@ use crate::scheduler::models::SchedulerState; use crate::scheduler::tests::test_types::SensorFairnessTestCase; use crate::sensors::models::SensorEvaluationOutcome; use crate::sensors::tests::helpers::{evaluations, snapshot}; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::begin::begin; use crate::store::main::commit::commit; use crate::store::main::migrate_store::migrate_store; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs index 51a15ed7ee..0546953fdf 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -6,14 +6,14 @@ use std::sync::LazyLock; use chrono::{DateTime, DurationRound, SecondsFormat, TimeDelta, Timelike, Utc}; use crate::declarations::main::list_triggers::list_triggers; -use crate::runs::main::list_runs::list_runs; +use crate::runs::main::listings::list_runs::list_runs; use crate::scheduler::main::_acquire_lease::acquire_lease; use crate::scheduler::main::_evaluate_trigger::evaluate_trigger; use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; use crate::scheduler::main::list_ticks::list_ticks; use crate::scheduler::main::run_scheduler::run_scheduler; use crate::scheduler::models::{SchedulerLease, SchedulerState, TickOutcome, TickRecord}; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::snapshots::models::{SnapshotInput, SnapshotNode}; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs index dd01a3869f..808abcb4e8 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs @@ -5,7 +5,7 @@ use std::time::Duration; use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; use crate::scheduler::main::run_scheduler::run_scheduler; use crate::scheduler::models::SchedulerState; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use uuid::Uuid; @@ -77,10 +77,13 @@ fn given_embedded_and_standalone_schedulers_when_taking_over_then_only_one_drain .1 .leader ); - let runs = - crate::runs::main::list_runs::list_runs(store, (location, "development"), 100) - .await - .expect("queued run"); + let runs = crate::runs::main::listings::list_runs::list_runs( + store, + (location, "development"), + 100, + ) + .await + .expect("queued run"); assert_eq!(runs.len(), test_case.expected_runs); let (stop, shutdown) = tokio::sync::watch::channel(false); let mut running = diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_precision.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_precision.rs index 860c124377..df7b6da1ed 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_precision.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_precision.rs @@ -9,7 +9,7 @@ use crate::declarations::main::list_triggers::list_triggers; use crate::scheduler::main::{_acquire_lease::acquire_lease, _evaluate_trigger::evaluate_trigger}; use crate::scheduler::tests::contracts::ticks::helpers::{set_trigger_state, snapshot, tick_times}; use crate::scheduler::tests::contracts::ticks::test_types::FirstTickPrecisionTestCase; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_progress.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_progress.rs index 713f0abfa4..5a1fdf9e71 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_progress.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_progress.rs @@ -13,7 +13,7 @@ use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; use crate::scheduler::main::{_acquire_lease::acquire_lease, _evaluate_trigger::evaluate_trigger}; use crate::scheduler::tests::contracts::ticks::helpers::{calendar_time, snapshot}; use crate::scheduler::tests::contracts::ticks::test_types::TriggerControlsTestCase; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::{begin::begin, commit::commit, migrate_store::migrate_store}; use crate::store::models::TransactionMode; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs index 71f3ed3af2..d243a9c5a6 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs @@ -8,7 +8,7 @@ use crate::credentials::models::AuditContext; use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::{change_trigger::change_trigger, list_triggers::list_triggers}; use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; -use crate::runs::main::list_runs::list_runs; +use crate::runs::main::listings::list_runs::list_runs; use crate::scheduler::_helpers::tick_retention::prune_ticks; use crate::scheduler::main::{ _acquire_lease::acquire_lease, _evaluate_trigger::evaluate_trigger, list_ticks::list_ticks, @@ -22,7 +22,7 @@ use crate::scheduler::tests::contracts::ticks::helpers::{ use crate::scheduler::tests::contracts::ticks::test_types::{ LatestCatchUpTestCase, TickRetentionTestCase, }; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs index b502477298..0801175ede 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs @@ -7,7 +7,7 @@ use crate::declarations::main::list_triggers::list_triggers; use crate::declarations::models::{TriggerChange, TriggerChangeRequest, TriggerStatus}; use crate::scheduler::main::_acquire_lease::acquire_lease; use crate::scheduler::main::_evaluate_trigger::evaluate_trigger; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use uuid::Uuid; @@ -248,7 +248,7 @@ fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_ #[test] fn given_a_calendar_with_unprocessed_due_times_when_refreshed_then_due_ticks_survive_and_only_future_times_are_replaced() { - use crate::runs::main::list_runs::list_runs; + use crate::runs::main::listings::list_runs::list_runs; use crate::scheduler::main::list_ticks::list_ticks; use serde_json::json; diff --git a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs index 334d1b9e2a..c72d94e524 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs @@ -7,8 +7,8 @@ use crate::declarations::main::change_trigger::change_trigger; use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; use crate::platform::models::PlatformEvent; use crate::runs::main::{ - advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, - request_job_run::request_job_run, + execution::advance_groups::advance_groups, execution::change_run::change_run, + execution::claim_next_run::claim_next_run, requests::request_job_run::request_job_run, }; use crate::runs::models::{ JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, @@ -262,3 +262,43 @@ pub(crate) async fn finish_claimed_run( .await .expect("complete parent"); } + +pub(crate) fn node_finished(node_key: &str, outcome: NodeOutcome) -> RunChange { + RunChange::NodeFinished { + node_key: node_key.into(), + outcome, + rows_affected: None, + error_code: None, + skipped_because_of: None, + error_message: None, + } +} + +/// Evaluates each trigger whose evaluation is due and returns how many runs each one requested. +pub(crate) async fn evaluate_sensors( + store: &ControlStore, + lease: &SchedulerLease, + triggers: &[crate::declarations::models::TriggerRecord], +) -> std::collections::BTreeMap { + let mut fired = std::collections::BTreeMap::new(); + for trigger in triggers { + let Some(requested) = + crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation( + store, + lease, + trigger.trigger_id, + ) + .await + .expect("begin evaluation") + else { + continue; + }; + let count = crate::sensors::main::complete_builtin_sensor::complete_builtin_sensor( + store, lease, &requested, + ) + .await + .expect("complete evaluation"); + *fired.entry(trigger.name.clone()).or_default() += count; + } + fired +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs index 1148fac76e..f7a7da35da 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs @@ -8,6 +8,8 @@ mod test_custom; mod test_delivery; #[path = "test_execution_bounds.rs"] mod test_execution_bounds; +#[path = "test_group_members.rs"] +mod test_group_members; #[path = "test_nodes.rs"] mod test_nodes; #[path = "test_person_state.rs"] diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs b/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs index d21d8b9706..593b2e8a92 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs @@ -11,13 +11,13 @@ use crate::credentials::main::tokens::issue_token::issue_token; use crate::credentials::models::{AuditContext, CredentialScope}; use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::list_triggers::list_triggers; -use crate::platform::main::connect_location_work::connect_location_work; -use crate::platform::main::report_location_health::report_location_health; +use crate::platform::main::health::report_location_health::report_location_health; +use crate::platform::main::work::connect_location_work::connect_location_work; use crate::platform::models::{ LocationHealth, LocationHealthReport, LocationWork, LocationWorkOutcome, LocationWorkPoll, LocationWorkResult, }; -use crate::runs::main::list_runs::list_runs; +use crate::runs::main::listings::list_runs::list_runs; use crate::runs::models::RunOrigin; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; @@ -25,7 +25,7 @@ use crate::sensors::main::complete_custom_sensor::complete_custom_sensor; use crate::sensors::models::{CustomSensorOutcome, SensorEvaluationOutcome}; use crate::sensors::tests::helpers::{evaluations, snapshot}; use crate::sensors::tests::test_types::{CustomSensorResultCase, SensorTestCase}; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_delivery.rs b/crates/sqlbuild-orchestration/tests/sensors/test_delivery.rs index 2116afd00d..bce1994270 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_delivery.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_delivery.rs @@ -9,8 +9,8 @@ use crate::credentials::models::AuditContext; use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::list_triggers::list_triggers; use crate::declarations::models::TriggerChange; -use crate::runs::main::list_runs::list_runs; -use crate::runs::main::request_job_run::request_job_run; +use crate::runs::main::listings::list_runs::list_runs; +use crate::runs::main::requests::request_job_run::request_job_run; use crate::runs::models::JobRunRequest; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; @@ -18,7 +18,7 @@ use crate::sensors::main::complete_builtin_sensor::complete_builtin_sensor; use crate::sensors::models::SensorEvaluationOutcome; use crate::sensors::tests::helpers::{append_events, change, evaluations, snapshot}; use crate::sensors::tests::test_types::SensorTestCase; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::fault_types::TransactionFault; diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs b/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs index 35b450e461..2a5598c1c8 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs @@ -9,8 +9,8 @@ use crate::credentials::models::AuditContext; use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::list_triggers::list_triggers; use crate::runs::main::{ - advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, - request_job_run::request_job_run, + execution::advance_groups::advance_groups, execution::change_run::change_run, + execution::claim_next_run::claim_next_run, requests::request_job_run::request_job_run, }; use crate::runs::models::{ JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, @@ -22,7 +22,7 @@ use crate::sensors::main::{ }; use crate::sensors::tests::helpers::snapshot; use crate::sensors::tests::test_types::SensorTestCase; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::lease_helpers::extend_lease_through_test; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_group_members.rs b/crates/sqlbuild-orchestration/tests/sensors/test_group_members.rs new file mode 100644 index 0000000000..445a617f1c --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_group_members.rs @@ -0,0 +1,566 @@ +//! A member job's outcome inside a group run is a run event of that job, as soon as the member ends. + +use std::collections::BTreeMap; +use std::time::Duration; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::runs::main::{ + execution::advance_groups::advance_groups, execution::change_run::change_run, + execution::claim_next_run::claim_next_run, listings::list_runs::list_runs, + requests::request_job_run::request_job_run, +}; +use crate::runs::models::{ + JobRunRequest, NodeOutcome, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use crate::runs::tests::claim_helpers::finish; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::tests::helpers::{evaluate_sensors, node_finished}; +use crate::sensors::tests::test_types::{ + GroupMemberSensorTestCase, SensorTestCase, SkippedMemberSensorTestCase, +}; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_member_job_watched_by_a_sensor_when_its_group_runs_then_the_member_outcome_fires_once_while_the_group_runs() + { + let test_cases = [ + GroupMemberSensorTestCase { + description: "SQLite member succeeds", + backend: Backend::Sqlite, + staging: (NodeOutcome::Success, RunOutcome::Success), + marts: (NodeOutcome::Success, RunOutcome::Success), + expected_firing: &["staging_success"], + expected_after_marts: &["staging_success"], + }, + GroupMemberSensorTestCase { + description: "PostgreSQL member succeeds", + backend: Backend::Postgres, + staging: (NodeOutcome::Success, RunOutcome::Success), + marts: (NodeOutcome::Success, RunOutcome::Success), + expected_firing: &["staging_success"], + expected_after_marts: &["staging_success"], + }, + GroupMemberSensorTestCase { + description: "SQLite member fails and its dependent is skipped", + backend: Backend::Sqlite, + staging: (NodeOutcome::Failed, RunOutcome::Failed), + marts: (NodeOutcome::Skipped, RunOutcome::Failed), + expected_firing: &["staging_failed"], + expected_after_marts: &["marts_failed", "staging_failed"], + }, + GroupMemberSensorTestCase { + description: "PostgreSQL member fails and its dependent is skipped", + backend: Backend::Postgres, + staging: (NodeOutcome::Failed, RunOutcome::Failed), + marts: (NodeOutcome::Skipped, RunOutcome::Failed), + expected_firing: &["staging_failed"], + expected_after_marts: &["marts_failed", "staging_failed"], + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = crate::runs::tests::helpers::snapshot(location); + input.nodes.get_mut("orders").expect("orders").definition["deps"] = + json!(["customers"]); + input.documents.insert( + "jobs".into(), + json!({ + "staging_job": {"description": "Stage customers", "selection": ["customers"]}, + "marts_job": {"description": "Build orders", "selection": ["orders"]}, + "report_job": {"description": "Report on inventory", "selection": ["inventory"]} + }), + ); + input.documents.insert( + "groups".into(), + json!({"nightly_job_group": { + "description": "Stage then build orders", "selection": ["customers", "orders"], + "jobs": {"staging_job": [], "marts_job": ["staging_job"]} + }}), + ); + let mut sensors = serde_json::Map::new(); + for (name, job, kind) in [ + ("staging_success", "staging_job", "success"), + ("staging_failed", "staging_job", "failed"), + ("marts_failed", "marts_job", "failed"), + ("group_completed", "nightly_job_group", "completed"), + ] { + sensors.insert( + name.into(), + json!({ + "description": "Report when a job ends", "job_key": "report_job", + "when": {"kind": "run_event", "job_key": job, "kinds": [kind]}, + "minimum_interval_seconds": 1 + }), + ); + } + input.documents.insert("triggers".into(), json!(sensors)); + publish_snapshot(store, &input).await.expect("publish"); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let group = Uuid::now_v7(); + request_job_run( + store, + (&context, None), + &JobRunRequest { + run_id: group, + location_id: location, + target: "development".into(), + job_key: "nightly_job_group".into(), + reason: "Build the nightly group".into(), + priority: 0, + window: None, + full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), + }, + ) + .await + .expect("request group"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::new(); + let scope = (location, "development"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("start group"); + let staging = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim staging") + .expect("staging ready"); + assert_eq!(staging.request.job_key.as_deref(), Some("staging_job")); + finish(store, &staging, test_case.staging).await; + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("advance group"); + + let triggers = list_triggers(store, location, "development") + .await + .expect("triggers"); + let mut fired: BTreeMap = BTreeMap::new(); + let evaluate = async |fired: &mut BTreeMap| { + evaluate_sensors(store, &lease, &triggers) + .await + .into_iter() + .for_each(|(name, count)| *fired.entry(name).or_default() += count); + }; + evaluate(&mut fired).await; + let firing: Vec<&str> = fired + .iter() + .filter(|(_, count)| **count > 0) + .map(|(name, _)| name.as_str()) + .collect(); + assert_eq!( + firing, test_case.expected_firing, + "{}: the member's outcome fires before the group ends", + test_case.description + ); + assert!( + fired.values().all(|count| *count <= 1), + "{}: {fired:?}", + test_case.description + ); + let runs = list_runs(store, scope, 100).await.expect("runs"); + assert_eq!( + runs.iter() + .find(|record| record.run_id == group) + .expect("group") + .status, + RunStatus::Running, + "{}: the group is still running", + test_case.description + ); + + let marts = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim marts") + .expect("marts ready"); + assert_eq!(marts.request.job_key.as_deref(), Some("marts_job")); + finish(store, &marts, test_case.marts).await; + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish group"); + tokio::time::sleep(Duration::from_millis(1100)).await; + evaluate(&mut fired).await; + let mut expected: Vec<&str> = test_case.expected_after_marts.to_vec(); + expected.push("group_completed"); + expected.sort_unstable(); + let firing: Vec<&str> = fired + .iter() + .filter(|(_, count)| **count > 0) + .map(|(name, _)| name.as_str()) + .collect(); + assert_eq!(firing, expected, "{}", test_case.description); + assert!( + fired.values().all(|count| *count <= 1), + "{}: each outcome fires once: {fired:?}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_member_retried_inside_its_group_when_sensors_watch_it_then_only_its_final_attempt_fires() +{ + let test_cases = [ + SensorTestCase { + description: "SQLite member retry fires once", + backend: Backend::Sqlite, + expected_runs: 3, + }, + SensorTestCase { + description: "PostgreSQL member retry fires once", + backend: Backend::Postgres, + expected_runs: 3, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = crate::runs::tests::helpers::snapshot(location); + input.documents.insert("jobs".into(), json!({ + "staging_job": {"description": "Stage customers and products", "selection": ["customers", "products"], + "on_failure": {"retries": 1, "retry_delay_seconds": 0}}, + "report_job": {"description": "Report on inventory", "selection": ["inventory"]} + })); + input.documents.insert( + "groups".into(), + json!({"nightly_job_group": { + "description": "Stage", "selection": ["customers", "products"], + "jobs": {"staging_job": []} + }}), + ); + let mut sensors = serde_json::Map::new(); + for (name, kind, nodes) in [ + ("staging_success", "success", None), + ("staging_failed", "failed", None), + ("staging_completed", "completed", None), + ( + "staging_executed", + "completed", + Some(["customers", "products"]), + ), + ] { + let mut when = + json!({"kind": "run_event", "job_key": "staging_job", "kinds": [kind]}); + when.as_object_mut() + .expect("when") + .extend(nodes.map(|nodes| ("executed_successfully".to_owned(), json!(nodes)))); + sensors.insert( + name.into(), + json!({ + "description": "Report when staging ends", "job_key": "report_job", + "when": when, "minimum_interval_seconds": 1 + }), + ); + } + input.documents.insert("triggers".into(), json!(sensors)); + publish_snapshot(store, &input).await.expect("publish"); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + request_job_run( + store, + (&context, None), + &JobRunRequest { + run_id: Uuid::now_v7(), + location_id: location, + target: "development".into(), + job_key: "nightly_job_group".into(), + reason: "Build the nightly group".into(), + priority: 0, + window: None, + full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), + }, + ) + .await + .expect("request group"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::new(); + let scope = (location, "development"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("start group"); + let first = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim staging") + .expect("staging ready"); + let worker = crate::runs::models::RunAuthority::Worker { + run_id: first.request.run_id, + claim_id: first.claim_id, + }; + for change in [ + RunChange::Started, + RunChange::NodeStarted { + node_key: "products".into(), + }, + node_finished("products", NodeOutcome::Success), + RunChange::NodeStarted { + node_key: "customers".into(), + }, + node_finished("customers", NodeOutcome::Failed), + RunChange::Finished { + outcome: RunOutcome::Partial, + reason: "customers failed".into(), + exit_code: Some(1), + }, + ] { + change_run( + store, + worker, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: first.request.run_id, + claim_id: Some(first.claim_id), + change, + }, + ) + .await + .expect("first attempt"); + } + let triggers = list_triggers(store, location, "development") + .await + .expect("triggers"); + let evaluate = async || { + let mut fired = evaluate_sensors(store, &lease, &triggers).await; + fired.retain(|_, count| *count > 0); + fired + }; + assert!( + evaluate().await.is_empty(), + "{}: an attempt the group retries fires nothing", + test_case.description + ); + let second = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim retry") + .expect("retry ready"); + assert_eq!(second.request.selection, ["customers".to_owned()].into()); + let worker = crate::runs::models::RunAuthority::Worker { + run_id: second.request.run_id, + claim_id: second.claim_id, + }; + for change in [ + RunChange::Started, + RunChange::NodeStarted { + node_key: "customers".into(), + }, + node_finished("customers", NodeOutcome::Success), + RunChange::Finished { + outcome: RunOutcome::Success, + reason: "retry succeeded".into(), + exit_code: Some(0), + }, + ] { + change_run( + store, + worker, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: second.request.run_id, + claim_id: Some(second.claim_id), + change, + }, + ) + .await + .expect("second attempt"); + } + tokio::time::sleep(Duration::from_millis(1100)).await; + let fired = evaluate().await; + assert_eq!( + fired, + BTreeMap::from([ + ("staging_completed".to_owned(), 1), + ("staging_executed".to_owned(), 1), + ("staging_success".to_owned(), 1), + ]), + "{}: the final attempt fires once, and nodes built by earlier attempts count", + test_case.description + ); + assert_eq!(fired.values().sum::(), test_case.expected_runs); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_member_the_group_skips_when_sensors_and_alerts_watch_it_then_it_is_skipped_not_failed() { + let test_cases = [ + SkippedMemberSensorTestCase { + description: "SQLite group skips its dependent member", + backend: Backend::Sqlite, + expected_marts_status: RunStatus::Skipped, + expected_fired: &[("marts_completed", 1), ("marts_skipped", 1)], + }, + SkippedMemberSensorTestCase { + description: "PostgreSQL group skips its dependent member", + backend: Backend::Postgres, + expected_marts_status: RunStatus::Skipped, + expected_fired: &[("marts_completed", 1), ("marts_skipped", 1)], + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = crate::runs::tests::helpers::snapshot(location); + input.nodes.get_mut("orders").expect("orders").definition["deps"] = + json!(["customers"]); + input.documents.insert( + "jobs".into(), + json!({ + "staging_job": {"description": "Stage customers", "selection": ["customers"]}, + "marts_job": {"description": "Build orders", "selection": ["orders"]}, + "report_job": {"description": "Report on inventory", "selection": ["inventory"]} + }), + ); + input.documents.insert( + "groups".into(), + json!({"nightly_job_group": { + "description": "Stage then build orders", "selection": ["customers", "orders"], + "jobs": {"staging_job": [], "marts_job": ["staging_job"]}, + "dependency_failure": {"marts_job": "skip"} + }}), + ); + let mut sensors = serde_json::Map::new(); + for kind in ["skipped", "failed", "completed"] { + sensors.insert( + format!("marts_{kind}"), + json!({ + "description": "Report when marts ends", "job_key": "report_job", + "when": {"kind": "run_event", "job_key": "marts_job", "kinds": [kind]}, + "minimum_interval_seconds": 1 + }), + ); + } + input.documents.insert("triggers".into(), json!(sensors)); + publish_snapshot(store, &input).await.expect("publish"); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + request_job_run( + store, + (&context, None), + &JobRunRequest { + run_id: Uuid::now_v7(), + location_id: location, + target: "development".into(), + job_key: "nightly_job_group".into(), + reason: "Build the nightly group".into(), + priority: 0, + window: None, + full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), + }, + ) + .await + .expect("request group"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::new(); + let scope = (location, "development"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("start group"); + let staging = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim staging") + .expect("staging ready"); + finish(store, &staging, (NodeOutcome::Failed, RunOutcome::Failed)).await; + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("skip marts"); + let runs = list_runs(store, scope, 100).await.expect("runs"); + let marts = runs + .iter() + .find(|record| record.job_key.as_deref() == Some("marts_job")) + .expect("marts run"); + assert_eq!( + marts.status, test_case.expected_marts_status, + "{}", + test_case.description + ); + let alerts = crate::runs::tests::helpers::rows( + store, + "SELECT run_id FROM recall_events WHERE event_type='platform_run_failed'", + vec![], + ) + .await; + assert_eq!( + alerts + .iter() + .map(|row| row["run_id"].as_str().unwrap_or_default().to_owned()) + .collect::>(), + vec![staging.request.run_id.to_string()], + "{}: only the failed member alerts", + test_case.description + ); + let triggers = list_triggers(store, location, "development") + .await + .expect("triggers"); + let mut fired = evaluate_sensors(store, &lease, &triggers).await; + fired.retain(|_, count| *count > 0); + assert_eq!( + fired, + test_case + .expected_fired + .iter() + .map(|(name, count)| ((*name).to_owned(), *count)) + .collect::>(), + "{}: a skipped member fires skipped, never failed", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs b/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs index ea5a94d84f..9b7f5d905f 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs @@ -11,8 +11,8 @@ use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::list_triggers::list_triggers; use crate::declarations::models::TriggerChange; use crate::runs::main::{ - change_run::change_run, claim_next_run::claim_next_run, list_runs::list_runs, - request_job_run::request_job_run, + execution::change_run::change_run, execution::claim_next_run::claim_next_run, + listings::list_runs::list_runs, requests::request_job_run::request_job_run, }; use crate::runs::models::{ JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, @@ -24,7 +24,7 @@ use crate::sensors::main::{ }; use crate::sensors::tests::helpers::{change, finish_run, snapshot}; use crate::sensors::tests::test_types::SensorTestCase; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::fault_types::TransactionFault; diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_person_state.rs b/crates/sqlbuild-orchestration/tests/sensors/test_person_state.rs index 511f861b17..3fba9fa415 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_person_state.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_person_state.rs @@ -19,7 +19,7 @@ use crate::sensors::main::read_person_sensor::read_person_sensor; use crate::sensors::models::PersonSensorCursor; use crate::sensors::tests::helpers::{append_events, snapshot}; use crate::sensors::tests::test_types::SensorTestCase; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::store::main::{begin::begin, commit::commit, query::query}; diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_recovery.rs b/crates/sqlbuild-orchestration/tests/sensors/test_recovery.rs index 7b8ba79ce8..ca6a1f1fcd 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_recovery.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_recovery.rs @@ -6,15 +6,15 @@ use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::list_triggers::list_triggers; use crate::declarations::models::{TriggerChange, TriggerStatus}; -use crate::platform::_helpers::retention::retain; -use crate::runs::main::list_runs::list_runs; +use crate::platform::_helpers::maintenance::retention::retain; +use crate::runs::main::listings::list_runs::list_runs; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; use crate::sensors::main::complete_builtin_sensor::complete_builtin_sensor; use crate::sensors::models::SensorEvaluationOutcome; use crate::sensors::tests::helpers::{append_events, change, evaluations, snapshot}; use crate::sensors::tests::test_types::SensorTestCase; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; use crate::store::main::{ begin::begin, commit::commit, migrate_store::migrate_store, query::query, diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs index 388b01aeb5..66b24b2fb4 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs @@ -10,8 +10,8 @@ use crate::credentials::models::AuditContext; use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::list_triggers::list_triggers; use crate::declarations::models::TriggerChange; -use crate::runs::main::list_runs::list_runs; -use crate::runs::main::request_rerun::request_rerun; +use crate::runs::main::listings::list_runs::list_runs; +use crate::runs::main::requests::request_rerun::request_rerun; use crate::runs::models::{NodeOutcome, RerunRequest, RunKind, RunOrigin, RunOutcome}; use crate::scheduler::tests::contracts::helpers::{acquire_lease, hold_lease}; use crate::sensors::main::{ @@ -20,7 +20,7 @@ use crate::sensors::main::{ }; use crate::sensors::tests::helpers::{change, finish_claimed_run, finish_run, snapshot}; use crate::sensors::tests::test_types::{SensorExecutionTestCase, SensorTestCase}; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::{ begin::begin, commit::commit, migrate_store::migrate_store, query::query, }; diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs index d16bcc357d..704ff052d5 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs @@ -8,9 +8,9 @@ use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::list_triggers::list_triggers; -use crate::platform::main::report_location_health::report_location_health; +use crate::platform::main::health::report_location_health::report_location_health; use crate::platform::models::{LocationHealth, LocationHealthReport}; -use crate::runs::main::list_runs::list_runs; +use crate::runs::main::listings::list_runs::list_runs; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::sensors::main::{ begin_sensor_evaluation::begin_sensor_evaluation, @@ -18,7 +18,7 @@ use crate::sensors::main::{ }; use crate::sensors::tests::helpers::snapshot; use crate::sensors::tests::test_types::SensorTestCase; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::snapshots::models::SnapshotNode; use crate::store::main::{ begin::begin, commit::commit, migrate_store::migrate_store, query::query, diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_transitions.rs b/crates/sqlbuild-orchestration/tests/sensors/test_transitions.rs index d3577f8581..c980b682b7 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_transitions.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_transitions.rs @@ -8,7 +8,7 @@ use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::list_triggers::list_triggers; use crate::declarations::models::TriggerChange; -use crate::runs::main::list_runs::list_runs; +use crate::runs::main::listings::list_runs::list_runs; use crate::runs::models::{NodeOutcome, RunOutcome}; use crate::scheduler::tests::contracts::helpers::{acquire_lease, hold_lease}; use crate::sensors::main::{ @@ -17,7 +17,7 @@ use crate::sensors::main::{ }; use crate::sensors::tests::helpers::{change, finish_run, snapshot}; use crate::sensors::tests::test_types::SensorTestCase; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::helpers::read_recall_events; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_types.rs b/crates/sqlbuild-orchestration/tests/sensors/test_types.rs index af27d0d35e..e500a1b0fb 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_types.rs @@ -1,4 +1,4 @@ -use crate::runs::models::{NodeOutcome, RunOutcome}; +use crate::runs::models::{NodeOutcome, RunOutcome, RunStatus}; use crate::sensors::models::CustomSensorOutcome; use crate::store::tests::store_test_types::Backend; @@ -22,3 +22,24 @@ pub(crate) struct SensorExecutionTestCase { pub(crate) expected_selector: &'static str, pub(crate) expected_filters: &'static [&'static str], } + +pub(crate) struct GroupMemberSensorTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + /// The node and run outcome the first member ends with. + pub(crate) staging: (NodeOutcome, RunOutcome), + /// The node and run outcome the dependent member ends with. + pub(crate) marts: (NodeOutcome, RunOutcome), + /// Sensors that fire once the first member ends, while the group still runs. + pub(crate) expected_firing: &'static [&'static str], + /// Member sensors that have fired once the dependent member ends too. + pub(crate) expected_after_marts: &'static [&'static str], +} + +pub(crate) struct SkippedMemberSensorTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_marts_status: RunStatus, + /// Sensors that fire, with how many runs each requested. + pub(crate) expected_fired: &'static [(&'static str, usize)], +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 7497a5a98c..29bfa22747 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -3,8 +3,8 @@ use std::collections::BTreeMap; use crate::snapshots::_helpers::preparation::prepare_snapshot; -use crate::snapshots::main::current_snapshot::current_snapshot; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; +use crate::snapshots::main::reads::current_snapshot::current_snapshot; use crate::snapshots::models::{SnapshotInput, SnapshotNode}; use crate::store::errors::ControlError; use crate::store::main::open_store::open_store; diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_connection_details.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_connection_details.rs new file mode 100644 index 0000000000..19147e6ab1 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_connection_details.rs @@ -0,0 +1,67 @@ +use serde_json::json; + +use crate::snapshots::main::reads::connection_details::connection_details; +use crate::snapshots::tests::test_types::ConnectionDetailsTestCase; + +#[test] +fn given_connection_settings_when_details_are_published_then_only_named_non_secret_fields_appear() { + let test_cases = [ + ConnectionDetailsTestCase { + description: "Snowflake key pair names the account, user, role, warehouse and database", + adapter: "snowflake", + settings: json!({"account": "orders-account", "user": "orders_loader", "role": "LOADER", + "warehouse": "ORDERS_WH", "database": "ORDERS", "schema": "PUBLIC", + "private_key_path": "/keys/orders.p8", "private_key_file_pwd": "hunter2"}), + expected_details: json!({"adapter": "snowflake", "account": "orders-account", "host": null, + "user": "orders_loader", "role": "LOADER", "warehouse": "ORDERS_WH", + "database": "ORDERS", "auth_method": "key_pair"}), + }, + ConnectionDetailsTestCase { + description: "a Snowflake authenticator names the method; unknown ones are kept as written", + adapter: "snowflake", + settings: json!({"account": "orders-account", "authenticator": "ExternalBrowser"}), + expected_details: json!({"adapter": "snowflake", "account": "orders-account", "host": null, + "user": null, "role": null, "warehouse": null, "database": null, + "auth_method": "external_browser"}), + }, + ConnectionDetailsTestCase { + description: "Postgres names its host, user and dbname; the password is only a method", + adapter: "postgres", + settings: json!({"host": "db.example.com", "user": "inventory", "dbname": "inventory", + "password": "hunter2", "port": "5432"}), + expected_details: json!({"adapter": "postgres", "account": null, "host": "db.example.com", + "user": "inventory", "role": null, "warehouse": null, "database": "inventory", + "auth_method": "password"}), + }, + ConnectionDetailsTestCase { + description: "a DuckDB file is the host, with no database or sign-in", + adapter: "duckdb", + settings: json!({"database": "warehouse/customers.duckdb"}), + expected_details: json!({"adapter": "duckdb", "account": null, "host": "warehouse/customers.duckdb", + "user": null, "role": null, "warehouse": null, "database": null, + "auth_method": "none"}), + }, + ConnectionDetailsTestCase { + description: "a token-only connection names the method and no value", + adapter: "motherduck", + settings: json!({"database": "md:customers", "token": "hunter2"}), + expected_details: json!({"adapter": "motherduck", "account": null, "host": null, + "user": null, "role": null, "warehouse": null, "database": "md:customers", + "auth_method": "token"}), + }, + ]; + for test_case in test_cases { + let settings = test_case.settings.as_object().expect("settings").clone(); + let published = json!(connection_details(test_case.adapter, &settings)); + assert_eq!( + published, test_case.expected_details, + "{}", + test_case.description + ); + assert!( + !published.to_string().contains("hunter2"), + "{}: never a secret", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs index b18477bf2f..51d112eb02 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs @@ -1,5 +1,7 @@ //! Snapshot contracts compiled within the snapshot owner. +#[path = "test_connection_details.rs"] +mod connection_details; #[path = "test_contract.rs"] mod contract; #[path = "helpers.rs"] diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_retention.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_retention.rs index 9e9e2e5d5f..462d7ea05e 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/test_retention.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_retention.rs @@ -6,8 +6,8 @@ use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::list_triggers::list_triggers; use crate::runs::tests::helpers::{request, request_run}; use crate::scheduler::tests::contracts::ticks::helpers::snapshot; -use crate::snapshots::main::prune_snapshots::prune_snapshots; -use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::main::publication::prune_snapshots::prune_snapshots; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::snapshots::models::SnapshotRetention; use crate::snapshots::tests::helpers::{retained_snapshot_state, retention_counts}; use crate::snapshots::tests::test_types::SnapshotRetentionTestCase; diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs index 997d311376..902f72a85f 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs @@ -27,3 +27,11 @@ pub(super) struct SnapshotRetentionTestCase { pub(super) backend: Backend, pub(super) expected_generations: &'static [i64], } + +#[derive(Debug)] +pub(super) struct ConnectionDetailsTestCase { + pub(super) description: &'static str, + pub(super) adapter: &'static str, + pub(super) settings: serde_json::Value, + pub(super) expected_details: serde_json::Value, +} diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_crash_recovery.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_crash_recovery.rs index b9a6af783c..d81df2ca63 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/test_crash_recovery.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_crash_recovery.rs @@ -1,6 +1,6 @@ //! Commits a killed process acknowledged survive, though closing a connection never checkpoints. -use crate::platform::_helpers::retention::wal_checkpoint; +use crate::platform::_helpers::maintenance::retention::wal_checkpoint; use crate::store::main::open_store::open_store; use crate::store::tests::schema::helpers::{ kill_writer_after, surviving_locations, wal_bytes_after_sequential_writes, diff --git a/crates/sqlbuild-orchestration/tests/warehouse_metadata/helpers.rs b/crates/sqlbuild-orchestration/tests/warehouse_metadata/helpers.rs new file mode 100644 index 0000000000..053834ced9 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/warehouse_metadata/helpers.rs @@ -0,0 +1,54 @@ +//! A small served copy and the state fixture that holds the same facts. + +use serde_json::{Value, json}; + +/// A copy as the server serves it: one relation, a fingerprint, a run, an observation and a consumption row. +pub(crate) fn served_copy() -> Value { + json!({ + "synced_at": "2026-03-01T12:00:00Z", + "relations": [{"state_database": "WAREHOUSE", "state_schema": "SHOP", "name": "ORDERS", + "relation_type": "TABLE", "columns": [{"name": "ID", "data_type": "NUMBER"}]}], + "fingerprints": [{"event_id": "fingerprint-1", "state_database": "WAREHOUSE", "state_schema": "SHOP", + "node_type": "model", "node_name": "orders", "target_database": "WAREHOUSE", + "target_schema": "SHOP", "target_name": "ORDERS", "run_id": "run-1", + "definition_hash": "definition", "version_hash": "version", "schema_fingerprint": "id:NUMBER", + "definition": "SELECT 1 AS id", "metadata_json": "{}", "ts": "2026-03-01T10:00:00Z"}], + "latest_runs": [{"state_database": "WAREHOUSE", "state_schema": "SHOP", "node_type": "model", + "node_name": "orders", "run_id": "run-1"}], + "observations": [{"state_database": "WAREHOUSE", "state_schema": "SHOP", + "observation_id": "observation-1", "kind": "cursor_max", "node": "orders", + "cursor_type": "integer", "value": "41", "run_id": "run-1", + "observed_at": "2026-03-01T10:00:01Z"}], + "consumption": [{"state_database": "WAREHOUSE", "state_schema": "SHOP", "execution_id": "execution-1", + "phase": "succeeded", "run_id": "run-1", "node_type": "model", "node_name": "orders", + "target_database": "WAREHOUSE", "target_schema": "SHOP", "target_name": "ORDERS", + "version_hash": "version", "input_count": 0, "input_kind": null, "input_name": null, + "input_version": null, "recorded_at": "2026-03-01T10:00:00Z"}], + "version": "mirror-7", + }) +} + +/// The same facts as a stored state fixture holds them. +pub(crate) fn stored_parts() -> Value { + let location = json!({"database": "WAREHOUSE", "schema": "SHOP"}); + json!({ + "provenance": {"synced_at": "2026-03-01T12:00:00Z", "version": "mirror-7"}, + "relations": [{"database": "WAREHOUSE", "schema": "SHOP", "name": "ORDERS", "relation_type": "TABLE", + "columns": [{"name": "ID", "data_type": "NUMBER"}]}], + "fingerprints": [{"location": location, "node_type": "model", "node_name": "orders", + "target_database": "WAREHOUSE", "target_schema": "SHOP", "target_name": "ORDERS", + "run_id": "run-1", "definition_hash": "definition", "version_hash": "version", + "schema_fingerprint": "id:NUMBER", "definition": "SELECT 1 AS id", "metadata_json": "{}", + "ts": "2026-03-01T10:00:00Z"}], + "latest_runs": [{"location": location, "node_type": "model", "node_name": "orders", "run_id": "run-1"}], + "observations": [{"location": location, "kind": "cursor_max", "node": "orders", + "cursor_type": "integer", "value": "41", "run_id": "run-1", + "observed_at": "2026-03-01T10:00:01Z"}], + "consumption": [{"location": location, "execution_id": "execution-1", "phase": "succeeded", + "run_id": "run-1", "node_type": "model", "node_name": "orders", + "target_database": "WAREHOUSE", "target_schema": "SHOP", "target_name": "ORDERS", + "version_hash": "version", "input_count": 0, "input_kind": null, "input_name": null, + "input_version": null, "recorded_at": "2026-03-01T10:00:00Z"}], + "has_consumption": true, + }) +} diff --git a/crates/sqlbuild-orchestration/tests/warehouse_metadata/test_contracts.rs b/crates/sqlbuild-orchestration/tests/warehouse_metadata/test_contracts.rs new file mode 100644 index 0000000000..801bd93736 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/warehouse_metadata/test_contracts.rs @@ -0,0 +1,8 @@ +//! Warehouse metadata contracts compiled within the warehouse metadata owner. + +#[path = "helpers.rs"] +mod helpers; +#[path = "test_preview_state.rs"] +mod preview_state; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/warehouse_metadata/test_preview_state.rs b/crates/sqlbuild-orchestration/tests/warehouse_metadata/test_preview_state.rs new file mode 100644 index 0000000000..4d88e3463c --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/warehouse_metadata/test_preview_state.rs @@ -0,0 +1,62 @@ +//! A location builds the same warehouse state from a served copy as a fixture stores. + +use sqlbuild_state::warehouse_state::main::warehouse_state::warehouse_state; +use sqlbuild_state::warehouse_state::models::{StateLocation, WarehouseState, WarehouseStateParts}; + +use crate::warehouse_metadata::main::location_reads::preview_state_parts::preview_state_parts; +use crate::warehouse_metadata::models::PreviewCopy; +use crate::warehouse_metadata::tests::helpers::{served_copy, stored_parts}; +use crate::warehouse_metadata::tests::test_types::{PreviewStateTestCase, StateAnswers}; + +#[test] +fn given_a_served_copy_and_a_stored_fixture_when_built_then_the_states_and_their_answers_match() { + let test_cases = [PreviewStateTestCase { + description: "one fact of every kind", + served: served_copy, + stored: stored_parts, + expected_answers: || { + ( + Some("41".into()), + vec!["run-1".into()], + 1, + Some("mirror-7".into()), + ) + }, + }]; + let shop = StateLocation { + database: Some("warehouse".into()), + schema: "shop".into(), + }; + let answers = |state: &WarehouseState| -> StateAnswers { + ( + state.target_max("orders").map(str::to_owned), + state + .latest_fingerprints(&shop) + .into_iter() + .map(|fingerprint| fingerprint.run_id.clone()) + .collect(), + state.consumption_rows(&shop).len(), + state.version().map(str::to_owned), + ) + }; + for test_case in test_cases { + let copy: Option = serde_json::from_value((test_case.served)()).ok(); + let fixture: Option = + serde_json::from_value((test_case.stored)()).ok(); + let from_copy: Option = + copy.map(|copy| warehouse_state(preview_state_parts(copy, true))); + let from_fixture: Option = fixture.map(warehouse_state); + assert_eq!( + ( + from_copy.as_ref().map(WarehouseState::parts), + from_copy.as_ref().map(answers), + ), + ( + from_fixture.as_ref().map(WarehouseState::parts), + Some((test_case.expected_answers)()), + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/warehouse_metadata/test_types.rs b/crates/sqlbuild-orchestration/tests/warehouse_metadata/test_types.rs new file mode 100644 index 0000000000..5973d893e3 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/warehouse_metadata/test_types.rs @@ -0,0 +1,14 @@ +//! Table cases of the warehouse state a location builds. + +use serde_json::Value; + +/// A state's cursor `MAX` of orders, newest shop fingerprint runs, consumption row count and version. +pub(crate) type StateAnswers = (Option, Vec, usize, Option); + +/// A served copy and a stored fixture of the same facts, and what both states answer. +pub(crate) struct PreviewStateTestCase { + pub(crate) description: &'static str, + pub(crate) served: fn() -> Value, + pub(crate) stored: fn() -> Value, + pub(crate) expected_answers: fn() -> StateAnswers, +} diff --git a/crates/sqlbuild-python-base/Cargo.toml b/crates/sqlbuild-python-base/Cargo.toml index aa9f59b91e..9addbdef9a 100644 --- a/crates/sqlbuild-python-base/Cargo.toml +++ b/crates/sqlbuild-python-base/Cargo.toml @@ -11,6 +11,7 @@ rust-version.workspace = true chrono.workspace = true mimalloc.workspace = true pyo3.workspace = true +rayon.workspace = true rusqlite.workspace = true rustc-hash.workspace = true serde.workspace = true diff --git a/crates/sqlbuild-python-base/src/native_host/_helpers/dialect.rs b/crates/sqlbuild-python-base/src/native_host/_helpers/dialect.rs new file mode 100644 index 0000000000..9b5326f8c7 --- /dev/null +++ b/crates/sqlbuild-python-base/src/native_host/_helpers/dialect.rs @@ -0,0 +1,48 @@ +//! Which adapter attributes a plan's SQL dialect may read. + +/// Adapter attributes that only write SQL or describe the dialect, by prefix. +const SYNTAX_PREFIXES: [&str; 3] = ["render_", "_render_", "supports_"]; +/// Adapter attributes that only describe the dialect or its defaults; none takes a connection. +const SYNTAX_ATTRIBUTES: [&str; 33] = [ + "adapter_name", + "allows_implicit_managed_write_schema", + "build_cursor_filter", + "collection_rendering", + "default_database", + "default_promotion_strategy", + "default_schema", + "default_table_promotion_mode", + "expression_inference_profile", + "format_row_diff_decimal_sql", + "lists_relations_authoritatively", + "max_identifier_length", + "max_statement_size", + "maximum_identifier_length", + "normalize_row_diff_numeric_type", + "persists_python_functions", + "python_functions_inherit_default_namespace", + "relation_grants_supported", + "relation_names_match", + "requires_derived_table_aliases", + "retention_state_from_relation", + "sql_analysis_dialect", + "sql_analysis_dialect_name", + "sql_lexical_syntax", + "star_exclude_keyword", + "state_timestamp_type", + "table_fn", + "table_type_entries", + "table_type_downgrade", + "untyped_column_type", + "value_renderer", + "views_bind_to_relation_identity", + "views_read_with_reader_access", +]; + +/// Whether `name` only writes SQL or describes the dialect. +pub(crate) fn syntax(name: &str) -> bool { + SYNTAX_PREFIXES + .iter() + .any(|prefix| name.starts_with(prefix)) + || SYNTAX_ATTRIBUTES.contains(&name) +} diff --git a/crates/sqlbuild-python-base/src/native_host/_helpers/host.rs b/crates/sqlbuild-python-base/src/native_host/_helpers/host.rs index dfd6cf90cb..27cd759b94 100644 --- a/crates/sqlbuild-python-base/src/native_host/_helpers/host.rs +++ b/crates/sqlbuild-python-base/src/native_host/_helpers/host.rs @@ -1,7 +1,7 @@ //! Resolve host entries: targets read once from `sqlbuild._native_host`, attributes read live. -use std::path::PathBuf; - +use crate::boundary::main::within::within; +use crate::build_progress::main::progress::write_persistent::write_persistent; use pyo3::PyErr; use pyo3::exceptions::PyImportError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; @@ -10,8 +10,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods}; use sqlbuild_host::entries::constants::{HOST_ENTRIES, HOST_MODULE, HOST_TARGETS_FUNCTION}; use sqlbuild_host::entries::models::{HostId, UserCodeHost}; use sqlbuild_host::entries::types::HostEntries; - -use crate::boundary::main::within::within; +use std::path::PathBuf; /// One entry's target module path and attribute name. type Target = (Py, Py); @@ -69,12 +68,10 @@ pub(crate) fn entry<'py>(py: Python<'py>, id: impl Into) -> PyResult, stream: &str, text: &str) -> PyResult<()> { let stream = py.import("sys")?.getattr(stream)?; - stream.call_method1("write", (text,))?; - stream.call_method0("flush")?; - Ok(()) + write_persistent(py, &stream, text) } /// Flush the interpreter's live console `stream` so text written before a child process appears first. diff --git a/crates/sqlbuild-python-base/src/native_host/_helpers/mod.rs b/crates/sqlbuild-python-base/src/native_host/_helpers/mod.rs index 7af1af0538..adf517e70e 100644 --- a/crates/sqlbuild-python-base/src/native_host/_helpers/mod.rs +++ b/crates/sqlbuild-python-base/src/native_host/_helpers/mod.rs @@ -1,4 +1,9 @@ pub(crate) mod connections; + pub(crate) mod host; + pub(crate) mod interrupt; + pub(crate) mod results; + +pub(crate) mod dialect; diff --git a/crates/sqlbuild-python-base/src/native_host/_helpers/results.rs b/crates/sqlbuild-python-base/src/native_host/_helpers/results.rs index 55e05d3c01..f6662f4c46 100644 --- a/crates/sqlbuild-python-base/src/native_host/_helpers/results.rs +++ b/crates/sqlbuild-python-base/src/native_host/_helpers/results.rs @@ -231,7 +231,8 @@ record!(SqlTestExecutionResult { record!(BuildExecutionResult { status, model_results, seed_results, function_results, load_results, test_results, source_audit_results, end_audit_results, success_count, failure_count, timeout_count, - skipped_count, warning_count, insufficient_count, audit_result_projection_attempted_count, + skipped_count, warning_count, insufficient_count, ignored_audit_failure_count, + audit_result_projection_attempted_count, audit_result_projection_written_count, audit_result_projection_failed_count, timings, }); @@ -336,6 +337,7 @@ impl<'py> ReadResult<'py> for AuditExecutionResult> { Some(mapping(&meta)?) }, skip_reason: ReadResult::read(&field(value, "skip_reason")?)?, + ignored: ReadResult::read(&field(value, "ignored")?)?, }) } } diff --git a/crates/sqlbuild-python-base/src/native_host/models.rs b/crates/sqlbuild-python-base/src/native_host/models.rs index 8e6757a6fa..8ce45cb900 100644 --- a/crates/sqlbuild-python-base/src/native_host/models.rs +++ b/crates/sqlbuild-python-base/src/native_host/models.rs @@ -1,8 +1,11 @@ //! Public records of native host. +use crate::native_host::_helpers::dialect::syntax; use crate::native_host::_helpers::interrupt::INTERRUPTED; use crate::native_host::_helpers::interrupt::SIGINT; use crate::native_host::_helpers::interrupt::restore_signal_handler; +use crate::native_host::types::Syntax; +use pyo3::exceptions::PyAttributeError; use pyo3::exceptions::PyKeyboardInterrupt; use pyo3::prelude::Bound; use pyo3::prelude::Py; @@ -11,6 +14,8 @@ use pyo3::prelude::PyAnyMethods; use pyo3::prelude::PyErr; use pyo3::prelude::PyResult; use pyo3::prelude::Python; +use pyo3::prelude::pyclass; +use pyo3::prelude::pymethods; use pyo3::sync::PyOnceLock; use pyo3::types::PyCFunction; use pyo3::types::PyDict; @@ -206,3 +211,59 @@ impl std::fmt::Debug for FamilyProjection { .finish_non_exhaustive() } } + +/// The adapter's SQL syntax for a plan; anything that would reach the warehouse is refused. +#[pyclass(frozen, module = "sqlbuild._native", name = "Dialect")] +#[derive(Debug)] +pub struct Dialect { + adapter: Py, +} + +impl Dialect { + /// The dialect of `adapter`. + pub fn of<'py>(adapter: &Bound<'py, PyAny>) -> PyResult> { + Bound::new( + adapter.py(), + Dialect { + adapter: adapter.clone().unbind(), + }, + ) + } + + /// A dialect passed as a keyword argument; an adapter or anything else is refused. + pub fn passed(value: Bound<'_, PyAny>) -> PyResult> { + Ok(value.cast_into::()?) + } + + /// One syntax attribute of the adapter, such as `render_qualified_name`. + pub fn attribute<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { + if !syntax(name) { + return Err(PyAttributeError::new_err(format!( + "a plan's SQL dialect writes SQL only; `{name}` would reach the warehouse" + ))); + } + self.adapter.bind(py).getattr(name) + } +} + +impl<'py> Syntax<'py> for Bound<'py, Dialect> { + fn syntax(&self, name: &str) -> PyResult> { + self.get().attribute(self.py(), name) + } +} + +#[pymethods] +impl Dialect { + #[new] + #[pyo3(signature = (adapter))] + fn new(adapter: Bound<'_, PyAny>) -> Self { + Dialect { + adapter: adapter.unbind(), + } + } + + /// Python code handed a dialect reads the same syntax attributes; anything else is refused. + fn __getattr__<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { + self.attribute(py, name) + } +} diff --git a/crates/sqlbuild-python-base/src/native_host/types.rs b/crates/sqlbuild-python-base/src/native_host/types.rs index 5d64f25885..5249a687f0 100644 --- a/crates/sqlbuild-python-base/src/native_host/types.rs +++ b/crates/sqlbuild-python-base/src/native_host/types.rs @@ -8,3 +8,9 @@ use pyo3::prelude::PyResult; pub trait ReadResult<'py>: Sized { fn read(value: &Bound<'py, PyAny>) -> PyResult; } + +/// Reading one syntax attribute of a bound dialect. +pub trait Syntax<'py> { + /// One syntax attribute, such as `render_qualified_name`; anything reaching the warehouse is refused. + fn syntax(&self, name: &str) -> PyResult>; +} diff --git a/crates/sqlbuild-python-base/src/native_threads/_helpers/compile_pool.rs b/crates/sqlbuild-python-base/src/native_threads/_helpers/compile_pool.rs new file mode 100644 index 0000000000..e873fd49b6 --- /dev/null +++ b/crates/sqlbuild-python-base/src/native_threads/_helpers/compile_pool.rs @@ -0,0 +1,19 @@ +//! The compile pool shared by model compilation and planning, started on first use. + +use std::sync::OnceLock; + +use rayon::{ThreadPool, ThreadPoolBuildError, ThreadPoolBuilder}; + +use crate::native_threads::constants::{PREPASS_WORKERS, WORKER_STACK_BYTES}; + +static POOL: OnceLock> = OnceLock::new(); + +pub(crate) fn compile_pool() -> &'static Result { + POOL.get_or_init(|| { + ThreadPoolBuilder::new() + .num_threads(PREPASS_WORKERS) + .stack_size(WORKER_STACK_BYTES) + .thread_name(|index| format!("sqlbuild-model-prepass-{index}")) + .build() + }) +} diff --git a/crates/sqlbuild-python-base/src/native_threads/_helpers/mod.rs b/crates/sqlbuild-python-base/src/native_threads/_helpers/mod.rs index 6f88f92bf0..d0a0aab372 100644 --- a/crates/sqlbuild-python-base/src/native_threads/_helpers/mod.rs +++ b/crates/sqlbuild-python-base/src/native_threads/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod compile_pool; pub(crate) mod exit_signal; pub(crate) mod interruptible_join; pub(crate) mod reentrant_lock; diff --git a/crates/sqlbuild-python-base/src/native_threads/constants.rs b/crates/sqlbuild-python-base/src/native_threads/constants.rs index a8ff53f00e..bdea9ddff2 100644 --- a/crates/sqlbuild-python-base/src/native_threads/constants.rs +++ b/crates/sqlbuild-python-base/src/native_threads/constants.rs @@ -2,3 +2,8 @@ /// Marks a thread whose writes command-output capture ignores, such as one running a command-output sink. pub const OUTPUT_SUPPRESSED_ATTRIBUTE: &str = "_sqlbuild_command_output_suppressed"; + +/// The compile pool's fixed worker count. +pub const PREPASS_WORKERS: usize = 4; +/// Each compile pool worker's stack size. +pub const WORKER_STACK_BYTES: usize = 16 * 1024 * 1024; diff --git a/crates/sqlbuild-python-base/src/native_threads/main/mod.rs b/crates/sqlbuild-python-base/src/native_threads/main/mod.rs index fcc7ac3c20..40cf8b5445 100644 --- a/crates/sqlbuild-python-base/src/native_threads/main/mod.rs +++ b/crates/sqlbuild-python-base/src/native_threads/main/mod.rs @@ -1,3 +1,5 @@ pub mod interpreter_exiting; pub mod join_interruptibly; +pub mod pooled; pub mod signal_interpreter_exit; +pub mod spawned; diff --git a/crates/sqlbuild-python-base/src/native_threads/main/pooled.rs b/crates/sqlbuild-python-base/src/native_threads/main/pooled.rs new file mode 100644 index 0000000000..89abe33366 --- /dev/null +++ b/crates/sqlbuild-python-base/src/native_threads/main/pooled.rs @@ -0,0 +1,13 @@ +//! Run work over items on the compile pool. + +use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; + +use crate::native_threads::_helpers::compile_pool::compile_pool; + +/// `each` of every item in order, run on the compile pool, or in turn when the pool cannot start. +pub fn pooled(items: &[T], each: impl Fn(&T) -> R + Sync + Send) -> Vec { + match compile_pool() { + Ok(pool) => pool.install(|| items.par_iter().map(&each).collect()), + Err(_) => items.iter().map(each).collect(), + } +} diff --git a/crates/sqlbuild-python-base/src/native_threads/main/spawned.rs b/crates/sqlbuild-python-base/src/native_threads/main/spawned.rs new file mode 100644 index 0000000000..c5beaf4c07 --- /dev/null +++ b/crates/sqlbuild-python-base/src/native_threads/main/spawned.rs @@ -0,0 +1,11 @@ +//! Hand a job to the compile pool. + +use crate::native_threads::_helpers::compile_pool::compile_pool; + +/// Run `job` on the compile pool while the caller continues, or now when the pool cannot start. +pub fn spawned(job: impl FnOnce() + Send + 'static) { + match compile_pool() { + Ok(pool) => pool.spawn(job), + Err(_) => job(), + } +} diff --git a/crates/sqlbuild-python-base/src/orchestration_reads/_helpers/orchestration_declarations.rs b/crates/sqlbuild-python-base/src/orchestration_reads/_helpers/orchestration_declarations.rs index 52fed366ca..45d5399dc7 100644 --- a/crates/sqlbuild-python-base/src/orchestration_reads/_helpers/orchestration_declarations.rs +++ b/crates/sqlbuild-python-base/src/orchestration_reads/_helpers/orchestration_declarations.rs @@ -1,27 +1,27 @@ //! The public authoring records enter native compilation here, never scheduler execution. +use crate::native_host::constants::AUTHORING_MODULE; +use crate::native_host::constants::CALENDAR_CLASS; +use crate::native_host::constants::CONSTRUCTING_MODULES; +use crate::native_host::constants::SENSOR_CLASS; +use crate::native_host::main::host_entry::host_entry as entry; use crate::orchestration_reads::models::ImportedDeclarations; use crate::orchestration_reads::models::RunControls; +use pyo3::exceptions::PyValueError; use pyo3::prelude::PyModule; use pyo3::prelude::PyModuleMethods; -use pyo3::wrap_pyfunction; -use std::collections::{BTreeMap, HashSet}; -use std::path::{Component, Path}; - -use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use pyo3::wrap_pyfunction; use serde_json::Value; use sqlbuild_discovery::orchestration_files::models::{self as files, RunFile}; use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_orchestration::declarations::models::DependencyFailure; use sqlbuild_orchestration::declarations::models::{DeclaredAt, OnFailure, Overlap, RunResources}; - -use crate::native_host::constants::AUTHORING_MODULE; -use crate::native_host::constants::CALENDAR_CLASS; -use crate::native_host::constants::CONSTRUCTING_MODULES; -use crate::native_host::constants::SENSOR_CLASS; -use crate::native_host::main::host_entry::host_entry as entry; +use std::collections::BTreeSet; +use std::collections::{BTreeMap, HashSet}; +use std::path::{Component, Path}; /// The file and line of the author's code constructing a declaration; `None` for code without a source file. #[pyfunction] @@ -92,9 +92,9 @@ pub(crate) fn run_controls(run: &RunFile) -> RunControls { files::Overlap::Queue => Overlap::Queue, files::Overlap::Skip => Overlap::Skip, }, - on_failure: run.on_failure.map(|policy| OnFailure { - retries: policy.retries, - retry_delay_seconds: policy.retry_delay_seconds, + on_failure: run.retry.map(|policy| OnFailure { + retries: policy.attempts.saturating_sub(1), + retry_delay_seconds: policy.delay_seconds, }), } } @@ -190,3 +190,26 @@ pub(crate) fn module_declarations<'py>( pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(declaration_site, module)?) } + +/// Each published member whose `[dependencies] on_failure`, its override or else the group's, is not `partial`. +pub(crate) fn dependency_failures( + dependencies: &files::GroupDependencies, + members: &BTreeSet, +) -> BTreeMap { + members + .iter() + .filter_map(|member| { + let written = dependencies + .members + .get(member) + .and_then(|settings| settings.on_failure) + .or(dependencies.defaults.on_failure)?; + let policy = match written { + files::DependencyFailure::Partial => return None, + files::DependencyFailure::Skip => DependencyFailure::Skip, + files::DependencyFailure::Run => DependencyFailure::Run, + }; + Some((member.clone(), policy)) + }) + .collect() +} diff --git a/crates/sqlbuild-python-base/src/orchestration_reads/main/dependency_failures.rs b/crates/sqlbuild-python-base/src/orchestration_reads/main/dependency_failures.rs new file mode 100644 index 0000000000..cc2b713e3b --- /dev/null +++ b/crates/sqlbuild-python-base/src/orchestration_reads/main/dependency_failures.rs @@ -0,0 +1,16 @@ +//! Each published member whose `[dependencies] on_failure`, its override or else the group's, is not `partial`. + +use crate::orchestration_reads::_helpers::orchestration_declarations; +use sqlbuild_discovery::orchestration_files::models::GroupDependencies; +use sqlbuild_orchestration::declarations::models::DependencyFailure; +use std::collections::BTreeMap; +use std::collections::BTreeSet; + +/// Each published member whose `[dependencies] on_failure`, its override or else the group's, is not `partial`. +#[inline] +pub fn dependency_failures( + dependencies: &GroupDependencies, + members: &BTreeSet, +) -> BTreeMap { + orchestration_declarations::dependency_failures(dependencies, members) +} diff --git a/crates/sqlbuild-python-base/src/orchestration_reads/main/mod.rs b/crates/sqlbuild-python-base/src/orchestration_reads/main/mod.rs index 0ae0a077a2..efeb08504a 100644 --- a/crates/sqlbuild-python-base/src/orchestration_reads/main/mod.rs +++ b/crates/sqlbuild-python-base/src/orchestration_reads/main/mod.rs @@ -1,5 +1,6 @@ pub mod callback_code_hash; pub mod declared_at; +pub mod dependency_failures; pub mod environment_hash; pub mod imported_declarations; pub mod missing_secrets; diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs index 9c1b64b6b8..530161c02c 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs @@ -399,6 +399,7 @@ const R_COMPILE_PIPELINE_OPTIONS: RecordSpec = RecordSpec { FieldSpec::new("concurrency", FieldDefault::None), FieldSpec::new("selection_diagnostics", FieldDefault::False), FieldSpec::new("plan_sql_tests", FieldDefault::True), + FieldSpec::new("ignore_audit_failures", FieldDefault::False), FieldSpec::new("record_migration_fingerprints", FieldDefault::True), FieldSpec::new("accepts_unit_test_selectors", FieldDefault::False), FieldSpec::new( diff --git a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs index e9bba6a067..d3fdf60b51 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs @@ -148,6 +148,7 @@ const R_TARGET_CONFIG: RecordSpec = RecordSpec { ), FieldSpec::new("concurrency", FieldDefault::None), FieldSpec::new("catalog_connection", FieldDefault::None), + FieldSpec::new("ignore_audit_failures", FieldDefault::None), ], }; const R_TARGET_SETTINGS: RecordSpec = RecordSpec { @@ -205,6 +206,7 @@ const R_LOCAL_TARGET_CONFIG: RecordSpec = RecordSpec { ), FieldSpec::new("concurrency", FieldDefault::None), FieldSpec::new("catalog_connection", FieldDefault::None), + FieldSpec::new("ignore_audit_failures", FieldDefault::None), ], }; const R_SETTINGS_CONFIG: RecordSpec = RecordSpec { @@ -233,6 +235,7 @@ const R_SETTINGS_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("delta_compile", FieldDefault::True), FieldSpec::new("allow_missing_secrets", FieldDefault::False), FieldSpec::new("allow_warehouse_round_trips", FieldDefault::False), + FieldSpec::new("ignore_audit_failures", FieldDefault::False), ], }; const R_SCOPES_CONFIG: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs b/crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs index 92fb5279c9..cd0e1c23f4 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs @@ -1,5 +1,7 @@ //! Construction, comparison and display of native record field values. +use crate::native_host::models::Dialect; +use crate::native_host::types::Syntax; use std::collections::{HashMap, HashSet}; use std::sync::{Mutex, OnceLock}; @@ -97,6 +99,8 @@ pub(crate) fn field<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult() { + return dialect.syntax(name); } strict_fallback(value, name)?; value.getattr(name) diff --git a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs index 9d56005e6d..c094aa7ba3 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs @@ -110,6 +110,7 @@ const R_PLANNER_POLICIES: RecordSpec = RecordSpec { FieldSpec::new("selection_diagnostics", FieldDefault::False), FieldSpec::new("plan_sql_tests", FieldDefault::True), FieldSpec::new("record_migration_fingerprints", FieldDefault::True), + FieldSpec::new("ignore_audit_failures", FieldDefault::False), ], }; const R_DIRECT_SOURCE_FRESHNESS_PLANNING_RESULT: RecordSpec = RecordSpec { @@ -782,6 +783,7 @@ const R_AUDIT_PLAN_ENTRY: RecordSpec = RecordSpec { FieldSpec::new("description", FieldDefault::None), FieldSpec::new("reads_outside_target_lineage", FieldDefault::False), FieldSpec::new("meta", FieldDefault::None), + FieldSpec::new("ignore_failure", FieldDefault::False), ], }; const R_CHAIN_STEP: RecordSpec = RecordSpec { @@ -932,6 +934,7 @@ const R_PLAN_OUTPUT: RecordSpec = RecordSpec { FieldSpec::new("cursor_overrides", FieldDefault::None), FieldSpec::new("existing_model_names", FieldDefault::EmptyFrozenset), FieldSpec::new("end_audit_source_keys", FieldDefault::EmptyFrozenset), + FieldSpec::new("ignore_audit_failures", FieldDefault::False), ], }; @@ -1086,6 +1089,7 @@ const R_WAREHOUSE_SNAPSHOT: RecordSpec = RecordSpec { FieldDefault::NativeCall(RecordId::WarehouseFingerprints), ), FieldSpec::new("cursor_snapshots", FieldDefault::EmptyDict), + FieldSpec::new("cursor_inputs_at_run", FieldDefault::False), FieldSpec::new( "source_freshness_state_schemas", FieldDefault::EmptyFrozenset, diff --git a/crates/sqlbuild-python-base/src/records/constants.rs b/crates/sqlbuild-python-base/src/records/constants.rs index 32bafbeb97..305b9ce34a 100644 --- a/crates/sqlbuild-python-base/src/records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/constants.rs @@ -262,6 +262,7 @@ const BUILD_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("cli_vars", FieldDefault::None), FieldSpec::new("run_tests", FieldDefault::True), FieldSpec::new("run_audits", FieldDefault::True), + FieldSpec::new("ignore_audit_failures", FieldDefault::None), FieldSpec::new("manifest", FieldDefault::False), FieldSpec::new("json_output", FieldDefault::False), FieldSpec::new("json_output_path", FieldDefault::None), @@ -576,6 +577,8 @@ const PLAN_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("no_cache", FieldDefault::False), FieldSpec::new("max_microbatches", FieldDefault::None), FieldSpec::new("selection_diagnostics", FieldDefault::False), + FieldSpec::new("timings", FieldDefault::False), + FieldSpec::new("ignore_audit_failures", FieldDefault::None), ], }; const PLAYGROUND_COMMAND_REQUEST: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-base/src/warehouse_access/_helpers/mod.rs b/crates/sqlbuild-python-base/src/warehouse_access/_helpers/mod.rs index 36e4f4ba72..8ecf9e3591 100644 --- a/crates/sqlbuild-python-base/src/warehouse_access/_helpers/mod.rs +++ b/crates/sqlbuild-python-base/src/warehouse_access/_helpers/mod.rs @@ -1,5 +1,4 @@ pub(crate) mod sqlite_connection; pub(crate) mod state_column_evolution; -pub(crate) mod synced_copy; pub(crate) mod warehouse_catalog; pub(crate) mod warehouse_statistics; diff --git a/crates/sqlbuild-python-base/src/warehouse_access/_helpers/synced_copy.rs b/crates/sqlbuild-python-base/src/warehouse_access/_helpers/synced_copy.rs deleted file mode 100644 index a96d0b15f7..0000000000 --- a/crates/sqlbuild-python-base/src/warehouse_access/_helpers/synced_copy.rs +++ /dev/null @@ -1,109 +0,0 @@ -//! A plan preview's adapter: warehouse reads come from the location's mirrored copy, nothing connects. - -use pyo3::PyErr; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyModuleMethods; - -use pyo3::create_exception; -use pyo3::exceptions::PyRuntimeError; -use pyo3::prelude::{Bound, PyResult, Python, pyclass, pymethods}; -use serde::Deserialize; - -create_exception!( - sqlbuild._native, - WarehouseAccessRefusedError, - PyRuntimeError, - "Raised when a plan preview reaches past the synced copy for the warehouse." -); - -/// Adapter methods that only describe the dialect, never reaching the warehouse. -pub(crate) const PASSTHROUGH_NAMES: [&str; 12] = [ - "_render_source_relation_cast_subquery_with_columns", - "adapter_name", - "default_database", - "default_schema", - "default_table_promotion_mode", - "expression_inference_profile", - "metadata_inspection_concurrency", - "python_functions_inherit_default_namespace", - "requires_derived_table_aliases", - "sql_analysis_dialect", - "sql_analysis_dialect_name", - "star_exclude_keyword", -]; -/// Rendering and capability methods pass through; every other unknown attribute is refused. -pub(crate) const PASSTHROUGH_PREFIXES: [&str; 2] = ["render_", "supports_"]; -/// Local table holding the copy's fingerprints; previews read it through per-location views. -pub(crate) const FINGERPRINT_STORE: &str = "__sqlbuild_synced_fingerprints"; -pub(crate) const FINGERPRINT_STORE_COLUMNS: [&str; 2] = ["state_database", "state_schema"]; -pub(crate) const MODEL_NODE_TYPE: &str = "model"; - -/// Items a preview reports as known only once the run plans live. -pub(crate) const PREVIEW_UNKNOWNS: [&str; 4] = [ - "source freshness", - "retention", - "column rename support", - "sources outside mirrored schemas", -]; - -/// A source's described columns, keyed to the exact SQL that was described. -#[derive(Deserialize)] -pub(crate) struct DescribedColumns { - pub(crate) sql_hash: String, - pub(crate) columns: Vec, -} - -pub(crate) fn refused(access: &str) -> pyo3::PyErr { - WarehouseAccessRefusedError::new_err(format!( - "plan previews read only the synced copy; refused warehouse access: {access}" - )) -} - -pub(crate) fn same(left: Option<&str>, right: Option<&str>) -> bool { - left.unwrap_or_default().to_lowercase() == right.unwrap_or_default().to_lowercase() -} - -/// Whether `wanted` is unset or holds `value`, compared as the warehouse folds case. -pub(crate) fn any_same(wanted: Option<&[String]>, value: &str) -> bool { - let Some(wanted) = wanted else { - return true; - }; - for item in wanted { - if same(Some(value), Some(item)) { - return true; - } - } - false -} - -pub(crate) fn literal(value: &str) -> String { - format!("'{}'", value.replace('\'', "''")) -} - -/// A connection that refuses every use; like the adapter, it lives in the adapter contract's module. -#[pyclass( - frozen, - module = "sqlbuild.adapter.contract", - name = "SyncedCopyConnection" -)] -pub(crate) struct SyncedCopyConnection; - -#[pymethods] -impl SyncedCopyConnection { - pub(crate) fn __getattr__(&self, name: &str) -> PyResult<()> { - Err(refused(&format!("connection.{name}"))) - } -} - -/// Register `WarehouseAccessRefusedError` on the native module. -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add( - "WarehouseAccessRefusedError", - module.py().get_type::(), - ) -} - -/// Whether `error` is a plan preview reaching past the synced copy for the warehouse. -pub(crate) fn is_warehouse_access_refused(py: Python<'_>, error: &PyErr) -> bool { - error.is_instance_of::(py) -} diff --git a/crates/sqlbuild-python-base/src/warehouse_access/_helpers/warehouse_catalog.rs b/crates/sqlbuild-python-base/src/warehouse_access/_helpers/warehouse_catalog.rs index 2501d41f88..21deab421a 100644 --- a/crates/sqlbuild-python-base/src/warehouse_access/_helpers/warehouse_catalog.rs +++ b/crates/sqlbuild-python-base/src/warehouse_access/_helpers/warehouse_catalog.rs @@ -4,7 +4,7 @@ use pyo3::IntoPyObjectExt; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_PAGE_RECORDS; use sqlbuild_orchestration::warehouse_metadata::models::{ CatalogRead, WarehouseCatalogColumn, WarehouseCatalogRecord, WarehouseHistoryPage, WarehouseHistoryRecord, @@ -18,13 +18,13 @@ pub(crate) fn read_catalog( (adapter, connection): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), request: &CatalogRead, ) -> PyResult { - let limit = request.limit.unwrap_or(MAX_HISTORY_RECORDS); + let limit = request.limit.unwrap_or(MAX_HISTORY_PAGE_RECORDS); if request.observed_since.is_some() { return Err(PyValueError::new_err( "catalog observations require full scans, not timestamp watermarks", )); } - if limit == 0 || limit > MAX_HISTORY_RECORDS { + if limit == 0 || limit > MAX_HISTORY_PAGE_RECORDS { return Err(PyValueError::new_err( "metadata page limit is outside the supported bounds", )); @@ -106,6 +106,7 @@ pub(crate) fn read_catalog( next_after: None, change_token: None, unchanged: false, + wakes_compute: false, }); } let keys = selected.iter().map(|(_, _, key)| key); @@ -168,5 +169,6 @@ pub(crate) fn read_catalog( next_after, change_token: None, unchanged: false, + wakes_compute: false, }) } diff --git a/crates/sqlbuild-python-base/src/warehouse_access/_helpers/warehouse_statistics.rs b/crates/sqlbuild-python-base/src/warehouse_access/_helpers/warehouse_statistics.rs index 544adb787c..b6247376e6 100644 --- a/crates/sqlbuild-python-base/src/warehouse_access/_helpers/warehouse_statistics.rs +++ b/crates/sqlbuild-python-base/src/warehouse_access/_helpers/warehouse_statistics.rs @@ -1,8 +1,9 @@ -//! Metadata estimates never execute exact counts or imply source-data currency. +//! Metadata estimates never execute exact counts; one adapter read serves each page of names. use pyo3::IntoPyObjectExt; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyTuple; use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use sqlbuild_orchestration::warehouse_metadata::models::{ StatisticsRead, WarehouseHistoryPage, WarehouseHistoryRecord, WarehouseStatisticsRecord, @@ -23,30 +24,31 @@ pub(crate) fn read_statistics( "statistics sampling needs at most 100 exact relation names", )); } + let estimates = call_with( + &adapter.getattr("estimate_relation_row_counts")?, + &[ + ("connection", connection.clone()), + ( + "database", + request.location.database.clone().into_bound_py_any(py)?, + ), + ( + "schema", + request.location.schema.clone().into_bound_py_any(py)?, + ), + ("names", PyTuple::new(py, &request.names)?.into_any()), + ], + )?; + let observed_at = chrono::Utc::now(); let mut records: Vec = Vec::new(); - for name in &request.names { - let estimate = call_with( - &adapter.getattr("estimate_relation_row_count")?, - &[ - ("connection", connection.clone()), - ( - "database", - request.location.database.clone().into_bound_py_any(py)?, - ), - ( - "schema", - request.location.schema.clone().into_bound_py_any(py)?, - ), - ("name", name.into_bound_py_any(py)?), - ("metadata_only", true.into_bound_py_any(py)?), - ], - )?; + for (name, estimate) in request.names.iter().zip(estimates.try_iter()?) { + let estimate = estimate?; records.push(WarehouseHistoryRecord::Statistics( WarehouseStatisticsRecord { state_database: request.location.database.clone(), state_schema: request.location.schema.clone(), name: name.clone(), - observed_at: chrono::Utc::now(), + observed_at, row_count: field(&estimate, "row_count")?.extract()?, detail: field(&estimate, "detail")?.extract()?, byte_count: field(&estimate, "byte_count")?.extract()?, @@ -54,10 +56,16 @@ pub(crate) fn read_statistics( }, )); } + if records.len() != request.names.len() { + return Err(PyValueError::new_err( + "the adapter estimated a different number of relations than were named", + )); + } Ok(WarehouseHistoryPage { records, next_after: request.next_after.map(|id| vec![Some(id.to_string())]), change_token: None, unchanged: false, + wakes_compute: false, }) } diff --git a/crates/sqlbuild-python-base/src/warehouse_access/main/is_warehouse_access_refused.rs b/crates/sqlbuild-python-base/src/warehouse_access/main/is_warehouse_access_refused.rs deleted file mode 100644 index fe53edaac4..0000000000 --- a/crates/sqlbuild-python-base/src/warehouse_access/main/is_warehouse_access_refused.rs +++ /dev/null @@ -1,11 +0,0 @@ -//! Whether `error` is a plan preview reaching past the synced copy for the warehouse. - -use crate::warehouse_access::_helpers::synced_copy; -use pyo3::PyErr; -use pyo3::prelude::Python; - -/// Whether `error` is a plan preview reaching past the synced copy for the warehouse. -#[inline] -pub fn is_warehouse_access_refused(py: Python<'_>, error: &PyErr) -> bool { - synced_copy::is_warehouse_access_refused(py, error) -} diff --git a/crates/sqlbuild-python-base/src/warehouse_access/main/mod.rs b/crates/sqlbuild-python-base/src/warehouse_access/main/mod.rs index e9d3d3e481..8ca9274e28 100644 --- a/crates/sqlbuild-python-base/src/warehouse_access/main/mod.rs +++ b/crates/sqlbuild-python-base/src/warehouse_access/main/mod.rs @@ -1,7 +1,5 @@ pub mod adapter_default_location; pub mod ensure_event_id_column; -pub mod is_warehouse_access_refused; pub mod read_catalog; pub mod read_statistics; -pub mod register_synced_copy; pub mod sqlite_params; diff --git a/crates/sqlbuild-python-base/src/warehouse_access/main/register_synced_copy.rs b/crates/sqlbuild-python-base/src/warehouse_access/main/register_synced_copy.rs deleted file mode 100644 index 1f72828bb4..0000000000 --- a/crates/sqlbuild-python-base/src/warehouse_access/main/register_synced_copy.rs +++ /dev/null @@ -1,12 +0,0 @@ -//! Register `WarehouseAccessRefusedError` on the native module. - -use crate::warehouse_access::_helpers::synced_copy; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; - -/// Register `WarehouseAccessRefusedError` on the native module. -#[inline] -pub fn register_synced_copy(module: &Bound<'_, PyModule>) -> PyResult<()> { - synced_copy::register(module) -} diff --git a/crates/sqlbuild-python-base/src/warehouse_access/mod.rs b/crates/sqlbuild-python-base/src/warehouse_access/mod.rs index d4bb633bca..01894c458e 100644 --- a/crates/sqlbuild-python-base/src/warehouse_access/mod.rs +++ b/crates/sqlbuild-python-base/src/warehouse_access/mod.rs @@ -1,4 +1,4 @@ -//! Warehouse catalog, statistics and state-column reads, the native SQLite connection and the synced copy. +//! Warehouse catalog, statistics and state-column reads and the native SQLite connection. pub(crate) mod _helpers; pub mod main; diff --git a/crates/sqlbuild-python-base/src/warehouse_access/models.rs b/crates/sqlbuild-python-base/src/warehouse_access/models.rs index c788096175..5c928acbf4 100644 --- a/crates/sqlbuild-python-base/src/warehouse_access/models.rs +++ b/crates/sqlbuild-python-base/src/warehouse_access/models.rs @@ -1,31 +1,11 @@ //! Public records of warehouse access. -use crate::boundary::main::call_with::call_with; use crate::boundary::main::imported::imported; -use crate::boundary::models::KeywordArguments; -use crate::native_host::main::host_entry::host_entry; -use crate::records::main::fields::field::field as native_field; use crate::warehouse_access::_helpers::sqlite_connection::database_error; -use crate::warehouse_access::_helpers::synced_copy::DescribedColumns; -use crate::warehouse_access::_helpers::synced_copy::FINGERPRINT_STORE; -use crate::warehouse_access::_helpers::synced_copy::FINGERPRINT_STORE_COLUMNS; -use crate::warehouse_access::_helpers::synced_copy::MODEL_NODE_TYPE; -use crate::warehouse_access::_helpers::synced_copy::PASSTHROUGH_NAMES; -use crate::warehouse_access::_helpers::synced_copy::PASSTHROUGH_PREFIXES; -use crate::warehouse_access::_helpers::synced_copy::PREVIEW_UNKNOWNS; -use crate::warehouse_access::_helpers::synced_copy::SyncedCopyConnection; -use crate::warehouse_access::_helpers::synced_copy::any_same; -use crate::warehouse_access::_helpers::synced_copy::literal; -use crate::warehouse_access::_helpers::synced_copy::refused; -use crate::warehouse_access::_helpers::synced_copy::same; -use chrono::SecondsFormat; -use pyo3::IntoPyObject; use pyo3::IntoPyObjectExt; use pyo3::exceptions::PyIndexError; -use pyo3::exceptions::PyRuntimeError; use pyo3::exceptions::PyTypeError; use pyo3::prelude::Bound; -use pyo3::prelude::Py; use pyo3::prelude::PyAny; use pyo3::prelude::PyAnyMethods; use pyo3::prelude::PyResult; @@ -33,33 +13,11 @@ use pyo3::prelude::Python; use pyo3::prelude::pyclass; use pyo3::prelude::pymethods; use pyo3::types::PyBytes; -use pyo3::types::PyDict; -use pyo3::types::PyDictMethods; -use pyo3::types::PyString; -use pyo3::types::PyTuple; use rusqlite::Connection; use rusqlite::OpenFlags; use rusqlite::params_from_iter; use rusqlite::types::Value; -use sqlbuild_analysis::fingerprint_storage::constants::FINGERPRINT_COLUMNS; -use sqlbuild_analysis::fingerprint_storage::main::read_latest_sql::read_latest_sql; -use sqlbuild_cache::digest::main::base64_text::base64_text; -use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; -use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; -use sqlbuild_core::constants::STATE_TABLE_NAMES; -use sqlbuild_host::entries::models::AdapterHost; -use sqlbuild_orchestration::warehouse_metadata::models::PreviewCopy; -use sqlbuild_orchestration::warehouse_metadata::models::WarehouseCatalogRecord; -use sqlbuild_state::consumption::models::ConsumedInput; -use sqlbuild_state::consumption::models::ConsumptionAppend; -use sqlbuild_state::consumption::models::ConsumptionRow; -use sqlbuild_state::consumption::models::NodeKey; -use sqlbuild_state::consumption::models::RelationKey; use sqlbuild_state::execution_history::constants::MEMORY_PATH; -use sqlbuild_state::observations::constants::CURSOR_MAX_KIND; -use sqlbuild_state::observations::constants::SOURCE_COLUMNS_KIND; -use std::collections::BTreeMap; -use std::collections::BTreeSet; use std::collections::VecDeque; use std::path::PathBuf; use std::sync::Arc; @@ -291,445 +249,3 @@ impl SqliteCursor { self.last_row_id } } - -/// Answers listing, column and fingerprint reads from the copy; dialect methods use `adapter`. -#[pyclass( - frozen, - module = "sqlbuild.adapter.contract", - name = "SyncedCopyAdapter" -)] -#[derive(Debug)] -pub struct SyncedCopyAdapter { - pub(crate) adapter: Py, - pub(crate) copy: PreviewCopy, - pub(crate) store: SqliteConnection, - pub(crate) views: Mutex, String)>>, - pub(crate) unknowns: Mutex>, - pub(crate) windows: Mutex>, -} - -impl SyncedCopyAdapter { - /// The facade over `adapter` answering from the copy written at `path`. - pub fn load(py: Python<'_>, adapter: Bound<'_, PyAny>, path: &str) -> PyResult { - let text: String = std::fs::read_to_string(path)?; - let copy: PreviewCopy = serde_json::from_str(&text) - .map_err(|error| PyRuntimeError::new_err(format!("invalid synced copy: {error}")))?; - let store: SqliteConnection = SqliteConnection::open(py, MEMORY_PATH)?; - let columns: Vec<&str> = FINGERPRINT_STORE_COLUMNS - .iter() - .chain(FINGERPRINT_COLUMNS.iter()) - .copied() - .collect(); - store.execute_native( - py, - &format!("CREATE TABLE {FINGERPRINT_STORE} ({})", columns.join(", ")), - &[], - )?; - let placeholders: String = vec!["?"; columns.len()].join(", "); - for record in ©.fingerprints { - let values: [Option; 14] = [ - record.state_database.clone(), - Some(record.state_schema.clone()), - Some(record.node_type.clone()), - Some(record.node_name.clone()), - record.target_database.clone(), - record.target_schema.clone(), - record.target_name.clone(), - Some(record.run_id.clone()), - Some(record.definition_hash.clone()), - Some(record.version_hash.clone()), - Some(record.schema_fingerprint.clone()), - Some(base64_text(record.definition.as_bytes())), - Some(base64_text(record.metadata_json.as_bytes())), - Some(record.ts.to_rfc3339_opts(SecondsFormat::AutoSi, true)), - ]; - let values: Vec = values - .into_iter() - .map(|value| { - value.map_or(rusqlite::types::Value::Null, rusqlite::types::Value::Text) - }) - .collect(); - store.execute_native( - py, - &format!("INSERT INTO {FINGERPRINT_STORE} VALUES ({placeholders})"), - &values, - )?; - } - Ok(Self { - adapter: adapter.unbind(), - copy, - store, - views: Mutex::new(Vec::new()), - unknowns: Mutex::new(BTreeSet::new()), - windows: Mutex::new(BTreeMap::new()), - }) - } - - /// When the copy was synced, as ISO 8601 text; `None` before every required table synced. - pub fn synced_at(&self) -> Option { - self.copy - .synced_at - .map(|at| at.to_rfc3339_opts(SecondsFormat::AutoSi, true)) - } - - /// What the preview could not know, in a stable order. - pub fn unknowns(&self) -> Vec { - let mut unknowns: Vec = PREVIEW_UNKNOWNS - .iter() - .map(|item| (*item).to_owned()) - .collect(); - unknowns.extend( - self.unknowns - .lock() - .unwrap_or_else(PoisonError::into_inner) - .iter() - .cloned(), - ); - unknowns - } - - /// Note one item the preview reports as unknown until the run plans live. - pub fn mark_unknown(&self, item: String) { - self.unknowns - .lock() - .unwrap_or_else(PoisonError::into_inner) - .insert(item); - } - - /// Note that `model` reads from `start` up to the latest data when the run starts. - pub fn mark_window(&self, model: &str, start: &str) { - self.windows - .lock() - .unwrap_or_else(PoisonError::into_inner) - .insert(model.to_owned(), start.to_owned()); - } - - /// Each model's window start, read up to the latest data when the run starts. - pub fn windows(&self) -> BTreeMap { - self.windows - .lock() - .unwrap_or_else(PoisonError::into_inner) - .clone() - } - - /// The model's cursor `MAX` text, only when observed by its newest successful build. - pub fn target_max(&self, model: &str) -> Option<&str> { - for run in &self.copy.latest_runs { - if run.node_type != MODEL_NODE_TYPE || run.node_name != model { - continue; - } - for observation in &self.copy.observations { - if observation.kind == CURSOR_MAX_KIND - && observation.node == model - && observation.run_id == run.run_id - && observation.state_schema == run.state_schema - && observation.state_database == run.state_database - { - return Some(observation.value.as_str()); - } - } - } - None - } - - /// Every copied consumption row of the table in `(database, schema)`. - pub fn consumption(&self, (database, schema): (Option<&str>, &str)) -> Vec { - let mut rows: Vec = Vec::new(); - for record in &self.copy.consumption { - if !same(Some(&record.state_schema), Some(schema)) - || !(database.is_none() - || record.state_database.is_none() - || same(record.state_database.as_deref(), database)) - { - continue; - } - let text = |value: &Option| value.clone().unwrap_or_default(); - rows.push(ConsumptionRow { - row: ConsumptionAppend { - execution_id: text(&record.execution_id), - phase: text(&record.phase), - run_id: text(&record.run_id), - node: NodeKey { - kind: text(&record.node_type), - name: text(&record.node_name), - }, - relation: RelationKey { - database: record.target_database.clone(), - schema: text(&record.target_schema), - name: text(&record.target_name), - }, - version_hash: text(&record.version_hash), - input_count: record.input_count.unwrap_or(-1), - input: record.input_kind.clone().map(|kind| ConsumedInput { - kind, - name: text(&record.input_name), - version: text(&record.input_version), - }), - }, - recorded_at: record.recorded_at.timestamp_micros(), - }); - } - rows - } - - /// The source's columns as last described, only when it described exactly `sql`. - pub fn described_columns(&self, source: &str, sql: &str) -> PyResult>> { - let hash: String = sha256_hex(sql.as_bytes()); - for observation in &self.copy.observations { - if observation.kind != SOURCE_COLUMNS_KIND || observation.node != source { - continue; - } - let described: DescribedColumns = - serde_json::from_str(&observation.value).map_err(|error| { - PyRuntimeError::new_err(format!( - "invalid source_columns observation for {source}: {error}" - )) - })?; - if described.sql_hash == hash { - return Ok(Some(described.columns)); - } - } - Ok(None) - } - - /// The relations a listing call named by its `database`, `schemas` and optional `names`. - pub(crate) fn listed_by<'py>( - &self, - py: Python<'py>, - function: &str, - options: Option>, - ) -> PyResult> { - let arguments: KeywordArguments<'_> = KeywordArguments::read( - py, - function, - options, - (&["connection", "database", "schemas"], &["names"]), - )?; - let database: Option = arguments.value("database")?.extract()?; - let schemas: Option> = arguments.value("schemas")?.extract()?; - let names: Option> = arguments.value("names")?.extract()?; - Ok(self.listed(database.as_deref(), schemas.as_deref(), names.as_deref())) - } - - pub(crate) fn listed( - &self, - database: Option<&str>, - schemas: Option<&[String]>, - names: Option<&[String]>, - ) -> Vec<&WarehouseCatalogRecord> { - let mut listed: Vec<&WarehouseCatalogRecord> = Vec::new(); - for record in &self.copy.relations { - if (STATE_TABLE_NAMES.contains(&record.name.as_str()) - && record.name != FINGERPRINT_TABLE_NAME) - || !(database.is_none() - || record.state_database.is_none() - || same(record.state_database.as_deref(), database)) - || !any_same(schemas, &record.state_schema) - || !any_same(names, &record.name) - { - continue; - } - listed.push(record); - } - listed - } - - pub(crate) fn relation_info<'py>( - &self, - py: Python<'py>, - record: &WarehouseCatalogRecord, - ) -> PyResult> { - call_with( - &host_entry(py, AdapterHost::RelationInfo)?, - &[ - ( - "database", - record - .state_database - .as_deref() - .into_pyobject(py)? - .into_any(), - ), - ("schema", PyString::new(py, &record.state_schema).into_any()), - ("name", PyString::new(py, &record.name).into_any()), - ( - "relation_type", - PyString::new(py, &record.relation_type).into_any(), - ), - ], - ) - } - - pub(crate) fn columns<'py>( - &self, - py: Python<'py>, - record: &WarehouseCatalogRecord, - ) -> PyResult> { - let column_info: Bound<'py, PyAny> = host_entry(py, AdapterHost::ColumnInfo)?; - let mut columns: Vec> = Vec::with_capacity(record.columns.len()); - for column in &record.columns { - columns.push(call_with( - &column_info, - &[ - ("name", PyString::new(py, &column.name).into_any()), - ("type", PyString::new(py, &column.data_type).into_any()), - ], - )?); - } - PyTuple::new(py, columns) - } -} - -#[pymethods] -impl SyncedCopyAdapter { - #[pyo3(signature = (connection_config = None))] - pub(crate) fn connect( - &self, - connection_config: Option>, - ) -> SyncedCopyConnection { - drop(connection_config); - SyncedCopyConnection - } - - #[pyo3(signature = (connection = None))] - pub(crate) fn close(&self, connection: Option>) { - drop(connection); - } - - #[pyo3(name = "list_relations", signature = (**options))] - pub(crate) fn relation_listing<'py>( - &self, - py: Python<'py>, - options: Option>, - ) -> PyResult> { - let relations: Vec> = self - .listed_by(py, "list_relations", options)? - .into_iter() - .map(|record| self.relation_info(py, record)) - .collect::>()?; - PyTuple::new(py, relations) - } - - #[pyo3(name = "get_all_columns", signature = (**options))] - pub(crate) fn all_columns<'py>( - &self, - py: Python<'py>, - options: Option>, - ) -> PyResult> { - let columns: Bound<'py, PyDict> = PyDict::new(py); - for record in self.listed_by(py, "get_all_columns", options)? { - columns.set_item(&record.name, self.columns(py, record)?)?; - } - Ok(columns) - } - - #[pyo3(name = "get_columns_for_relations", signature = (*, connection, relations))] - pub(crate) fn columns_by_relation<'py>( - &self, - py: Python<'py>, - connection: Bound<'py, PyAny>, - relations: Bound<'py, PyAny>, - ) -> PyResult> { - drop(connection); - let found: Bound<'py, PyDict> = PyDict::new(py); - for relation in relations.try_iter()? { - let relation: Bound<'py, PyAny> = relation?; - let database: Option = native_field(&relation, "database")?.extract()?; - let schema: Option = native_field(&relation, "schema")?.extract()?; - let name: String = native_field(&relation, "name")?.extract()?; - let schemas: Option> = schema.clone().map(|schema| vec![schema]); - let names: [String; 1] = [name.clone()]; - if let Some(record) = self - .listed(database.as_deref(), schemas.as_deref(), Some(&names)) - .first() - { - found.set_item((database, schema, name), self.columns(py, record)?)?; - } - } - Ok(found) - } - - /// The latest-row read over this location's copied fingerprints, through a local view. - #[pyo3(signature = (*, database, schema))] - pub(crate) fn render_read_latest_fingerprints_sql( - &self, - py: Python<'_>, - database: Option, - schema: String, - ) -> PyResult { - let mut views = self.views.lock().unwrap_or_else(PoisonError::into_inner); - let key: (Option, String) = (database.clone(), schema.clone()); - let position: usize = match views.iter().position(|view| *view == key) { - Some(position) => position, - None => { - let mut condition: String = - format!("lower(state_schema) = lower({})", literal(&schema)); - if let Some(database) = &database { - condition.push_str(&format!( - " AND (state_database IS NULL OR lower(state_database) = lower({}))", - literal(database) - )); - } - self.store.execute_native( - py, - &format!( - "CREATE VIEW {FINGERPRINT_STORE}_{} AS SELECT {} FROM {FINGERPRINT_STORE} \ - WHERE {condition}", - views.len(), - FINGERPRINT_COLUMNS.join(", ") - ), - &[], - )?; - views.push(key); - views.len() - 1 - } - }; - Ok(read_latest_sql(&format!("{FINGERPRINT_STORE}_{position}"))) - } - - /// Only reads of the copied fingerprints run; any other statement would reach the warehouse. - #[pyo3(signature = (*, connection, sql))] - pub(crate) fn execute( - &self, - py: Python<'_>, - connection: Bound<'_, PyAny>, - sql: &str, - ) -> PyResult { - drop(connection); - if !sql.contains(&format!("{FINGERPRINT_STORE}_")) { - return Err(refused(sql.trim().lines().next().unwrap_or_default())); - } - self.store.execute_native(py, sql, &[]) - } - - #[pyo3(signature = (*, connection, sql))] - pub(crate) fn query_column_names( - &self, - connection: Bound<'_, PyAny>, - sql: &str, - ) -> PyResult<()> { - drop(connection); - Err(refused(&format!( - "DESCRIBE {}", - sql.trim().lines().next().unwrap_or_default() - ))) - } - - /// Values and dialect methods come from the real adapter; any other method is refused. - pub(crate) fn __getattr__<'py>( - &self, - py: Python<'py>, - name: &str, - ) -> PyResult> { - let value: Bound<'py, PyAny> = native_field(self.adapter.bind(py), name)?; - if !value.is_callable() - || PASSTHROUGH_NAMES.contains(&name) - || PASSTHROUGH_PREFIXES - .iter() - .any(|prefix| name.starts_with(prefix)) - { - return Ok(value); - } - Err(refused(&format!("adapter.{name}"))) - } -} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs index 04d9a882b3..89cbcc4650 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs @@ -1,15 +1,7 @@ //! The compile analysis sequence: discovery, graph, lineage, contracts, selection and rules. -use crate::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; -use std::time::Instant; - -use pyo3::IntoPyObject; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods}; -use sqlbuild_host::entries::models::ExecutorHost; -use sqlbuild_rules::lint_text::main::counts_toward_exit::counts_toward_exit; - use crate::compile_driver::_helpers::setup::project::build_project_graph_with_analysis_selection; +use crate::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use crate::lineage::main::build_project_column_lineage::build_project_column_lineage; use crate::rules::constants::RULE_FIX_AVAILABLE_NOTE; use crate::rules::main::checking::rules_progress::rules_progress; @@ -18,8 +10,12 @@ use crate::rules::main::rule::evaluate_rules::evaluate_rules; use crate::rules::models::EarlyRulesScope; use crate::rules::models::EarlyRulesSettings; use crate::rules::models::EarlySqlLint; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods}; use sqlbuild_analysis::graph::constants::PLANNER_DEFAULT_CODE; use sqlbuild_analysis::graph::errors::SelectorError; +use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -43,13 +39,19 @@ use sqlbuild_python_project::planner_identity::main::execution_cycle::execution_ use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; use sqlbuild_python_project::project_snapshot::main::job_group_cycles::job_group_cycles; +use sqlbuild_rules::lint_text::main::counts_toward_exit::counts_toward_exit; +use std::time::Instant; const RICH_LINEAGE: &str = "rich"; + const FAST_LINEAGE: &str = "fast"; + const RICH_COLUMN_LINEAGE: &str = "RICH"; + const FAST_COLUMN_LINEAGE: &str = "FAST"; const GENERIC_DIALECT: &str = "generic"; + const TARGET_DIRECTORY: &str = "target"; /// `ColumnLineageMode.`. @@ -570,6 +572,50 @@ impl<'py> Driver<'_, 'py> { } } +/// The core diagnostics followed by one rule-phase error per rule finding; lint-unavailable ones only warn. +fn with_rule_diagnostics<'py>( + py: Python<'py>, + core: &Bound<'py, PyTuple>, + rules_result: &Bound<'py, PyAny>, +) -> PyResult> { + let phase: Bound<'py, PyAny> = enum_member(py, RecordId::DiagnosticPhase, "RULE")?; + let severity: Bound<'py, PyAny> = enum_member(py, RecordId::DiagnosticSeverity, "ERROR")?; + let unavailable: Bound<'py, PyAny> = enum_member(py, RecordId::DiagnosticSeverity, "WARNING")?; + let fix_note: Bound<'py, PyAny> = PyString::new(py, RULE_FIX_AVAILABLE_NOTE).into_any(); + let mut diagnostics: Vec> = core.iter().collect(); + for fault in native_field(rules_result, "findings")?.try_iter()? { + let fault: Bound<'py, PyAny> = fault?; + let notes: Bound<'py, PyTuple> = if native_field(&fault, "fixable")?.is_truthy()? { + PyTuple::new(py, [fix_note.clone()])? + } else { + PyTuple::empty(py) + }; + diagnostics.push(new_record_with( + py, + RecordId::CompilerDiagnostic, + &[ + ("phase", phase.clone()), + ( + "severity", + match counts_toward_exit(&native_field(&fault, "code")?.extract::()?) { + true => severity.clone(), + false => unavailable.clone(), + }, + ), + ("code", native_field(&fault, "code")?), + ("message", native_field(&fault, "message")?), + ("path", native_field(&fault, "path")?), + ("line", native_field(&fault, "line")?), + ("column", native_field(&fault, "column")?), + ("help", native_field(&fault, "remediation")?), + ("notes", notes.into_any()), + ("affected_rules", native_field(&fault, "affected_rules")?), + ], + )?); + } + PyTuple::new(py, diagnostics) +} + /// Compile errors for what every plan refuses: an execution-order cycle and the first SQL/Python boundary error. fn boundary_diagnostics<'py>( graph: &Bound<'py, PyAny>, @@ -614,47 +660,3 @@ fn boundary_diagnostics<'py>( }) .collect() } - -/// The core diagnostics followed by one rule-phase error per rule finding; lint-unavailable ones only warn. -fn with_rule_diagnostics<'py>( - py: Python<'py>, - core: &Bound<'py, PyTuple>, - rules_result: &Bound<'py, PyAny>, -) -> PyResult> { - let phase: Bound<'py, PyAny> = enum_member(py, RecordId::DiagnosticPhase, "RULE")?; - let severity: Bound<'py, PyAny> = enum_member(py, RecordId::DiagnosticSeverity, "ERROR")?; - let unavailable: Bound<'py, PyAny> = enum_member(py, RecordId::DiagnosticSeverity, "WARNING")?; - let fix_note: Bound<'py, PyAny> = PyString::new(py, RULE_FIX_AVAILABLE_NOTE).into_any(); - let mut diagnostics: Vec> = core.iter().collect(); - for fault in native_field(rules_result, "findings")?.try_iter()? { - let fault: Bound<'py, PyAny> = fault?; - let notes: Bound<'py, PyTuple> = if native_field(&fault, "fixable")?.is_truthy()? { - PyTuple::new(py, [fix_note.clone()])? - } else { - PyTuple::empty(py) - }; - diagnostics.push(new_record_with( - py, - RecordId::CompilerDiagnostic, - &[ - ("phase", phase.clone()), - ( - "severity", - match counts_toward_exit(&native_field(&fault, "code")?.extract::()?) { - true => severity.clone(), - false => unavailable.clone(), - }, - ), - ("code", native_field(&fault, "code")?), - ("message", native_field(&fault, "message")?), - ("path", native_field(&fault, "path")?), - ("line", native_field(&fault, "line")?), - ("column", native_field(&fault, "column")?), - ("help", native_field(&fault, "remediation")?), - ("notes", notes.into_any()), - ("affected_rules", native_field(&fault, "affected_rules")?), - ], - )?); - } - PyTuple::new(py, diagnostics) -} diff --git a/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_host.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_host.rs index a3ecefdb66..f7ed339cb0 100644 --- a/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_host.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_host.rs @@ -32,15 +32,17 @@ use sqlbuild_python_front::render::models::ExpandedDeclarations; use sqlbuild_python_front::render::models::ExpansionOutput; use sqlbuild_python_front::render::types::SpanRow; use sqlbuild_python_front::sqltext::models::ScannedReference; -use sqlbuild_python_project::compiled_project::constants::{CONTRACT_KEY, INCREMENTAL_MODE_KEY}; +use sqlbuild_python_project::compiled_project::constants::CONTRACT_KEY; +use sqlbuild_python_project::compiled_project::constants::INCREMENTAL_MODE_KEY; use sqlbuild_python_project::compiled_project::main::config_values::config_values; use sqlbuild_python_project::compiled_project::main::declarations::declarations; use sqlbuild_python_project::compiled_project::main::materialization::materialization; use sqlbuild_python_project::compiled_project::models::DeclarationPy; -use sqlbuild_python_project::compiled_views::models::{ - InputProjections, ModelInputHandles, SchemaHandles, -}; -use sqlbuild_python_project::compiled_views::types::{ModelInputRecord, SchemaRecord}; +use sqlbuild_python_project::compiled_views::models::InputProjections; +use sqlbuild_python_project::compiled_views::models::ModelInputHandles; +use sqlbuild_python_project::compiled_views::models::SchemaHandles; +use sqlbuild_python_project::compiled_views::types::ModelInputRecord; +use sqlbuild_python_project::compiled_views::types::SchemaRecord; const HEADER_KEYS_ATTRIBUTE: &str = "model_header_keys"; const LAYER_SCHEMA_ATTRIBUTE: &str = "layer_schema"; diff --git a/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_opt_out.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_opt_out.rs index df8b43801b..777d7de817 100644 --- a/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_opt_out.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_opt_out.rs @@ -23,9 +23,9 @@ use sqlbuild_cache::digest::types::ContentDigest; use sqlbuild_core::constants::PANIC_MESSAGE; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; -use crate::input_loops::_helpers::model_loop::model_loop_prepass::spawned; use sqlbuild_python_base::cache::main::parse_key::parse_key; use sqlbuild_python_base::cache::models::NativeParseStore; +use sqlbuild_python_base::native_threads::main::spawned::spawned; use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; use crate::input_loops::_helpers::model::model_loop::LoopContext; diff --git a/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_prepass.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_prepass.rs index db5025480a..548d40372e 100644 --- a/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_prepass.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model_loop/model_loop_prepass.rs @@ -1,21 +1,14 @@ -//! The model loop's compile pool off the GIL: header facts first, then work the loop hands off. +//! The model loop's header facts, computed on the compile pool off the GIL. -use std::sync::{Arc, OnceLock}; +use std::sync::Arc; -use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; -use rayon::{ThreadPool, ThreadPoolBuildError, ThreadPoolBuilder}; use sqlbuild_discovery::model_files::main::header_facts::header_facts; use sqlbuild_discovery::model_files::models::HeaderFacts; +use sqlbuild_python_base::native_threads::main::pooled::pooled; use sqlbuild_sqltext::compiler::models::AuthoredValue; use crate::input_loops::_helpers::model::model_loop_host::ModelLoopRow; -/// The compile pools' fixed worker count. -const PREPASS_WORKERS: usize = 4; -const WORKER_STACK_BYTES: usize = 16 * 1024 * 1024; - -static POOL: OnceLock> = OnceLock::new(); - /// A model's native header values and the relative path parse messages name. pub(crate) type NativeHeader = (Arc<[(String, AuthoredValue)]>, String); @@ -24,35 +17,6 @@ pub(crate) fn all_header_facts(headers: &[Option]) -> Vec( - items: &[T], - each: impl Fn(&T) -> R + Sync + Send, -) -> Vec { - match compile_pool() { - Ok(pool) => pool.install(|| items.par_iter().map(&each).collect()), - Err(_) => items.iter().map(each).collect(), - } -} - -/// Run `job` on the compile pool while the caller continues, or now when the pool cannot start. -pub(crate) fn spawned(job: impl FnOnce() + Send + 'static) { - match compile_pool() { - Ok(pool) => pool.spawn(job), - Err(_) => job(), - } -} - -fn compile_pool() -> &'static Result { - POOL.get_or_init(|| { - ThreadPoolBuilder::new() - .num_threads(PREPASS_WORKERS) - .stack_size(WORKER_STACK_BYTES) - .thread_name(|index| format!("sqlbuild-model-prepass-{index}")) - .build() - }) -} - fn facts_of(header: &Option) -> Option { let (values, path) = header.as_ref()?; Some(header_facts(values, path)) diff --git a/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/audit_loop.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/audit_loop.rs index 39e8d839b0..bfa98184c6 100644 --- a/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/audit_loop.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/audit_loop.rs @@ -44,9 +44,9 @@ use sqlbuild_python_front::model_config::models::HeaderColumns; use sqlbuild_python_front::model_config::models::NativeModelValidator; use sqlbuild_python_front::model_config::models::PyNode; use sqlbuild_python_front::model_config::models::{AuditInstance, HeaderAudits}; -use sqlbuild_python_project::compiled_project::models::{ - AuditBlockRow, AuditFileRow, AuditSettings, -}; +use sqlbuild_python_project::compiled_project::models::AuditBlockRow; +use sqlbuild_python_project::compiled_project::models::AuditFileRow; +use sqlbuild_python_project::compiled_project::models::AuditSettings; use sqlbuild_python_project::resource_views::models::AuditHandles; use sqlbuild_python_project::resource_views::types::AttachedAudit; diff --git a/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/function_loop.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/function_loop.rs index c29484c4f4..71c32a6ba7 100644 --- a/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/function_loop.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/function_loop.rs @@ -47,7 +47,8 @@ use sqlbuild_python_base::compile_context::main::delta::record_source_reads::rec use sqlbuild_python_front::attachments::main::parsed_header::parsed_header; use sqlbuild_python_front::model_config::models::NativeModelValidator; use sqlbuild_python_project::analysis_session::main::query_columns_error::query_columns_error; -use sqlbuild_python_project::compiled_project::models::{FunctionRow, FunctionSettings}; +use sqlbuild_python_project::compiled_project::models::FunctionRow; +use sqlbuild_python_project::compiled_project::models::FunctionSettings; use sqlbuild_python_project::resource_views::models::FunctionHandles; use sqlbuild_python_project::resource_views::types::AttachedFunction; diff --git a/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/resource_attachment.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/resource_attachment.rs index 43edebe9bb..bfb9223a6d 100644 --- a/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/resource_attachment.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/resource_attachment.rs @@ -10,14 +10,17 @@ use crate::input_loops::models::AttachmentOwner; use crate::input_loops::models::NativeModelLoop; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_project::models::AuditFileRow; +use sqlbuild_python_project::compiled_project::models::AuditSettings; +use sqlbuild_python_project::compiled_project::models::FunctionRow; +use sqlbuild_python_project::compiled_project::models::FunctionSettings; use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; -use sqlbuild_python_project::compiled_project::models::{ - AuditFileRow, AuditSettings, FunctionRow, FunctionSettings, ScenarioRow, SqlTestSettings, - TestFileRow, -}; -use sqlbuild_python_project::compiled_project::types::{ - AttachedSqlTests, SourceRow, SourceSettings, -}; +use sqlbuild_python_project::compiled_project::models::ScenarioRow; +use sqlbuild_python_project::compiled_project::models::SqlTestSettings; +use sqlbuild_python_project::compiled_project::models::TestFileRow; +use sqlbuild_python_project::compiled_project::types::AttachedSqlTests; +use sqlbuild_python_project::compiled_project::types::SourceRow; +use sqlbuild_python_project::compiled_project::types::SourceSettings; use sqlbuild_python_project::resource_views::models::NativeAuditInput; use sqlbuild_python_project::resource_views::models::NativeFunctionInput; use sqlbuild_python_project::resource_views::models::NativeSourceInput; diff --git a/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/resource_diagnostics.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/resource_diagnostics.rs index 0c222902e9..2a83ee67a4 100644 --- a/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/resource_diagnostics.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/resource_diagnostics.rs @@ -30,11 +30,13 @@ use sqlbuild_python_base::compile_context::main::delta::injection::injection; use sqlbuild_python_base::compile_context::models::DeltaInjection; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::discovery::models::NativeModelFile; +use sqlbuild_python_project::compiled_project::models::DiagnosticSettings; +use sqlbuild_python_project::compiled_project::models::HookDescriptionRow; use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; -use sqlbuild_python_project::compiled_project::models::{ - DiagnosticSettings, HookDescriptionRow, PythonResourceRow, ScenarioDescriptionRow, -}; -use sqlbuild_python_project::compiled_project::types::{LoaderRow, ResourceDiagnostics}; +use sqlbuild_python_project::compiled_project::models::PythonResourceRow; +use sqlbuild_python_project::compiled_project::models::ScenarioDescriptionRow; +use sqlbuild_python_project::compiled_project::types::LoaderRow; +use sqlbuild_python_project::compiled_project::types::ResourceDiagnostics; use sqlbuild_python_project::resource_views::models::SeedParts; use sqlbuild_python_project::resource_views::types::SeedViews; diff --git a/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/source_loop.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/source_loop.rs index a244800c61..4a48713050 100644 --- a/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/source_loop.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/source_loop.rs @@ -32,7 +32,8 @@ use sqlbuild_python_front::scopes::models::NativeDeclarationContexts; use sqlbuild_python_front::sqltext::main::declaration_scan::declaration_scan; use sqlbuild_python_front::sqltext::main::interpolated::interpolated; use sqlbuild_python_front::sqltext::models::InterpolationSources; -use sqlbuild_python_project::compiled_project::types::{SourceRow, SourceSettings}; +use sqlbuild_python_project::compiled_project::types::SourceRow; +use sqlbuild_python_project::compiled_project::types::SourceSettings; use sqlbuild_python_project::resource_views::main::source_views::replaced::replaced; use sqlbuild_python_project::resource_views::models::SourceHandles; use sqlbuild_python_project::resource_views::types::AttachedSource; diff --git a/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/sql_test_loop.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/sql_test_loop.rs index 85fe3fcf93..486708799d 100644 --- a/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/sql_test_loop.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/sql_test_loop.rs @@ -49,9 +49,9 @@ use sqlbuild_python_base::cache::models::SqlTestScanStore; use sqlbuild_python_base::compile_context::main::delta::record_test_source_reads::record_test_source_reads; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::render::main::sql_macros::without_with_list_calls::without_with_list_calls; -use sqlbuild_python_project::compiled_project::models::{ - SqlTestSettings, TestBlockRow, TestFileRow, -}; +use sqlbuild_python_project::compiled_project::models::SqlTestSettings; +use sqlbuild_python_project::compiled_project::models::TestBlockRow; +use sqlbuild_python_project::compiled_project::models::TestFileRow; use sqlbuild_python_project::compiled_project::types::TestCaseRow; use sqlbuild_python_project::resource_views::main::source_views::replaced::replaced; diff --git a/crates/sqlbuild-python-compile/src/input_loops/main/mod.rs b/crates/sqlbuild-python-compile/src/input_loops/main/mod.rs index ecd159c3e5..bdd05222c6 100644 --- a/crates/sqlbuild-python-compile/src/input_loops/main/mod.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/main/mod.rs @@ -1,7 +1,6 @@ pub mod attach_checks; pub mod attach_resources; pub mod model_loop_classes; -pub mod pooled; pub mod register_attachment; pub mod register_expanded_offsets; pub mod register_model_loop; diff --git a/crates/sqlbuild-python-compile/src/input_loops/main/pooled.rs b/crates/sqlbuild-python-compile/src/input_loops/main/pooled.rs deleted file mode 100644 index c8100b7377..0000000000 --- a/crates/sqlbuild-python-compile/src/input_loops/main/pooled.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! `each` of every item in order, run on the compile pool, or in turn when the pool cannot start. - -use crate::input_loops::_helpers::model_loop::model_loop_prepass; - -/// `each` of every item in order, run on the compile pool, or in turn when the pool cannot start. -#[inline] -pub fn pooled(items: &[T], each: impl Fn(&T) -> R + Sync + Send) -> Vec { - model_loop_prepass::pooled(items, each) -} diff --git a/crates/sqlbuild-python-compile/src/project_assembly/main/syntax_error_messages.rs b/crates/sqlbuild-python-compile/src/project_assembly/main/syntax_error_messages.rs index 69aedb8c97..58abf32a25 100644 --- a/crates/sqlbuild-python-compile/src/project_assembly/main/syntax_error_messages.rs +++ b/crates/sqlbuild-python-compile/src/project_assembly/main/syntax_error_messages.rs @@ -1,12 +1,12 @@ //! Each check's `syntax_error_message`, in order, parsed together on the compile pool. -use crate::input_loops::main::pooled::pooled; use crate::project_assembly::_helpers::resources::syntax_value_error; use pyo3::prelude::{PyResult, Python}; use sqlbuild_analysis::assembly::project::main::sql_syntax_error::sql_syntax_error as check_syntax_error; use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxMode}; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::native_threads::main::pooled::pooled; /// Each check's `syntax_error_message`, in order, parsed together on the compile pool. pub fn syntax_error_messages( diff --git a/crates/sqlbuild-python-front/src/command_runtime/main/mod.rs b/crates/sqlbuild-python-front/src/command_runtime/main/mod.rs index 7482b588aa..8d8c59cb22 100644 --- a/crates/sqlbuild-python-front/src/command_runtime/main/mod.rs +++ b/crates/sqlbuild-python-front/src/command_runtime/main/mod.rs @@ -1,2 +1,3 @@ +pub mod published_concurrency; pub mod resolve_concurrency; pub mod resolve_configured_concurrency; diff --git a/crates/sqlbuild-python-front/src/command_runtime/main/published_concurrency.rs b/crates/sqlbuild-python-front/src/command_runtime/main/published_concurrency.rs new file mode 100644 index 0000000000..391ab27b93 --- /dev/null +++ b/crates/sqlbuild-python-front/src/command_runtime/main/published_concurrency.rs @@ -0,0 +1,44 @@ +//! The concurrency a published job without its own setting runs with on the compiled `target`, and its layer. + +use std::path::Path; + +use pyo3::IntoPyObject; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_config::effective::models::ConcurrencySource as Layer; +use sqlbuild_config::project::main::load_project_setting_keys::load_project_setting_keys; +use sqlbuild_config::project::main::project_config_paths::project_config_paths; +use sqlbuild_orchestration::declarations::models::{ConcurrencySource, ResolvedConcurrency}; + +use crate::command_runtime::_helpers::concurrency::CONCURRENCY; +use crate::command_runtime::main::resolve_concurrency::resolve_concurrency; + +/// The concurrency a published job without its own setting runs with on the compiled `target`, and its layer. +pub fn published_concurrency( + project_dir: &Path, + discovered: &Bound<'_, PyAny>, + target: Option<&str>, +) -> PyResult { + let py = discovered.py(); + let effective = resolve_concurrency( + &py.None().into_bound(py), + discovered, + target.into_pyobject(py)?.as_any(), + )?; + let project_sets = || -> PyResult { + let keys = load_project_setting_keys(&project_config_paths(project_dir)).map_err(|_| { + PyValueError::new_err("cannot read [settings] of sqlbuild_project.toml") + })?; + Ok(keys.iter().any(|key| key == CONCURRENCY)) + }; + let source = match effective.source { + Layer::CommandLine => ConcurrencySource::Job, + Layer::Target(_) => ConcurrencySource::Target, + Layer::LocalSettings => ConcurrencySource::Local, + Layer::ProjectSettings if project_sets()? => ConcurrencySource::Project, + Layer::ProjectSettings => ConcurrencySource::Default, + }; + let limit = u64::try_from(effective.limit) + .map_err(|_| PyValueError::new_err("concurrency must be at least 1"))?; + Ok(ResolvedConcurrency { limit, source }) +} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_config.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_config.rs index 2aaacb4b95..58df15eacc 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_config.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_config.rs @@ -1,14 +1,13 @@ //! Load the project and local configuration files into their Python config objects. -use std::path::PathBuf; - use pyo3::IntoPyObjectExt; use pyo3::exceptions::{PyOSError, PyValueError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; use sqlbuild_config::project::errors::{ConfigFailure, FailureClass, LoadFailure}; +use sqlbuild_config::project::main::load_pool_limits::load_pool_limits; +use sqlbuild_config::project::models::PoolLimits; use sqlbuild_config::project::models::{Built, ConfigFile, ConfigPaths}; - use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_python_base::boundary::main::config_value_to_python::config_value_to_python; use sqlbuild_python_base::records::main::construct::error_class::error_class; @@ -16,6 +15,7 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::registry::record_named::record_named; use sqlbuild_python_base::records::types::ErrorId; +use std::path::PathBuf; const FILE_NAMES: [&str; 4] = [ PROJECT_CONFIG_FILENAME, @@ -23,6 +23,7 @@ const FILE_NAMES: [&str; 4] = [ "sqlbuild_local.toml", "sqlbuild_local.yml", ]; + /// The Python module of each `StrEnum` a loaded config holds that is not native. const PYTHON_ENUM_MODULES: [(&str, &str); 1] = [("AuditSeverity", "sqlbuild.compiler.auditing.types")]; @@ -171,3 +172,12 @@ pub(crate) fn configured( let files: Files<'_> = files(project_dir)?; load(&files.native).or_else(|failure| Err(load_error(project_dir.py(), &failure, &files)?)) } + +/// `[pools]` of `project_dir` and each target's pool limits. +pub(crate) fn project_pool_limits( + py: Python<'_>, + project_dir: &Bound<'_, PyAny>, +) -> PyResult { + let files = files(project_dir)?; + load_pool_limits(&files.native).or_else(|failure| Err(load_error(py, &failure, &files)?)) +} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs index 7b412f439f..16cbfbccca 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs @@ -3,17 +3,21 @@ use crate::discovery::main::project::project_facts::project_facts; use crate::discovery::main::project::validation_error::validation_error; use crate::discovery::main::sources::orchestration_declarations::orchestration_declarations; +use crate::discovery::models::NodeRow; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::pyfunction; +use sqlbuild_discovery::orchestration_files::models::{FilePosition, OrchestrationInventory}; use sqlbuild_discovery::project_validation::main::validate_project_inputs::validate_project_inputs; use sqlbuild_discovery::project_validation::models::ProjectFacts; - -use crate::discovery::models::NodeRow; +use sqlbuild_python_base::native_host::main::write_console::write_console; use sqlbuild_python_base::records::main::fields::field::field as native_field; pub(crate) const CSV_MODULE: &str = "csv"; + pub(crate) const BUILTINS_MODULE: &str = "builtins"; + pub(crate) const PATHLIB_MODULE: &str = "pathlib"; + /// The exceptions a CSV dialect may raise, by class name. pub(crate) const CSV_DIALECT_ERRORS: [&str; 2] = ["TypeError", "ValueError"]; @@ -34,7 +38,37 @@ pub(crate) fn validate_discovered_project_inputs( let mut facts: ProjectFacts = project_facts(py, &inputs, nodes)?; let project_dir: Bound<'_, PyAny> = native_field(&inputs, "project_dir")?; if !project_dir.is_none() { - facts.orchestration = orchestration_declarations(&project_dir)?.names(); + let inventory = orchestration_declarations(&project_dir)?; + warn_deprecated_orchestration(py, &inventory)?; + facts.orchestration = inventory.names(); } validate_project_inputs(&facts).map_err(|failure| validation_error(py, failure)) } + +/// Write a warning line to stderr for each deprecated name a job or job group file still uses. +fn warn_deprecated_orchestration( + py: Python<'_>, + inventory: &OrchestrationInventory, +) -> PyResult<()> { + let runs = inventory + .jobs + .iter() + .map(|(path, job)| (path, &job.run)) + .chain( + inventory + .groups + .iter() + .map(|(path, group)| (path, &group.run)), + ); + for (path, run) in runs { + for deprecation in &run.deprecations { + let FilePosition { line, column } = deprecation.position; + write_console( + py, + "stderr", + &format!("Warning: {path}:{line}:{column}: {}\n", deprecation.message), + )?; + } + } + Ok(()) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs index 3f64b315b8..f508e7dd0d 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs @@ -1,9 +1,11 @@ //! Every job, job group, calendar and sensor of `project_dir`, or `OrchestrationDeclarationError` (D017). use crate::discovery::_helpers::failures::raised::failure_error; +use crate::discovery::_helpers::project::project_config::project_pool_limits; use crate::discovery::_helpers::sources::orchestration_files::orchestration_error; use crate::discovery::_helpers::sources::orchestration_values::check_orchestration_values; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_discovery::orchestration_files::main::check_pools::check_pools; use sqlbuild_discovery::orchestration_files::main::orchestration_inventory::orchestration_inventory; use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; use sqlbuild_discovery::tree::models::ProjectTree; @@ -20,6 +22,9 @@ pub fn orchestration_declarations( let inventory = py.detach(|| orchestration_inventory(&tree, (version.major, version.minor))); let inventory = inventory.map_err(|failure| orchestration_error(&tree, project_dir, failure))?; + let limits = project_pool_limits(py, project_dir)?; + check_pools(&inventory, &limits) + .map_err(|failure| orchestration_error(&tree, project_dir, failure))?; check_orchestration_values(py, &inventory).map_err(|failure| failure_error(py, failure))?; Ok(inventory) } diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs index 0329261a25..5516229985 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs @@ -5,7 +5,6 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_host::entries::models::FacadeHost; - use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -25,6 +24,7 @@ const CONTRACT_CLASSES: [(&str, RecordId); 6] = [ RecordId::MeasurementThresholdBound, ), ]; + const YAML_ENTRY_CLASSES: [(&str, RecordId); 9] = [ ("schema_seed_entry", RecordId::SchemaSeedEntry), ("seed_csv_settings", RecordId::SeedCsvSettings), @@ -45,7 +45,9 @@ const YAML_ENTRY_CLASSES: [(&str, RecordId); 9] = [ RecordId::SourceFreshnessValueKind, ), ]; + pub(crate) static CONTRACT_CLASS_TABLE: PyOnceLock> = PyOnceLock::new(); + pub(crate) static YAML_ENTRY_CLASS_TABLE: PyOnceLock> = PyOnceLock::new(); pub(crate) fn built_contract_classes(py: Python<'_>) -> PyResult> { diff --git a/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_io.rs b/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_io.rs new file mode 100644 index 0000000000..2f722c4def --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_io.rs @@ -0,0 +1,187 @@ +//! The consumption table through the adapter: its reads and writes. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_core::constants::CONSUMPTION_TABLE_NAME; +use sqlbuild_state::consumption::constants::{CONSUMPTION_COLUMN_TYPES, RECENT_EXECUTIONS}; +use sqlbuild_state::consumption::main::executions_by_id_sql::executions_by_id_sql; +use sqlbuild_state::consumption::main::recent_executions_sql::recent_executions_sql; +use sqlbuild_state::consumption::models::{ConsumptionAppend, ConsumptionRow}; +use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; + +use crate::native_host::_helpers::consumption_state::{EXECUTION_ID_BATCH, read_row, row_cells}; +use crate::native_host::constants::ROWS_PER_STATEMENT; +use crate::native_host::main::state_table_name::state_table_name; +use crate::native_host::models::Location; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +fn table_of(adapter: &Bound<'_, PyAny>, location: &Location) -> PyResult> { + let py: Python<'_> = adapter.py(); + let database: Bound<'_, PyAny> = location + .database + .as_deref() + .map_or_else(|| py.None().into_bound(py), |database| text(py, database)); + state_table_name( + &native_field(adapter, "render_qualified_name")?, + (&database, &text(py, &location.schema)), + CONSUMPTION_TABLE_NAME, + ) +} + +pub(crate) fn execute<'py>( + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + sql: &str, +) -> PyResult> { + call_with( + &native_field(adapter, "execute")?, + &[ + ("connection", connection.clone()), + ("sql", text(adapter.py(), sql)), + ], + ) +} + +/// Create the schema, table and indexes in `location`; `started` rows precede schema preparation. +pub(crate) fn ensure_table( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + location: &Location, +) -> PyResult<()> { + let py: Python<'_> = adapter.py(); + let keywords = location.keywords(py); + for statement in + call_with(&native_field(adapter, "render_create_schema")?, &keywords)?.try_iter()? + { + execute(adapter, connection, &statement?.extract::()?)?; + } + let create: String = call_with( + &native_field(adapter, "render_create_consumption_table_sql")?, + &keywords, + )? + .extract()?; + if !create.is_empty() { + execute(adapter, connection, &create)?; + } + for index in call_with( + &native_field(adapter, "render_create_consumption_index_sqls")?, + &keywords, + )? + .try_iter()? + { + execute(adapter, connection, &index?.extract::()?)?; + } + Ok(()) +} + +/// Whether the warehouse catalog lists the consumption table in `location`. +pub(crate) fn table_exists( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + location: &Location, +) -> PyResult { + let py: Python<'_> = adapter.py(); + let mut keywords = location.keywords(py); + keywords.push(("connection", connection.clone())); + keywords.push(("name", text(py, CONSUMPTION_TABLE_NAME))); + call_with(&native_field(adapter, "get_columns")?, &keywords)?.is_truthy() +} + +/// The consumption table of `location` when `wanted` and the warehouse lists it. +fn readable_table( + (adapter, connection): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + location: &Location, + wanted: bool, +) -> PyResult> { + let Some(table) = table_of(adapter, location)? else { + return Ok(None); + }; + if !wanted || !table_exists(adapter, connection, location)? { + return Ok(None); + } + Ok(Some(table)) +} + +/// The consumption rows `sql` reads. +fn fetched_rows( + (adapter, connection): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + sql: &str, +) -> PyResult> { + let result: Bound<'_, PyAny> = execute(adapter, connection, sql)?; + call_with(&native_field(&result, "fetchall")?, &[])? + .try_iter()? + .map(|row| read_row(&row?)) + .collect() +} + +/// Rows of the newest executions per relation in `location`; none when the table is missing. +pub(crate) fn read_recent( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + location: &Location, +) -> PyResult> { + match readable_table((adapter, connection), location, true)? { + Some(table) => fetched_rows( + (adapter, connection), + &recent_executions_sql(&table, RECENT_EXECUTIONS), + ), + None => Ok(Vec::new()), + } +} + +/// Rows of the named executions in `location`, read in batches; none when the table is missing. +pub(crate) fn read_executions( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + location: &Location, + execution_ids: &[&str], +) -> PyResult> { + let Some(table) = readable_table((adapter, connection), location, !execution_ids.is_empty())? + else { + return Ok(Vec::new()); + }; + let mut rows: Vec = Vec::new(); + for batch in execution_ids.chunks(EXECUTION_ID_BATCH) { + rows.extend(fetched_rows( + (adapter, connection), + &executions_by_id_sql(&table, batch), + )?); + } + Ok(rows) +} + +/// Append `rows` to `location`'s table, in statements no longer than every warehouse accepts. +pub(crate) fn append( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + location: &Location, + rows: &[ConsumptionAppend], +) -> PyResult<()> { + let py: Python<'_> = adapter.py(); + let Some(table) = table_of(adapter, location)? else { + return Ok(()); + }; + let recorded_at: String = + call_with(&native_field(adapter, "render_current_utc_timestamp")?, &[])?.extract()?; + let columns: Vec<&str> = CONSUMPTION_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .collect(); + for chunk in rows.chunks(ROWS_PER_STATEMENT) { + let cells: Vec> = chunk + .iter() + .map(|row| row_cells(py, row, &recorded_at)) + .collect::>()?; + execute( + adapter, + connection, + &values_insert_sql(&table, &columns, &cells), + )?; + } + Ok(()) +} diff --git a/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs b/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs index 3c72bab1e6..c85dda8d0c 100644 --- a/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs +++ b/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs @@ -1,13 +1,10 @@ -//! The consumption table through the adapter, and the connection and callbacks a recording build holds. +//! The consumption table's rows, SQL and locations, and the callbacks a recording build holds. -use crate::native_host::constants::ROWS_PER_STATEMENT; -use crate::native_host::main::execute::execute; use crate::native_host::main::fixed_state_table_sql::fixed_state_table_sql; use crate::native_host::main::integer_literal::integer_literal; use crate::native_host::main::literal::literal; use crate::native_host::main::optional_text::optional_text; use crate::native_host::main::state_table_name::state_table_name; -use crate::native_host::models::Location; use chrono::{DateTime, NaiveDateTime, Utc}; use pyo3::exceptions::PyValueError; @@ -15,23 +12,16 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyInt, PyString}; use sqlbuild_core::constants::CONSUMPTION_TABLE_NAME; -use sqlbuild_state::consumption::constants::{ - CONSUMPTION_COLUMN_TYPES, NULLABLE_COLUMNS, RECENT_EXECUTIONS, -}; -use sqlbuild_state::consumption::main::executions_by_id_sql::executions_by_id_sql; +use sqlbuild_state::consumption::constants::{CONSUMPTION_COLUMN_TYPES, NULLABLE_COLUMNS}; use sqlbuild_state::consumption::main::prune_executions_sql::prune_executions_sql; -use sqlbuild_state::consumption::main::recent_executions_sql::recent_executions_sql; use sqlbuild_state::consumption::models::{ ConsumedInput, ConsumptionAppend, ConsumptionRow, NodeKey, RelationKey, }; -use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; -use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::contracts::main::render_state_sql_literal::render_state_sql_literal; -use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Execution IDs per `IN` list when reading executions back by ID. -const EXECUTION_ID_BATCH: usize = 500; +pub(crate) const EXECUTION_ID_BATCH: usize = 500; /// Warehouse timestamp texts with a UTC offset, as drivers render aware values. const OFFSET_TIME_FORMATS: [&str; 2] = ["%Y-%m-%dT%H:%M:%S%.f%:z", "%Y-%m-%d %H:%M:%S%.f%:z"]; /// Warehouse timestamp texts without an offset, which the consumption table stores as UTC. @@ -41,19 +31,6 @@ pub(crate) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { PyString::new(py, value).into_any() } -fn table_of(adapter: &Bound<'_, PyAny>, location: &Location) -> PyResult> { - let py: Python<'_> = adapter.py(); - let database: Bound<'_, PyAny> = location - .database - .as_deref() - .map_or_else(|| py.None().into_bound(py), |database| text(py, database)); - state_table_name( - &native_field(adapter, "render_qualified_name")?, - (&database, &text(py, &location.schema)), - CONSUMPTION_TABLE_NAME, - ) -} - /// Create-if-missing DDL for the consumption table, with adapter-rendered column types. #[pyfunction] #[pyo3(signature = (*, database, schema, render_qualified_name, render_framework_type))] @@ -92,98 +69,6 @@ pub(crate) fn build_consumption_prune_sql( .unwrap_or_default()) } -/// Create the schema, table and indexes in `location`; `started` rows precede schema preparation. -pub(crate) fn ensure_table( - adapter: &Bound<'_, PyAny>, - connection: &Bound<'_, PyAny>, - location: &Location, -) -> PyResult<()> { - let py: Python<'_> = adapter.py(); - let keywords = location.keywords(py); - for statement in - call_with(&native_field(adapter, "render_create_schema")?, &keywords)?.try_iter()? - { - execute(adapter, connection, &statement?.extract::()?)?; - } - let create: String = call_with( - &native_field(adapter, "render_create_consumption_table_sql")?, - &keywords, - )? - .extract()?; - if !create.is_empty() { - execute(adapter, connection, &create)?; - } - for index in call_with( - &native_field(adapter, "render_create_consumption_index_sqls")?, - &keywords, - )? - .try_iter()? - { - execute(adapter, connection, &index?.extract::()?)?; - } - Ok(()) -} - -/// Whether the warehouse catalog lists the consumption table in `location`. -pub(crate) fn table_exists( - adapter: &Bound<'_, PyAny>, - connection: &Bound<'_, PyAny>, - location: &Location, -) -> PyResult { - let py: Python<'_> = adapter.py(); - let mut keywords = location.keywords(py); - keywords.push(("connection", connection.clone())); - keywords.push(("name", text(py, CONSUMPTION_TABLE_NAME))); - call_with(&native_field(adapter, "get_columns")?, &keywords)?.is_truthy() -} - -/// Rows of the newest executions per relation in `location`; none when the table is missing. -pub(crate) fn read_recent( - adapter: &Bound<'_, PyAny>, - connection: &Bound<'_, PyAny>, - location: &Location, -) -> PyResult> { - let Some(table) = table_of(adapter, location)? else { - return Ok(Vec::new()); - }; - if !table_exists(adapter, connection, location)? { - return Ok(Vec::new()); - } - let result: Bound<'_, PyAny> = execute( - adapter, - connection, - &recent_executions_sql(&table, RECENT_EXECUTIONS), - )?; - call_with(&native_field(&result, "fetchall")?, &[])? - .try_iter()? - .map(|row| read_row(&row?)) - .collect() -} - -/// Rows of the named executions in `location`, read in batches; none when the table is missing. -pub(crate) fn read_executions( - adapter: &Bound<'_, PyAny>, - connection: &Bound<'_, PyAny>, - location: &Location, - execution_ids: &[&str], -) -> PyResult> { - let Some(table) = table_of(adapter, location)? else { - return Ok(Vec::new()); - }; - if execution_ids.is_empty() || !table_exists(adapter, connection, location)? { - return Ok(Vec::new()); - } - let mut rows: Vec = Vec::new(); - for batch in execution_ids.chunks(EXECUTION_ID_BATCH) { - let result: Bound<'_, PyAny> = - execute(adapter, connection, &executions_by_id_sql(&table, batch))?; - for row in call_with(&native_field(&result, "fetchall")?, &[])?.try_iter()? { - rows.push(read_row(&row?)?); - } - } - Ok(rows) -} - fn required_text(value: &Bound<'_, PyAny>) -> PyResult { Ok(optional_text(value)?.unwrap_or_default()) } @@ -207,7 +92,7 @@ fn epoch_micros(value: &Bound<'_, PyAny>) -> PyResult { ))) } -fn read_row(row: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn read_row(row: &Bound<'_, PyAny>) -> PyResult { let cell = |index: usize| row.get_item(index); let input_kind: Option = optional_text(&cell(10)?)?; let input: Option = match input_kind { @@ -261,7 +146,11 @@ pub(crate) fn state_literal( render_state_sql_literal(py, &value, declared_type, None) } -fn row_cells(py: Python<'_>, row: &ConsumptionAppend, recorded_at: &str) -> PyResult> { +pub(crate) fn row_cells( + py: Python<'_>, + row: &ConsumptionAppend, + recorded_at: &str, +) -> PyResult> { let input = row.input.as_ref(); Ok(vec![ literal(py, Some(&row.execution_id))?, @@ -280,34 +169,3 @@ fn row_cells(py: Python<'_>, row: &ConsumptionAppend, recorded_at: &str) -> PyRe recorded_at.to_owned(), ]) } - -/// Append `rows` to `location`'s table, in statements no longer than every warehouse accepts. -pub(crate) fn append( - adapter: &Bound<'_, PyAny>, - connection: &Bound<'_, PyAny>, - location: &Location, - rows: &[ConsumptionAppend], -) -> PyResult<()> { - let py: Python<'_> = adapter.py(); - let Some(table) = table_of(adapter, location)? else { - return Ok(()); - }; - let recorded_at: String = - call_with(&native_field(adapter, "render_current_utc_timestamp")?, &[])?.extract()?; - let columns: Vec<&str> = CONSUMPTION_COLUMN_TYPES - .iter() - .map(|(column, _)| *column) - .collect(); - for chunk in rows.chunks(ROWS_PER_STATEMENT) { - let cells: Vec> = chunk - .iter() - .map(|row| row_cells(py, row, &recorded_at)) - .collect::>()?; - execute( - adapter, - connection, - &values_insert_sql(&table, &columns, &cells), - )?; - } - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/native_host/_helpers/mod.rs b/crates/sqlbuild-python-front/src/native_host/_helpers/mod.rs index 378e185cd4..efbf0ab752 100644 --- a/crates/sqlbuild-python-front/src/native_host/_helpers/mod.rs +++ b/crates/sqlbuild-python-front/src/native_host/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod audit_results_state; +pub(crate) mod consumption_io; pub(crate) mod consumption_state; pub(crate) mod observations_state; diff --git a/crates/sqlbuild-python-front/src/native_host/main/append_replaced_consumption.rs b/crates/sqlbuild-python-front/src/native_host/main/append_replaced_consumption.rs index 075e9464b6..841980adf4 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/append_replaced_consumption.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/append_replaced_consumption.rs @@ -7,7 +7,7 @@ use sqlbuild_state::consumption::main::replaced_append::replaced_append; use sqlbuild_state::consumption::models::{ConsumptionAppend, NodeKey, RelationKey}; use uuid::Uuid; -use crate::native_host::_helpers::consumption_state::{append, table_exists}; +use crate::native_host::_helpers::consumption_io::{append, table_exists}; use crate::native_host::models::Location; /// Append a never-completed execution for each of `replaced` where its schema has a consumption table. diff --git a/crates/sqlbuild-python-front/src/native_host/main/execute.rs b/crates/sqlbuild-python-front/src/native_host/main/execute.rs index b0b3f91534..c3aab6eee8 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/execute.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/execute.rs @@ -1,9 +1,9 @@ //! Execute. -use crate::native_host::_helpers::consumption_state::text; -use pyo3::prelude::{Bound, PyAny, PyResult}; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::records::main::fields::field::field as native_field; +use crate::native_host::_helpers::consumption_io; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; /// Execute. pub fn execute<'py>( @@ -11,11 +11,5 @@ pub fn execute<'py>( connection: &Bound<'py, PyAny>, sql: &str, ) -> PyResult> { - call_with( - &native_field(adapter, "execute")?, - &[ - ("connection", connection.clone()), - ("sql", text(adapter.py(), sql)), - ], - ) + consumption_io::execute(adapter, connection, sql) } diff --git a/crates/sqlbuild-python-front/src/native_host/mod.rs b/crates/sqlbuild-python-front/src/native_host/mod.rs index d3f58713b5..25dd4f94c9 100644 --- a/crates/sqlbuild-python-front/src/native_host/mod.rs +++ b/crates/sqlbuild-python-front/src/native_host/mod.rs @@ -4,3 +4,4 @@ pub(crate) mod _helpers; pub mod constants; pub mod main; pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-front/src/native_host/models.rs b/crates/sqlbuild-python-front/src/native_host/models.rs index d4fb5a61ea..f2ce6baf6c 100644 --- a/crates/sqlbuild-python-front/src/native_host/models.rs +++ b/crates/sqlbuild-python-front/src/native_host/models.rs @@ -1,10 +1,11 @@ //! Public records of native host. -use crate::native_host::_helpers::consumption_state::append; -use crate::native_host::_helpers::consumption_state::ensure_table; -use crate::native_host::_helpers::consumption_state::read_executions; -use crate::native_host::_helpers::consumption_state::read_recent; +use crate::native_host::_helpers::consumption_io::append; +use crate::native_host::_helpers::consumption_io::ensure_table; +use crate::native_host::_helpers::consumption_io::read_executions; +use crate::native_host::_helpers::consumption_io::read_recent; use crate::native_host::_helpers::consumption_state::text; +use crate::native_host::types::ConsumptionReads; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::Bound; use pyo3::prelude::Py; @@ -13,9 +14,6 @@ use pyo3::prelude::PyAnyMethods; use pyo3::prelude::PyResult; use pyo3::prelude::Python; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; -use sqlbuild_state::consumption::constants::RECENT_EXECUTIONS; -use sqlbuild_state::consumption::main::recent_executions::recent_executions; use sqlbuild_state::consumption::models::ConsumptionAppend; use sqlbuild_state::consumption::models::ConsumptionRow; use sqlbuild_state::consumption::models::RelationKey; @@ -89,14 +87,6 @@ impl ConsumptionConnection { py: Python<'_>, location: &Location, ) -> PyResult> { - if let Ok(copy) = self.adapter.bind(py).downcast::() { - return Ok(recent_executions( - © - .get() - .consumption((location.database.as_deref(), &location.schema)), - RECENT_EXECUTIONS, - )); - } read_recent(self.adapter.bind(py), &self.connected(py)?, location) } @@ -106,13 +96,6 @@ impl ConsumptionConnection { location: &Location, execution_ids: &[&str], ) -> PyResult> { - if let Ok(copy) = self.adapter.bind(py).downcast::() { - let mut rows: Vec = copy - .get() - .consumption((location.database.as_deref(), &location.schema)); - rows.retain(|row| execution_ids.contains(&row.row.execution_id.as_str())); - return Ok(rows); - } read_executions( self.adapter.bind(py), &self.connected(py)?, @@ -144,6 +127,21 @@ impl ConsumptionConnection { } } +impl ConsumptionReads for ConsumptionConnection { + fn read_recent(&self, py: Python<'_>, location: &Location) -> PyResult> { + ConsumptionConnection::read_recent(self, py, location) + } + + fn read_executions( + &self, + py: Python<'_>, + location: &Location, + execution_ids: &[&str], + ) -> PyResult> { + ConsumptionConnection::read_executions(self, py, location, execution_ids) + } +} + /// The build's own node-completion callback, which the recorder forwards every result to. #[derive(Debug)] pub struct NodeCompleteHook(Py); diff --git a/crates/sqlbuild-python-front/src/native_host/types.rs b/crates/sqlbuild-python-front/src/native_host/types.rs new file mode 100644 index 0000000000..0d0b83f623 --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/types.rs @@ -0,0 +1,19 @@ +//! Public traits of native host. + +use crate::native_host::models::Location; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_state::consumption::models::ConsumptionRow; + +/// Reading recorded consumption: from the warehouse table, or from a synced state. +pub trait ConsumptionReads { + /// The newest executions per relation recorded in `location`. + fn read_recent(&self, py: Python<'_>, location: &Location) -> PyResult>; + /// The named executions recorded in `location`. + fn read_executions( + &self, + py: Python<'_>, + location: &Location, + execution_ids: &[&str], + ) -> PyResult>; +} diff --git a/crates/sqlbuild-python-output/src/build_progress/_helpers/result_text.rs b/crates/sqlbuild-python-output/src/build_progress/_helpers/result_text.rs index e99382c092..969cee0698 100644 --- a/crates/sqlbuild-python-output/src/build_progress/_helpers/result_text.rs +++ b/crates/sqlbuild-python-output/src/build_progress/_helpers/result_text.rs @@ -94,6 +94,7 @@ fn audit_result_text(audit: &AuditExecutionResult>) -> AuditRes }, execution_error: audit.execution_error.clone(), skip_reason: audit.skip_reason.clone(), + ignored: audit.ignored, } } diff --git a/crates/sqlbuild-python-output/src/result_output/_helpers/results/aggregation.rs b/crates/sqlbuild-python-output/src/result_output/_helpers/results/aggregation.rs index 9c38b88447..f1a2b9583e 100644 --- a/crates/sqlbuild-python-output/src/result_output/_helpers/results/aggregation.rs +++ b/crates/sqlbuild-python-output/src/result_output/_helpers/results/aggregation.rs @@ -1,8 +1,10 @@ //! Build `BuildExecutionResult` with aggregate counts and overall status from typed results. +use crate::result_output::_helpers::results::projection::Projection; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyDict; use pyo3::{IntoPyObjectExt, pyfunction}; +use sqlbuild_core::build_aggregate::constants::IGNORED_AUDIT_FAILURE; use sqlbuild_core::build_aggregate::main::aggregate_build_counts::aggregate_build_counts; use sqlbuild_core::build_aggregate::models::{BuildCounts, BuildResultFacts, ModelResultFacts}; use sqlbuild_host::entries::models::ExecutorHost; @@ -11,8 +13,6 @@ use sqlbuild_host::models::{ ModelExecutionResult, SeedExecutionResult, SqlTestExecutionResult, }; use sqlbuild_host::types::ResultEnum; - -use crate::result_output::_helpers::results::projection::Projection; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -68,11 +68,7 @@ impl<'a, V> RetainedBuildResults<'a, V> { pub(crate) fn counts(&self) -> BuildCounts { let mut models: Vec = Vec::new(); for model in &self.models { - let audit_outcomes: Vec = model - .audit_results - .iter() - .map(|audit| audit.outcome.value().to_owned()) - .collect(); + let audit_outcomes: Vec = model.audit_results.iter().map(audit_fact).collect(); models.push(ModelResultFacts { status: model.status.value().to_owned(), error_code: model.error_code.clone(), @@ -110,12 +106,12 @@ impl<'a, V> RetainedBuildResults<'a, V> { source_audit_outcomes: self .source_audits .iter() - .map(|result| result.outcome.value().to_owned()) + .map(|result| audit_fact(result)) .collect(), end_audit_outcomes: self .end_audits .iter() - .map(|result| result.outcome.value().to_owned()) + .map(|result| audit_fact(result)) .collect(), }) } @@ -147,7 +143,7 @@ fn facts(sections: &[Bound<'_, PyAny>]) -> PyResult { status: native_field(&model, "status")?.str()?.extract()?, error_code: native_field(&model, "error_code")?.extract()?, warning_count: native_field(&model, "warning_messages")?.len()?, - audit_outcomes: texts(&native_field(&model, "audit_results")?, "outcome")?, + audit_outcomes: audit_facts(&native_field(&model, "audit_results")?)?, }) }) .collect::>()?; @@ -167,8 +163,8 @@ fn facts(sections: &[Bound<'_, PyAny>]) -> PyResult { functions, load_statuses: texts(§ions[3], "status")?, test_outcomes: texts(§ions[4], "outcome")?, - source_audit_outcomes: texts(§ions[5], "outcome")?, - end_audit_outcomes: texts(§ions[6], "outcome")?, + source_audit_outcomes: audit_facts(§ions[5])?, + end_audit_outcomes: audit_facts(§ions[6])?, }) } @@ -192,6 +188,10 @@ pub(crate) fn aggregated_build_result<'py>( "insufficient_count", counts.insufficient.into_bound_py_any(py)?, ), + ( + "ignored_audit_failure_count", + counts.ignored_audit_failures.into_bound_py_any(py)?, + ), ]); call_with( &host_entry(py, ExecutorHost::BuildExecutionResult)?, @@ -212,3 +212,27 @@ pub(crate) fn aggregate_build_result<'py>( .collect::>()?; aggregated_build_result(request.py(), §ions) } + +/// An audit's outcome as the aggregation counts it: an ignored failure is its own outcome. +fn audit_fact(audit: &AuditExecutionResult) -> String { + if audit.ignored { + return IGNORED_AUDIT_FAILURE.to_owned(); + } + audit.outcome.value().to_owned() +} + +/// Each audit result's outcome as the aggregation counts it. +fn audit_facts(results: &Bound<'_, PyAny>) -> PyResult> { + results + .try_iter()? + .map(|result| audit_result_fact(&result?)) + .collect() +} + +/// A Python audit result's outcome as [`audit_fact`] counts it. +fn audit_result_fact(result: &Bound<'_, PyAny>) -> PyResult { + if native_field(result, "ignored")?.is_truthy()? { + return Ok(IGNORED_AUDIT_FAILURE.to_owned()); + } + native_field(result, "outcome")?.str()?.extract() +} diff --git a/crates/sqlbuild-python-planner/Cargo.toml b/crates/sqlbuild-python-planner/Cargo.toml index 2495999310..d587689ca4 100644 --- a/crates/sqlbuild-python-planner/Cargo.toml +++ b/crates/sqlbuild-python-planner/Cargo.toml @@ -8,6 +8,7 @@ publish.workspace = true rust-version.workspace = true [dependencies] +chrono.workspace = true pyo3.workspace = true serde_json.workspace = true sqlbuild-analysis.workspace = true diff --git a/crates/sqlbuild-python-planner/src/consumption/_helpers/facts.rs b/crates/sqlbuild-python-planner/src/consumption/_helpers/facts.rs index ac4fe2ff2a..d564c2eeb5 100644 --- a/crates/sqlbuild-python-planner/src/consumption/_helpers/facts.rs +++ b/crates/sqlbuild-python-planner/src/consumption/_helpers/facts.rs @@ -1,14 +1,20 @@ //! The facts `--changed` and the consumption recorder read from the plan and compiled project. use crate::consumption::main::relation::relation; -use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::collections::HashMap; +use std::collections::HashSet; +use std::collections::VecDeque; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyDict; use sqlbuild_core::fingerprint_identity::main::model_local_identity_hash::model_local_identity_hash; use sqlbuild_state::consumption::models::{ForcedRun, InputFact, NodeFacts, NodeKey, RelationKey}; -use crate::consumption::_helpers::relation_reads::{Locations, sql_reads_undeclared}; +use crate::consumption::_helpers::relation_reads::Locations; +use crate::consumption::_helpers::relation_reads::prefetch_reads; +use crate::consumption::_helpers::relation_reads::sql_reads_undeclared; use crate::planner_identity::main::function_local_hashes::function_local_hashes; use crate::planner_identity::main::hook_version_hashes::hook_version_hashes; use crate::planner_identity::main::model_version_identity_metadata_json::model_version_identity_metadata_json; @@ -312,6 +318,7 @@ pub(crate) fn build_facts( checked: HashMap::new(), }); let mut nodes: Vec = Vec::new(); + let mut reads: Vec = Vec::new(); let mut pending: VecDeque = selected.keys().cloned().collect(); let mut visited: BTreeSet = BTreeSet::new(); while let Some(key) = pending.pop_front() { @@ -346,34 +353,66 @@ pub(crate) fn build_facts( pending.push_back(upstream.clone()); } } - let mut forced: Vec = selected.get(&key).cloned().unwrap_or_default(); let destination: Bound<'_, PyAny> = node.destination()?; - let relation: Option = (!native_field(&destination, "schema")?.is_none()) - .then(|| relation(&destination)) - .transpose()?; - if let (Compiled::Model(model), Some(locations), Some(function_checks)) = - (node, &locations, &mut function_checks) - && (sql_reads_undeclared( - py, - query_sql(model)?.extract()?, - (&node_inputs, relation.as_ref()), - (locations, dialect.as_deref()), - )? || function_checks.undeclared(py, &deps)?) - { - forced.push(ForcedRun::InputsNotEnumerable); - } - let Some(relation) = relation else { + if native_field(&destination, "schema")?.is_none() { continue; - }; + } + if let (Compiled::Model(model), Some(_)) = (node, &locations) { + reads.push(ReadCheck { + position: nodes.len(), + sql: query_sql(model)?.extract()?, + deps, + }); + } nodes.push(NodeFacts { selected: selected.contains_key(&key), + forced: selected.get(&key).cloned().unwrap_or_default(), key, - relation, + relation: relation(&destination)?, version_hash, - forced, inputs: node_inputs, }); } - runnable.retain(|key| nodes.iter().any(|node| node.key == *key)); + if let (Some(locations), Some(function_checks)) = (&locations, &mut function_checks) { + undeclared_reads(py, (&mut nodes, reads), (locations, function_checks))?; + } + let walked: HashSet = nodes.iter().map(|node| node.key.clone()).collect(); + runnable.retain(|key| walked.contains(key)); Ok(BuildFacts { nodes, runnable }) } + +/// A walked model whose compiled SQL must read only its declared inputs. +struct ReadCheck { + position: usize, + sql: String, + deps: Vec<(String, String)>, +} + +/// Force each checked model whose SQL or functions read beyond its declared inputs. +fn undeclared_reads( + py: Python<'_>, + (nodes, reads): (&mut [NodeFacts], Vec), + (locations, function_checks): (&Locations, &mut FunctionReads<'_>), +) -> PyResult<()> { + let dialect: Option<&str> = function_checks.dialect; + prefetch_reads( + py, + reads.iter().map(|read| read.sql.clone()).collect(), + dialect, + )?; + for read in reads { + let Some(node) = nodes.get_mut(read.position) else { + continue; + }; + if sql_reads_undeclared( + py, + read.sql, + (&node.inputs, Some(&node.relation)), + (locations, dialect), + )? || function_checks.undeclared(py, &read.deps)? + { + node.forced.push(ForcedRun::InputsNotEnumerable); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-python-planner/src/consumption/_helpers/recorder.rs b/crates/sqlbuild-python-planner/src/consumption/_helpers/recorder.rs index 9afb032387..56d1ab5278 100644 --- a/crates/sqlbuild-python-planner/src/consumption/_helpers/recorder.rs +++ b/crates/sqlbuild-python-planner/src/consumption/_helpers/recorder.rs @@ -3,6 +3,7 @@ use crate::consumption::models::ChangedDecision; use crate::consumption::models::ConsumptionRecorder; use crate::consumption::models::Prepared; +use sqlbuild_python_front::native_host::types::ConsumptionReads; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::Mutex; use std::time::{Duration, Instant}; @@ -165,9 +166,9 @@ pub(crate) fn prepare<'py>( } /// Recent executions the build reads, with older ones read back when deciding needs them. -pub(crate) fn read_history( +pub(crate) fn read_history( py: Python<'_>, - connection: &ConsumptionConnection, + connection: &R, (facts, policy): (&BuildFacts, Option<&ChangedPolicy>), ) -> (ConsumptionHistory, Vec) { let mut warnings: Vec = Vec::new(); @@ -225,9 +226,9 @@ fn needs_read_back(policy: &ChangedPolicy) -> bool { } /// `history` plus every older upstream execution its builds read, followed back until none is left. -fn with_read_back( +fn with_read_back( py: Python<'_>, - connection: &ConsumptionConnection, + connection: &R, history: ConsumptionHistory, nodes: &[NodeFacts], ) -> (ConsumptionHistory, Vec) { @@ -241,9 +242,9 @@ fn with_read_back( } /// The rows of these executions; a schema that cannot be read leaves its rebuilds counted. -fn read_references( +fn read_references( py: Python<'_>, - connection: &ConsumptionConnection, + connection: &R, references: &[(RelationKey, String)], ) -> (Vec, Vec) { let mut by_location: BTreeMap> = BTreeMap::new(); diff --git a/crates/sqlbuild-python-planner/src/consumption/_helpers/relation_reads.rs b/crates/sqlbuild-python-planner/src/consumption/_helpers/relation_reads.rs index 25bb384cb0..e9ddf99c0a 100644 --- a/crates/sqlbuild-python-planner/src/consumption/_helpers/relation_reads.rs +++ b/crates/sqlbuild-python-planner/src/consumption/_helpers/relation_reads.rs @@ -4,12 +4,15 @@ use std::collections::{BTreeSet, HashMap}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_analysis::auditing::main::replace_reference_calls::replace_reference_calls; +use sqlbuild_analysis::relation_names::main::extract_relation_names::extract_relation_names; use sqlbuild_state::consumption::models::{InputFact, NodeKey, RelationKey}; +use sqlbuild_python_base::native_threads::main::pooled::pooled; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_front::native_host::main::optional_text::optional_text; use sqlbuild_python_project::analysis::main::extract_sql_relation_names::extract_sql_relation_names; +use sqlbuild_python_project::analysis::main::prefetch_sql_relation_names::prefetch_sql_relation_names; use sqlbuild_python_project::analysis::types::RelationRow; /// Function calls, `__udf("name")(...)`, parsed as plain calls; their reads are declared inputs. @@ -115,6 +118,30 @@ fn names((name, schema, database): &RelationRow, location: &RelationKey) -> bool }) } +/// `sql` with function calls as plain calls, as the read check parses it. +fn plain_calls(sql: String) -> String { + FUNCTION_CALLS.iter().fold(sql, |sql, function| { + replace_reference_calls(&sql, function, |_| { + Ok::<_, std::convert::Infallible>(Some((*function).to_owned())) + }) + .unwrap_or_else(|never| match never {}) + }) +} + +/// Parse the relations each of `sqls` reads, in parallel, ahead of their read checks. +pub(crate) fn prefetch_reads( + py: Python<'_>, + sqls: Vec, + dialect: Option<&str>, +) -> PyResult<()> { + prefetch_sql_relation_names( + py, + sqls.into_iter().map(plain_calls).collect(), + dialect, + |missing: &[String]| pooled(missing, |sql: &String| extract_relation_names(sql, dialect)), + ) +} + /// True when `sql` cannot be parsed or names an ambiguous relation or one no input writes. pub(crate) fn sql_reads_undeclared( py: Python<'_>, @@ -122,12 +149,7 @@ pub(crate) fn sql_reads_undeclared( (inputs, own): (&[InputFact], Option<&RelationKey>), (locations, dialect): (&Locations, Option<&str>), ) -> PyResult { - let sql: String = FUNCTION_CALLS.iter().fold(sql, |sql, function| { - replace_reference_calls(&sql, function, |_| { - Ok::<_, std::convert::Infallible>(Some((*function).to_owned())) - }) - .unwrap_or_else(|never| match never {}) - }); + let sql: String = plain_calls(sql); let Some((relations, temporary)) = extract_sql_relation_names(py, &sql, dialect)? else { return Ok(true); }; diff --git a/crates/sqlbuild-python-planner/src/consumption/main/preview_changed.rs b/crates/sqlbuild-python-planner/src/consumption/main/preview_changed.rs index 0f5cb9af37..86bc064471 100644 --- a/crates/sqlbuild-python-planner/src/consumption/main/preview_changed.rs +++ b/crates/sqlbuild-python-planner/src/consumption/main/preview_changed.rs @@ -5,20 +5,18 @@ use crate::consumption::_helpers::recorder::changed_decision; use crate::consumption::_helpers::recorder::read_history; use crate::consumption::models::ChangedDecision; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use sqlbuild_python_front::native_host::models::ConsumptionConnection; +use sqlbuild_python_front::native_host::types::ConsumptionReads; use sqlbuild_state::consumption::models::{ChangedPolicy, ConsumptionHistory}; /// What `plan --changed` previews: what a build would run now, read without recording anything. -pub fn preview_changed( +pub fn preview_changed( (plan, project): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - (adapter, connection_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + reads: &R, (policy, full_refresh): (&ChangedPolicy, bool), ) -> PyResult<(ChangedDecision, Vec)> { let py: Python<'_> = plan.py(); let facts: BuildFacts = build_facts((plan, project), (true, full_refresh))?; - let connection = ConsumptionConnection::new(adapter, connection_config); let (history, warnings): (ConsumptionHistory, Vec) = - read_history(py, &connection, (&facts, Some(policy))); - connection.close(py)?; + read_history(py, reads, (&facts, Some(policy))); Ok((changed_decision(&facts, &history, policy), warnings)) } diff --git a/crates/sqlbuild-python-planner/src/freshness/_helpers/freshness_observe.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/freshness_observe.rs index 37f08a5d38..aebcfb9910 100644 --- a/crates/sqlbuild-python-planner/src/freshness/_helpers/freshness_observe.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/freshness_observe.rs @@ -1,19 +1,23 @@ //! Observe the current freshness of selected sources and compare it with previous state. use crate::freshness::models::PreviousState; +use crate::live_reads::models::LiveReads; +use sqlbuild_python_base::native_host::models::Dialect; use std::collections::{HashMap, HashSet}; use pyo3::exceptions::{PyException, PyValueError}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyString}; use sqlbuild_core::freshness_text::main::freshness_json::freshness_json; use sqlbuild_host::entries::models::ExecutorHost; -use crate::freshness::_helpers::observation::{ - observation_record, observation_source, observe_adapter_batch, observe_configured, strategy, -}; +use crate::freshness::_helpers::observation::observation_record; +use crate::freshness::_helpers::observation::observation_source; +use crate::freshness::_helpers::observation::observe_adapter_batch; +use crate::freshness::_helpers::observation::observe_configured; +use crate::freshness::_helpers::observation::strategy; use crate::freshness::models::Observer; use sqlbuild_core::freshness_text::main::freshness_status::freshness_status; use sqlbuild_core::freshness_text::models::{ @@ -40,12 +44,15 @@ fn optional_text(value: Bound<'_, PyAny>) -> PyResult> { Ok(Some(value.str()?.extract()?)) } -/// `str(exc)` of an `Exception`, or the error itself for anything else. -fn exception_message(py: Python<'_>, error: PyErr) -> PyResult { - if !error.is_instance_of::(py) { - return Err(error); +/// `result`, with an `Exception` caught as its `str(exc)`; anything else still raises. +fn caught(py: Python<'_>, result: PyResult) -> PyResult> { + match result { + Ok(value) => Ok(Ok(value)), + Err(error) if error.is_instance_of::(py) => { + Ok(Err(error.value(py).str()?.extract()?)) + } + Err(error) => Err(error), } - error.value(py).str()?.extract() } fn target_result( @@ -136,18 +143,6 @@ fn observed_result( Ok(result) } -/// Adapter-strategy observations in one batch, or the message of the batch failure. -fn adapter_batch<'py>( - observer: &Observer<'py>, - sources: &[Bound<'py, PyAny>], -) -> PyResult, String>> { - let py: Python<'py> = observer.adapter.py(); - match observe_adapter_batch(observer, sources) { - Ok(batch) => Ok(Ok(batch)), - Err(error) => Ok(Err(exception_message(py, error)?)), - } -} - /// One adapter-strategy source's observation, or its unknown or error result. fn adapter_observation<'py>( batch: &Result, String>, @@ -195,13 +190,12 @@ fn configured_observation<'py>( observer: &Observer<'py>, (name, source): (&str, &Bound<'py, PyAny>), ) -> PyResult, FreshnessSourceResult>> { - let py: Python<'py> = source.py(); - match observe_configured(observer, source) { + match caught(source.py(), observe_configured(observer, source))? { Ok(observation) => Ok(Ok(observation)), - Err(error) => Ok(Err(target_result( + Err(message) => Ok(Err(target_result( source, (name, FreshnessStatus::Error), - Some(exception_message(py, error)?), + Some(message), )?)), } } @@ -217,7 +211,7 @@ fn observe_sources( let mut adapter_sources: Vec> = Vec::new(); for source in sources { let name: String = native_field(source, "name")?.extract()?; - let Some(observed_source) = observation_source(&observer.adapter, source)? else { + let Some(observed_source) = observation_source(&observer.dialect, source)? else { results.push(target_result( source, (&name, FreshnessStatus::Unknown), @@ -236,7 +230,10 @@ fn observe_sources( } } } - let batch = adapter_batch(observer, &adapter_sources)?; + let batch = caught( + observer.reads.py(), + observe_adapter_batch(observer, &adapter_sources), + )?; for (name, source) in &observed { let observation = if strategy(source)?.as_deref() == Some(ADAPTER_STRATEGY) { adapter_observation(&batch, (name, source))? @@ -258,7 +255,7 @@ pub(crate) fn observe_freshness( (sources, select, exclude): (&Bound<'_, PyAny>, &[String], &[String]), (run_id, state): (Option, &PreviousState<'_>), ) -> PyResult> { - let py: Python<'_> = observer.adapter.py(); + let py: Python<'_> = observer.reads.py(); let selected: HashSet<&str> = select.iter().map(String::as_str).collect(); let excluded: HashSet<&str> = exclude.iter().map(String::as_str).collect(); let mut chosen: Vec> = Vec::new(); @@ -314,9 +311,11 @@ pub(crate) fn observe_source_freshness_json( } value.extract() }; + let adapter: Bound<'_, PyAny> = arguments.value("adapter")?; let observer: Observer<'_> = Observer { - adapter: arguments.value("adapter")?, - connection: arguments.value("connection")?, + dialect: Dialect::of(&adapter)?, + reads: LiveReads::new(&adapter, &arguments.value("connection")?, None) + .handle(adapter.py())?, observed_at: arguments.value("observed_at")?, }; let previous: Bound<'_, PyAny> = arguments.value("previous_records")?; diff --git a/crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs index cc734a6bcf..c5724ca86e 100644 --- a/crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs @@ -2,10 +2,15 @@ use crate::freshness::main::presumed_utc_datetime::presumed_utc_datetime; use crate::freshness::models::Observer; +use crate::planner_warehouse::main::reads::refused_access::refused_access; +use crate::planner_warehouse::models::PlannerReads; +use crate::planner_warehouse::models::{AdapterRead, Read}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; +use sqlbuild_python_base::native_host::models::Dialect; +use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_state::freshness::main::single_value_problem::single_value_problem; use sqlbuild_state::freshness::models::QueryShape; @@ -50,7 +55,7 @@ pub(crate) fn strategy(source: &Bound<'_, PyAny>) -> PyResult> { /// The source as observed: itself with freshness config, an adapter-metadata copy, or none. pub(crate) fn observation_source<'py>( - adapter: &Bound<'py, PyAny>, + dialect: &Bound<'py, Dialect>, source: &Bound<'py, PyAny>, ) -> PyResult>> { let py: Python<'py> = source.py(); @@ -59,8 +64,9 @@ pub(crate) fn observation_source<'py>( } if !native_field(source, "expression")?.is_none() || native_field(source, "table")?.is_none() - || !adapter - .call_method0("supports_table_freshness_metadata")? + || !dialect + .syntax("supports_table_freshness_metadata")? + .call0()? .is_truthy()? { return Ok(None); @@ -191,9 +197,11 @@ fn observe_configured_inner<'py>( } else { format!(" WHERE {}", filter.str()?) }; - let relation: String = render_source_read(source, &observer.adapter)?; + let relation: String = render_source_read(source, &observer.dialect)?; call_with( - &native_field(&observer.adapter, "render_source_freshness_max_query")?, + &observer + .dialect + .syntax("render_source_freshness_max_query")?, &[ ("column", column), ("source_relation", text(py, &relation)), @@ -235,15 +243,16 @@ fn single_data_version<'py>( ) -> PyResult> { let py: Python<'py> = source_name.py(); let name: String = source_name.str()?.extract()?; - let result: Bound<'py, PyAny> = match call_with( - &native_field(&observer.adapter, "query")?, - &[ - ("connection", observer.connection.clone()), - ("sql", sql), - ("limit", py.None().into_bound(py)), - ], - ) { - Ok(result) => result, + let result: Bound<'py, PyAny> = match observer + .reads + .get() + .reads() + .adapter_read(py, AdapterRead::FreshnessQuery(sql)) + { + Ok(Read::Known(result)) => result, + Ok(Read::Unknown(_)) => { + return Err(refused_access(&format!("freshness query of source {name}"))); + } Err(error) => { let Some(message) = exception_text(py, &error)? else { return Err(error); @@ -278,16 +287,17 @@ fn single_data_version<'py>( } fn unsupported_message( - adapter: &Bound<'_, PyAny>, + dialect: &Bound<'_, Dialect>, source: &Bound<'_, PyAny>, ) -> PyResult> { - if !adapter - .call_method0("supports_table_freshness_metadata")? + if !dialect + .syntax("supports_table_freshness_metadata")? + .call0()? .is_truthy()? { return Ok(Some(format!( "adapter '{}' does not support table freshness metadata", - native_field(adapter, "adapter_name")?.str()? + dialect.syntax("adapter_name")?.str()? ))); } if !native_field(source, "expression")?.is_none() || native_field(source, "table")?.is_none() { @@ -378,18 +388,25 @@ fn observe_adapter_source<'py>( source: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = source.py(); - if let Some(message) = unsupported_message(&observer.adapter, source)? { + if let Some(message) = unsupported_message(&observer.dialect, source)? { return Err(observation_error(py, &message)); } - let metadata: Bound<'py, PyAny> = call_with( - &native_field(&observer.adapter, "get_table_freshness_metadata")?, - &[ - ("connection", observer.connection.clone()), - ("database", native_field(source, "database")?), - ("schema", native_field(source, "schema")?), - ("name", native_field(source, "table")?), - ], - )?; + let metadata: Bound<'py, PyAny> = match observer.reads.get().reads().adapter_read( + py, + AdapterRead::SingleTableFreshness([ + native_field(source, "database")?, + native_field(source, "schema")?, + native_field(source, "table")?, + ]), + )? { + Read::Known(metadata) => metadata, + Read::Unknown(_) => { + let name: String = native_field(source, "name")?.str()?.extract()?; + return Err(refused_access(&format!( + "freshness metadata of source {name}" + ))); + } + }; let name: Bound<'py, PyAny> = native_field(source, "name")?; if let Some(unknown) = metadata_unknown(&name, &metadata)? { return Err(observation_error( @@ -407,7 +424,7 @@ pub(crate) fn observe_adapter_batch<'py>( observer: &Observer<'py>, sources: &[Bound<'py, PyAny>], ) -> PyResult> { - let py: Python<'py> = observer.adapter.py(); + let py: Python<'py> = observer.reads.py(); if sources.is_empty() { return new_record(py, RecordId::AdapterSourceFreshnessBatch, (), None); } @@ -433,13 +450,13 @@ fn observe_adapter_batch_inner<'py>( observer: &Observer<'py>, sources: &[Bound<'py, PyAny>], ) -> PyResult> { - let py: Python<'py> = observer.adapter.py(); + let py: Python<'py> = observer.reads.py(); let unknown_sources: Bound<'py, PyDict> = PyDict::new(py); let requests: Bound<'py, PyDict> = PyDict::new(py); let request_class: Bound<'py, PyAny> = host_entry(py, AdapterHost::TableFreshnessRequest)?; for source in sources { let name: Bound<'py, PyAny> = native_field(source, "name")?; - if let Some(message) = unsupported_message(&observer.adapter, source)? { + if let Some(message) = unsupported_message(&observer.dialect, source)? { unknown_sources .set_item(&name, unknown(&name, ("UNAVAILABLE", text(py, &message)))?)?; continue; @@ -467,13 +484,16 @@ fn observe_adapter_batch_inner<'py>( for request in requests.values() { unique.set_item(request, py.None())?; } - let metadata_by_request: Bound<'py, PyAny> = call_with( - &native_field(&observer.adapter, "get_tables_freshness_metadata")?, - &[ - ("connection", observer.connection.clone()), - ("requests", PyTuple::new(py, unique.keys())?.into_any()), - ], - )?; + let unique_requests: Bound<'py, PyAny> = PyTuple::new(py, unique.keys())?.into_any(); + let metadata_by_request: Bound<'py, PyAny> = match observer + .reads + .get() + .reads() + .adapter_read(py, AdapterRead::TableFreshness(unique_requests))? + { + Read::Known(metadata) => metadata, + Read::Unknown(item) => return Err(refused_access(&item)), + }; let metadata_class: Bound<'py, PyAny> = host_entry(py, AdapterHost::TableFreshnessMetadata)?; let observations: Bound<'py, PyDict> = PyDict::new(py); for (name, request) in requests.iter() { @@ -548,16 +568,16 @@ pub(crate) fn observation_record<'py>( /// Observe one source freshness config and return a comparable data version. #[pyfunction] -#[pyo3(signature = (*, adapter, connection, source, observed_at))] +#[pyo3(signature = (*, dialect, reads, source, observed_at))] pub(crate) fn observe_configured_source_freshness<'py>( - adapter: Bound<'py, PyAny>, - connection: Bound<'py, PyAny>, + dialect: Bound<'py, Dialect>, + reads: Bound<'py, PlannerReads>, source: &Bound<'py, PyAny>, observed_at: Bound<'py, PyAny>, ) -> PyResult> { let observer: Observer<'py> = Observer { - adapter, - connection, + dialect, + reads, observed_at, }; observe_configured(&observer, source) @@ -565,16 +585,16 @@ pub(crate) fn observe_configured_source_freshness<'py>( /// Observe adapter metadata freshness per source, batching the warehouse lookups. #[pyfunction] -#[pyo3(signature = (*, adapter, connection, sources, observed_at))] +#[pyo3(signature = (*, dialect, reads, sources, observed_at))] pub(crate) fn observe_adapter_sources_freshness<'py>( - adapter: Bound<'py, PyAny>, - connection: Bound<'py, PyAny>, + dialect: Bound<'py, Dialect>, + reads: Bound<'py, PlannerReads>, sources: &Bound<'py, PyAny>, observed_at: Bound<'py, PyAny>, ) -> PyResult> { let observer: Observer<'py> = Observer { - adapter, - connection, + dialect, + reads, observed_at, }; let sources: Vec> = sources.try_iter()?.collect::>()?; diff --git a/crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs index 4829e50f12..f8894e6c07 100644 --- a/crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs @@ -13,23 +13,27 @@ use sqlbuild_host::entries::models::AdapterHost; use sqlbuild_state::freshness::main::configuration_error::configuration_error; use crate::freshness::_helpers::exception_types::SourceFreshnessObservationError; -use crate::freshness::_helpers::observation::{ - observation_record, observation_source, observe_adapter_batch, observe_configured, strategy, - unknown, -}; -use crate::freshness::_helpers::state_io::read_latest; +use crate::freshness::_helpers::observation::observation_record; +use crate::freshness::_helpers::observation::observation_source; +use crate::freshness::_helpers::observation::observe_adapter_batch; +use crate::freshness::_helpers::observation::observe_configured; +use crate::freshness::_helpers::observation::strategy; +use crate::freshness::_helpers::observation::unknown; use crate::freshness::_helpers::support::text; use crate::freshness::models::Observer; -use crate::freshness::models::StateRead; +use crate::planner_warehouse::main::reads::is_refused_access::is_refused_access; +use crate::planner_warehouse::models::PlannerReads; +use crate::planner_warehouse::models::Read; +use crate::planner_warehouse::models::StateHistoryRead; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::models::Dialect; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_base::warehouse_access::main::is_warehouse_access_refused::is_warehouse_access_refused; use sqlbuild_python_front::state_rows::main::event_rows::age_policy::age_policy; use sqlbuild_python_front::state_rows::main::event_rows::equivalent::equivalent; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; @@ -39,7 +43,7 @@ const NO_CONFIG_MESSAGE: &str = "no freshness config and adapter metadata unavai /// The message of an adapter, freshness observation or preview access error, if it is one. fn handled_message<'py>(py: Python<'py>, error: &PyErr) -> PyResult>> { - if is_warehouse_access_refused(py, error) { + if is_refused_access(py, error) { return Ok(Some(text(py, PREVIEW_MESSAGE))); } let value: Bound<'py, PyAny> = error.value(py).clone().into_any(); @@ -59,24 +63,27 @@ fn freshness_strategy_is_adapter(source: &Bound<'_, PyAny>) -> PyResult { /// Latest previous records by identity across the state schemas, newest observation winning. fn previous_records<'py>( arguments: &KeywordArguments<'py>, - (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + reads: &Bound<'py, PlannerReads>, source_names: Option<&[String]>, ) -> PyResult> { - let py: Python<'py> = adapter.py(); - let read: StateRead<'py> = StateRead { - connection: connection.clone(), - execute: native_field(adapter, "execute")?, - render_qualified_name: arguments.value("render_qualified_name")?, - render_read_latest_sql: native_field(adapter, "render_read_latest_source_freshness_sql")?, - }; + let py: Python<'py> = reads.py(); let database: Bound<'py, PyAny> = arguments.value("state_database")?; let exists_by_schema: Bound<'py, PyAny> = arguments.value("state_table_exists_by_schema")?; let merged: Bound<'py, PyDict> = PyDict::new(py); for schema in arguments.value("state_schemas")?.try_iter()? { let schema: Bound<'py, PyAny> = schema?; let exists: bool = mapping_get_or(&exists_by_schema, &schema, false)?.is_truthy()?; - let previous: Bound<'py, PyAny> = - read_latest(&read, (&database, &schema), exists, source_names)?; + let Read::Known(previous) = reads.get().reads().state_history( + py, + StateHistoryRead::SourceFreshness { + location: (&database, &schema), + table_exists: exists, + source_names, + }, + )? + else { + continue; + }; for (identity, candidate) in native_field(&previous, "records")? .downcast::()? .iter() @@ -123,7 +130,7 @@ fn adapter_batch<'py>( observer: &Observer<'py>, sources: &[Bound<'py, PyAny>], ) -> PyResult> { - let py: Python<'py> = observer.adapter.py(); + let py: Python<'py> = observer.reads.py(); let error: PyErr = match observe_adapter_batch(observer, sources) { Ok(batch) => return Ok(batch), Err(error) => error, @@ -156,8 +163,8 @@ pub(crate) fn build_direct_source_freshness_planning_result<'py>( options, ( &[ - "adapter", - "connection", + "dialect", + "reads", "sources", "state_database", "state_schemas", @@ -170,8 +177,8 @@ pub(crate) fn build_direct_source_freshness_planning_result<'py>( ), )?; let observer: Observer<'py> = Observer { - adapter: arguments.value("adapter")?, - connection: arguments.value("connection")?, + dialect: Dialect::passed(arguments.value("dialect")?)?, + reads: PlannerReads::passed(arguments.value("reads")?)?, observed_at: arguments.value("observed_at")?, }; let run_id: Bound<'py, PyAny> = arguments.value("run_id")?; @@ -190,11 +197,8 @@ pub(crate) fn build_direct_source_freshness_planning_result<'py>( } else { None }; - let previous: Bound<'py, PyDict> = previous_records( - &arguments, - (&observer.adapter, &observer.connection), - source_names.as_deref(), - )?; + let previous: Bound<'py, PyDict> = + previous_records(&arguments, &observer.reads, source_names.as_deref())?; let unknown_sources: Bound<'py, PyDict> = PyDict::new(py); let observed_sources: Bound<'py, PyDict> = PyDict::new(py); let mut adapter_sources: Vec> = Vec::new(); @@ -203,7 +207,7 @@ pub(crate) fn build_direct_source_freshness_planning_result<'py>( continue; } let name: Bound<'py, PyAny> = native_field(source, "name")?; - let Some(observed) = observation_source(&observer.adapter, source)? else { + let Some(observed) = observation_source(&observer.dialect, source)? else { unknown_sources.set_item( &name, unknown(&name, ("NO_FRESHNESS_CONFIG", text(py, NO_CONFIG_MESSAGE)))?, diff --git a/crates/sqlbuild-python-planner/src/freshness/_helpers/state_io.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/state_io.rs index 6bcbeed0ce..2fa1d3789e 100644 --- a/crates/sqlbuild-python-planner/src/freshness/_helpers/state_io.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/state_io.rs @@ -13,7 +13,10 @@ use sqlbuild_state::freshness::constants::{ use sqlbuild_state::freshness::main::read_latest_sql::read_latest_sql; use sqlbuild_state::freshness::main::relevant_sources_sql::relevant_sources_sql; -use crate::freshness::_helpers::support::{INPUT_ERROR, exception_text, freshness_error, text}; +use crate::freshness::_helpers::support::INPUT_ERROR; +use crate::freshness::_helpers::support::exception_text; +use crate::freshness::_helpers::support::freshness_error; +use crate::freshness::_helpers::support::text; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; diff --git a/crates/sqlbuild-python-planner/src/freshness/_helpers/support.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/support.rs index f4b87f48bf..f2e9ffbc44 100644 --- a/crates/sqlbuild-python-planner/src/freshness/_helpers/support.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/support.rs @@ -5,9 +5,8 @@ use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyString; -use crate::freshness::_helpers::exception_types::{ - SourceFreshnessInputError, SourceFreshnessObservationError, -}; +use crate::freshness::_helpers::exception_types::SourceFreshnessInputError; +use crate::freshness::_helpers::exception_types::SourceFreshnessObservationError; use sqlbuild_python_front::state_rows::main::event_rows::date_time_typed::date_time_typed; pub(crate) const OBSERVATION_ERROR: fn(String) -> PyErr = SourceFreshnessObservationError::new_err; diff --git a/crates/sqlbuild-python-planner/src/freshness/models.rs b/crates/sqlbuild-python-planner/src/freshness/models.rs index ba73434ddc..beab1775ac 100644 --- a/crates/sqlbuild-python-planner/src/freshness/models.rs +++ b/crates/sqlbuild-python-planner/src/freshness/models.rs @@ -1,7 +1,9 @@ //! Public records of freshness. +use crate::planner_warehouse::models::PlannerReads; use pyo3::prelude::Bound; use pyo3::prelude::PyAny; +use sqlbuild_python_base::native_host::models::Dialect; /// Previous state records by identity; absent means no comparison. #[derive(Debug)] @@ -9,11 +11,11 @@ pub struct PreviousState<'py> { pub by_identity: Option>, } -/// The adapter, connection and observation time every source is observed with. +/// The SQL dialect, warehouse reads and observation time every source is observed with. #[derive(Debug)] pub struct Observer<'py> { - pub adapter: Bound<'py, PyAny>, - pub connection: Bound<'py, PyAny>, + pub dialect: Bound<'py, Dialect>, + pub reads: Bound<'py, PlannerReads>, pub observed_at: Bound<'py, PyAny>, } diff --git a/crates/sqlbuild-python-planner/src/lib.rs b/crates/sqlbuild-python-planner/src/lib.rs index ee655109a6..a2ea78bc3b 100644 --- a/crates/sqlbuild-python-planner/src/lib.rs +++ b/crates/sqlbuild-python-planner/src/lib.rs @@ -5,6 +5,7 @@ pub mod consumption; pub mod freshness; pub mod lifecycle_events; +pub mod live_reads; pub mod microbatches; pub mod observations; pub mod plan_output; diff --git a/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs index 31720fe93d..e7b96e0bf5 100644 --- a/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs @@ -17,10 +17,12 @@ use sqlbuild_core::lifecycle_events::models::{HookFacts, SequenceFact}; use sqlbuild_python_base::boundary::main::optional_text::optional_text; use crate::lifecycle_events::_helpers::payloads::validation_error; -use crate::lifecycle_events::_helpers::validation::{ - frozen_json, known_event_check, optional_text_check, required_text_check, schema_version_check, - timestamp_check, -}; +use crate::lifecycle_events::_helpers::validation::frozen_json; +use crate::lifecycle_events::_helpers::validation::known_event_check; +use crate::lifecycle_events::_helpers::validation::optional_text_check; +use crate::lifecycle_events::_helpers::validation::required_text_check; +use crate::lifecycle_events::_helpers::validation::schema_version_check; +use crate::lifecycle_events::_helpers::validation::timestamp_check; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; diff --git a/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/validation.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/validation.rs index b2bc109d64..a27eb28597 100644 --- a/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/validation.rs +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/validation.rs @@ -6,9 +6,10 @@ use pyo3::types::{PyBool, PyInt, PyString}; use sqlbuild_core::lifecycle_events::main::validate_known_event::validate_known_event; use sqlbuild_core::lifecycle_events::models::KnownEventFacts; -use crate::lifecycle_events::_helpers::payloads::{ - PathSegment, frozen, object_entries, validation_error, -}; +use crate::lifecycle_events::_helpers::payloads::PathSegment; +use crate::lifecycle_events::_helpers::payloads::frozen; +use crate::lifecycle_events::_helpers::payloads::object_entries; +use crate::lifecycle_events::_helpers::payloads::validation_error; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-planner/src/live_reads/_helpers/bounded.rs b/crates/sqlbuild-python-planner/src/live_reads/_helpers/bounded.rs new file mode 100644 index 0000000000..ddf86cf94e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/live_reads/_helpers/bounded.rs @@ -0,0 +1,172 @@ +//! Warehouse reads answered live through the adapter. + +use crate::live_reads::constants::QUERY_RECORD; +use crate::live_reads::constants::RECORD_QUERY; +use crate::live_reads::constants::RUN_RECORDED_QUERY; +use crate::planner_warehouse::constants::RUN_BOUNDED; +use crate::planner_warehouse::models::EligibleCursorQuery; +use pyo3::IntoPyObject; +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyCFunction, PyDict, PyFloat, PyString, PyTuple}; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; +use std::time::Instant; + +pub(crate) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +/// Run one cursor statement; a failure leaves only that relation unavailable. +pub(crate) fn read_cursor_query<'py>( + py: Python<'py>, + sql: &str, + (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let started = Instant::now(); + let outcome = call_with( + execute, + &[("connection", connection.clone()), ("sql", text(py, sql))], + ) + .and_then(|result| result.call_method0("fetchall")); + let elapsed = started.elapsed().as_secs_f64(); + let (row, error, count) = match outcome { + Ok(rows) => { + let count = rows.len()?; + let row = if count > 0 { + rows.get_item(0)? + } else { + py.None().into_bound(py) + }; + ( + row, + py.None().into_bound(py), + count.into_pyobject(py)?.into_any(), + ) + } + Err(error) if error.is_instance_of::(py) => { + let value = error.value(py).clone().into_any(); + (py.None().into_bound(py), value, py.None().into_bound(py)) + } + Err(error) => return Err(error), + }; + let error_text = if error.is_none() { + py.None().into_bound(py) + } else { + error.str()?.into_any() + }; + let record = call_with( + &host_entry(py, QUERY_RECORD)?, + &[ + ("sql", text(py, sql)), + ("elapsed_seconds", PyFloat::new(py, elapsed).into_any()), + ("row_count", count), + ("error", error_text), + ], + )?; + call_with(&host_entry(py, RECORD_QUERY)?, &[("record", record)])?; + Ok(PyTuple::new(py, [row, PyFloat::new(py, elapsed).into_any(), error])?.into_any()) +} + +/// A bounded-inspection task running one cursor statement. +pub(crate) fn bound_task<'py>( + py: Python<'py>, + sql: String, + (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let connection: Py = connection.clone().unbind(); + let execute: Py = execute.clone().unbind(); + Ok(PyCFunction::new_closure( + py, + None, + None, + move |args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>| + -> PyResult> { + let py = args.py(); + Ok(read_cursor_query(py, &sql, (connection.bind(py), execute.bind(py)))?.unbind()) + }, + )? + .into_any()) +} + +/// A bounded-inspection task reading one model's highest eligible target cursor. +pub(crate) fn eligible_task<'py>( + py: Python<'py>, + query: EligibleCursorQuery, + (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let connection: Py = connection.clone().unbind(); + let execute: Py = execute.clone().unbind(); + Ok(PyCFunction::new_closure( + py, + None, + None, + move |args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>| -> PyResult> { + let py = args.py(); + let run_connection = connection.clone_ref(py); + let run_execute = execute.clone_ref(py); + let run_sql = query.sql.clone(); + let run = PyCFunction::new_closure( + py, + None, + None, + move |args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>| -> PyResult> { + let py = args.py(); + Ok(run_execute + .bind(py) + .call((), Some(&keywords(py, &[ + ("connection", run_connection.bind(py).clone()), + ("sql", text(py, &run_sql)), + ])?))? + .call_method0("fetchall")? + .unbind()) + }, + )?; + let rows = call_with( + &host_entry(py, RUN_RECORDED_QUERY)?, + &[("sql", text(py, &query.sql)), ("run", run.into_any())], + ); + match rows { + Ok(rows) => Ok(rows.unbind()), + Err(error) if error.is_instance_of::(py) => { + let wrapped: PyErr = planner_error( + py, + &format!( + "model '{}': failed to query highest eligible target cursor for {}.{}: {}", + query.model_name, + query.target, + query.cursor_column, + error.value(py).str()? + ), + (None, None), + ); + wrapped.set_cause(py, Some(error)); + Err(wrapped) + } + Err(error) => Err(error), + } + }, + )? + .into_any()) +} + +pub(crate) fn run_bounded<'py>( + py: Python<'py>, + tasks: Vec>, + concurrency: i64, + on_complete: Option>, +) -> PyResult>> { + let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("tasks", PyTuple::new(py, tasks)?.into_any()), + ("concurrency", concurrency.into_pyobject(py)?.into_any()), + ]; + if let Some(on_complete) = on_complete { + fields.push(("on_complete", on_complete)); + } + call_with(&host_entry(py, RUN_BOUNDED)?, &fields)? + .try_iter()? + .collect() +} diff --git a/crates/sqlbuild-python-planner/src/live_reads/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/live_reads/_helpers/mod.rs new file mode 100644 index 0000000000..b3886f7bf0 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/live_reads/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod bounded; diff --git a/crates/sqlbuild-python-planner/src/live_reads/constants.rs b/crates/sqlbuild-python-planner/src/live_reads/constants.rs new file mode 100644 index 0000000000..3c29abdd37 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/live_reads/constants.rs @@ -0,0 +1,20 @@ +//! Public constants of live warehouse reads. + +use sqlbuild_host::entries::models::AdapterHost; + +pub const ACTIVE_CATALOG: AdapterHost = AdapterHost::ActiveInspectionCatalog; + +pub const INSPECTION_CATALOG: AdapterHost = AdapterHost::OpenInspectionCatalog; + +pub const RELATIONS_MAIN_LIST: AdapterHost = AdapterHost::ListRelationsForInspection; + +pub const RELATIONS_MAIN_COLUMNS: AdapterHost = AdapterHost::GetColumnsForInspection; + +pub const RECORD_QUERY: AdapterHost = AdapterHost::RecordInspectionQuery; + +pub const RUN_RECORDED_QUERY: AdapterHost = AdapterHost::RunRecordedInspectionQuery; + +pub const QUERY_RECORD: AdapterHost = AdapterHost::InspectionQueryRecord; + +/// The adapter method that runs one statement on a connection. +pub const EXECUTE: &str = "execute"; diff --git a/crates/sqlbuild-python-planner/src/live_reads/main/_live_reads.rs b/crates/sqlbuild-python-planner/src/live_reads/main/_live_reads.rs new file mode 100644 index 0000000000..f3d1e734dd --- /dev/null +++ b/crates/sqlbuild-python-planner/src/live_reads/main/_live_reads.rs @@ -0,0 +1,23 @@ +//! Live warehouse reads for planner functions called from Python. + +use crate::live_reads::models::LiveReads; +use crate::planner_warehouse::models::PlannerReads; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use std::sync::Arc; + +/// Live warehouse reads for planner functions called from Python, optionally observing statements. +#[pyfunction] +#[pyo3(signature = (*, adapter, connection, connection_config = None, execute = None))] +pub(in crate::live_reads) fn live_reads<'py>( + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + connection_config: Option>, + execute: Option>, +) -> PyResult> { + let mut reads = LiveReads::new(adapter, connection, connection_config.as_ref()); + reads.execute = execute + .filter(|execute| !execute.is_none()) + .map(Bound::unbind); + Bound::new(adapter.py(), PlannerReads::new(Arc::new(reads))) +} diff --git a/crates/sqlbuild-python-planner/src/live_reads/main/bound_task.rs b/crates/sqlbuild-python-planner/src/live_reads/main/bound_task.rs new file mode 100644 index 0000000000..8230420d86 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/live_reads/main/bound_task.rs @@ -0,0 +1,14 @@ +//! A bounded warehouse read task over a planning connection. + +use crate::live_reads::_helpers::bounded; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; + +/// A bounded warehouse read task running `sql` over a planning connection. +#[inline] +pub fn bound_task<'py>( + py: Python<'py>, + sql: String, + (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + bounded::bound_task(py, sql, (connection, execute)) +} diff --git a/crates/sqlbuild-python-planner/src/live_reads/main/mod.rs b/crates/sqlbuild-python-planner/src/live_reads/main/mod.rs new file mode 100644 index 0000000000..3fbfcb0b8e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/live_reads/main/mod.rs @@ -0,0 +1,3 @@ +pub(super) mod _live_reads; +pub mod bound_task; +pub mod register_live_reads; diff --git a/crates/sqlbuild-python-planner/src/live_reads/main/register_live_reads.rs b/crates/sqlbuild-python-planner/src/live_reads/main/register_live_reads.rs new file mode 100644 index 0000000000..2fd1ec95db --- /dev/null +++ b/crates/sqlbuild-python-planner/src/live_reads/main/register_live_reads.rs @@ -0,0 +1,12 @@ +//! Register the live warehouse reads. + +use crate::live_reads::main::_live_reads::live_reads; +use crate::planner_warehouse::models::PlannerReads; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +pub fn register_live_reads(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(live_reads, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-planner/src/live_reads/mod.rs b/crates/sqlbuild-python-planner/src/live_reads/mod.rs new file mode 100644 index 0000000000..19d4b07806 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/live_reads/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-planner/src/live_reads/models.rs b/crates/sqlbuild-python-planner/src/live_reads/models.rs new file mode 100644 index 0000000000..030c294279 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/live_reads/models.rs @@ -0,0 +1,479 @@ +//! Public records of live warehouse reads. + +use crate::freshness::main::read_latest::read_latest; +use crate::freshness::models::StateRead; +use crate::live_reads::_helpers::bounded::eligible_task; +use crate::live_reads::_helpers::bounded::read_cursor_query; +use crate::live_reads::_helpers::bounded::run_bounded; +use crate::live_reads::_helpers::bounded::text; +use crate::live_reads::constants::ACTIVE_CATALOG; +use crate::live_reads::constants::EXECUTE; +use crate::live_reads::constants::INSPECTION_CATALOG; +use crate::live_reads::constants::RELATIONS_MAIN_COLUMNS; +use crate::live_reads::constants::RELATIONS_MAIN_LIST; +use crate::live_reads::main::bound_task::bound_task; +use crate::observations::main::capture_source_columns::capture_source_columns; +use crate::planner_warehouse::models::{ + AdapterRead, EligibleCursorQuery, FingerprintRead, PlannerReads, Read, StateHistoryRead, +}; +use crate::planner_warehouse::types::CursorProgress; +use crate::planner_warehouse::types::WarehouseReads; +use pyo3::IntoPyObject; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyBool; +use pyo3::types::PyTuple; +use sqlbuild_host::entries::models::AdapterHost; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::native_host::models::ConsumptionConnection; +use sqlbuild_python_front::native_host::models::Location; +use sqlbuild_python_front::state_rows::main::migrations::read_events::read_events; +use sqlbuild_python_front::state_rows::main::migrations::stored_old_name_columns::stored_old_name_columns; +use sqlbuild_python_front::state_rows::models::StateAccess; +use sqlbuild_python_project::fingerprints::main::read_latest_fingerprints::read_latest_fingerprints; +use sqlbuild_state::consumption::models::ConsumptionRow; +use std::sync::Arc; + +/// The reads of one live plan: its adapter and planning connection, read only when asked. +#[derive(Debug)] +pub struct LiveReads { + adapter: Py, + connection: Py, + /// Runs SQL on the connection: the adapter's `execute` unless a caller observes statements. + pub(crate) execute: Option>, + consumption: Option, +} + +impl LiveReads { + /// Reads through `connection`; consumption history connects with `connection_config` when given. + pub fn new( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + connection_config: Option<&Bound<'_, PyAny>>, + ) -> Self { + Self { + adapter: adapter.clone().unbind(), + connection: connection.clone().unbind(), + execute: None, + consumption: connection_config + .filter(|config| !config.is_none()) + .map(|config| ConsumptionConnection::new(adapter, config)), + } + } + + /// These reads as the handle a plan holds. + pub fn handle(self, py: Python<'_>) -> PyResult> { + Bound::new(py, PlannerReads::new(Arc::new(self))) + } + + /// Call a host entry with this adapter and connection, plus `arguments`. + fn host_call<'py>( + &self, + py: Python<'py>, + entry: AdapterHost, + arguments: Vec<(&str, Bound<'py, PyAny>)>, + ) -> PyResult> { + let mut keywords = vec![ + ("adapter", self.adapter(py).clone()), + ("connection", self.connection(py)), + ]; + keywords.extend(arguments); + call_with(&host_entry(py, entry)?, &keywords) + } + + /// Call an adapter method on this connection, plus `arguments`; the warehouse answers it. + fn known_call<'py>( + &self, + py: Python<'py>, + method: &str, + arguments: Vec<(&str, Bound<'py, PyAny>)>, + ) -> PyResult>> { + let mut keywords = vec![("connection", self.connection(py))]; + keywords.extend(arguments); + Ok(Read::Known(call_with( + &self.method(py, method)?, + &keywords, + )?)) + } + + fn adapter<'py>(&self, py: Python<'py>) -> &Bound<'py, PyAny> { + self.adapter.bind(py) + } + + fn connection<'py>(&self, py: Python<'py>) -> Bound<'py, PyAny> { + self.connection.bind(py).clone() + } + + fn method<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { + if name == EXECUTE + && let Some(execute) = &self.execute + { + return Ok(execute.bind(py).clone()); + } + native_field(self.adapter(py), name) + } + + /// The inspection catalog open for this connection, or `None`. + fn active_catalog<'py>(&self, py: Python<'py>) -> PyResult> { + self.host_call(py, ACTIVE_CATALOG, Vec::new()) + } + + fn state_access<'py>(&self, py: Python<'py>) -> PyResult> { + Ok(StateAccess { + connection: self.connection(py), + execute: self.method(py, "execute")?, + render_qualified_name: self.method(py, "render_qualified_name")?, + }) + } + + fn consumption(&self) -> PyResult<&ConsumptionConnection> { + self.consumption.as_ref().ok_or_else(|| { + PyRuntimeError::new_err("live reads without a connection config read no consumption") + }) + } +} + +impl WarehouseReads for LiveReads { + fn relations<'py>( + &self, + py: Python<'py>, + database: &Bound<'py, PyAny>, + (schemas, names): (Bound<'py, PyAny>, Bound<'py, PyAny>), + ) -> PyResult> { + self.host_call( + py, + RELATIONS_MAIN_LIST, + vec![ + ("database", database.clone()), + ("schemas", schemas), + ("names", names), + ], + ) + } + + fn columns<'py>( + &self, + py: Python<'py>, + relations: Bound<'py, PyAny>, + ) -> PyResult> { + self.host_call(py, RELATIONS_MAIN_COLUMNS, vec![("relations", relations)]) + } + + fn fingerprints<'py>( + &self, + py: Python<'py>, + read: FingerprintRead<'_>, + ) -> PyResult> { + let node_names: Bound<'py, PyAny> = match &read.node_names { + None => py.None().into_bound(py), + Some(names) => PyTuple::new(py, names)?.into_any(), + }; + read_latest_fingerprints(Some(&keywords( + py, + &[ + ("connection", self.connection(py)), + ("execute", self.method(py, "execute")?), + ( + "table_exists", + PyBool::new(py, read.table_exists).to_owned().into_any(), + ), + ("database", read.database.into_pyobject(py)?.into_any()), + ("schema", text(py, read.schema)), + ( + "render_qualified_name", + self.method(py, "render_qualified_name")?, + ), + ( + "render_read_latest_sql", + self.method(py, "render_read_latest_fingerprints_sql")?, + ), + ("node_names", node_names), + ( + "filtered_node_types", + PyTuple::new(py, &read.filtered_node_types)?.into_any(), + ), + ], + )?)) + } + + fn within_inspection<'py>( + &self, + py: Python<'py>, + plan: &mut dyn FnMut() -> PyResult>, + ) -> PyResult> { + let catalog: Bound<'py, PyAny> = self.host_call(py, INSPECTION_CATALOG, Vec::new())?; + within(&catalog, |_| plan()) + } + + fn inspection_concurrency(&self, py: Python<'_>) -> PyResult { + let catalog = self.active_catalog(py)?; + if catalog.is_none() { + return Ok(1); + } + native_field(&catalog, "concurrency")?.extract() + } + + fn prefetch_schema_listings<'py>( + &self, + py: Python<'py>, + scopes: Bound<'py, PyAny>, + ) -> PyResult<()> { + let catalog = self.active_catalog(py)?; + if catalog.is_none() { + return Ok(()); + } + call_with( + &native_field(&catalog, "prefetch_schema_listings")?, + &[ + ("scopes", scopes), + ("best_effort", PyBool::new(py, true).to_owned().into_any()), + ], + )?; + Ok(()) + } + + fn lists_by_schema(&self, py: Python<'_>) -> PyResult { + let catalog = self.active_catalog(py)?; + Ok(!catalog.is_none() && native_field(&catalog, "schema_scoped")?.is_truthy()?) + } + + fn cursor_bounds<'py>( + &self, + py: Python<'py>, + statements: &dyn Fn() -> PyResult>, + progress: &dyn CursorProgress<'py>, + ) -> PyResult>>> { + let statements: Vec = statements()?; + let connection: Bound<'py, PyAny> = self.connection(py); + let execute: Bound<'py, PyAny> = self.method(py, "execute")?; + let concurrency: i64 = self.inspection_concurrency(py)?; + let total: usize = statements.len(); + if concurrency <= 1 || total <= 1 { + let mut outcomes: Vec> = Vec::with_capacity(total); + for (index, sql) in statements.iter().enumerate() { + progress.starting(index)?; + let outcome = read_cursor_query(py, sql, (&connection, &execute))?; + progress.finished(index, &outcome)?; + outcomes.push(outcome); + } + return Ok(Read::Known(outcomes)); + } + progress.concurrent((concurrency as usize).min(total))?; + let tasks: Vec> = statements + .into_iter() + .map(|sql| bound_task(py, sql, (&connection, &execute))) + .collect::>()?; + Ok(Read::Known(run_bounded( + py, + tasks, + concurrency, + Some(progress.completion_callback(py)?), + )?)) + } + + fn eligible_cursor_maxes<'py>( + &self, + py: Python<'py>, + queries: Vec, + ) -> PyResult>>> { + let connection: Bound<'py, PyAny> = self.connection(py); + let execute: Bound<'py, PyAny> = self.method(py, "execute")?; + let tasks: Vec> = queries + .into_iter() + .map(|query| eligible_task(py, query, (&connection, &execute))) + .collect::>()?; + Ok(Read::Known(run_bounded( + py, + tasks, + self.inspection_concurrency(py)?, + None, + )?)) + } + + fn target_max(&self, model: &str) -> Read { + Read::Unknown(format!("cursor position of {model}").into()) + } + + fn note_window(&self, _model: &str, _start: &str) {} + + fn note_unknown(&self, _item: String) {} + + fn described_columns( + &self, + py: Python<'_>, + source: &str, + sql: &str, + ) -> PyResult>> { + let names = call_with( + &self.method(py, "query_column_names")?, + &[("connection", self.connection(py)), ("sql", text(py, sql))], + )?; + let described: Vec = names + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?; + capture_source_columns(source, sql, &described); + Ok(Read::Known(described)) + } + + fn read_access<'py>( + &self, + py: Python<'py>, + relations: &dyn Fn() -> PyResult>>, + ) -> PyResult>>> { + let relations: Vec> = relations()?; + let probe: Bound<'py, PyAny> = self.method(py, "probe_relation_read")?; + let partial: Bound<'py, PyAny> = imported(py, "functools", "partial")?; + let tasks: Vec> = relations + .into_iter() + .map(|relation| { + partial.call( + (&probe,), + Some(&keywords( + py, + &[("connection", self.connection(py)), ("relation", relation)], + )?), + ) + }) + .collect::>()?; + let concurrency: i64 = imported(py, "builtins", "int")? + .call1((self.method(py, "metadata_inspection_concurrency")?,))? + .extract()?; + Ok(Read::Known(run_bounded( + py, + tasks, + concurrency.max(1), + None, + )?)) + } + + fn state_history<'py>( + &self, + py: Python<'py>, + read: StateHistoryRead<'_, 'py>, + ) -> PyResult>> { + Ok(Read::Known(match read { + StateHistoryRead::SourceFreshness { + location, + table_exists, + source_names, + } => { + let read: StateRead<'py> = StateRead { + connection: self.connection(py), + execute: self.method(py, "execute")?, + render_qualified_name: self.method(py, "render_qualified_name")?, + render_read_latest_sql: self + .method(py, "render_read_latest_source_freshness_sql")?, + }; + read_latest(&read, location, table_exists, source_names)? + } + StateHistoryRead::MigrationTables { database, filter } => { + self.relations(py, database, filter)? + } + StateHistoryRead::MigrationEvents { + layout, + location, + stored, + } => read_events(&self.state_access(py)?, layout, location, stored)?.into_any(), + })) + } + + fn adapter_read<'py>( + &self, + py: Python<'py>, + read: AdapterRead<'py>, + ) -> PyResult>> { + let (method, arguments) = match read { + AdapterRead::TableFreshness(requests) => ( + "get_tables_freshness_metadata", + vec![("requests", requests)], + ), + AdapterRead::SingleTableFreshness([database, schema, name]) => ( + "get_table_freshness_metadata", + vec![("database", database), ("schema", schema), ("name", name)], + ), + AdapterRead::FreshnessQuery(sql) => ( + "query", + vec![("sql", sql), ("limit", py.None().into_bound(py))], + ), + AdapterRead::ColumnRenameUnavailable(destination) => ( + "column_rename_unavailable_reason", + vec![("destination", destination)], + ), + }; + self.known_call(py, method, arguments) + } + + fn table_types(&self) -> Read<()> { + Read::Known(()) + } + + fn retention<'py>(&self, py: Python<'py>) -> PyResult>> { + Ok(Read::Known(imported(py, "functools", "partial")?.call( + (self.method(py, "inspect_retention")?,), + Some(&keywords(py, &[("connection", self.connection(py))])?), + )?)) + } + + fn old_name_view_columns<'py>( + &self, + py: Python<'py>, + location: (Bound<'py, PyAny>, Bound<'py, PyAny>), + ) -> PyResult>> { + Ok(Read::Known(stored_old_name_columns( + self.adapter(py), + &self.connection(py), + location, + )?)) + } + + fn deferred_relations<'py>( + &self, + py: Python<'py>, + (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), + ) -> PyResult> { + call_with( + &self.method(py, "list_relations")?, + &[ + ("connection", self.connection(py)), + ("database", database), + ("schemas", PyTuple::new(py, [schema])?.into_any()), + ], + ) + } + + fn recent_consumption( + &self, + py: Python<'_>, + location: &Location, + ) -> PyResult> { + self.consumption()?.read_recent(py, location) + } + + fn consumption_executions( + &self, + py: Python<'_>, + location: &Location, + execution_ids: &[&str], + ) -> PyResult> { + self.consumption()? + .read_executions(py, location, execution_ids) + } + + fn close_consumption(&self, py: Python<'_>) -> PyResult<()> { + match &self.consumption { + Some(consumption) => consumption.close(py), + None => Ok(()), + } + } +} diff --git a/crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs b/crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs index 1323a18b41..99d7928178 100644 --- a/crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs +++ b/crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs @@ -13,7 +13,9 @@ use sqlbuild_state::microbatches::models::{ ReplayState, ReplayVersions, RequirementFact, TextInterval, }; -use crate::microbatches::_helpers::conversion::{BoundParser, enum_text, optional_text}; +use crate::microbatches::_helpers::conversion::BoundParser; +use crate::microbatches::_helpers::conversion::enum_text; +use crate::microbatches::_helpers::conversion::optional_text; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; diff --git a/crates/sqlbuild-python-planner/src/microbatches/_helpers/statements.rs b/crates/sqlbuild-python-planner/src/microbatches/_helpers/statements.rs index 688f4379bb..13eabb8415 100644 --- a/crates/sqlbuild-python-planner/src/microbatches/_helpers/statements.rs +++ b/crates/sqlbuild-python-planner/src/microbatches/_helpers/statements.rs @@ -14,7 +14,8 @@ use sqlbuild_state::microbatches::models::{MicrobatchEventIdentity, ScopeRead}; use sqlbuild_state::state_tables::main::create_table_sql::create_table_sql; use sqlbuild_state::state_tables::models::TableColumn; -use crate::microbatches::_helpers::conversion::{optional_text, state_error}; +use crate::microbatches::_helpers::conversion::optional_text; +use crate::microbatches::_helpers::conversion::state_error; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::contracts::main::render_state_sql_literal::render_state_sql_literal; diff --git a/crates/sqlbuild-python-planner/src/observations/_helpers/publication.rs b/crates/sqlbuild-python-planner/src/observations/_helpers/publication.rs index f1971faabe..de75677e17 100644 --- a/crates/sqlbuild-python-planner/src/observations/_helpers/publication.rs +++ b/crates/sqlbuild-python-planner/src/observations/_helpers/publication.rs @@ -9,11 +9,12 @@ use sqlbuild_analysis::planner_warehouse::models::CursorTagRequest; use sqlbuild_state::observations::constants::{CURSOR_MAX_KIND, SOURCE_COLUMNS_KIND}; use crate::consumption::main::relation::relation; +use crate::live_reads::main::bound_task::bound_task; use crate::planner_warehouse::constants::RUN_BOUNDED; -use crate::planner_warehouse::main::inspection::adapter_cursor_sql::adapter_cursor_sql; -use crate::planner_warehouse::main::inspection::bound_task::bound_task; +use crate::planner_warehouse::main::inspection::dialect_cursor_sql::dialect_cursor_sql; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::models::Dialect; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::native_host::main::append_observations::append_observations; use sqlbuild_python_front::native_host::models::Location; @@ -73,7 +74,7 @@ pub(crate) fn cursor_reads( }; reads.push(CursorRead { location: Location::of(&relation(&destination)?), - sql: adapter_cursor_sql(adapter, &query)?, + sql: dialect_cursor_sql(&Dialect::of(adapter)?, &query)?, cursor_type: query.cursor_type, model: name, }); diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs index b8bc601539..c3e8b9d137 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs @@ -1,10 +1,16 @@ //! Convert public migration models at the Python boundary for native presentation. use crate::plan_output::_helpers::plan_text::display_limit; -use crate::plan_output::_helpers::values::{iso, optional_text, relation_name, texts}; +use crate::plan_output::_helpers::values::iso; +use crate::plan_output::_helpers::values::optional_text; +use crate::plan_output::_helpers::values::relation_name; +use crate::plan_output::_helpers::values::texts; use crate::plan_output::main::items::items; use crate::plan_output::main::text::text; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; use pyo3::pyfunction; use sqlbuild_core::cli_text::models::{ CliStyle, ColumnMigrationText, MigrationDecisionText, MigrationText, MigrationTextReport, diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs index 58f9c36c29..6e450f2ef4 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs @@ -25,6 +25,8 @@ const RENAMED_REASON: &str = "renamed"; const RENAMED_DECISION: &str = "renamed"; const VIEW_MATERIALIZATION: &str = "view"; const CHANGED_IDENTITY: &str = "changed"; +/// The plan document field saying audit failures hold nothing back; present only when they do. +pub(crate) const IGNORE_AUDIT_FAILURES_KEY: &str = "ignore_audit_failures"; fn token<'py>(bound: Bound<'py, PyAny>) -> PyResult> { Ok(PyString::new(bound.py(), &sentinel_to_token(&bound)?).into_any()) diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs index 2d3bf62336..95f586c794 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs @@ -13,18 +13,20 @@ use sqlbuild_core::cli_text::models::{ }; use sqlbuild_core::plan_display::main::format_query_diff::format_query_diff; -use crate::plan_output::_helpers::cursor_details::{ - CursorDetails, DEFERRED, RESOLVED, cursor_details, runtime_inputs_clause, -}; -use crate::plan_output::_helpers::display::{ - format_no_input_rows, format_waiting_on_empty_inputs, model_materialization_label, -}; +use crate::plan_output::_helpers::cursor_details::CursorDetails; +use crate::plan_output::_helpers::cursor_details::DEFERRED; +use crate::plan_output::_helpers::cursor_details::RESOLVED; +use crate::plan_output::_helpers::cursor_details::cursor_details; +use crate::plan_output::_helpers::cursor_details::runtime_inputs_clause; +use crate::plan_output::_helpers::display::format_no_input_rows; +use crate::plan_output::_helpers::display::format_waiting_on_empty_inputs; +use crate::plan_output::_helpers::display::model_materialization_label; use crate::plan_output::_helpers::migrations::pre_build_lines; use crate::plan_output::_helpers::plan_json::unified_diff; -use crate::plan_output::_helpers::python_identity::{ - definition_source_text, dependency_source_text, -}; -use crate::plan_output::_helpers::values::{optional_text, texts}; +use crate::plan_output::_helpers::python_identity::definition_source_text; +use crate::plan_output::_helpers::python_identity::dependency_source_text; +use crate::plan_output::_helpers::values::optional_text; +use crate::plan_output::_helpers::values::texts; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -514,5 +516,6 @@ pub(crate) fn report(arguments: &KeywordArguments<'_>) -> PyResult, options: Option>) -> P if arguments.passed(CHANGED_KEY)? { result.set_item(CHANGED_KEY, arguments.value(CHANGED_KEY)?)?; } + if native_field(&plan, IGNORE_AUDIT_FAILURES_KEY)?.is_truthy()? { + result.set_item(IGNORE_AUDIT_FAILURES_KEY, true)?; + } indented_json(result.as_any()) } diff --git a/crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs index d22a1fa85d..bd6322ca35 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs @@ -34,9 +34,10 @@ use sqlbuild_analysis::planner_changes::models::{ }; use crate::planner_changes::_helpers::renames::origin_names; -use crate::planner_changes::_helpers::results::{ - backfill_result, change_kind, replay_policy_error, schema_findings, -}; +use crate::planner_changes::_helpers::results::backfill_result; +use crate::planner_changes::_helpers::results::change_kind; +use crate::planner_changes::_helpers::results::replay_policy_error; +use crate::planner_changes::_helpers::results::schema_findings; use crate::planner_identity::main::compiled_function_fingerprint_sql::compiled_function_fingerprint_sql; use crate::planner_identity::main::function_local_hashes::function_local_hashes; use crate::planner_identity::main::model_declared_columns_hash::model_declared_columns_hash; @@ -331,11 +332,13 @@ pub(crate) fn model_change<'py>( let mut query_changed: bool = false; if let (Some(hash), Some(recorded)) = (&compiled_query_hash, &recorded) { query_changed = origins.is_none() && definition_changed; - log_query_check( - (model, &model_name, &fingerprint), - hash, - (&recorded.definition_hash, query_changed), - )?; + if query_changed { + log_query_check( + (model, &model_name, &fingerprint), + hash, + &recorded.definition_hash, + )?; + } } let findings: Vec = model_schema_findings(model, snapshot, &model_name, &dialect)?; @@ -386,10 +389,11 @@ fn recorded_fingerprint(fingerprint: &Bound<'_, PyAny>) -> PyResult, &Bound<'_, PyAny>, &Bound<'_, PyAny>), compiled_query_hash: &str, - (definition_hash, query_changed): (&str, bool), + definition_hash: &str, ) -> PyResult<()> { let py: Python<'_> = model.py(); let logger: Bound<'_, PyAny> = @@ -403,22 +407,14 @@ fn log_query_check( "message", text(&format!( "fingerprint comparison compiled_query_hash={compiled_query_hash} \ - fingerprint_definition_hash={definition_hash} query_changed={}", - if query_changed { "True" } else { "False" } + fingerprint_definition_hash={definition_hash} query_changed=True" )), ), ("sqlbuild_subject", text("model")), ("sqlbuild_name", model_name.clone()), ("sqlbuild_event", text("query_change_check")), ("sqlbuild_phase", text("planner")), - ( - "sqlbuild_status", - text(if query_changed { - "changed" - } else { - "unchanged" - }), - ), + ("sqlbuild_status", text("changed")), ], )?; let log_sql: Bound<'_, PyAny> = host_entry(py, ExecutorHost::LogSql)?; diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/contexts.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/contexts.rs index 601e006dbb..b5c5eb107c 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/contexts.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/contexts.rs @@ -5,9 +5,9 @@ use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::pymethods; use pyo3::types::{PyDict, PyFrozenSet}; -use crate::planner_entries::models::{ - ModelPlanContext, PlanEntryBuildInputs, PlannerRelationsContext, -}; +use crate::planner_entries::models::ModelPlanContext; +use crate::planner_entries::models::PlanEntryBuildInputs; +use crate::planner_entries::models::PlannerRelationsContext; use sqlbuild_python_base::boundary::models::KeywordArguments; /// The argument `name`, or `default()` when it was not passed. diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/getters.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/getters.rs index c98bdf79f5..589c6e3b93 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/getters.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/getters.rs @@ -3,9 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::planner_entries::models::{ - ModelPlanContext, PlanEntryBuildInputs, PlannerRelationsContext, -}; +use crate::planner_entries::models::ModelPlanContext; +use crate::planner_entries::models::PlanEntryBuildInputs; +use crate::planner_entries::models::PlannerRelationsContext; pub(crate) fn read_model_plan_context<'py>( value: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs index 49b574f80d..bb20834b7b 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs @@ -37,6 +37,8 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::native_host::models::Dialect; +use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; @@ -45,7 +47,8 @@ use sqlbuild_python_front::discovery::main::failures::microbatch_guidance::micro use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::compiled_views::main::project::functions::functions; use sqlbuild_python_project::planner_phases::main::object::object; -use sqlbuild_python_project::planner_strategy::models::{CursorInputLookups, ModelEntryDecisions}; +use sqlbuild_python_project::planner_strategy::models::CursorInputLookups; +use sqlbuild_python_project::planner_strategy::models::ModelEntryDecisions; use sqlbuild_python_project::semantic_shapes::main::source_binding_diagnostics::source_binding_diagnostics; const MICROBATCH_COMPLEXITY_CODE: &str = "MICROBATCH_CONCURRENCY_COMPLEXITY"; @@ -61,7 +64,7 @@ pub(crate) struct ContextReads<'py> { impl<'py> ContextReads<'py> { pub(crate) fn of( context: Bound<'py, PyAny>, - adapter: Bound<'py, PyAny>, + dialect: Bound<'py, Dialect>, external_resolver: Bound<'py, PyAny>, ) -> PyResult { let py = context.py(); @@ -77,7 +80,7 @@ impl<'py> ContextReads<'py> { seed_locations: field(&fields.seed_locations), runtime_cursor_producer_names: field(&fields.runtime_cursor_producer_names), }, - resolution: Resolution::of_context(adapter, &context, external_resolver)?, + resolution: Resolution::of_context(dialect, &context, external_resolver)?, invocation_time: field(&fields.invocation_time), context, }) @@ -103,7 +106,7 @@ pub(crate) fn plan_model_from_change<'py>( let model = &request.model; let reads = request.context; let context = &reads.context; - let adapter = &reads.resolution.adapter; + let dialect = &reads.resolution.dialect; let py = model.py(); let backfill = if request.full_refresh { native( @@ -155,7 +158,7 @@ pub(crate) fn plan_model_from_change<'py>( Some(reads.lookups.functions_by_name.clone()), )?; let cursor_input_relations = - build_cursor_input_relations((model, adapter), &cursor_column, &reads.lookups)?.into_any(); + build_cursor_input_relations((model, dialect), &cursor_column, &reads.lookups)?.into_any(); let mut any_runtime_owned = false; for relation in cursor_input_relations.try_iter()? { if native_field(&relation?, "is_runtime_owned")?.is_truthy()? { @@ -181,7 +184,7 @@ pub(crate) fn plan_model_from_change<'py>( let range = microbatch_range(model, &window)?; let (entry, entry_warnings) = assemble_model_plan_entry(&ModelEntryDecisions { model: model.clone(), - adapter: adapter.clone(), + dialect: dialect.clone(), snapshot: request.snapshot.clone(), change_result: request.change_result.clone(), action, @@ -215,7 +218,7 @@ pub(crate) fn plan_model_from_change<'py>( /// The binding-diagnostic failure that stops planning, if any diagnostic is an error. fn check_binding_diagnostics<'py>( project: &Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, + dialect: &Bound<'py, Dialect>, relations: &Bound<'py, PyAny>, selected_keys: &Bound<'py, PyAny>, ) -> PyResult<()> { @@ -244,7 +247,7 @@ fn check_binding_diagnostics<'py>( ), ( "profile", - adapter.call_method0("expression_inference_profile")?, + dialect.syntax("expression_inference_profile")?.call0()?, ), ("selected_keys", selected_keys.clone()), ], @@ -297,7 +300,7 @@ fn names_of(entries: &[Bound<'_, PyAny>]) -> PyResult { /// Concurrent microbatch configuration checks, and the warning concurrency complexity earns. fn check_microbatch_concurrency<'py>( project: &Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, + dialect: &Bound<'py, Dialect>, entries: &[Bound<'py, PyAny>], concurrency: i64, ) -> PyResult>> { @@ -328,17 +331,18 @@ fn check_microbatch_concurrency<'py>( )); } if !configured.is_empty() - && !adapter - .call_method0("supports_concurrent_microbatch_dml")? + && !dialect + .syntax("supports_concurrent_microbatch_dml")? + .call0()? .is_truthy()? { return Err(error_of( py, ErrorId::CompileInput, &format!( - "adapter '{}' does not support concurrent microbatch delete/insert DML; set \ + "dialect '{}' does not support concurrent microbatch delete/insert DML; set \ batch_concurrency = 1", - native_field(adapter, "adapter_name")?.str()? + dialect.syntax("adapter_name")?.str()? ), &[], )); @@ -392,7 +396,7 @@ pub(crate) fn plan_entries<'py>( ) -> PyResult> { let PlanEntriesRequest { project, - adapter, + dialect, scope, snapshot, relations, @@ -411,7 +415,7 @@ pub(crate) fn plan_entries<'py>( let build = inputs.cast::()?.get(); let relations_context = relations.cast::()?.get(); let selected_keys = native_field(&scope, "selected_keys")?; - check_binding_diagnostics(&project, &adapter, &relations, &selected_keys)?; + check_binding_diagnostics(&project, &dialect, &relations, &selected_keys)?; let freshness_blocked = build.source_freshness_blocked_model_names.bind(py).clone(); let external_blocked = build.external_blocked_model_names.bind(py).clone(); let start_override = build.start_cursor_override.bind(py).clone(); @@ -452,7 +456,7 @@ pub(crate) fn plan_entries<'py>( .into_any(); let reads = ContextReads::of( context, - adapter.clone(), + dialect.clone(), native_field(&project, "external_sql_reference_resolver")?, )?; let sql_analysis_enabled = native_field(&settings, "sql_analysis")?.is_truthy()?; @@ -532,7 +536,7 @@ pub(crate) fn plan_entries<'py>( warnings.push(warning?); } } - if let Some(warning) = check_microbatch_concurrency(&project, &adapter, &entries, concurrency)? + if let Some(warning) = check_microbatch_concurrency(&project, &dialect, &entries, concurrency)? { warnings.push(warning); } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/relations.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/relations.rs index e6525f08cd..238bd99f52 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/relations.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/relations.rs @@ -1,7 +1,8 @@ //! Relation locations and source metadata for plan entry construction. use crate::planner_entries::_helpers::entries::values::planner_model; -use crate::planner_entries::models::{PlannerRelationsContext, RelationsRequest}; +use crate::planner_entries::models::PlannerRelationsContext; +use crate::planner_entries::models::RelationsRequest; use crate::planner_resolution::main::references::apply_deferred_locations::apply_deferred_locations; use crate::planner_resolution::main::references::build_function_locations::build_function_locations; use crate::planner_resolution::main::references::build_model_locations::build_model_locations; @@ -11,12 +12,19 @@ use crate::planner_warehouse::main::sources::build_source_read_map::build_source use crate::planner_warehouse::main::sources::gather_source_inspection::gather_source_inspection; use crate::planner_warehouse::main::sources::semantic_source_columns::semantic_source_columns; use crate::planner_warehouse::main::sources::with_declared_source_reads::with_declared_source_reads; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use crate::planner_warehouse::models::PlannerReads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::models::Dialect; +use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; use sqlbuild_python_base::records::types::RecordId; @@ -51,7 +59,7 @@ fn selected_audit_relation_keys( /// Reuse already-gathered source columns for the read map, or gather them once. fn source_warehouse_columns<'py>( - (project, adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + (project, reads): (&Bound<'py, PyAny>, Option<&Bound<'py, PlannerReads>>), source_read_map: &Bound<'py, PyAny>, known_source_columns: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { @@ -67,6 +75,12 @@ fn source_warehouse_columns<'py>( let names = PyFrozenSet::new(py, known.keys())?.into_any(); return Ok((known.into_any(), names)); } + let Some(reads) = reads else { + return Ok(( + PyDict::new(py).into_any(), + PyFrozenSet::empty(py)?.into_any(), + )); + }; let entries = PyTuple::new( py, mapping_values(source_read_map)? @@ -75,8 +89,7 @@ fn source_warehouse_columns<'py>( )?; let gathered = gather_source_inspection( project, - adapter, - connection, + reads, entries.into_any().extract::>>()?, )?; Ok((gathered.get_item(0)?, gathered.get_item(1)?)) @@ -88,8 +101,8 @@ pub(crate) fn planner_relations_context<'py>( ) -> PyResult> { let RelationsRequest { project, - adapter, - connection, + dialect, + reads, scope, deferral, project_config, @@ -178,7 +191,7 @@ pub(crate) fn planner_relations_context<'py>( .into_any(); let python_source_read_map = read_map(&python_source_map, Some(python_names))?; let (columns, listed_source_names) = source_warehouse_columns( - (&project, &adapter, &connection), + (&project, reads.as_ref()), &source_read_map, &known_source_columns, )?; @@ -188,8 +201,10 @@ pub(crate) fn planner_relations_context<'py>( py, &[ ("project", project.clone()), - ("adapter", adapter.clone()), - ("connection", connection), + ( + "reads", + reads.map_or_else(|| py.None().into_bound(py), Bound::into_any), + ), ("source_read_map", source_read_map.clone()), ("selected_keys", selected_keys), ("columns", columns), @@ -207,7 +222,7 @@ pub(crate) fn planner_relations_context<'py>( source_read_map: source_read_map.unbind(), python_source_read_map: python_source_read_map.unbind(), source_warehouse_columns: semantic_columns.unbind(), - star_exclude_keyword: adapter.call_method0("star_exclude_keyword")?.unbind(), + star_exclude_keyword: dialect.syntax("star_exclude_keyword")?.call0()?.unbind(), listed_source_names: listed_source_names.unbind(), }, )? @@ -226,7 +241,7 @@ pub(crate) fn build_planner_relations_context<'py>( "build_planner_relations_context", kwargs, ( - &["project", "adapter", "connection", "scope"], + &["project", "dialect", "reads", "scope"], &[ "deferral", "project_config", @@ -238,8 +253,8 @@ pub(crate) fn build_planner_relations_context<'py>( )?; planner_relations_context(RelationsRequest { project: args.value("project")?, - adapter: args.value("adapter")?, - connection: args.value("connection")?, + dialect: Dialect::passed(args.value("dialect")?)?, + reads: Some(PlannerReads::passed(args.value("reads")?)?), scope: args.value("scope")?, deferral: args.value("deferral")?, project_config: args.value("project_config")?, diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs index 37cfc3b0c6..f47081b779 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs @@ -1,8 +1,9 @@ //! Clone planning: destination and origin entries, and deferred-clone graph traversal. -use crate::planner_entries::_helpers::entries::values::{ - native, planner_model, planner_type, string, -}; +use crate::planner_entries::_helpers::entries::values::native; +use crate::planner_entries::_helpers::entries::values::planner_model; +use crate::planner_entries::_helpers::entries::values::planner_type; +use crate::planner_entries::_helpers::entries::values::string; use crate::planner_entries::models::ModelPlanContext; use crate::planner_entries::types::PhaseTimings; use crate::planner_identity::main::build_compiled_function_fingerprint_sql::build_compiled_function_fingerprint_sql; @@ -15,6 +16,7 @@ use crate::planner_resolution::models::ModelSqlRequest; use crate::planner_resolution::models::Resolution; use crate::planner_strategy::main::strategy::materialization_of::materialization_of; use crate::planner_warehouse::main::sources::gather_source_columns::gather_source_columns; +use crate::planner_warehouse::models::PlannerReads; use pyo3::PyRef; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; @@ -25,6 +27,8 @@ use sqlbuild_core::graph_walk::models::ClonePolicy; use sqlbuild_model_config::config_values::models::ConfigNode; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::models::Dialect; +use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -181,7 +185,7 @@ fn resolved_source_entries<'py>( fn clone_resolution<'py>( (project, plan): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), source_project: &Bound<'py, PyAny>, - (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + reads: &Bound<'py, PlannerReads>, ) -> PyResult> { let py = project.py(); let model_locations = object(py, build_model_locations(&compiled_models(project)?)?)?; @@ -209,8 +213,7 @@ fn clone_resolution<'py>( py, gather_source_columns( source_project, - adapter, - connection, + reads, Some(resolved_source_entries((project, plan), source_project)?), )?, )?; @@ -257,7 +260,7 @@ fn model_entry<'py>( fn clone_model_entries<'py>( (project, plan): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - adapter: &Bound<'py, PyAny>, + dialect: &Bound<'py, Dialect>, resolution: &CloneResolution<'py>, ) -> PyResult> { let py = project.py(); @@ -295,7 +298,7 @@ fn clone_model_entries<'py>( function_locations: resolution.function_locations.clone().unbind(), source_map: resolution.source_map.clone().unbind(), source_warehouse_columns: resolution.source_warehouse_columns.clone().unbind(), - star_exclude_keyword: adapter.call_method0("star_exclude_keyword")?.unbind(), + star_exclude_keyword: dialect.syntax("star_exclude_keyword")?.call0()?.unbind(), runtime_cursor_producer_names: PyFrozenSet::empty(py)?.into_any().unbind(), future_cursor_config: py.None(), start_cursor_config: py.None(), @@ -304,7 +307,7 @@ fn clone_model_entries<'py>( )? .into_any(); let sql = resolve_model_sql( - &Resolution::of_context(adapter.clone(), &context, py.None().into_bound(py))?, + &Resolution::of_context(dialect.clone(), &context, py.None().into_bound(py))?, &context, &ModelSqlRequest { model: model.clone(), @@ -338,7 +341,7 @@ fn clone_model_entries<'py>( fn clone_source_entries<'py>( project: &Bound<'py, PyAny>, (selected, execution_order): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - adapter: &Bound<'py, PyAny>, + dialect: &Bound<'py, Dialect>, ) -> PyResult> { let py = project.py(); let entries = PyDict::new(py); @@ -365,7 +368,9 @@ fn clone_source_entries<'py>( kwargs.set_item("database", native_field(&entry, "database")?)?; kwargs.set_item("schema", native_field(&entry, "schema")?)?; kwargs.set_item("name", &relation_name)?; - let qualified = native_field(adapter, "render_qualified_name")?.call((), Some(&kwargs))?; + let qualified = dialect + .syntax("render_qualified_name")? + .call((), Some(&kwargs))?; let destination = new_record( py, RecordId::CompiledRelationLocation, @@ -407,7 +412,7 @@ fn clone_source_entries<'py>( fn clone_function_entries<'py>( (project, plan): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - adapter: &Bound<'py, PyAny>, + dialect: &Bound<'py, Dialect>, resolution: &CloneResolution<'py>, ) -> PyResult> { let py = project.py(); @@ -428,7 +433,7 @@ fn clone_function_entries<'py>( Some(keywords( py, &[ - ("adapter", adapter.clone()), + ("dialect", dialect.clone().into_any()), ("function", function.clone()), ("model_locations", resolution.model_locations.clone()), ("seed_locations", resolution.seed_locations.clone()), @@ -440,7 +445,7 @@ fn clone_function_entries<'py>( ), ( "star_exclude_keyword", - adapter.call_method0("star_exclude_keyword")?, + dialect.syntax("star_exclude_keyword")?.call0()?, ), ], )?), @@ -546,7 +551,7 @@ fn names_of<'py>(entries: &Bound<'py, PyAny>) -> PyResult> { fn origin_source_entries<'py>( project: &Bound<'py, PyAny>, selected_names: &Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, + dialect: &Bound<'py, Dialect>, ) -> PyResult> { let py = project.py(); let mut keys: Vec> = Vec::new(); @@ -570,7 +575,7 @@ fn origin_source_entries<'py>( &PyFrozenSet::new(py, keys)?.into_any(), &tuple_of(py, order)?, ), - adapter, + dialect, ) } @@ -589,8 +594,8 @@ pub(crate) fn timed_clone_planning<'py>( "project", "select", "exclude", - "adapter", - "connection", + "dialect", + "reads", "origin_project", "origin_source_project", "destination_source_project", @@ -599,20 +604,16 @@ pub(crate) fn timed_clone_planning<'py>( ), )?; let project = args.value("project")?; - let adapter = args.value("adapter")?; - let connection = args.value("connection")?; + let dialect = Dialect::passed(args.value("dialect")?)?; + let reads = PlannerReads::passed(args.value("reads")?)?; let destination_sources = args.value("destination_source_project")?; let plan = clone_plan_output(&project, &args.value("select")?, &args.value("exclude")?)?; let inspection_started = std::time::Instant::now(); - let resolution = clone_resolution( - (&project, &plan), - &destination_sources, - (&adapter, &connection), - )?; + let resolution = clone_resolution((&project, &plan), &destination_sources, &reads)?; let inspection_seconds = inspection_started.elapsed().as_secs_f64(); - let models = clone_model_entries((&project, &plan), &adapter, &resolution)?; + let models = clone_model_entries((&project, &plan), &dialect, &resolution)?; let seeds = seed_entries(&project, "key", &native_field(&plan, "selected_keys")?)?; - let functions = clone_function_entries((&project, &plan), &adapter, &resolution)?; + let functions = clone_function_entries((&project, &plan), &dialect, &resolution)?; let mut sources = PyTuple::empty(py).into_any(); if native_field(&destination_sources, "effective_target_name")? .eq(native_field(&project, "effective_target_name")?)? @@ -623,7 +624,7 @@ pub(crate) fn timed_clone_planning<'py>( &native_field(&plan, "selected_keys")?, &native_field(&plan, "execution_order")?, ), - &adapter, + &dialect, )?; } let origin_project = args.value("origin_project")?; @@ -632,7 +633,7 @@ pub(crate) fn timed_clone_planning<'py>( let origin_sources = origin_source_entries( &args.value("origin_source_project")?, &names_of(&sources)?, - &adapter, + &dialect, )?; let origin_by_name = PyDict::new(py); for entry in origin_sources.try_iter()? { diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs index 88473cfaf3..6773dfc41e 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs @@ -1,31 +1,30 @@ //! Static plans for commands that do not build models, and direct seed identity reads. -use crate::planner_entries::_helpers::entries::values::{ - boolean, none, planner_model, planner_type, -}; +use crate::planner_entries::_helpers::entries::values::{none, planner_model, planner_type}; use crate::planner_entries::_helpers::plans::plan_output::{ function_entry, selected_test_entries, test_selected, }; use crate::planner_identity::main::build_compiled_function_fingerprint_sql::build_compiled_function_fingerprint_sql; use crate::planner_identity::main::build_seed_identity::build_seed_identity; use crate::planner_strategy::main::model_entries::extract_seed_columns::extract_seed_columns; +use crate::planner_warehouse::models::FingerprintRead; +use crate::planner_warehouse::models::PlannerReads; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{ - PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PySet, PySetMethods, PyString, - PyTuple, + PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PySet, PySetMethods, PyTuple, }; use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_SEED; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; -use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::models::Dialect; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; use sqlbuild_python_project::compiled_views::main::project::sql_tests::sql_tests; -use sqlbuild_python_project::fingerprints::main::read_latest_fingerprints::read_latest_fingerprints; use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_phases::main::optional_text::optional_text; use sqlbuild_python_project::resource_views::main::function_reads::deps::deps; use sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::schema_entry; use sqlbuild_python_project::resource_views::main::seed_reads::seed_file::seed_file; @@ -141,18 +140,18 @@ pub(crate) fn build_test_command_plan<'py>( "build_test_command_plan", kwargs, ( - &["project", "adapter", "scope", "relations"], + &["project", "dialect", "scope", "relations"], &["case_name"], ), )?; let project = args.value("project")?; - let adapter = args.value("adapter")?; + let dialect = Dialect::passed(args.value("dialect")?)?; let scope = args.value("scope")?; let relations = args.value("relations")?; let selected_keys = native_field(&scope, "selected_keys")?; let selection = native_field(&scope, "sql_test_selection")?; let selected = selected_test_entries( - (&project, &adapter), + (&project, &dialect), &selected_keys, &selection, &args.value("case_name")?, @@ -189,7 +188,7 @@ pub(crate) fn build_test_command_plan<'py>( let fingerprint_sql = object(py, build_compiled_function_fingerprint_sql(&function)?)?; functions.push(function_entry( &function, - &adapter, + &dialect, &relations, ( fingerprint_sql, @@ -300,13 +299,13 @@ type Target = (Option, String); /// Read only fingerprint state required by selected direct seeds. #[pyfunction] -#[pyo3(signature = (*, adapter, connection, seeds))] +#[pyo3(signature = (*, reads, seeds))] pub(crate) fn read_selected_seed_fingerprints<'py>( - adapter: &Bound<'py, PyAny>, - connection: &Bound<'py, PyAny>, + reads: &Bound<'py, PlannerReads>, seeds: &Bound<'py, PyAny>, ) -> PyResult> { - let py = adapter.py(); + let py = reads.py(); + let warehouse = reads.get().reads(); let table_name: &str = FINGERPRINT_TABLE_NAME; let mut names_by_target: BTreeMap<(String, String), (Target, BTreeSet)> = BTreeMap::new(); @@ -334,14 +333,14 @@ pub(crate) fn read_selected_seed_fingerprints<'py>( let fingerprints = PyDict::new(py); let mut relations_by_target: Vec> = Vec::new(); for ((database, schema), _) in names_by_target.values() { - let names = PyTuple::new(py, [table_name])?; - let schemas = PyTuple::new(py, [schema.as_str()])?; - let kwargs = PyDict::new(py); - kwargs.set_item("connection", connection)?; - kwargs.set_item("database", database)?; - kwargs.set_item("schemas", schemas)?; - kwargs.set_item("names", names)?; - relations_by_target.push(native_field(adapter, "list_relations")?.call((), Some(&kwargs))?); + relations_by_target.push(warehouse.relations( + py, + &optional_text(py, database.as_deref()), + ( + PyTuple::new(py, [schema.as_str()])?.into_any(), + PyTuple::new(py, [table_name])?.into_any(), + ), + )?); } for (((database, schema), names), relations) in names_by_target.values().zip(relations_by_target) @@ -354,31 +353,16 @@ pub(crate) fn read_selected_seed_fingerprints<'py>( break; } } - let node_names = PyTuple::new(py, names.iter())?; - let latest = read_latest_fingerprints(Some(&keywords( + let latest = warehouse.fingerprints( py, - &[ - ("connection", connection.clone()), - ("execute", native_field(adapter, "execute")?), - ("table_exists", boolean(py, table_exists)?), - ( - "database", - database - .as_deref() - .map_or_else(|| none(py), |name| PyString::new(py, name).into_any()), - ), - ("schema", PyString::new(py, schema).into_any()), - ( - "render_qualified_name", - native_field(adapter, "render_qualified_name")?, - ), - ( - "render_read_latest_sql", - native_field(adapter, "render_read_latest_fingerprints_sql")?, - ), - ("node_names", node_names.into_any()), - ], - )?))?; + FingerprintRead { + database: database.clone(), + schema, + table_exists, + node_names: Some(names.iter().cloned().collect()), + filtered_node_types: Vec::new(), + }, + )?; let items = PyList::new( py, native_field(&latest, "fingerprints")? diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs index 4a069d1b96..77c64a33bc 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs @@ -1,11 +1,17 @@ //! Display-only plans for external plan surfaces when warehouse planning cannot resolve. -use crate::planner_entries::_helpers::entries::values::{ - boolean, native, none, planner_model, planner_type, string, -}; +use crate::planner_entries::_helpers::entries::values::boolean; +use crate::planner_entries::_helpers::entries::values::native; +use crate::planner_entries::_helpers::entries::values::none; +use crate::planner_entries::_helpers::entries::values::planner_model; +use crate::planner_entries::_helpers::entries::values::planner_type; +use crate::planner_entries::_helpers::entries::values::string; use crate::planner_phases::main::rebuilds::resolve_model_full_refresh::resolve_model_full_refresh; use crate::planner_strategy::main::strategy::materialization_of::materialization_of; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs index 1be05282ab..ff6617a080 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs @@ -1,7 +1,16 @@ //! Top-level planner orchestration producing an execution plan. +use crate::planner_phases::constants::ASSEMBLING; +use crate::planner_phases::constants::BUILDING_ENTRIES; +use crate::planner_phases::constants::DETECTING_CHANGES; +use crate::planner_phases::constants::READING_WAREHOUSE; +use crate::planner_phases::constants::SOURCE_FRESHNESS; +use crate::planner_phases::constants::TEST_FIXTURES; +use crate::planner_phases::main::timing::record_step::record_step; +use crate::planner_phases::main::timing::timed::timed; +use crate::planner_warehouse::models::PlannerReads; use sqlbuild_analysis::planner_scope::constants::MODEL; -use sqlbuild_host::entries::models::AdapterHost; +use sqlbuild_python_base::native_host::models::Dialect; use crate::planner_changes::main::detect_changes::detect_changes; use crate::planner_changes::main::resolve_model_actions::resolve_model_actions; @@ -38,7 +47,6 @@ use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; use std::time::Instant; -pub(crate) const INSPECTION_CATALOG: AdapterHost = AdapterHost::OpenInspectionCatalog; /// Selected scopes, and the overrides with cursor overrides typed for the selected models. pub(crate) fn scopes_and_cursor_overrides<'py>( args: &KeywordArguments<'py>, @@ -120,8 +128,8 @@ fn deferred_migration_origins<'py>( defer_to_migration_origins( ( &args.value("project")?, - &args.value("adapter")?, - &args.value("connection")?, + &Dialect::passed(args.value("dialect")?)?, + &PlannerReads::passed(args.value("reads")?)?, ), ( locations.downcast::()?, @@ -143,9 +151,12 @@ pub(crate) fn planned_output<'py>( let deferral = args.value("deferral")?; let policies = args.value("policies")?; let origin_failures = deferred_migration_origins(args, scopes, &deferral)?; - let warehouse = gather_planner_warehouse_state(runtime, scopes, overrides, &deferral)?; + let warehouse = timed(READING_WAREHOUSE, || { + gather_planner_warehouse_state(runtime, scopes, overrides, &deferral) + })?; let started = Instant::now(); let snapshot = warehouse_field(&warehouse, |warehouse| &warehouse.snapshot)?; + let changes_started = Instant::now(); let identities = build_planner_identity_context( &project, (scopes, &snapshot), @@ -160,12 +171,19 @@ pub(crate) fn planned_output<'py>( )?; let inspection_scope = class_field(scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?; let resolved_actions = resolve_model_actions(&inspection_scope, &changes)?; - let source_freshness = build_planner_source_freshness_result( - (&project, &inspection_scope), - (&args.value("adapter")?, &args.value("connection")?), - &warehouse_field(&warehouse, |warehouse| &warehouse.inspection_relations)?, - &native_field(&snapshot, "source_freshness_state_schemas")?, - )?; + record_step(DETECTING_CHANGES, changes_started.elapsed().as_secs_f64()); + let source_freshness = timed(SOURCE_FRESHNESS, || { + build_planner_source_freshness_result( + (&project, &inspection_scope), + ( + &Dialect::passed(args.value("dialect")?)?, + &PlannerReads::passed(args.value("reads")?)?, + ), + &warehouse_field(&warehouse, |warehouse| &warehouse.inspection_relations)?, + &native_field(&snapshot, "source_freshness_state_schemas")?, + ) + })?; + let entries_started = Instant::now(); let reconciliation = reconcile_execution_changes( (&inspection_scope, &snapshot), &identities, @@ -174,11 +192,13 @@ pub(crate) fn planned_output<'py>( )?; let entries = build_planner_entry_results( (runtime, &warehouse), - (overrides, &deferral), + overrides, (&inspection_scope, &reconciliation), &source_freshness, )?; - let planned_sql_tests = test_planning.result(py)?.into_bound(py); + record_step(BUILDING_ENTRIES, entries_started.elapsed().as_secs_f64()); + let planned_sql_tests = timed(TEST_FIXTURES, || test_planning.result(py))?.into_bound(py); + let assembling_started = Instant::now(); let output = base_plan_output( PlanOutputContext { runtime, @@ -216,6 +236,7 @@ pub(crate) fn planned_output<'py>( let output = add_deferred_origin_notes(output, (&project, scopes), (&deferral, &origin_failures))?; let plan_output = add_plan_metadata(output, &source_freshness, &policies)?.build(py)?; + record_step(ASSEMBLING, assembling_started.elapsed().as_secs_f64()); let on_progress = args.value("on_progress")?; if !on_progress.is_none() { on_progress.call1((format!( diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs index 327003a020..fb7e92c43d 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs @@ -13,6 +13,7 @@ use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::models::Dialect; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; @@ -348,7 +349,7 @@ fn require_intermediate_loader_targets<'py>( /// Build one canonical function entry from resolved relation inputs. pub(crate) fn function_entry<'py>( function: &Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, + dialect: &Bound<'py, Dialect>, relations: &Bound<'py, PyAny>, (fingerprint_query_sql, previous_query_sql, reason): ( Bound<'py, PyAny>, @@ -362,7 +363,7 @@ pub(crate) fn function_entry<'py>( Some(keywords( py, &[ - ("adapter", adapter.clone()), + ("dialect", dialect.clone().into_any()), ("function", function.clone()), ( "model_locations", @@ -420,7 +421,7 @@ pub(crate) fn function_entry<'py>( } fn function_entries<'py>( - (project, adapter, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + (project, dialect, snapshot): (&Bound<'py, PyAny>, &Bound<'py, Dialect>, &Bound<'py, PyAny>), relations: &Bound<'py, PyAny>, changes: &Bound<'py, PyAny>, selected_keys: &Bound<'py, PyAny>, @@ -449,7 +450,7 @@ fn function_entries<'py>( }; entries.push(function_entry( &function, - adapter, + dialect, relations, ( native_field(&change, "fingerprint_sql")?, @@ -488,7 +489,7 @@ pub(crate) struct SelectedTests<'py> { /// Selected SQL test entries and their warnings, failing on invalid fixtures. pub(crate) fn selected_test_entries<'py>( - (project, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (project, dialect): (&Bound<'py, PyAny>, &Bound<'py, Dialect>), selected_keys: &Bound<'py, PyAny>, selection: &Bound<'py, PyAny>, case_name: &Bound<'py, PyAny>, @@ -518,7 +519,7 @@ pub(crate) fn selected_test_entries<'py>( }; let results = assemble_sql_test_plans( &selected, - (project, adapter), + (project, dialect), &SqlTestAssembly { sql_analysis_enabled: analysis, validate_fixtures: true, @@ -554,12 +555,12 @@ pub(crate) fn selected_test_entries<'py>( /// SQL tests selected by name or overlapping one selection, planned into a reusable result. fn planned_sql_tests<'py>( project: &Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, + dialect: &Bound<'py, Dialect>, selected_keys: &Bound<'py, PyAny>, selection: &Bound<'py, PyAny>, ) -> PyResult> { let py = project.py(); - let selected = selected_test_entries((project, adapter), selected_keys, selection, &none(py))?; + let selected = selected_test_entries((project, dialect), selected_keys, selection, &none(py))?; native( py, RecordId::PlannedSqlTests, @@ -593,7 +594,7 @@ pub(crate) fn plan_output_fields<'py>( ) -> PyResult)>> { let PlanOutputInputs { project, - adapter, + dialect, scope, snapshot, relations, @@ -628,7 +629,7 @@ pub(crate) fn plan_output_fields<'py>( }; require_intermediate_loader_targets(&project, &snapshot, &source_map, &source_load_entries)?; let functions = function_entries( - (&project, &adapter, &snapshot), + (&project, &dialect, &snapshot), &relations, &changes, &selected_keys, @@ -646,7 +647,7 @@ pub(crate) fn plan_output_fields<'py>( py, &[ ("project", project.clone()), - ("adapter", adapter.clone()), + ("dialect", dialect.clone().into_any()), ("scope", scope.clone()), ("relations", relations.clone()), ("model_materializations", materializations.into_any()), @@ -659,7 +660,7 @@ pub(crate) fn plan_output_fields<'py>( || native_field(&planned, "selected_keys")?.ne(&selected_keys)? || native_field(&planned, "sql_test_selection")?.ne(&selection)? { - planned = planned_sql_tests(&project, &adapter, &selected_keys, &selection)?; + planned = planned_sql_tests(&project, &dialect, &selected_keys, &selection)?; } let test_entries = native_field(&planned, "entries")?; let mut test_keys: Vec> = Vec::new(); @@ -761,11 +762,11 @@ pub(crate) fn plan_output_fields<'py>( /// Plan SQL tests selected by name or overlapping one selection into a reusable result. pub(crate) fn plan_selected_sql_tests<'py>( project: &Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, + dialect: &Bound<'py, Dialect>, selected_keys: &Bound<'py, PyAny>, sql_test_selection: &Bound<'py, PyAny>, ) -> PyResult> { - planned_sql_tests(project, adapter, selected_keys, sql_test_selection) + planned_sql_tests(project, dialect, selected_keys, sql_test_selection) } /// Selected SQL test entries and warnings, optionally narrowed to one parameterized case. @@ -780,12 +781,15 @@ pub(crate) fn build_selected_test_entries<'py>( "build_selected_test_entries", kwargs, ( - &["project", "adapter", "selected_keys", "sql_test_selection"], + &["project", "dialect", "selected_keys", "sql_test_selection"], &["case_name"], ), )?; let selected = selected_test_entries( - (&args.value("project")?, &args.value("adapter")?), + ( + &args.value("project")?, + &Dialect::passed(args.value("dialect")?)?, + ), &args.value("selected_keys")?, &args.value("sql_test_selection")?, &args.value("case_name")?, diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs index 1fbe363f6f..bbaab1bd4e 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs @@ -1,5 +1,6 @@ //! SQL test planning on a native thread, joined by the planner or, at interpreter exit, within a bound. +use sqlbuild_python_base::native_host::models::Dialect; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender, channel}; use std::sync::{Mutex, PoisonError}; @@ -151,7 +152,7 @@ impl SqlTestPlanning { /// The inputs one planning run reads, owned so they can move to the planning thread. pub(crate) struct PlanningRequest { pub(crate) project: Py, - pub(crate) adapter: Py, + pub(crate) dialect: Py, pub(crate) selected_keys: Py, pub(crate) sql_test_selection: Py, } @@ -161,7 +162,7 @@ fn plan(py: Python<'_>, request: &PlanningRequest) -> Planned { compiler_guard(|| { plan_selected_sql_tests( request.project.bind(py), - request.adapter.bind(py), + request.dialect.bind(py), request.selected_keys.bind(py), request.sql_test_selection.bind(py), ) @@ -233,7 +234,7 @@ pub(crate) fn start_sql_test_planning<'py>( ( &[ "project", - "adapter", + "dialect", "selected_keys", "sql_test_selection", "enabled", @@ -245,7 +246,7 @@ pub(crate) fn start_sql_test_planning<'py>( py, PlanningRequest { project: args.value("project")?.unbind(), - adapter: args.value("adapter")?.unbind(), + dialect: Dialect::passed(args.value("dialect")?)?.unbind(), selected_keys: args.value("selected_keys")?.unbind(), sql_test_selection: args.value("sql_test_selection")?.unbind(), }, diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/plan_model.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/plan_model.rs index de7eebbf54..968f545a76 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/plan_model.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/plan_model.rs @@ -9,6 +9,7 @@ use pyo3::pyfunction; use pyo3::types::PyDict; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::models::Dialect; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Detect changes and build a plan entry and warnings for a single model. @@ -26,7 +27,7 @@ pub fn plan_model<'py>( &[ "model", "snapshot", - "adapter", + "dialect", "context", "sql_analysis_enabled", "query_change_tracking", @@ -60,7 +61,7 @@ pub fn plan_model<'py>( snapshot, context: &ContextReads::of( args.value("context")?, - args.value("adapter")?, + Dialect::passed(args.value("dialect")?)?, args.value("external_sql_reference_resolver")?, )?, sql_analysis_enabled: args.flag("sql_analysis_enabled")?, diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_audit_command_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_audit_command_plan.rs index 9ae83b6857..c1d3d70e33 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_audit_command_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_audit_command_plan.rs @@ -8,6 +8,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::native_host::models::Dialect; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; @@ -15,10 +16,10 @@ use sqlbuild_python_project::planner_phases::main::object::object; /// Project selected audits from canonical static state. #[pyfunction] -#[pyo3(signature = (*, project, adapter, scope, relations))] +#[pyo3(signature = (*, project, dialect, scope, relations))] pub fn build_audit_command_plan<'py>( project: &Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, + dialect: &Bound<'py, Dialect>, scope: &Bound<'py, PyAny>, relations: &Bound<'py, PyAny>, ) -> PyResult> { @@ -37,7 +38,7 @@ pub fn build_audit_command_plan<'py>( py, &[ ("project", project.clone()), - ("adapter", adapter.clone()), + ("dialect", dialect.clone().into_any()), ("scope", scope.clone()), ("relations", relations.clone()), ("model_materializations", materializations.into_any()), diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs index 3c15840c05..ff8ea412c4 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs @@ -1,23 +1,21 @@ //! Build the execution plan for one selection against live warehouse state. -use crate::planner_entries::_helpers::plans::execution_plan::INSPECTION_CATALOG; use crate::planner_entries::_helpers::plans::execution_plan::planned_output; use crate::planner_entries::_helpers::plans::execution_plan::scopes_and_cursor_overrides; use crate::planner_entries::_helpers::plans::test_planning::{PlanningRequest, start_planning}; use crate::planner_phases::main::inputs::class_field::class_field; use crate::planner_phases::models::PlannerRuntime; +use crate::planner_warehouse::models::PlannerReads; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::PyDict; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::build_command::main::utc_now::utc_now; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::models::Dialect; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; -/// Build the execution plan for one selection against live warehouse state. +/// Build the execution plan for one selection through the plan's warehouse reads. #[pyfunction] #[pyo3(signature = (**kwargs))] pub fn build_execution_plan<'py>( @@ -31,8 +29,8 @@ pub fn build_execution_plan<'py>( ( &[ "project", - "adapter", - "connection", + "dialect", + "reads", "selection", "overrides", "deferral", @@ -47,14 +45,14 @@ pub fn build_execution_plan<'py>( ), )?; let project = args.value("project")?; - let adapter = args.value("adapter")?; - let connection = args.value("connection")?; + let dialect = Dialect::passed(args.value("dialect")?)?; + let reads = PlannerReads::passed(args.value("reads")?)?; let runtime = Bound::new( py, PlannerRuntime { project: project.clone().unbind(), - adapter: adapter.clone().unbind(), - connection: connection.clone().unbind(), + dialect: dialect.clone().unbind(), + reads: reads.clone().unbind(), project_config: args.value("project_config")?.unbind(), local_config: args.value("local_config")?.unbind(), on_progress: args.value("on_progress")?.unbind(), @@ -69,21 +67,15 @@ pub fn build_execution_plan<'py>( py, PlanningRequest { project: project.clone().unbind(), - adapter: adapter.clone().unbind(), + dialect: dialect.unbind(), selected_keys: native_field(&inspection, "selected_keys")?.unbind(), sql_test_selection: native_field(&inspection, "sql_test_selection")?.unbind(), }, native_field(&args.value("policies")?, "plan_sql_tests")?.is_truthy()?, )?; - let planned = (|| { - let catalog = call_with( - &host_entry(py, INSPECTION_CATALOG)?, - &[("adapter", adapter), ("connection", connection)], - )?; - within(&catalog, |_| { - planned_output(&args, &runtime, (&scopes, &overrides), test_planning.get()) - }) - })(); + let planned = reads.get().reads().within_inspection(py, &mut || { + planned_output(&args, &runtime, (&scopes, &overrides), test_planning.get()) + }); let joined = test_planning.get().result(py); let planned = planned?; joined?; diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/read_selected_seed_fingerprints.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/read_selected_seed_fingerprints.rs index 2a833b1c09..9ce7cb7576 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/read_selected_seed_fingerprints.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/read_selected_seed_fingerprints.rs @@ -1,15 +1,15 @@ //! Read only fingerprint state required by selected direct seeds. use crate::planner_entries::_helpers::plans::command_plans; +use crate::planner_warehouse::models::PlannerReads; use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyDict; /// Read only fingerprint state required by selected direct seeds. #[inline] pub fn read_selected_seed_fingerprints<'py>( - adapter: &Bound<'py, PyAny>, - connection: &Bound<'py, PyAny>, + reads: &Bound<'py, PlannerReads>, seeds: &Bound<'py, PyAny>, ) -> PyResult> { - command_plans::read_selected_seed_fingerprints(adapter, connection, seeds) + command_plans::read_selected_seed_fingerprints(reads, seeds) } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_relation_context.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_relation_context.rs index 63cc2accfc..90a6eed32d 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_relation_context.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_relation_context.rs @@ -1,12 +1,12 @@ //! Resolve canonical locations and source routing without warehouse inventory. use crate::planner_entries::_helpers::entries::relations::planner_relations_context; -use crate::planner_entries::_helpers::entries::values::none; use crate::planner_entries::models::RelationsRequest; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::PyDict; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::models::Dialect; /// Resolve canonical locations and source routing without warehouse inventory. #[pyfunction] @@ -20,7 +20,7 @@ pub fn resolve_static_relation_context<'py>( "resolve_static_relation_context", kwargs, ( - &["project", "adapter", "scope"], + &["project", "dialect", "scope"], &[ "deferral", "project_config", @@ -31,8 +31,8 @@ pub fn resolve_static_relation_context<'py>( )?; planner_relations_context(RelationsRequest { project: args.value("project")?, - adapter: args.value("adapter")?, - connection: none(py), + dialect: Dialect::passed(args.value("dialect")?)?, + reads: None, scope: args.value("scope")?, deferral: args.value("deferral")?, project_config: args.value("project_config")?, diff --git a/crates/sqlbuild-python-planner/src/planner_entries/models.rs b/crates/sqlbuild-python-planner/src/planner_entries/models.rs index 91867bcf9b..367bb18a6f 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/models.rs @@ -1,52 +1,55 @@ //! Public records of planner entries. +use crate::planner_warehouse::models::PlannerReads; use pyo3::prelude::{Bound, Py, PyAny}; use pyo3::pyclass; +use sqlbuild_python_base::native_host::models::Dialect; use sqlbuild_python_project::native_host::models::ModelLookup; /// What resolving relation locations and source metadata for one scope reads. #[derive(Debug)] pub struct RelationsRequest<'py> { - pub project: Bound<'py, PyAny>, - pub adapter: Bound<'py, PyAny>, - pub connection: Bound<'py, PyAny>, - pub scope: Bound<'py, PyAny>, - pub deferral: Bound<'py, PyAny>, - pub project_config: Bound<'py, PyAny>, - pub local_config: Bound<'py, PyAny>, - pub known_source_columns: Bound<'py, PyAny>, - pub relation_keys: Bound<'py, PyAny>, + pub(crate) project: Bound<'py, PyAny>, + pub(crate) dialect: Bound<'py, Dialect>, + /// The warehouse reads, or `None` to resolve from canonical static state only. + pub(crate) reads: Option>, + pub(crate) scope: Bound<'py, PyAny>, + pub(crate) deferral: Bound<'py, PyAny>, + pub(crate) project_config: Bound<'py, PyAny>, + pub(crate) local_config: Bound<'py, PyAny>, + pub(crate) known_source_columns: Bound<'py, PyAny>, + pub(crate) relation_keys: Bound<'py, PyAny>, } /// What building model plan entries for one execution scope reads. #[derive(Debug)] pub struct PlanEntriesRequest<'py> { - pub project: Bound<'py, PyAny>, - pub adapter: Bound<'py, PyAny>, - pub scope: Bound<'py, PyAny>, - pub snapshot: Bound<'py, PyAny>, - pub relations: Bound<'py, PyAny>, - pub resolved_models: Bound<'py, PyAny>, - pub cursor_overrides: Bound<'py, PyAny>, - pub full_refresh: bool, - pub build_inputs: Bound<'py, PyAny>, + pub(crate) project: Bound<'py, PyAny>, + pub(crate) dialect: Bound<'py, Dialect>, + pub(crate) scope: Bound<'py, PyAny>, + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) relations: Bound<'py, PyAny>, + pub(crate) resolved_models: Bound<'py, PyAny>, + pub(crate) cursor_overrides: Bound<'py, PyAny>, + pub(crate) full_refresh: bool, + pub(crate) build_inputs: Bound<'py, PyAny>, /// The effective worker limit execution will use. - pub concurrency: i64, + pub(crate) concurrency: i64, } /// What assembling the plan output's fields reads. #[derive(Debug)] pub struct PlanOutputInputs<'py> { - pub project: Bound<'py, PyAny>, - pub adapter: Bound<'py, PyAny>, - pub scope: Bound<'py, PyAny>, - pub snapshot: Bound<'py, PyAny>, - pub relations: Bound<'py, PyAny>, - pub changes: Bound<'py, PyAny>, - pub model_entries: Vec>, - pub model_warnings: Vec>, - pub reload_sources: Bound<'py, PyAny>, - pub extras: Bound<'py, PyAny>, + pub(crate) project: Bound<'py, PyAny>, + pub(crate) dialect: Bound<'py, Dialect>, + pub(crate) scope: Bound<'py, PyAny>, + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) relations: Bound<'py, PyAny>, + pub(crate) changes: Bound<'py, PyAny>, + pub(crate) model_entries: Vec>, + pub(crate) model_warnings: Vec>, + pub(crate) reload_sources: Bound<'py, PyAny>, + pub(crate) extras: Bound<'py, PyAny>, } /// Resolved relation and source inputs for plan entry construction. diff --git a/crates/sqlbuild-python-planner/src/planner_identity/_helpers/identities.rs b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/identities.rs index 68240b5a93..c3a5e95296 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/_helpers/identities.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/identities.rs @@ -16,7 +16,8 @@ use sqlbuild_analysis::planner_identity::main::graph_write_identity_hashes::grap use sqlbuild_analysis::planner_identity::models::{GraphKey, IdentityGraph, IdentityNode}; use sqlbuild_core::fingerprint_identity::main::model_local_identity_hash::model_local_identity_hash; -use crate::planner_identity::_helpers::metadata::{function_local_hashes, seed_identity}; +use crate::planner_identity::_helpers::metadata::function_local_hashes; +use crate::planner_identity::_helpers::metadata::seed_identity; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs index 8e0c01fe50..430da13459 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs @@ -1,8 +1,12 @@ //! Gather live candidate evidence without changing adapter inspection ownership. -use crate::planner_migrations::_helpers::column::column_objects::{config, definition_hash, lower}; +use crate::planner_migrations::_helpers::column::column_objects::config; +use crate::planner_migrations::_helpers::column::column_objects::definition_hash; +use crate::planner_migrations::_helpers::column::column_objects::lower; use crate::planner_migrations::_helpers::projections::query_shapes::parse_query_shape; -use crate::planner_migrations::models::{ColumnCandidate, ColumnMigrationInputs, MigrationPhase}; +use crate::planner_migrations::models::ColumnCandidate; +use crate::planner_migrations::models::ColumnMigrationInputs; +use crate::planner_migrations::models::MigrationPhase; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs index 18b11be9ee..436d400ded 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs @@ -1,14 +1,19 @@ //! Materialize column decisions from native proofs and live evidence. +use crate::planner_phases::main::inputs::runtime_reads::runtime_reads; +use crate::planner_warehouse::models::{AdapterRead, Read}; use sqlbuild_host::entries::models::ExecutorHost; use crate::planner_migrations::_helpers::column::column_candidates::declared_names; -use crate::planner_migrations::_helpers::column::column_objects::{lower, model, replace}; +use crate::planner_migrations::_helpers::column::column_objects::lower; +use crate::planner_migrations::_helpers::column::column_objects::model; +use crate::planner_migrations::_helpers::column::column_objects::replace; use crate::planner_migrations::_helpers::column::columns::identical_renames; use crate::planner_migrations::_helpers::column::columns::rename_hints; -use crate::planner_migrations::constants::{AUTOMATIC, DONE}; -use crate::planner_migrations::models::{CheckedColumns, ColumnCandidate}; -use crate::planner_phases::main::inputs::runtime_field::runtime_field; +use crate::planner_migrations::constants::AUTOMATIC; +use crate::planner_migrations::constants::DONE; +use crate::planner_migrations::models::CheckedColumns; +use crate::planner_migrations::models::ColumnCandidate; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; use sqlbuild_analysis::planner_migrations::main::column_decision::column_decision; @@ -177,19 +182,14 @@ pub(crate) fn checked<'py>( } else { native_field(&destination, "name")? }; - let reason = call_with( - &native_field( - &native_field(runtime, "adapter")?, - "column_rename_unavailable_reason", - )?, - &[ - ( - "connection", - runtime_field(runtime, |runtime| &runtime.connection)?, - ), - ("destination", relation), - ], - )?; + let reason = match runtime_reads(runtime)? + .get() + .reads() + .adapter_read(runtime.py(), AdapterRead::ColumnRenameUnavailable(relation))? + { + Read::Known(reason) => reason, + Read::Unknown(_) => runtime.py().None().into_bound(runtime.py()), + }; if reason.is_none() { return Ok(CheckedColumns { entries, diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs index 3d9eecefc4..3c80cdda67 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs @@ -1,11 +1,14 @@ //! Project column renames onto the existing snapshot and redirect cursor reads. -use crate::planner_migrations::_helpers::column::column_objects::{ - config, definition_hash, lower, native, replace, -}; +use crate::planner_migrations::_helpers::column::column_objects::config; +use crate::planner_migrations::_helpers::column::column_objects::definition_hash; +use crate::planner_migrations::_helpers::column::column_objects::lower; +use crate::planner_migrations::_helpers::column::column_objects::native; +use crate::planner_migrations::_helpers::column::column_objects::replace; use crate::planner_migrations::_helpers::projections::query_shapes::renames_explain_change; use crate::planner_migrations::models::ColumnCandidate; -use crate::planner_migrations::models::{ColumnMigrationInputs, MigrationPhase}; +use crate::planner_migrations::models::ColumnMigrationInputs; +use crate::planner_migrations::models::MigrationPhase; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::boundary::main::imported::imported; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_planning.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_planning.rs index 32d4707cb3..2c1c02a3bd 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_planning.rs @@ -5,12 +5,13 @@ use crate::planner_migrations::_helpers::column::known_input_columns::KnownInput use crate::planner_migrations::models::{ ColumnMigrationInputs, ColumnMigrationPlanning, MigrationPhase, }; -use crate::planner_phases::main::inputs::runtime_field::runtime_field; +use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyDict, PySet, PySetMethods}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination; @@ -66,10 +67,10 @@ pub(crate) fn plan_column_migrations<'py>( let py = phase.runtime.py(); let runtime = phase.runtime.clone(); let snapshot = phase.snapshot.clone(); - let adapter = runtime_field(&runtime, |runtime| &runtime.adapter)?; - let candidates = crate::planner_migrations::_helpers::column::column_candidates::candidates( + let dialect = runtime_dialect(&runtime)?; + let candidates = super::column_candidates::candidates( (phase, columns), - &adapter.call_method0("sql_analysis_dialect")?, + &dialect.syntax("sql_analysis_dialect")?.call0()?, )?; if candidates.is_empty() { return Ok(ColumnMigrationPlanning { @@ -105,22 +106,15 @@ pub(crate) fn plan_column_migrations<'py>( let mut warnings: Vec> = Vec::new(); let mut hints: Vec> = Vec::new(); for candidate in &candidates { - let decided = crate::planner_migrations::_helpers::column::column_entries::decide( + let decided = super::column_entries::decide( &runtime, candidate, &native_field(&state, "column_events")?, &inputs, )?; let physical = mapping_get(&columns.physical_relations, name_object(&candidate.model)?)?; - let checked = crate::planner_migrations::_helpers::column::column_entries::checked( - &runtime, candidate, decided, &physical, - )?; - for hint in crate::planner_migrations::_helpers::column::column_entries::hints( - candidate, - &checked.entries, - )? - .try_iter()? - { + let checked = super::column_entries::checked(&runtime, candidate, decided, &physical)?; + for hint in super::column_entries::hints(candidate, &checked.entries)?.try_iter()? { hints.push(hint?); } entries.extend(checked.entries); @@ -132,23 +126,12 @@ pub(crate) fn plan_column_migrations<'py>( )?; for entry in &mut entries { *entry = replace(entry, &[("missing_origin_policy", policy.clone())])?; - if let Some(warning) = - crate::planner_migrations::_helpers::column::column_warnings::entry_warning(entry)? - { + if let Some(warning) = super::column_warnings::entry_warning(entry)? { warnings.push(warning); } } - let overlaid = crate::planner_migrations::_helpers::column::column_overlay::overlay( - &snapshot, - &candidates, - &entries, - &inputs, - )?; - let snapshot = crate::planner_migrations::_helpers::column::column_overlay::cursors( - (phase, columns), - overlaid, - &entries, - )?; + let overlaid = super::column_overlay::overlay(&snapshot, &candidates, &entries, &inputs)?; + let snapshot = super::column_overlay::cursors((phase, columns), overlaid, &entries)?; Ok(ColumnMigrationPlanning { snapshot, entries, diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_warnings.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_warnings.rs index 58375264e8..865c9af4f0 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_warnings.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_warnings.rs @@ -2,10 +2,10 @@ use crate::planner_migrations::_helpers::column::column_objects::native; use crate::planner_migrations::_helpers::column::column_objects::text; -use crate::planner_migrations::constants::{ - DONE, MISSING_ORIGIN_ALLOW_OUTCOME, MISSING_ORIGIN_CONFIRMATION_OUTCOME, - MISSING_ORIGIN_DENY_OUTCOME, -}; +use crate::planner_migrations::constants::DONE; +use crate::planner_migrations::constants::MISSING_ORIGIN_ALLOW_OUTCOME; +use crate::planner_migrations::constants::MISSING_ORIGIN_CONFIRMATION_OUTCOME; +use crate::planner_migrations::constants::MISSING_ORIGIN_DENY_OUTCOME; use pyo3::exceptions::PyKeyError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::planner_migrations::main::column_warning::column_warning; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs index 48b937d2e1..5fde1c4193 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs @@ -2,10 +2,12 @@ use crate::planner_migrations::_helpers::column::column_objects::config; use crate::planner_migrations::_helpers::column::column_objects::native; -use crate::planner_migrations::_helpers::discovery::discovery_events::{ - identity, key, newest, relation, -}; -use crate::planner_migrations::models::{ModelMigrationDeclaration, ModelMigrationRequest}; +use crate::planner_migrations::_helpers::discovery::discovery_events::identity; +use crate::planner_migrations::_helpers::discovery::discovery_events::key; +use crate::planner_migrations::_helpers::discovery::discovery_events::newest; +use crate::planner_migrations::_helpers::discovery::discovery_events::relation; +use crate::planner_migrations::models::ModelMigrationDeclaration; +use crate::planner_migrations::models::ModelMigrationRequest; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{ PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PySet, PySetMethods, PyString, PyTuple, diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/model_discovery.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/model_discovery.rs index 6f81614411..b23d4059aa 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/model_discovery.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/model_discovery.rs @@ -3,15 +3,16 @@ use crate::planner_migrations::_helpers::discovery::discovery_events::renames; use crate::planner_migrations::_helpers::discovery::discovery_models::ordered; use crate::planner_migrations::_helpers::discovery::discovery_requests::automatic; -use crate::planner_migrations::_helpers::planning::matching::{ - match_migration_fingerprints, migration_destination_fingerprints, -}; -use crate::planner_migrations::models::{ - DiscoveryRequest, FingerprintInputs, ModelMigrationDiscovery, -}; +use crate::planner_migrations::_helpers::planning::matching::match_migration_fingerprints; +use crate::planner_migrations::_helpers::planning::matching::migration_destination_fingerprints; +use crate::planner_migrations::models::DiscoveryRequest; +use crate::planner_migrations::models::FingerprintInputs; +use crate::planner_migrations::models::ModelMigrationDiscovery; +use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; use pyo3::prelude::{PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple, PyTupleMethods}; use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use std::collections::BTreeSet; @@ -54,7 +55,9 @@ pub(crate) fn discover_model_migrations<'py>( )?; let inputs = FingerprintInputs { metadata_jsons: metadata.into_any(), - dialect: native_field(&runtime, "adapter")?.call_method0("sql_analysis_dialect")?, + dialect: runtime_dialect(&runtime)? + .syntax("sql_analysis_dialect")? + .call0()?, fingerprints: cache, }; call_with( diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs index 6429ff9e66..5fa9706ce6 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs @@ -1,6 +1,7 @@ //! Read-only warehouse evidence gathered once for all model migrations in one plan. use crate::planner_migrations::models::MigrationStateInspection; +use crate::planner_phases::main::inputs::runtime_reads::runtime_reads; use std::collections::HashSet; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; @@ -8,7 +9,6 @@ use pyo3::types::{PyString, PyTuple}; use sqlbuild_state::events::main::newest_fingerprint::newest_fingerprint; use sqlbuild_state::events::models::{EventRelation, FingerprintKey, FingerprintOrder}; -use crate::planner_phases::main::inputs::runtime_field::runtime_field; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// A lowercased schema and relation name. @@ -100,8 +100,7 @@ pub(crate) fn inspection<'py>( known_fingerprints: Option<&Bound<'py, PyAny>>, ) -> PyResult> { let state = MigrationStateInspection::new( - &runtime_field(runtime, |runtime| &runtime.adapter)?, - &runtime_field(runtime, |runtime| &runtime.connection)?, + &runtime_reads(runtime)?, &crate::planner_migrations::_helpers::planning::planning_origin::planning_database( runtime, )?, diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs index 79828786c6..66f9710514 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs @@ -2,12 +2,16 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::planner_migrations::_helpers::column::column_objects::{lower, model, replace, text}; -use crate::planner_migrations::_helpers::model::model_handover::{ - applies, empty_version, handover, -}; +use crate::planner_migrations::_helpers::column::column_objects::lower; +use crate::planner_migrations::_helpers::column::column_objects::model; +use crate::planner_migrations::_helpers::column::column_objects::replace; +use crate::planner_migrations::_helpers::column::column_objects::text; +use crate::planner_migrations::_helpers::model::model_handover::applies; +use crate::planner_migrations::_helpers::model::model_handover::empty_version; +use crate::planner_migrations::_helpers::model::model_handover::handover; use crate::planner_migrations::_helpers::planning::compatibility::check_migration_compatibility; -use crate::planner_migrations::models::{CompatibilityCheck, MigrationDecisionRequest}; +use crate::planner_migrations::models::CompatibilityCheck; +use crate::planner_migrations::models::MigrationDecisionRequest; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyBool; use sqlbuild_analysis::planner_migrations::main::model_decision::model_decision; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_handover.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_handover.rs index e63588b412..cc87ead457 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_handover.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_handover.rs @@ -1,6 +1,7 @@ //! Adapt immutable migration fingerprints without changing their persisted representation. -use crate::planner_migrations::_helpers::column::column_objects::{replace, text}; +use crate::planner_migrations::_helpers::column::column_objects::replace; +use crate::planner_migrations::_helpers::column::column_objects::text; use crate::planner_migrations::constants::MIGRATION_MODEL_NAME_METADATA_KEY; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_overlay.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_overlay.rs index 709519a2ca..01a95ea529 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_overlay.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_overlay.rs @@ -1,7 +1,8 @@ //! Project inherited fingerprints and live relation facts before normal planning. +use crate::planner_migrations::_helpers::column::column_objects::config; use crate::planner_migrations::_helpers::column::column_objects::native; -use crate::planner_migrations::_helpers::column::column_objects::{config, replace}; +use crate::planner_migrations::_helpers::column::column_objects::replace; use crate::planner_migrations::models::MigrationPhase; use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs index 4cc0a60a9a..df3650bc04 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs @@ -1,15 +1,18 @@ //! Model migration orchestration over Python-owned live inspection. use crate::planner_changes::main::plan_reference_renames::plan_reference_renames; -use crate::planner_migrations::_helpers::column::column_objects::{config, replace}; +use crate::planner_migrations::_helpers::column::column_objects::config; +use crate::planner_migrations::_helpers::column::column_objects::replace; use crate::planner_migrations::_helpers::discovery::model_discovery::discover_model_migrations; -use crate::planner_migrations::_helpers::planning::planning_origin::{ - require_same_database, resolve_origin, -}; -use crate::planner_migrations::models::{ - DiscoveryRequest, MigrationDecisionRequest, MigrationPhase, ModelMigrationDeclaration, - ModelMigrationDiscovery, ModelMigrationPlanning, ModelMigrationRequest, -}; +use crate::planner_migrations::_helpers::planning::planning_origin::require_same_database; +use crate::planner_migrations::_helpers::planning::planning_origin::resolve_origin; +use crate::planner_migrations::models::DiscoveryRequest; +use crate::planner_migrations::models::MigrationDecisionRequest; +use crate::planner_migrations::models::MigrationPhase; +use crate::planner_migrations::models::ModelMigrationDeclaration; +use crate::planner_migrations::models::ModelMigrationDiscovery; +use crate::planner_migrations::models::ModelMigrationPlanning; +use crate::planner_migrations::models::ModelMigrationRequest; use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyString, PyTuple}; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_warnings.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_warnings.rs index c29e3a9b4f..74223c718a 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_warnings.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_warnings.rs @@ -1,8 +1,7 @@ //! Adapt diagnostic display facts while preserving policy and code ordering. -use crate::planner_migrations::_helpers::column::column_warnings::{ - missing_origin_outcome, warning, -}; +use crate::planner_migrations::_helpers::column::column_warnings::missing_origin_outcome; +use crate::planner_migrations::_helpers::column::column_warnings::warning; use crate::planner_migrations::constants::HIDDEN_ORIGIN_REMEDY; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyString; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs index 18f8f935b7..693488e50e 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs @@ -1,19 +1,25 @@ //! Assemble new and resumed old-name view steps from recorded facts. -use crate::planner_migrations::_helpers::column::column_objects::{model, text}; -use crate::planner_migrations::_helpers::old_name::old_name_objects::{ - expires, location, relation, retention, stores_history, -}; +use crate::planner_migrations::_helpers::column::column_objects::model; +use crate::planner_migrations::_helpers::column::column_objects::text; +use crate::planner_migrations::_helpers::old_name::old_name_objects::expires; +use crate::planner_migrations::_helpers::old_name::old_name_objects::location; +use crate::planner_migrations::_helpers::old_name::old_name_objects::relation; +use crate::planner_migrations::_helpers::old_name::old_name_objects::retention; +use crate::planner_migrations::_helpers::old_name::old_name_objects::stores_history; use crate::planner_migrations::_helpers::old_name::old_names::declared_column_aliases; use crate::planner_migrations::_helpers::old_name::old_names::old_name_view_skip_reason; -use crate::planner_migrations::constants::{PENDING_ARCHIVE, PENDING_VIEW}; +use crate::planner_migrations::constants::PENDING_ARCHIVE; +use crate::planner_migrations::constants::PENDING_VIEW; use crate::planner_migrations::models::OldNamePlanner; +use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyBool, PyList, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::record_class::record_class; @@ -238,9 +244,7 @@ impl<'py> OldNamePlanner<'py> { ), ( "grants_supported", - self.runtime - .getattr("adapter")? - .getattr("relation_grants_supported")?, + runtime_dialect(&self.runtime)?.syntax("relation_grants_supported")?, ), ]); } @@ -370,9 +374,7 @@ impl<'py> OldNamePlanner<'py> { ), ( "grants_supported", - self.runtime - .getattr("adapter")? - .getattr("relation_grants_supported")?, + runtime_dialect(&self.runtime)?.syntax("relation_grants_supported")?, ), ( "archived", diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs index 7518d1ea03..cb3a98cf8c 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs @@ -1,8 +1,11 @@ //! Old-name planner object conversions and existing native helper calls. +use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::native_host::types::Syntax; -use crate::planner_migrations::_helpers::column::column_objects::{config, model}; +use crate::planner_migrations::_helpers::column::column_objects::config; +use crate::planner_migrations::_helpers::column::column_objects::model; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -27,7 +30,7 @@ pub(crate) fn location<'py>( ("name", native_field(relation, "name")?), ]; let qualified = call_with( - &native_field(&native_field(runtime, "adapter")?, "render_qualified_name")?, + &runtime_dialect(runtime)?.syntax("render_qualified_name")?, &fields, )?; fields.push(("qualified_name", qualified)); diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs index 5f591d5791..871c2e9c66 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs @@ -5,7 +5,8 @@ use sqlbuild_host::entries::models::AdapterHost; use crate::planner_migrations::_helpers::old_name::old_name_objects::recorded_view; use crate::planner_migrations::_helpers::old_name::old_names::old_name_claim_errors; use crate::planner_migrations::_helpers::old_name::old_names::old_name_schemas_to_read; -use crate::planner_migrations::models::{MigrationPhase, OldNameViewPlanning}; +use crate::planner_migrations::models::MigrationPhase; +use crate::planner_migrations::models::OldNameViewPlanning; use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PySet, PySetMethods, PyTuple}; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_references.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_references.rs index 4607ee3b2d..8b0e57d3f1 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_references.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_references.rs @@ -2,7 +2,8 @@ use sqlbuild_host::entries::models::UserCodeHost; -use crate::planner_migrations::_helpers::column::column_objects::{native, text}; +use crate::planner_migrations::_helpers::column::column_objects::native; +use crate::planner_migrations::_helpers::column::column_objects::text; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/compatibility.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/compatibility.rs index 37c9a370ea..afcaf2a97d 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/compatibility.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/compatibility.rs @@ -1,7 +1,8 @@ //! Convert existing planner contracts for origin compatibility checks. use crate::planner_changes::main::detect_model_changes::detect_model_changes; -use crate::planner_migrations::models::{CompatibilityCheck, MigrationCompatibilityResult}; +use crate::planner_migrations::models::CompatibilityCheck; +use crate::planner_migrations::models::MigrationCompatibilityResult; use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/matching.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/matching.rs index 26d7df98bf..13d8a64ed0 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/matching.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/matching.rs @@ -1,6 +1,7 @@ //! Fingerprint cache callbacks around native one-to-one migration matching. -use crate::planner_migrations::models::{FingerprintInputs, ModelMigrationRequest}; +use crate::planner_migrations::models::FingerprintInputs; +use crate::planner_migrations::models::ModelMigrationRequest; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_boundary.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_boundary.rs index 9436b99e4c..7e12a8e95a 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_boundary.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_boundary.rs @@ -2,7 +2,9 @@ use crate::planner_migrations::_helpers::column::column_objects::native; use crate::planner_migrations::models::MigrationPhase; +use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; use crate::planner_phases::main::inputs::runtime_field::runtime_field; +use crate::planner_phases::main::inputs::runtime_reads::runtime_reads; use crate::planner_phases::main::rebuilds::effective_target_config::effective_target_config; use crate::planner_phases::main::rebuilds::effectively_full_refreshed_model_names::effectively_full_refreshed_model_names; use crate::planner_warehouse::main::inspection::gather_redirected_cursor_snapshots::gather_redirected_cursor_snapshots; @@ -68,14 +70,8 @@ pub(crate) fn redirected_cursor_snapshots<'py>( py, &[ ("project", project), - ( - "adapter", - runtime_field(runtime, |runtime| &runtime.adapter)?, - ), - ( - "connection", - runtime_field(runtime, |runtime| &runtime.connection)?, - ), + ("dialect", runtime_dialect(runtime)?.into_any()), + ("reads", runtime_reads(runtime)?.into_any()), ("existing_relations", existing_relations), ("target_relations", target_relations), ("cursor_scope", cursor_scope), diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_execution.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_execution.rs index bf918e5010..a013e0c8be 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_execution.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_execution.rs @@ -1,9 +1,10 @@ //! Preserve adapter overrides when describing migration execution. +use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; use sqlbuild_host::entries::models::AdapterHost; +use sqlbuild_python_base::native_host::types::Syntax; use crate::planner_migrations::_helpers::column::column_objects::replace; -use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyBool; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -25,15 +26,16 @@ fn migration_promotion<'py>( runtime: &Bound<'py, PyAny>, decision: &Bound<'py, PyAny>, ) -> PyResult> { - let adapter = runtime_field(runtime, |runtime| &runtime.adapter)?; + let dialect = runtime_dialect(runtime)?; let promotion = record_class(runtime.py(), RecordId::MigrationPromotion)?; - if adapter - .call_method0("supports_transactional_ddl")? + if dialect + .syntax("supports_transactional_ddl")? + .call0()? .is_truthy()? { return native_field(&promotion, "TRANSACTIONAL_RENAME"); } - if !decision.eq("migrate")? && native_field(&adapter, "adapter_name")?.eq("snowflake")? { + if !decision.eq("migrate")? && dialect.syntax("adapter_name")?.eq("snowflake")? { return native_field(&promotion, "SWAP"); } native_field(&promotion, "RENAME") @@ -46,8 +48,8 @@ pub(crate) fn migration_with_execution<'py>( entry: &Bound<'py, PyAny>, ) -> PyResult> { let py = runtime.py(); - let adapter = runtime_field(runtime, |runtime| &runtime.adapter)?; - let transient = if native_field(&adapter, "adapter_name")?.eq("snowflake")? { + let dialect = runtime_dialect(runtime)?; + let transient = if dialect.syntax("adapter_name")?.eq("snowflake")? { PyBool::new( py, native_field( @@ -62,7 +64,7 @@ pub(crate) fn migration_with_execution<'py>( py.None().into_bound(py) }; let stage = call_with( - &native_field(&adapter, "render_migration_stage")?, + &dialect.syntax("render_migration_stage")?, &[ ("origin", label(&native_field(entry, "origin")?)?), ("stage", label(&native_field(entry, "destination")?)?), diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs index c303de0661..a57764dc40 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs @@ -2,11 +2,13 @@ use crate::planner_migrations::constants::QUALIFIED_RELATION_MAX_PARTS; use crate::planner_migrations::models::ModelMigrationRequest; +use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::PyErr; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PySet, PySetMethods}; use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -20,7 +22,7 @@ pub(crate) fn location<'py>( fields: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { let qualified = call_with( - &native_field(&native_field(runtime, "adapter")?, "render_qualified_name")?, + &runtime_dialect(runtime)?.syntax("render_qualified_name")?, fields, )?; let mut fields = fields.to_vec(); diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/constants.rs b/crates/sqlbuild-python-planner/src/planner_migrations/constants.rs index 06519f2cb1..8c51022a8e 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/constants.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/constants.rs @@ -2,7 +2,6 @@ use crate::planner_migrations::_helpers::projections::getters::read_migration_state_inspection; use crate::planner_migrations::models::MigrationStateInspection; -use sqlbuild_host::entries::models::AdapterHost; use sqlbuild_python_base::records::main::registry::class_of::class_of; use sqlbuild_python_base::records::models::GetterEntry; @@ -14,11 +13,6 @@ pub(crate) const PENDING_ARCHIVE: &str = "pending_archive"; pub(crate) const PENDING_VIEW: &str = "pending_view"; -pub(crate) const LIST_RELATIONS_FOR_INSPECTION: AdapterHost = - AdapterHost::ListRelationsForInspection; - -pub(crate) const GET_COLUMNS_FOR_INSPECTION: AdapterHost = AdapterHost::GetColumnsForInspection; - pub(crate) const QUALIFIED_RELATION_MAX_PARTS: usize = 3; pub(crate) const MIGRATION_MODEL_NAME_METADATA_KEY: &str = "model_name"; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/models.rs b/crates/sqlbuild-python-planner/src/planner_migrations/models.rs index f188887cb0..24f03dbbbf 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/models.rs @@ -7,8 +7,12 @@ use crate::planner_migrations::_helpers::discovery::state_inspection::pending; use crate::planner_migrations::_helpers::discovery::state_inspection::relation_key; use crate::planner_migrations::_helpers::discovery::state_inspection::schema_or_empty; use crate::planner_migrations::_helpers::discovery::state_inspection::tuple_of; -use crate::planner_migrations::constants::GET_COLUMNS_FOR_INSPECTION; -use crate::planner_migrations::constants::LIST_RELATIONS_FOR_INSPECTION; +use crate::planner_warehouse::main::inspection::columns_for::columns_for; +use crate::planner_warehouse::main::inspection::list_relations::list_relations; +use crate::planner_warehouse::models::FingerprintRead; +use crate::planner_warehouse::models::PlannerReads; +use crate::planner_warehouse::models::Read; +use crate::planner_warehouse::models::StateHistoryRead; use pyo3::prelude::Py; use pyo3::prelude::PyAnyMethods; use pyo3::prelude::PyRef; @@ -18,7 +22,6 @@ use pyo3::prelude::Python; use pyo3::prelude::pyclass; use pyo3::prelude::pymethods; use pyo3::prelude::{Bound, PyAny}; -use pyo3::types::PyBool; use pyo3::types::PyDict; use pyo3::types::PyString; use pyo3::types::PyTuple; @@ -28,9 +31,6 @@ use sqlbuild_core::constants::COLUMN_MIGRATION_TABLE_NAME; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; use sqlbuild_core::constants::MIGRATION_TABLE_NAME; use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; @@ -39,10 +39,6 @@ use sqlbuild_python_front::state_rows::constants::MODEL_KIND; use sqlbuild_python_front::state_rows::constants::OLD_NAME_KIND; use sqlbuild_python_front::state_rows::main::migrations::layout::layout; use sqlbuild_python_front::state_rows::main::migrations::old_name_views::old_name_views; -use sqlbuild_python_front::state_rows::main::migrations::read_events::read_events; -use sqlbuild_python_front::state_rows::main::migrations::stored_old_name_columns::stored_old_name_columns; -use sqlbuild_python_front::state_rows::models::StateAccess; -use sqlbuild_python_project::fingerprints::main::read_latest_fingerprints::read_latest_fingerprints; use sqlbuild_state::events::models::EventRelation; use std::collections::HashMap; use std::collections::HashSet; @@ -200,55 +196,78 @@ pub(crate) struct DiscoveryRequest<'py> { #[pyclass(module = "sqlbuild._native", name = "MigrationStateInspection")] #[derive(Debug)] pub struct MigrationStateInspection { - pub(crate) adapter: Py, - pub(crate) connection: Py, - pub(crate) database: Py, - pub(crate) known_fingerprints: Py, - pub(crate) read_schemas: HashSet, + reads: Py, + database: Py, + known_fingerprints: Py, + read_schemas: HashSet, /// Model fingerprints per lowercased schema, in first-read order. - pub(crate) fingerprints: Vec<(String, Vec>)>, - pub(crate) events: Vec>, - pub(crate) event_schemas: HashSet, - pub(crate) relations: HashMap>, - pub(crate) columns: HashMap>, - pub(crate) column_event_schemas: HashSet, - pub(crate) column_events: Vec>, - pub(crate) old_name_schemas: HashSet, - pub(crate) old_name_moves: Vec>, - pub(crate) old_name_facts: Vec>, + fingerprints: Vec<(String, Vec>)>, + /// Positions in `fingerprints` (schema, fingerprint) by node name, in the order `fingerprints` lists them. + fingerprints_by_name: HashMap>, + events: Vec>, + event_schemas: HashSet, + relations: HashMap>, + columns: HashMap>, + column_event_schemas: HashSet, + column_events: Vec>, + old_name_schemas: HashSet, + old_name_moves: Vec>, + old_name_facts: Vec>, } impl MigrationStateInspection { - pub(crate) fn access<'py>(&self, py: Python<'py>) -> PyResult> { - let adapter: &Bound<'py, PyAny> = self.adapter.bind(py); - Ok(StateAccess { - connection: self.connection.bind(py).clone(), - execute: native_field(adapter, "execute")?, - render_qualified_name: native_field(adapter, "render_qualified_name")?, - }) + fn reads<'py>(&self, py: Python<'py>) -> Bound<'py, PlannerReads> { + self.reads.bind(py).clone() } - pub(crate) fn list_relations<'py>( + fn list_relations<'py>( &self, py: Python<'py>, schemas: Vec>, names: Bound<'py, PyAny>, ) -> PyResult>> { - call_with( - &host_entry(py, LIST_RELATIONS_FOR_INSPECTION)?, - &[ - ("adapter", self.adapter.bind(py).clone()), - ("connection", self.connection.bind(py).clone()), - ("database", self.database.bind(py).clone()), - ("schemas", PyTuple::new(py, schemas)?.into_any()), - ("names", names), - ], + list_relations( + self.reads.bind(py), + self.database.bind(py), + PyTuple::new(py, schemas)?.into_any(), + names, )? .try_iter()? .collect() } - pub(crate) fn model_fingerprints<'py>( + /// The listed migration state tables of `schemas`, or `None` when the reads do not know them. + fn migration_state_tables<'py>( + &self, + py: Python<'py>, + schemas: Vec>, + names: Bound<'py, PyAny>, + ) -> PyResult>>> { + match self.reads.get().reads().state_history( + py, + StateHistoryRead::MigrationTables { + database: self.database.bind(py), + filter: (PyTuple::new(py, schemas)?.into_any(), names), + }, + )? { + Read::Known(listed) => Ok(Some(listed.try_iter()?.collect::>()?)), + Read::Unknown(_) => Ok(None), + } + } + + /// Every inspected schema's model fingerprint positions by node name, in the order `fingerprints` lists them. + fn by_name(&self, py: Python<'_>) -> PyResult>> { + let mut by_name: HashMap> = HashMap::new(); + for (schema, (_, fingerprints)) in self.fingerprints.iter().enumerate() { + for (position, fingerprint) in fingerprints.iter().enumerate() { + let name: String = native_field(fingerprint.bind(py), "node_name")?.extract()?; + by_name.entry(name).or_default().push((schema, position)); + } + } + Ok(by_name) + } + + fn model_fingerprints<'py>( &self, py: Python<'py>, schema: &Bound<'py, PyAny>, @@ -256,25 +275,17 @@ impl MigrationStateInspection { ) -> PyResult>> { let known: Bound<'py, PyAny> = mapping_get(self.known_fingerprints.bind(py), key)?; let latest: Bound<'py, PyAny> = if known.is_none() { - let adapter: &Bound<'py, PyAny> = self.adapter.bind(py); - let fingerprint_set: Bound<'py, PyAny> = read_latest_fingerprints(Some(&keywords( + let schema: String = schema.extract()?; + let fingerprint_set: Bound<'py, PyAny> = self.reads.get().reads().fingerprints( py, - &[ - ("connection", self.connection.bind(py).clone()), - ("execute", native_field(adapter, "execute")?), - ("table_exists", PyBool::new(py, true).to_owned().into_any()), - ("database", self.database.bind(py).clone()), - ("schema", schema.clone()), - ( - "render_qualified_name", - native_field(adapter, "render_qualified_name")?, - ), - ( - "render_read_latest_sql", - native_field(adapter, "render_read_latest_fingerprints_sql")?, - ), - ], - )?))?; + FingerprintRead { + database: self.database.bind(py).extract()?, + schema: &schema, + table_exists: true, + node_names: None, + filtered_node_types: Vec::new(), + }, + )?; let by_identity: Bound<'py, PyAny> = native_field(&fingerprint_set, "fingerprints_by_identity")?; if by_identity.is_truthy()? { @@ -299,14 +310,13 @@ impl MigrationStateInspection { #[pymethods] impl MigrationStateInspection { #[new] - #[pyo3(signature = (*, adapter, connection, database, known_fingerprints = None))] + #[pyo3(signature = (*, reads, database, known_fingerprints = None))] pub(crate) fn new( - adapter: &Bound<'_, PyAny>, - connection: &Bound<'_, PyAny>, + reads: &Bound<'_, PlannerReads>, database: &Bound<'_, PyAny>, known_fingerprints: Option<&Bound<'_, PyAny>>, ) -> PyResult { - let py: Python<'_> = adapter.py(); + let py: Python<'_> = reads.py(); let known: Bound<'_, PyAny> = match known_fingerprints { Some(known) if !known.is_none() @@ -317,12 +327,12 @@ impl MigrationStateInspection { _ => PyDict::new(py).into_any(), }; Ok(Self { - adapter: adapter.clone().unbind(), - connection: connection.clone().unbind(), + reads: reads.clone().unbind(), database: database.clone().unbind(), known_fingerprints: known.unbind(), read_schemas: HashSet::new(), fingerprints: Vec::new(), + fingerprints_by_name: HashMap::new(), events: Vec::new(), event_schemas: HashSet::new(), relations: HashMap::new(), @@ -337,25 +347,25 @@ impl MigrationStateInspection { /// The database whose namespaces this inspection reads. #[getter] - pub fn database(&self, py: Python<'_>) -> Py { + pub(crate) fn database(&self, py: Python<'_>) -> Py { self.database.clone_ref(py) } /// Every migration event read so far. #[getter] - pub fn events<'py>(&self, py: Python<'py>) -> PyResult> { + pub(crate) fn events<'py>(&self, py: Python<'py>) -> PyResult> { tuple_of(py, &self.events) } /// Every column migration event read so far. #[getter] - pub fn column_events<'py>(&self, py: Python<'py>) -> PyResult> { + pub(crate) fn column_events<'py>(&self, py: Python<'py>) -> PyResult> { tuple_of(py, &self.column_events) } /// The old-name history of every move read by `inspect_old_name_views`. #[getter] - pub fn old_name_histories<'py>(&self, py: Python<'py>) -> PyResult> { + pub(crate) fn old_name_histories<'py>(&self, py: Python<'py>) -> PyResult> { old_name_views( tuple_of(py, &self.old_name_moves)?.as_any(), tuple_of(py, &self.old_name_facts)?.as_any(), @@ -364,10 +374,7 @@ impl MigrationStateInspection { /// Read the column migration state table once for every schema not inspected yet. #[pyo3(signature = (*, schemas))] - pub(crate) fn inspect_column_events( - slf: &Bound<'_, Self>, - schemas: &Bound<'_, PyAny>, - ) -> PyResult<()> { + fn inspect_column_events(slf: &Bound<'_, Self>, schemas: &Bound<'_, PyAny>) -> PyResult<()> { let py: Python<'_> = slf.py(); let pending = pending(schemas, &slf.borrow().column_event_schemas)?; if pending.is_empty() { @@ -376,11 +383,14 @@ impl MigrationStateInspection { slf.borrow_mut() .column_event_schemas .extend(pending.iter().map(|(_, key)| key.clone())); - let listed: Vec> = slf.borrow().list_relations( + let Some(listed) = slf.borrow().migration_state_tables( py, pending.iter().map(|(schema, _)| schema.clone()).collect(), PyTuple::new(py, [COLUMN_MIGRATION_TABLE_NAME])?.into_any(), - )?; + )? + else { + return Ok(()); + }; let mut present: HashSet = HashSet::new(); for relation in &listed { if lowered(&native_field(relation, "name")?)? == COLUMN_MIGRATION_TABLE_NAME { @@ -393,24 +403,31 @@ impl MigrationStateInspection { if !present.contains(&key) { continue; } - let (access, database) = { + let (reads, database) = { let this: PyRef<'_, Self> = slf.borrow(); - (this.access(py)?, this.database.bind(py).clone()) + (this.reads(py), this.database.bind(py).clone()) + }; + let Read::Known(events) = reads.get().reads().state_history( + py, + StateHistoryRead::MigrationEvents { + layout: &layout(COLUMN_KIND)?, + location: (database, schema), + stored: None, + }, + )? + else { + return Ok(()); }; - let events = read_events(&access, &layout(COLUMN_KIND)?, (database, schema), None)?; slf.borrow_mut() .column_events - .extend(events.iter().map(Bound::unbind)); + .extend(events.downcast::()?.iter().map(Bound::unbind)); } Ok(()) } /// Read moves and old-name facts once for every schema not inspected yet. #[pyo3(signature = (*, schemas))] - pub(crate) fn inspect_old_name_views( - slf: &Bound<'_, Self>, - schemas: &Bound<'_, PyAny>, - ) -> PyResult<()> { + fn inspect_old_name_views(slf: &Bound<'_, Self>, schemas: &Bound<'_, PyAny>) -> PyResult<()> { let py: Python<'_> = slf.py(); let pending = pending(schemas, &slf.borrow().old_name_schemas)?; if pending.is_empty() { @@ -419,11 +436,14 @@ impl MigrationStateInspection { slf.borrow_mut() .old_name_schemas .extend(pending.iter().map(|(_, key)| key.clone())); - let listed: Vec> = slf.borrow().list_relations( + let Some(listed) = slf.borrow().migration_state_tables( py, pending.iter().map(|(schema, _)| schema.clone()).collect(), PyTuple::new(py, [MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME])?.into_any(), - )?; + )? + else { + return Ok(()); + }; let state_tables: HashSet = listed.iter().map(relation_key).collect::>()?; for (schema, key) in pending { @@ -432,42 +452,51 @@ impl MigrationStateInspection { { continue; } - let (access, adapter, database) = { + let (reads, database) = { let this: PyRef<'_, Self> = slf.borrow(); - ( - this.access(py)?, - this.adapter.bind(py).clone(), - this.database.bind(py).clone(), - ) + (this.reads(py), this.database.bind(py).clone()) + }; + let reads = reads.get().reads(); + let Read::Known(stored) = + reads.old_name_view_columns(py, (database.clone(), schema.clone()))? + else { + return Ok(()); + }; + let Read::Known(facts) = reads.state_history( + py, + StateHistoryRead::MigrationEvents { + layout: &layout(OLD_NAME_KIND)?, + location: (database.clone(), schema.clone()), + stored: Some(&stored), + }, + )? + else { + return Ok(()); }; - let stored: Vec = stored_old_name_columns( - &adapter, - &access.connection, - (database.clone(), schema.clone()), - )?; - let facts = read_events( - &access, - &layout(OLD_NAME_KIND)?, - (database.clone(), schema.clone()), - Some(&stored), - )?; slf.borrow_mut() .old_name_facts - .extend(facts.iter().map(Bound::unbind)); - let moves = read_events(&access, &layout(MODEL_KIND)?, (database, schema), None)?; + .extend(facts.downcast::()?.iter().map(Bound::unbind)); + let Read::Known(moves) = reads.state_history( + py, + StateHistoryRead::MigrationEvents { + layout: &layout(MODEL_KIND)?, + location: (database, schema), + stored: None, + }, + )? + else { + return Ok(()); + }; slf.borrow_mut() .old_name_moves - .extend(moves.iter().map(Bound::unbind)); + .extend(moves.downcast::()?.iter().map(Bound::unbind)); } Ok(()) } /// Read migration events once from schemas known to hold the migration table. #[pyo3(signature = (*, schemas))] - pub(crate) fn inspect_migration_events( - slf: &Bound<'_, Self>, - schemas: &Bound<'_, PyAny>, - ) -> PyResult<()> { + fn inspect_migration_events(slf: &Bound<'_, Self>, schemas: &Bound<'_, PyAny>) -> PyResult<()> { let py: Python<'_> = slf.py(); let mut sorted: Vec<(String, Bound<'_, PyAny>)> = Vec::new(); for schema in schemas.try_iter()? { @@ -480,24 +509,31 @@ impl MigrationStateInspection { if !slf.borrow_mut().event_schemas.insert(key) { continue; } - let (access, database) = { + let (reads, database) = { let this: PyRef<'_, Self> = slf.borrow(); - (this.access(py)?, this.database.bind(py).clone()) + (this.reads(py), this.database.bind(py).clone()) + }; + let Read::Known(events) = reads.get().reads().state_history( + py, + StateHistoryRead::MigrationEvents { + layout: &layout(MODEL_KIND)?, + location: (database, schema), + stored: None, + }, + )? + else { + return Ok(()); }; - let events = read_events(&access, &layout(MODEL_KIND)?, (database, schema), None)?; slf.borrow_mut() .events - .extend(events.iter().map(Bound::unbind)); + .extend(events.downcast::()?.iter().map(Bound::unbind)); } Ok(()) } /// Read state tables once for every schema not inspected yet. #[pyo3(signature = (*, schemas))] - pub(crate) fn inspect_schemas( - slf: &Bound<'_, Self>, - schemas: &Bound<'_, PyAny>, - ) -> PyResult<()> { + fn inspect_schemas(slf: &Bound<'_, Self>, schemas: &Bound<'_, PyAny>) -> PyResult<()> { let py: Python<'_> = slf.py(); let pending = pending(schemas, &slf.borrow().read_schemas)?; if pending.is_empty() { @@ -513,6 +549,7 @@ impl MigrationStateInspection { )?; let state_tables: HashSet = listed.iter().map(relation_key).collect::>()?; + let mut read_fingerprints: bool = false; for (schema, key) in pending { if state_tables.contains(&(key.clone(), MIGRATION_TABLE_NAME.to_owned())) { let single = PyTuple::new(py, [schema.clone()])?; @@ -529,17 +566,19 @@ impl MigrationStateInspection { Some((_, stored)) => *stored = models, None => this.fingerprints.push((key, models)), } + read_fingerprints = true; } } + if read_fingerprints { + let by_name: HashMap> = slf.borrow().by_name(py)?; + slf.borrow_mut().fingerprints_by_name = by_name; + } Ok(()) } /// List candidate relations and their columns with one metadata read. #[pyo3(signature = (*, locations))] - pub(crate) fn inspect_relations( - slf: &Bound<'_, Self>, - locations: &Bound<'_, PyAny>, - ) -> PyResult<()> { + fn inspect_relations(slf: &Bound<'_, Self>, locations: &Bound<'_, PyAny>) -> PyResult<()> { let py: Python<'_> = slf.py(); let mut wanted: HashSet = HashSet::new(); let mut schemas: Vec = Vec::new(); @@ -582,20 +621,10 @@ impl MigrationStateInspection { let columns: Bound<'_, PyAny> = if matched.is_empty() { PyDict::new(py).into_any() } else { - let this: PyRef<'_, Self> = slf.borrow(); - let adapter: Bound<'_, PyAny> = this.adapter.bind(py).clone(); - let connection: Bound<'_, PyAny> = this.connection.bind(py).clone(); - drop(this); - call_with( - &host_entry(py, GET_COLUMNS_FOR_INSPECTION)?, - &[ - ("adapter", adapter), - ("connection", connection), - ( - "relations", - PyTuple::new(py, matched.iter().map(|(_, relation)| relation))?.into_any(), - ), - ], + let reads: Bound<'_, PlannerReads> = slf.borrow().reads(py); + columns_for( + &reads, + PyTuple::new(py, matched.iter().map(|(_, relation)| relation))?.into_any(), )? }; let empty: Bound<'_, PyTuple> = PyTuple::empty(py); @@ -610,11 +639,7 @@ impl MigrationStateInspection { } /// The listed relation for a location. - pub(crate) fn relation( - &self, - py: Python<'_>, - location: &Bound<'_, PyAny>, - ) -> PyResult> { + fn relation(&self, py: Python<'_>, location: &Bound<'_, PyAny>) -> PyResult> { Ok(match self.relations.get(&relation_key(location)?) { Some(relation) => relation.clone_ref(py), None => py.None(), @@ -622,11 +647,7 @@ impl MigrationStateInspection { } /// Listed columns for a location. - pub(crate) fn relation_columns( - &self, - py: Python<'_>, - location: &Bound<'_, PyAny>, - ) -> PyResult> { + fn relation_columns(&self, py: Python<'_>, location: &Bound<'_, PyAny>) -> PyResult> { Ok(match self.columns.get(&relation_key(location)?) { Some(columns) => columns.clone_ref(py), None => PyTuple::empty(py).into_any().unbind(), @@ -635,7 +656,7 @@ impl MigrationStateInspection { /// The latest model fingerprint that built the location or named model. #[pyo3(signature = (*, location, model_name))] - pub(crate) fn model_fingerprint<'py>( + fn model_fingerprint<'py>( &self, py: Python<'py>, location: &Bound<'py, PyAny>, @@ -656,21 +677,29 @@ impl MigrationStateInspection { /// The newest model fingerprint with this node name in any inspected schema. #[pyo3(signature = (*, model_name))] - pub(crate) fn named_fingerprint<'py>( + fn named_fingerprint<'py>( &self, py: Python<'py>, model_name: &str, ) -> PyResult>> { - let mut all: Vec> = Vec::new(); - for (_, fingerprints) in &self.fingerprints { - all.extend(fingerprints.iter().map(|item| item.clone_ref(py))); - } - newest(py, &all, (None, Some(model_name))) + let named: Vec> = self + .fingerprints_by_name + .get(model_name) + .into_iter() + .flatten() + .filter_map(|(schema, position)| { + self.fingerprints + .get(*schema) + .and_then(|(_, fingerprints)| fingerprints.get(*position)) + }) + .map(|fingerprint| fingerprint.clone_ref(py)) + .collect(); + newest(py, &named, (None, Some(model_name))) } /// Latest model fingerprints whose names are no longer in the project. #[pyo3(signature = (*, project_model_names))] - pub(crate) fn orphan_fingerprints<'py>( + fn orphan_fingerprints<'py>( &self, py: Python<'py>, project_model_names: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs index f215690bac..9d15634920 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs @@ -3,9 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::planner_phases::models::{ - PlannerIdentityContext, PlannerRuntime, PlannerWarehouseState, -}; +use crate::planner_phases::models::PlannerIdentityContext; +use crate::planner_phases::models::PlannerRuntime; +use crate::planner_phases::models::PlannerWarehouseState; use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; use sqlbuild_python_project::planner_scope::models::RetainedScope; @@ -50,8 +50,8 @@ pub(crate) fn read_planner_runtime<'py>( }; Some(match name { "project" => (&this.project).into_bound_py_any(py), - "adapter" => (&this.adapter).into_bound_py_any(py), - "connection" => (&this.connection).into_bound_py_any(py), + "dialect" => (&this.dialect).into_bound_py_any(py), + "reads" => (&this.reads).into_bound_py_any(py), "project_config" => (&this.project_config).into_bound_py_any(py), "local_config" => (&this.local_config).into_bound_py_any(py), "on_progress" => (&this.on_progress).into_bound_py_any(py), diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs index f2f9136fc3..283cea4ba4 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs @@ -5,9 +5,12 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pymethods; use pyo3::types::{PyDict, PyTuple}; -use crate::planner_phases::models::{PlannerRuntime, PlannerWarehouseState}; +use crate::planner_phases::models::PlannerRuntime; +use crate::planner_phases::models::PlannerWarehouseState; +use crate::planner_warehouse::models::PlannerReads; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::build_command::main::utc_now::utc_now; +use sqlbuild_python_base::native_host::models::Dialect; #[pymethods] impl PlannerRuntime { @@ -19,7 +22,7 @@ impl PlannerRuntime { "PlannerRuntime", kwargs, ( - &["project", "adapter", "connection"], + &["project", "dialect", "reads"], &[ "project_config", "local_config", @@ -36,8 +39,8 @@ impl PlannerRuntime { }; Ok(Self { project: args.value("project")?.unbind(), - adapter: args.value("adapter")?.unbind(), - connection: args.value("connection")?.unbind(), + dialect: Dialect::passed(args.value("dialect")?)?.unbind(), + reads: PlannerReads::passed(args.value("reads")?)?.unbind(), project_config: args.value("project_config")?.unbind(), local_config: args.value("local_config")?.unbind(), on_progress: args.value("on_progress")?.unbind(), @@ -57,6 +60,28 @@ pub(crate) fn runtime_field<'py>( .clone()) } +/// The warehouse reads of a planner runtime passed as a Python object. +pub(crate) fn runtime_reads<'py>( + runtime: &Bound<'py, PyAny>, +) -> PyResult> { + Ok(runtime + .cast::()? + .get() + .reads + .bind(runtime.py()) + .clone()) +} + +/// The SQL dialect of a planner runtime passed as a Python object. +pub(crate) fn runtime_dialect<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { + Ok(runtime + .cast::()? + .get() + .dialect + .bind(runtime.py()) + .clone()) +} + #[pymethods] impl PlannerWarehouseState { #[new] diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs index 444a59987c..4c5adbe3cd 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs @@ -5,9 +5,12 @@ use crate::planner_entries::models::RelationsRequest; use crate::planner_migrations::main::plan_column_migrations::plan_column_migrations; use crate::planner_migrations::main::plan_model_migrations::plan_model_migrations; use crate::planner_migrations::main::plan_old_name_views::plan_old_name_views as plan_old_name_migration_views; -use crate::planner_migrations::models::{ColumnMigrationInputs, MigrationPhase}; +use crate::planner_migrations::models::ColumnMigrationInputs; +use crate::planner_migrations::models::MigrationPhase; use crate::planner_phases::_helpers::inputs::contexts::class_field; use crate::planner_phases::_helpers::inputs::runtime::runtime_field; +use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; +use crate::planner_phases::main::inputs::runtime_reads::runtime_reads; use crate::planner_phases::main::rebuilds::effectively_full_refreshed_model_names::effectively_full_refreshed_model_names; use crate::planner_phases::models::{PlannerRuntime, PlannerWarehouseState}; use crate::planner_warehouse::main::inspection::warehouse_snapshot::warehouse_snapshot; @@ -80,7 +83,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( ), ], )?; - let adapter = runtime_field(runtime, |runtime| &runtime.adapter)?; + let reads = runtime_reads(runtime)?; let stale_scope = scope_data(&class_field(scopes, |s: &PlannerScopeResolution| { &s.stale_warning_scope })?)?; @@ -95,9 +98,8 @@ pub(crate) fn gather_planner_warehouse_state<'py>( .into_any(); let snapshot = warehouse_snapshot(&SnapshotRequest { project: project.clone(), - adapter: adapter.clone(), - connection: runtime_field(runtime, |runtime| &runtime.connection)?, - execute: native_field(&adapter, "execute")?, + dialect: runtime_dialect(runtime)?, + reads: reads.clone(), selected_keys: stale_keys, full_refresh: false, full_refresh_model_names: effectively_full_refreshed_model_names(&project, full_refresh)?, @@ -121,8 +123,8 @@ pub(crate) fn gather_planner_warehouse_state<'py>( fingerprints.call_method0("persist")?; let inspection_relations = planner_relations_context(RelationsRequest { project: project.clone(), - adapter: adapter.clone(), - connection: runtime_field(runtime, |runtime| &runtime.connection)?, + dialect: runtime_dialect(runtime)?, + reads: Some(reads), scope: class_field(scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?, deferral: deferral.clone(), project_config: config, diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/mod.rs index 68672218c6..fb562f6103 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/mod.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod inputs; pub(crate) mod outputs; pub(crate) mod rebuilds; +pub(crate) mod timing; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs index d8e27b2193..eb7288ec73 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs @@ -4,6 +4,8 @@ use crate::planner_phases::_helpers::inputs::contexts::class_field; use crate::planner_phases::_helpers::inputs::runtime::runtime_field; use crate::planner_phases::_helpers::inputs::runtime::warehouse_field; use crate::planner_phases::constants::MISSING_UPSTREAM_CODE; +use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; +use crate::planner_phases::main::inputs::runtime_reads::runtime_reads; use crate::planner_warehouse::main::sources::check_selected_source_tables_exist::check_selected_source_tables_exist; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyString, PyTuple}; @@ -89,10 +91,7 @@ pub(crate) fn check_selected_scope_buildability<'py>( } check_selected_source_tables_exist( &project, - ( - &runtime_field(runtime, |runtime| &runtime.adapter)?, - &runtime_field(runtime, |runtime| &runtime.connection)?, - ), + (&runtime_dialect(runtime)?, &runtime_reads(runtime)?), &selected_scope, &warehouse_field(warehouse, |warehouse| &warehouse.inspection_relations)?, ) diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/output_assembly.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/output_assembly.rs index f3c26b7b1f..cf337e003b 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/output_assembly.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/output_assembly.rs @@ -8,6 +8,10 @@ use crate::planner_phases::_helpers::inputs::contexts::class_field; use crate::planner_phases::_helpers::inputs::runtime::runtime_field; use crate::planner_phases::_helpers::inputs::runtime::warehouse_field; use crate::planner_phases::_helpers::rebuilds::retention::{plan_retention, plan_table_types}; +use crate::planner_phases::constants::AUDIT_ENTRIES; +use crate::planner_phases::constants::IGNORE_AUDIT_FAILURES; +use crate::planner_phases::constants::IGNORE_FAILURE; +use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; use crate::planner_phases::models::PlannerIdentityContext; use crate::planner_phases::models::{ PlanOutputContext, PlanOutputFields, PlanWarningContext, PlannedModelEntries, @@ -17,9 +21,11 @@ use crate::planner_warehouse::main::sources::source_freshness_unknown_warnings:: use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; +use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_project::native_host::models::ModelLookup; use sqlbuild_python_project::planner_phases::main::native::native; @@ -111,7 +117,7 @@ pub(crate) fn base_plan_output<'py>( let mut output = PlanOutputFields { fields: plan_output_fields(PlanOutputInputs { project: runtime_field(runtime, |runtime| &runtime.project)?, - adapter: runtime_field(runtime, |runtime| &runtime.adapter)?, + dialect: runtime_dialect(runtime)?, scope: execution_scope.clone(), snapshot: warehouse_field(warehouse, |warehouse| &warehouse.snapshot)?, relations: warehouse_field(warehouse, |warehouse| &warehouse.inspection_relations)?, @@ -184,7 +190,9 @@ pub(crate) fn add_storage_policies<'py>( &entries, &ModelLookup::Native(scope_data(&scope)?), &warehouse_field(warehouse, |warehouse| &warehouse.migration_fingerprints)?, - &native_field(runtime, "adapter")?.call_method0("sql_analysis_dialect")?, + &runtime_dialect(runtime)? + .syntax("sql_analysis_dialect")? + .call0()?, )?, )?; } @@ -358,3 +366,28 @@ pub(crate) fn freshness_metadata<'py>( metadata.set_item("blocked_model_names", propagated("blocked_model_names")?)?; Ok(metadata.into_any()) } + +/// Mark every planned audit's failure as one that holds nothing back, and the plan as saying so. +pub(crate) fn ignoring_audit_failures<'py>( + py: Python<'py>, + output: PlanOutputFields<'py>, +) -> PyResult> { + let ignored = PyBool::new(py, true).to_owned().into_any(); + let mut fields: Vec<(&'static str, Bound<'py, PyAny>)> = Vec::new(); + for (name, value) in output.fields { + if name != AUDIT_ENTRIES { + fields.push((name, value)); + continue; + } + let mut entries: Vec> = Vec::new(); + for entry in value.try_iter()? { + entries.push(replaced(&entry?, &[(IGNORE_FAILURE, ignored.clone())])?); + } + fields.push((name, PyTuple::new(py, entries)?.into_any())); + } + fields.push((IGNORE_AUDIT_FAILURES, ignored)); + Ok(PlanOutputFields { + fields, + metadata: output.metadata, + }) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/deferred_origins.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/deferred_origins.rs index 9abc53ca3c..023e854878 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/deferred_origins.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/deferred_origins.rs @@ -1,8 +1,11 @@ //! Deferred locations of renamed models the deferred target holds only under their old name. +use crate::planner_warehouse::models::PlannerReads; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use sqlbuild_python_base::native_host::models::Dialect; +use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -58,7 +61,7 @@ type ListedSchemas<'py> = Vec<(Option, String, Bound<'py, PyDict>)>; /// Relations of one deferred `database.schema`, listing it on first use, with the cache. fn schema_relations<'py>( - (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + reads: &Bound<'py, PlannerReads>, (database, schema): (&Option, &str), mut listed: ListedSchemas<'py>, ) -> (PyResult>, ListedSchemas<'py>) { @@ -67,18 +70,20 @@ fn schema_relations<'py>( }) { return (Ok(found.clone()), listed); } - let py: Python<'py> = adapter.py(); + let py: Python<'py> = reads.py(); let found = (|| -> PyResult> { let found: Bound<'py, PyDict> = PyDict::new(py); - for relation in call_with( - &native_field(adapter, "list_relations")?, - &[ - ("connection", connection.clone()), - ("database", database.clone().into_pyobject(py)?.into_any()), - ("schemas", PyTuple::new(py, [schema])?.into_any()), - ], - )? - .try_iter()? + for relation in reads + .get() + .reads() + .deferred_relations( + py, + ( + database.clone().into_pyobject(py)?.into_any(), + PyString::new(py, schema).into_any(), + ), + )? + .try_iter()? { let relation: Bound<'py, PyAny> = relation?; found.set_item(native_field(&relation, "name")?, relation)?; @@ -108,7 +113,11 @@ fn migration_origin_relation( /// Redirect unselected `models` unbuilt in the deferred target to their `migrate_from` relation. pub(crate) fn defer_to_migration_origins<'py>( - (project, adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + (project, dialect, reads): ( + &Bound<'py, PyAny>, + &Bound<'py, Dialect>, + &Bound<'py, PlannerReads>, + ), (locations, relations): (&Bound<'py, PyDict>, &Bound<'py, PyDict>), models: &[String], ) -> PyResult> { @@ -138,8 +147,7 @@ pub(crate) fn defer_to_migration_origins<'py>( lookups.push((origin_database, origin_schema, origin_name)); let mut origin: Option> = None; for (index, (database, schema, relation_name)) in lookups.iter().enumerate() { - let (schema_found, updated) = - schema_relations((adapter, connection), (database, schema), listed); + let (schema_found, updated) = schema_relations(reads, (database, schema), listed); listed = updated; let hit: Option> = match schema_found { Ok(schema_found) => relation_named(&schema_found, relation_name)?, @@ -182,7 +190,7 @@ pub(crate) fn defer_to_migration_origins<'py>( let schema: Bound<'py, PyAny> = native_field(&origin, "schema")?; let relation_name: Bound<'py, PyAny> = native_field(&origin, "name")?; let qualified_name: Bound<'py, PyAny> = call_with( - &native_field(adapter, "render_qualified_name")?, + &dialect.syntax("render_qualified_name")?, &[ ("database", database.clone()), ("schema", schema.clone()), diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs index 582a32a21f..03d6b014b0 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs @@ -2,11 +2,13 @@ use crate::planner_entries::main::entries::default_microbatch_limit_action::default_microbatch_limit_action; use crate::planner_entries::main::entries::plan_entries::plan_entries; -use crate::planner_entries::main::entries::planner_relations_context::planner_relations_context; -use crate::planner_entries::models::{PlanEntriesRequest, PlanEntryBuildInputs, RelationsRequest}; +use crate::planner_entries::models::PlanEntriesRequest; +use crate::planner_entries::models::PlanEntryBuildInputs; use crate::planner_phases::_helpers::inputs::runtime::runtime_field; use crate::planner_phases::_helpers::inputs::runtime::warehouse_field; -use crate::planner_phases::models::{ChangeReconciliation, ModelEntryResults}; +use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; +use crate::planner_phases::models::ChangeReconciliation; +use crate::planner_phases::models::ModelEntryResults; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyFrozenSet; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -47,30 +49,17 @@ fn planned_concurrency( .limit) } -/// Model plan entries for the pruned execution scope, with the freshness and override inputs they need. +/// Model plan entries for the execution scope over the relations its warehouse phase resolved. pub(crate) fn build_planner_entry_results<'py>( (runtime, warehouse): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - (overrides, deferral): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + overrides: &Bound<'py, PyAny>, (execution_scope, reconciliation): (&Bound<'py, PyAny>, &ChangeReconciliation<'py>), source_freshness: &Bound<'py, PyAny>, ) -> PyResult> { let py = runtime.py(); let project = runtime_field(runtime, |runtime| &runtime.project)?; let config = runtime_field(runtime, |runtime| &runtime.project_config)?; - let relations = planner_relations_context(RelationsRequest { - project: project.clone(), - adapter: runtime_field(runtime, |runtime| &runtime.adapter)?, - connection: runtime_field(runtime, |runtime| &runtime.connection)?, - scope: execution_scope.clone(), - deferral: deferral.clone(), - project_config: config.clone(), - local_config: runtime_field(runtime, |runtime| &runtime.local_config)?, - known_source_columns: native_field( - &native_field(warehouse, "inspection_relations")?, - "source_warehouse_columns", - )?, - relation_keys: py.None().into_bound(py), - })?; + let relations = native_field(warehouse, "inspection_relations")?; let propagation = native_field(source_freshness, "propagation")?; let blocked = if propagation.is_none() { PyFrozenSet::empty(py)?.into_any() @@ -125,7 +114,7 @@ pub(crate) fn build_planner_entry_results<'py>( )?; plan_entries(PlanEntriesRequest { project, - adapter: runtime_field(runtime, |runtime| &runtime.adapter)?, + dialect: runtime_dialect(runtime)?, scope: execution_scope.clone(), snapshot: warehouse_field(warehouse, |warehouse| &warehouse.snapshot)?, relations, diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs index 14cb257a33..6703a803c0 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs @@ -1,14 +1,22 @@ //! Warehouse-aware time-travel retention and Snowflake table-type planning. +use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; use crate::planner_phases::main::rebuilds::effective_target_config::effective_target_config; use sqlbuild_host::entries::models::AdapterHost; +use sqlbuild_python_base::native_host::types::Syntax; use crate::planner_phases::_helpers::inputs::runtime::runtime_field; -use crate::planner_phases::constants::{ - BIGQUERY_ADAPTER, DECREASE_WARNING, DOWNGRADE_HISTORY_DAYS, PERMANENT_TABLE_TYPE, - RETENTION_DECREASE, RETENTION_INCREASE, RETENTION_MATCH, SNOWFLAKE_ADAPTER, - TRANSIENT_TABLE_TYPE, -}; +use crate::planner_phases::constants::BIGQUERY_ADAPTER; +use crate::planner_phases::constants::DECREASE_WARNING; +use crate::planner_phases::constants::DOWNGRADE_HISTORY_DAYS; +use crate::planner_phases::constants::PERMANENT_TABLE_TYPE; +use crate::planner_phases::constants::RETENTION_DECREASE; +use crate::planner_phases::constants::RETENTION_INCREASE; +use crate::planner_phases::constants::RETENTION_MATCH; +use crate::planner_phases::constants::SNOWFLAKE_ADAPTER; +use crate::planner_phases::constants::TRANSIENT_TABLE_TYPE; +use crate::planner_phases::main::inputs::runtime_reads::runtime_reads; +use crate::planner_warehouse::models::Read; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; @@ -21,9 +29,9 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::typed_values::typed_values; +use sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination; use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; @@ -70,7 +78,7 @@ fn selected_models<'py>( } fn adapter_is(runtime: &Bound<'_, PyAny>, name: &str) -> PyResult { - native_field(&native_field(runtime, "adapter")?, "adapter_name")?.eq(name) + runtime_dialect(runtime)?.syntax("adapter_name")?.eq(name) } /// Entries for selected Snowflake table-type drift. @@ -113,11 +121,10 @@ pub(crate) fn plan_table_types<'py>( if !snowflake { return tuple(py, Vec::new()); } - let adapter = runtime_field(runtime, |runtime| &runtime.adapter)?; - if let Ok(facade) = adapter.downcast::() { - facade.get().mark_unknown("table types".to_owned()); + if let Read::Unknown(_) = runtime_reads(runtime)?.get().reads().table_types() { return tuple(py, Vec::new()); } + let dialect = runtime_dialect(runtime)?; let target = effective_target_config(runtime)?; let relations = native_field(&native_field(warehouse, "snapshot")?, "existing_relations")?; let materialization = record_class(py, RecordId::MaterializationType)?; @@ -150,9 +157,10 @@ pub(crate) fn plan_table_types<'py>( continue; } let downgrade = actual == Some(false) && desired_transient; - let destination = sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?; - let limit: i64 = adapter - .call_method0("maximum_identifier_length")? + let destination = destination(&model)?; + let limit: i64 = dialect + .syntax("maximum_identifier_length")? + .call0()? .extract()?; let target_name: String = native_field(&destination, "name")?.extract()?; entries.push(planner_model( @@ -269,31 +277,16 @@ fn render_changes<'py>( fields.push(("state", state.clone())); } call_with( - &native_field( - &native_field(runtime, "adapter")?, - "render_retention_changes", - )?, + &runtime_dialect(runtime)?.syntax("render_retention_changes")?, &fields, ) } fn inspect<'py>( - runtime: &Bound<'py, PyAny>, + inspect_retention: &Bound<'py, PyAny>, request: &Bound<'py, PyAny>, ) -> PyResult> { - call_with( - &native_field( - &runtime_field(runtime, |runtime| &runtime.adapter)?, - "inspect_retention", - )?, - &[ - ( - "connection", - runtime_field(runtime, |runtime| &runtime.connection)?, - ), - ("request", request.clone()), - ], - ) + call_with(inspect_retention, &[("request", request.clone())]) } fn state_direction(state: &Bound<'_, PyAny>, desired_days: i64) -> PyResult<&'static str> { @@ -319,7 +312,7 @@ fn transient_error(py: Python<'_>, name: &str, desired_days: i64, convert: &str) } fn relation_retention<'py>( - runtime: &Bound<'py, PyAny>, + (runtime, inspect_retention): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), warehouse: &Bound<'py, PyAny>, model: &Bound<'py, PyAny>, ) -> PyResult>> { @@ -397,17 +390,14 @@ fn relation_retention<'py>( let mut listed = py.None().into_bound(py); if native_field(&relation, "identity")?.eq(&lookup)? { listed = call_with( - &native_field( - &native_field(runtime, "adapter")?, - "retention_state_from_relation", - )?, + &runtime_dialect(runtime)?.syntax("retention_state_from_relation")?, &[("request", request.clone()), ("relation", relation.clone())], )?; } let state = if listed.is_truthy()? { listed } else { - inspect(runtime, &request)? + inspect(inspect_retention, &request)? }; if native_field(&state, "is_transient")?.is_truthy()? && desired_days > 1 @@ -456,7 +446,7 @@ fn relation_retention<'py>( } fn bigquery_retention<'py>( - runtime: &Bound<'py, PyAny>, + (runtime, inspect_retention): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), models: &[Bound<'py, PyAny>], ) -> PyResult>> { let py = runtime.py(); @@ -555,7 +545,7 @@ fn bigquery_retention<'py>( ("desired_days", desired_days.into_pyobject(py)?.into_any()), ], )?; - let state = inspect(runtime, &request)?; + let state = inspect(inspect_retention, &request)?; let base = RetentionEntry { request: request.clone(), model_names: sorted_names(py, names)?, @@ -612,17 +602,23 @@ pub(crate) fn plan_retention<'py>( models.push(model); } } - if models.is_empty() - || runtime_field(runtime, |runtime| &runtime.adapter)?.is_instance_of::() - { + if models.is_empty() { return tuple(py, Vec::new()); } + let Read::Known(inspect_retention) = runtime_reads(runtime)?.get().reads().retention(py)? + else { + return tuple(py, Vec::new()); + }; let entries = if adapter_is(runtime, BIGQUERY_ADAPTER)? { - bigquery_retention(runtime, &models)? + bigquery_retention((runtime, &inspect_retention), &models)? } else { let mut entries: Vec<_> = Vec::new(); for model in &models { - entries.extend(relation_retention(runtime, warehouse, model)?); + entries.extend(relation_retention( + (runtime, &inspect_retention), + warehouse, + model, + )?); } entries }; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/timing/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/timing/mod.rs new file mode 100644 index 0000000000..9a4ac362a9 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/timing/mod.rs @@ -0,0 +1 @@ +pub(crate) mod timings; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/timing/timings.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/timing/timings.rs new file mode 100644 index 0000000000..eaf51c55ca --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/timing/timings.rs @@ -0,0 +1,34 @@ +//! The planning steps recorded on this thread for one plan command. + +use crate::planner_phases::constants::OTHER; +use crate::planner_phases::models::PlanTimings; +use std::cell::RefCell; + +thread_local! { + pub(crate) static STEPS: RefCell>> = const { RefCell::new(None) }; +} + +/// `recorded` steps within `total_seconds`, with `Other` added as what they leave uncovered. +pub(crate) fn timings_of( + mut recorded: Vec<(&'static str, f64)>, + total_seconds: f64, +) -> PlanTimings { + let covered: f64 = recorded.iter().map(|(_, seconds)| seconds).sum(); + recorded.push((OTHER, (total_seconds - covered).max(0.0))); + PlanTimings { + total_seconds, + steps: recorded, + } +} + +/// Add `seconds` to `step` for the plan command collecting on this thread, if one is. +pub(crate) fn record(step: &'static str, seconds: f64) { + STEPS.with(|steps| { + if let Some(steps) = steps.borrow_mut().as_mut() { + match steps.iter_mut().find(|(name, _)| *name == step) { + Some((_, total)) => *total += seconds, + None => steps.push((step, seconds)), + } + } + }); +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/constants.rs b/crates/sqlbuild-python-planner/src/planner_phases/constants.rs index a482e9ba20..0a6b6af84f 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/constants.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/constants.rs @@ -62,3 +62,43 @@ pub const GETTERS: &[GetterEntry] = &[ read: read_planner_warehouse_state, }, ]; +/// A step's share of the plan's total time is shown as a percentage. +pub const PERCENT: f64 = 100.0; +/// Plan document timings are rounded to the millisecond. +pub const MILLISECONDS: f64 = 1000.0; +/// The planner policy, and the plan field, saying audit failures hold nothing back. +pub const IGNORE_AUDIT_FAILURES: &str = "ignore_audit_failures"; +/// The plan field holding the planned audits. +pub const AUDIT_ENTRIES: &str = "audit_entries"; +/// An audit plan entry's field saying its failure holds nothing back. +pub const IGNORE_FAILURE: &str = "ignore_failure"; + +/// Compiling the project before planning; a location planning its published project skips it. +pub const COMPILING: &str = "Compiling the project"; + +/// The warehouse snapshot, model migrations and source relations, live or from a synced state. +pub const READING_WAREHOUSE: &str = "Reading warehouse state"; + +/// Model identities, buildability and the changes each selected node has. +pub const DETECTING_CHANGES: &str = "Detecting changes"; + +/// Observing selected sources' freshness and comparing it to recorded state. +pub const SOURCE_FRESHNESS: &str = "Checking source freshness"; + +/// Reconciling changes into actions and building each node's plan entry. +pub const BUILDING_ENTRIES: &str = "Building plan entries"; + +/// Waiting for SQL test planning, which runs alongside the other steps. +pub const TEST_FIXTURES: &str = "Preparing test fixtures"; + +/// Storage policies, warnings and metadata around the plan entries. +pub const ASSEMBLING: &str = "Assembling the plan"; + +/// `--changed`: reading recorded consumption to decide what runs. +pub const BUILD_HISTORY: &str = "Reading build history"; + +/// Rendering the plan as text or JSON. +pub const WRITING: &str = "Writing the plan"; + +/// Time inside the plan that no step covers. +pub const OTHER: &str = "Other"; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/mod.rs index b02e5cc9c9..67e2b2f8e0 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/mod.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/mod.rs @@ -1,4 +1,6 @@ pub mod class_field; pub mod gather_planner_warehouse_state; +pub mod runtime_dialect; pub mod runtime_field; +pub mod runtime_reads; pub mod warehouse_field; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/runtime_dialect.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/runtime_dialect.rs new file mode 100644 index 0000000000..a6b2b0de5f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/runtime_dialect.rs @@ -0,0 +1,11 @@ +//! The SQL dialect of a planner runtime passed as a Python object. + +use crate::planner_phases::_helpers::inputs::runtime; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::native_host::models::Dialect; + +/// The SQL dialect of a planner runtime passed as a Python object. +#[inline] +pub fn runtime_dialect<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { + runtime::runtime_dialect(runtime) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/runtime_reads.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/runtime_reads.rs new file mode 100644 index 0000000000..4bea35b140 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/runtime_reads.rs @@ -0,0 +1,11 @@ +//! The warehouse reads of a planner runtime passed as a Python object. + +use crate::planner_phases::_helpers::inputs::runtime; +use crate::planner_warehouse::models::PlannerReads; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// The warehouse reads of a planner runtime passed as a Python object. +#[inline] +pub fn runtime_reads<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { + runtime::runtime_reads(runtime) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/mod.rs index c73411eada..5dec40a364 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/mod.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/mod.rs @@ -1,3 +1,4 @@ pub mod inputs; pub mod outputs; pub mod rebuilds; +pub mod timing; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_metadata.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_metadata.rs index c6605b8674..b6da309952 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_metadata.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_metadata.rs @@ -1,7 +1,10 @@ //! Add the direct-mode source freshness, remaining stale models and selection diagnostics metadata. use crate::planner_phases::_helpers::outputs::output_assembly::freshness_metadata; +use crate::planner_phases::_helpers::outputs::output_assembly::ignoring_audit_failures; +use crate::planner_phases::constants::IGNORE_AUDIT_FAILURES; use crate::planner_phases::models::PlanOutputFields; +use pyo3::prelude::PyAnyMethods; use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -29,5 +32,8 @@ pub fn add_plan_metadata<'py>( output .metadata .push(("selection_diagnostics", diagnostics.into_any())); + if native_field(policies, IGNORE_AUDIT_FAILURES)?.is_truthy()? { + return ignoring_audit_failures(py, output); + } Ok(output) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/build_planner_entry_results.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/build_planner_entry_results.rs index f13fa09f2f..488c021d5a 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/build_planner_entry_results.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/build_planner_entry_results.rs @@ -1,20 +1,21 @@ //! Model plan entries for the pruned execution scope, with the freshness and override inputs they need. use crate::planner_phases::_helpers::rebuilds::execution_changes; -use crate::planner_phases::models::{ChangeReconciliation, ModelEntryResults}; +use crate::planner_phases::models::ChangeReconciliation; +use crate::planner_phases::models::ModelEntryResults; use pyo3::prelude::{Bound, PyAny, PyResult}; /// Model plan entries for the pruned execution scope, with the freshness and override inputs they need. #[inline] pub fn build_planner_entry_results<'py>( (runtime, warehouse): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - (overrides, deferral): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + overrides: &Bound<'py, PyAny>, (execution_scope, reconciliation): (&Bound<'py, PyAny>, &ChangeReconciliation<'py>), source_freshness: &Bound<'py, PyAny>, ) -> PyResult> { execution_changes::build_planner_entry_results( (runtime, warehouse), - (overrides, deferral), + overrides, (execution_scope, reconciliation), source_freshness, ) diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/defer_to_migration_origins.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/defer_to_migration_origins.rs index b119e3161f..89f6d43292 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/defer_to_migration_origins.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/defer_to_migration_origins.rs @@ -1,18 +1,24 @@ //! Redirect unselected `models` unbuilt in the deferred target to their `migrate_from` relation. use crate::planner_phases::_helpers::rebuilds::deferred_origins; +use crate::planner_warehouse::models::PlannerReads; use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyDict; +use sqlbuild_python_base::native_host::models::Dialect; /// Redirect unselected `models` unbuilt in the deferred target to their `migrate_from` relation. #[inline] pub fn defer_to_migration_origins<'py>( - (project, adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + (project, dialect, reads): ( + &Bound<'py, PyAny>, + &Bound<'py, Dialect>, + &Bound<'py, PlannerReads>, + ), (locations, relations): (&Bound<'py, PyDict>, &Bound<'py, PyDict>), models: &[String], ) -> PyResult> { deferred_origins::defer_to_migration_origins( - (project, adapter, connection), + (project, dialect, reads), (locations, relations), models, ) diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/timing/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/timing/mod.rs new file mode 100644 index 0000000000..e041f6f7b6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/timing/mod.rs @@ -0,0 +1,2 @@ +pub mod record_step; +pub mod timed; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/timing/record_step.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/timing/record_step.rs new file mode 100644 index 0000000000..212255f31d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/timing/record_step.rs @@ -0,0 +1,9 @@ +//! Record a planning step timed by its caller. + +use crate::planner_phases::_helpers::timing::timings; + +/// Add `seconds` to `step` for the plan command collecting on this thread, if one is. +#[inline] +pub fn record_step(step: &'static str, seconds: f64) { + timings::record(step, seconds); +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/timing/timed.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/timing/timed.rs new file mode 100644 index 0000000000..f3c9316223 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/timing/timed.rs @@ -0,0 +1,12 @@ +//! Time one planning step. + +use crate::planner_phases::_helpers::timing::timings::record; +use std::time::Instant; + +/// Run `body`, recording its wall time as `step`. +pub fn timed(step: &'static str, body: impl FnOnce() -> T) -> T { + let started: Instant = Instant::now(); + let result: T = body(); + record(step, started.elapsed().as_secs_f64()); + result +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/models.rs b/crates/sqlbuild-python-planner/src/planner_phases/models.rs index 14219134aa..ebe7df27dc 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/models.rs @@ -1,8 +1,16 @@ //! Public records of planner phases. +use crate::planner_phases::_helpers::timing::timings::STEPS; +use crate::planner_phases::_helpers::timing::timings::timings_of; +use crate::planner_phases::constants::MILLISECONDS; +use crate::planner_phases::constants::PERCENT; +use crate::planner_warehouse::models::PlannerReads; use pyo3::prelude::{Bound, Py, PyAny}; use pyo3::pyclass; +use serde_json::{Value, json}; +use sqlbuild_python_base::native_host::models::Dialect; use std::collections::BTreeMap; +use std::time::Instant; /// The plan output's fields and metadata entries, each set before the output is built. #[derive(Debug)] @@ -65,15 +73,15 @@ pub struct PlanWarningContext<'a, 'py> { #[pyclass(module = "sqlbuild._native", frozen, get_all)] #[derive(Debug)] pub struct PlannerRuntime { - pub project: Py, - pub adapter: Py, - pub connection: Py, - pub project_config: Py, - pub local_config: Py, - pub on_progress: Py, - pub invocation_time: Py, + pub(crate) project: Py, + pub(crate) dialect: Py, + pub(crate) reads: Py, + pub(crate) project_config: Py, + pub(crate) local_config: Py, + pub(crate) on_progress: Py, + pub(crate) invocation_time: Py, /// Python nodes' `(database, schema)`, where their results and fingerprints live; `None` when unknown. - pub node_results_location: Option<(Option, Option)>, + pub(crate) node_results_location: Option<(Option, Option)>, } /// Warehouse snapshot, inspection relations and migration plans gathered once per plan. @@ -106,3 +114,87 @@ pub(crate) struct IdentityWork { pub identity_builds: u64, pub stale_detections: BTreeMap, } + +/// While held, this thread's planning steps are recorded for one plan command. +#[derive(Debug)] +pub struct PlanTimingScope { + started: Instant, + previous: Option>, +} + +/// A plan's steps in pipeline order with `Other` last, and its total, in seconds. +#[derive(Clone, Debug, PartialEq)] +pub struct PlanTimings { + pub(crate) total_seconds: f64, + pub(crate) steps: Vec<(&'static str, f64)>, +} + +impl PlanTimingScope { + pub fn enter() -> Self { + Self { + started: Instant::now(), + previous: STEPS.replace(Some(Vec::new())), + } + } + + /// The steps recorded so far and the time since the scope opened; `Other` is the remainder. + pub fn timings(&self) -> PlanTimings { + let total_seconds: f64 = self.started.elapsed().as_secs_f64(); + let recorded: Vec<(&'static str, f64)> = + STEPS.with(|steps| steps.borrow().clone().unwrap_or_default()); + timings_of(recorded, total_seconds) + } +} + +impl Drop for PlanTimingScope { + fn drop(&mut self) { + STEPS.set(self.previous.take()); + } +} + +impl PlanTimings { + /// The `--timings` table: each step's seconds and share of the total, then the total. + pub fn table(&self) -> String { + const HEADER: &str = "Plan timings"; + let name_width: usize = self + .steps + .iter() + .map(|(name, _)| name.len() + 2) + .max() + .unwrap_or(0) + .max(HEADER.len()); + let mut lines: Vec = vec![format!( + "{HEADER:7} {:>5}", + "seconds", "share" + )]; + let row_width: usize = name_width - 2; + for (name, seconds) in &self.steps { + let share: f64 = Some(seconds / self.total_seconds * PERCENT) + .filter(|share| share.is_finite()) + .unwrap_or_default(); + lines.push(format!( + " {name:7.1} {:>4.0}%", + share + )); + } + lines.push(format!( + " {:7.1}", + "Total", self.total_seconds + )); + lines.join("\n") + } + + /// The plan document's `timings`: total and steps in seconds, and what the plan read. + pub fn json(&self, planned_from: &str) -> Value { + let rounded = |seconds: f64| (seconds * MILLISECONDS).round() / MILLISECONDS; + json!({ + "total_seconds": rounded(self.total_seconds), + "steps": self + .steps + .iter() + .map(|(name, seconds)| json!({"name": name, "seconds": rounded(*seconds)})) + .collect::>(), + "planned_from": planned_from, + }) + } +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_model.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_model.rs index f518f20798..a5ba16028f 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_model.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_model.rs @@ -6,9 +6,8 @@ use crate::plan_output::main::resolve_bounded_cursor_override::resolve_bounded_c use crate::planner_entries::models::ModelPlanContext; use crate::planner_phases::main::inputs::class_field::class_field; use crate::planner_resolution::_helpers::cursor::cursor_bounds::compute_cursor_bounds; -use crate::planner_resolution::_helpers::cursor::cursor_policies::{ - future_cursor_config, start_cursor_config, -}; +use crate::planner_resolution::_helpers::cursor::cursor_policies::future_cursor_config; +use crate::planner_resolution::_helpers::cursor::cursor_policies::start_cursor_config; use crate::planner_resolution::_helpers::cursor::cursor_windows::cursor_inputs_without_rows; use crate::planner_resolution::_helpers::cursor::cursor_windows::empty_cursor_window; use crate::planner_resolution::_helpers::cursor::cursor_windows::empty_input_rebuild_refusal; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_overrides.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_overrides.rs index a6b2ad573a..e7636cd665 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_overrides.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_overrides.rs @@ -1,7 +1,13 @@ //! Selected-model inspection and untyped override conversion. -use crate::planner_resolution::_helpers::references::values::{method, string}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use crate::planner_resolution::_helpers::references::values::method; +use crate::planner_resolution::_helpers::references::values::string; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyErr; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; use pyo3::pyfunction; use pyo3::types::PyDict; use sqlbuild_cursor::planner::main::override_kind::override_kind; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs index 31a7c01bb8..3664304ecf 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs @@ -11,13 +11,18 @@ use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_e use crate::plan_output::main::resolve_bounded_cursor_override::resolve_bounded_cursor_override; use crate::planner_resolution::_helpers::cursor::cursor_bounds::compute_cursor_bounds; use crate::planner_resolution::_helpers::cursor::cursor_bounds::duration; -use crate::planner_resolution::_helpers::cursor::cursor_safety::{replace, utc_clock}; +use crate::planner_resolution::_helpers::cursor::cursor_safety::replace; +use crate::planner_resolution::_helpers::cursor::cursor_safety::utc_clock; use crate::planner_resolution::_helpers::cursor::cursor_windows::normalize_cursor_snapshot_grain; use crate::planner_resolution::_helpers::cursor::cursor_windows::without_destination_cursor; use crate::planner_resolution::_helpers::references::values::string; use crate::planner_resolution::_helpers::safety::future_safety::resolve_future_cursor_safety; use crate::planner_strategy::main::model_entries::resolve_microbatch_lookback::resolve_microbatch_lookback; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; use pyo3::types::{PyDict, PyString, PyTuple}; use pyo3::{IntoPyObject, PyErr, pyfunction}; use sqlbuild_cursor::algebra::main::compare::compare; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs index 5fab639f0e..30fcbb6af1 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs @@ -13,10 +13,11 @@ use pyo3::{PyErr, pyfunction}; use sqlbuild_model_config::config_values::models::ConfigNode; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::models::Dialect; +use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; -use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; use sqlbuild_python_front::attachments::main::resolve_cursor_intrinsics::resolve_cursor_intrinsics; use sqlbuild_python_front::cursor::main::scalars::sentinel_to_token::sentinel_to_token; use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; @@ -41,7 +42,7 @@ pub(crate) fn resolve_model_sql_of<'py>( kwargs, ( &[ - "adapter", + "dialect", "model", "snapshot", "context", @@ -58,7 +59,7 @@ pub(crate) fn resolve_model_sql_of<'py>( )?; let context = args.value("context")?; let resolution = Resolution::of_context( - args.value("adapter")?, + Dialect::passed(args.value("dialect")?)?, &context, args.value("external_sql_reference_resolver")?, )?; @@ -114,14 +115,14 @@ pub(crate) fn resolve_model_sql<'py>( let bounds = match bounds { bounds if !bounds.is_none() => bounds, _ if cause.is_none() - && resolution.adapter.is_instance_of::() + && native_field(&request.snapshot, "cursor_inputs_at_run")?.is_truthy()? && native_field(&roles, "watermark_inputs")?.is_truthy()? => { unbounded(py)? } _ => return Err(unresolved_bounds_error(py, &name, cause.as_deref())?), }; - sql = intrinsic_bounds(&sql, &bounds, &cursor_type, &resolution.adapter)?; + sql = intrinsic_bounds(&sql, &bounds, &cursor_type, &resolution.dialect)?; } resolution.assert_resolved(&sql, &format!("Model '{name}'"))?; Ok(sql) @@ -144,12 +145,12 @@ fn intrinsic_bounds<'py>( sql: &str, bounds: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, + dialect: &Bound<'py, Dialect>, ) -> PyResult { - let py = adapter.py(); + let py = dialect.py(); let literal = |field: &str| -> PyResult { call_with( - &native_field(adapter, "render_cursor_bound_literal")?, + &dialect.syntax("render_cursor_bound_literal")?, &[ ( "value", @@ -182,13 +183,13 @@ fn unresolved_bounds_error(py: Python<'_>, name: &str, cause: Option<&str>) -> P } #[pyfunction] -#[pyo3(signature = (*, model, adapter, context, external_sql_reference_resolver))] +#[pyo3(signature = (*, model, dialect, context, external_sql_reference_resolver))] pub(crate) fn resolve_model_hook_entries<'py>( model: &Bound<'py, PyAny>, - adapter: Bound<'py, PyAny>, + dialect: Bound<'py, Dialect>, context: Bound<'py, PyAny>, external_sql_reference_resolver: Bound<'py, PyAny>, ) -> PyResult> { - let resolution = Resolution::of_context(adapter, &context, external_sql_reference_resolver)?; + let resolution = Resolution::of_context(dialect, &context, external_sql_reference_resolver)?; model_hook_entries(model, &resolution) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs index 23aff09270..8db885ebe1 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs @@ -1,10 +1,13 @@ //! Reference lookup and adapter callbacks for planner SQL. use sqlbuild_host::entries::models::AdapterHost; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::models::Dialect; -use crate::planner_resolution::_helpers::references::values::{ - cursor_wrap, filtering_of, method, string, -}; +use crate::planner_resolution::_helpers::references::values::cursor_wrap; +use crate::planner_resolution::_helpers::references::values::filtering_of; +use crate::planner_resolution::_helpers::references::values::method; +use crate::planner_resolution::_helpers::references::values::string; use crate::planner_resolution::models::CursorFiltering; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -12,7 +15,6 @@ use pyo3::pyfunction; use pyo3::types::PyDict; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_front::sqltext::models::SqlCodeScanner; @@ -35,7 +37,7 @@ pub(crate) fn resolve_ref_references( "seed_locations", "cursor_bounds", "cursor_filter_inputs", - "adapter", + "dialect", "cursor_type", "lower_bound_inclusive", ], @@ -48,7 +50,7 @@ pub(crate) fn resolve_ref_references( &args.value("model_locations")?, &args.value("seed_locations")?, ), - &args.value("adapter")?, + &Dialect::passed(args.value("dialect")?)?, &filtering_of(&args)?, ) } @@ -57,10 +59,10 @@ pub(crate) fn resolve_ref_references( pub(crate) fn resolve_refs( original: &str, (model_locations, seed_locations): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - adapter: &Bound<'_, PyAny>, + dialect: &Bound<'_, Dialect>, filtering: &CursorFiltering<'_>, ) -> PyResult { - let py = adapter.py(); + let py = dialect.py(); let mut sql = original.to_owned(); for (kind, locations) in [("ref", model_locations), ("seed", seed_locations)] { let mut output = String::new(); @@ -73,7 +75,10 @@ pub(crate) fn resolve_refs( } else { let relation: String = call_with( &host_entry(py, AdapterHost::ResolveRelationLocationQualifiedName)?, - &[("adapter", adapter.clone()), ("location", target)], + &[ + ("adapter", dialect.clone().into_any()), + ("location", target), + ], )? .extract()?; let char_end = sql[..span.end].chars().count(); @@ -82,7 +87,7 @@ pub(crate) fn resolve_refs( .nth(char_end) .map_or(original.len(), |(offset, _)| offset); output.push_str(&cursor_wrap( - adapter, + dialect, filtering, relation, (&span.name, !ref_alias(original, original_end)), @@ -134,10 +139,10 @@ pub(crate) fn resolve_dbt_ref_references( fn resolve_function_calls( query_sql: &str, locations: &Bound<'_, PyAny>, - adapter: &Bound<'_, PyAny>, + dialect: &Bound<'_, Dialect>, table: bool, ) -> PyResult { - let py = adapter.py(); + let py = dialect.py(); let kind = if table { "table_fn" } else { "udf" }; let spans = reference_spans(query_sql, kind); if spans.is_empty() { @@ -190,7 +195,7 @@ fn resolve_function_calls( "render_udf_call" }; let rendered: String = method( - adapter, + dialect, hook, &[ ("target", qualified), @@ -213,21 +218,21 @@ fn resolve_function_calls( } #[pyfunction] -#[pyo3(signature = (*, query_sql, function_locations, adapter))] +#[pyo3(signature = (*, query_sql, function_locations, dialect))] pub(crate) fn resolve_udf_references( query_sql: &str, function_locations: &Bound<'_, PyAny>, - adapter: &Bound<'_, PyAny>, + dialect: &Bound<'_, Dialect>, ) -> PyResult { - resolve_function_calls(query_sql, function_locations, adapter, false) + resolve_function_calls(query_sql, function_locations, dialect, false) } #[pyfunction] -#[pyo3(signature = (*, query_sql, function_locations, adapter))] +#[pyo3(signature = (*, query_sql, function_locations, dialect))] pub(crate) fn resolve_table_function_references( query_sql: &str, function_locations: &Bound<'_, PyAny>, - adapter: &Bound<'_, PyAny>, + dialect: &Bound<'_, Dialect>, ) -> PyResult { - resolve_function_calls(query_sql, function_locations, adapter, true) + resolve_function_calls(query_sql, function_locations, dialect, true) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/sources.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/sources.rs index a650168767..68ecae8b1c 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/sources.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/sources.rs @@ -1,8 +1,9 @@ //! Source contract validation and adapter-controlled projection rendering. -use crate::planner_resolution::_helpers::references::values::{ - cursor_wrap, filtering_of, method, string, -}; +use crate::planner_resolution::_helpers::references::values::cursor_wrap; +use crate::planner_resolution::_helpers::references::values::filtering_of; +use crate::planner_resolution::_helpers::references::values::method; +use crate::planner_resolution::_helpers::references::values::string; use crate::planner_resolution::models::CursorFiltering; use crate::planner_strategy::main::values::input_error::input_error; use pyo3::IntoPyObject; @@ -10,6 +11,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyTuple}; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::models::Dialect; +use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; @@ -40,7 +43,7 @@ fn rows(value: &Bound<'_, PyAny>) -> PyResult)>> { fn check_columns( entry: &Bound<'_, PyAny>, columns: &Bound<'_, PyAny>, - adapter: &Bound<'_, PyAny>, + dialect: &Bound<'_, Dialect>, relation: &str, ) -> PyResult<()> { let py = entry.py(); @@ -82,11 +85,8 @@ fn check_columns( if actual.is_empty() { continue; } - let equal: bool = types_equal( - &native_field(adapter, "sql_analysis_dialect_name")?, - actual, - &kind, - )?; + let equal: bool = + types_equal(&dialect.syntax("sql_analysis_dialect_name")?, actual, &kind)?; if !equal { return Err(source_error( py, @@ -101,24 +101,24 @@ fn check_columns( } fn cast_projection<'py>( - adapter: &Bound<'py, PyAny>, + dialect: &Bound<'py, Dialect>, name: &str, declared: &str, kind: &str, ) -> PyResult> { - let py = adapter.py(); + let py = dialect.py(); method( - adapter, + dialect, "render_source_expression_cast", &[ ( "expression", - adapter.call_method1("render_identifier", (name,))?, + dialect.syntax("render_identifier")?.call1((name,))?, ), ("target_type", string(py, kind)), ( "alias", - adapter.call_method1("render_identifier", (declared,))?, + dialect.syntax("render_identifier")?.call1((declared,))?, ), ], ) @@ -127,7 +127,7 @@ fn cast_projection<'py>( fn cast_source( entry: &Bound<'_, PyAny>, columns: &Bound<'_, PyAny>, - adapter: &Bound<'_, PyAny>, + dialect: &Bound<'_, Dialect>, relation: String, ) -> PyResult { let declared = rows(&native_field(entry, "columns")?)?; @@ -182,13 +182,15 @@ fn cast_source( let mut projections: Vec> = Vec::new(); for projection in &plan.projections { projections.push(if let Some((declared, kind)) = &projection.cast { - cast_projection(adapter, &projection.name, declared, kind)? + cast_projection(dialect, &projection.name, declared, kind)? } else { - adapter.call_method1("render_identifier", (&projection.name,))? + dialect + .syntax("render_identifier")? + .call1((&projection.name,))? }); } return method( - adapter, + dialect, "render_source_expression_cast_subquery", &[ ("source_relation", string(py, &relation)), @@ -203,11 +205,11 @@ fn cast_source( let mut projections: Vec> = Vec::new(); for projection in &plan.projections { if let Some((name, kind)) = &projection.cast { - projections.push(cast_projection(adapter, &projection.name, name, kind)?); + projections.push(cast_projection(dialect, &projection.name, name, kind)?); } } method( - adapter, + dialect, "_render_source_relation_cast_subquery_with_columns", &[ ("source_relation", string(py, &relation)), @@ -240,17 +242,17 @@ fn cast_source( } #[pyfunction] -#[pyo3(signature = (*, adapter, source_entry))] +#[pyo3(signature = (*, dialect, source_entry))] pub(crate) fn render_source_read_relation( - adapter: &Bound<'_, PyAny>, + dialect: &Bound<'_, Dialect>, source_entry: &Bound<'_, PyAny>, ) -> PyResult { - render_source_read(source_entry, adapter) + render_source_read(source_entry, dialect) } fn resolve_one( (source_map, warehouse_columns): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - adapter: &Bound<'_, PyAny>, + dialect: &Bound<'_, Dialect>, filtering: &CursorFiltering<'_>, (name, original): (&str, &str), ) -> PyResult { @@ -258,8 +260,8 @@ fn resolve_one( if entry.is_none() { return Ok(original.to_owned()); } - let adapter = adapter.clone(); - let mut relation = render_source_read_relation(&adapter, &entry)?; + let dialect = dialect.clone(); + let mut relation = render_source_read_relation(&dialect, &entry)?; let columns = mapping_get(warehouse_columns, name)?; let expression = !native_field(&entry, "expression")?.is_none(); if columns.is_truthy()? @@ -274,12 +276,12 @@ fn resolve_one( } else { relation.clone() }; - check_columns(&entry, &columns, &adapter, &label)?; + check_columns(&entry, &columns, &dialect, &label)?; } if native_field(&entry, "type_enforcement")?.is_truthy()? { - relation = cast_source(&entry, &columns, &adapter, relation)?; + relation = cast_source(&entry, &columns, &dialect, relation)?; } - cursor_wrap(&adapter, filtering, relation, (name, false)) + cursor_wrap(&dialect, filtering, relation, (name, false)) } #[pyfunction] @@ -300,7 +302,7 @@ pub(crate) fn resolve_source_references( "star_exclude_keyword", "cursor_bounds", "cursor_filter_inputs", - "adapter", + "dialect", "cursor_type", "lower_bound_inclusive", ], @@ -313,7 +315,7 @@ pub(crate) fn resolve_source_references( &args.value("source_map")?, &args.value("source_warehouse_columns")?, ), - &args.value("adapter")?, + &Dialect::passed(args.value("dialect")?)?, &filtering_of(&args)?, ) } @@ -322,18 +324,19 @@ pub(crate) fn resolve_source_references( pub(crate) fn resolve_sources( sql: &str, lookups: (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - adapter: &Bound<'_, PyAny>, + dialect: &Bound<'_, Dialect>, filtering: &CursorFiltering<'_>, ) -> PyResult { - let aliases = adapter - .call_method0("requires_derived_table_aliases")? + let aliases = dialect + .syntax("requires_derived_table_aliases")? + .call0()? .is_truthy()?; let mut output = String::new(); let mut last = 0; for span in reference_spans(sql, "source") { let mut relation = resolve_one( lookups, - adapter, + dialect, filtering, (&span.name, &sql[span.start..span.end]), )?; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/values.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/values.rs index 689cb59b7a..ec6709a011 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/values.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/values.rs @@ -5,6 +5,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyString; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::models::Dialect; +use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_front::cursor::main::scalars::sentinel_to_token::sentinel_to_token; @@ -24,7 +26,7 @@ pub(crate) fn method<'py>( } pub(crate) fn cursor_wrap( - adapter: &Bound<'_, PyAny>, + dialect: &Bound<'_, Dialect>, filtering: &CursorFiltering<'_>, relation: String, (name, inject_alias): (&str, bool), @@ -37,12 +39,12 @@ pub(crate) fn cursor_wrap( let Some(column) = column else { return Ok(relation); }; - let py = adapter.py(); + let py = dialect.py(); let cursor_type = &filtering.cursor_type; let literal = |bound: &str| -> PyResult { let token = sentinel_to_token(&native_field(bounds, bound)?)?; method( - adapter, + dialect, "render_cursor_bound_literal", &[ ("value", string(py, &token)), @@ -53,10 +55,11 @@ pub(crate) fn cursor_wrap( }; let start = literal("start")?; let end = literal("end")?; - let operand = cursor_operand(adapter, &column, cursor_type)?; + let operand = cursor_operand(dialect, &column, cursor_type)?; let alias = inject_alias - && adapter - .call_method0("requires_derived_table_aliases")? + && dialect + .syntax("requires_derived_table_aliases")? + .call0()? .is_truthy()?; Ok(cursor_subquery( &relation, @@ -73,16 +76,16 @@ pub(crate) fn cursor_wrap( /// The adapter's expression for a cursor column compared with bounds, or the column itself. fn cursor_operand( - adapter: &Bound<'_, PyAny>, + dialect: &Bound<'_, Dialect>, column: &str, cursor_type: &Bound<'_, PyAny>, ) -> PyResult { - let py = adapter.py(); - if !adapter.hasattr("render_cursor_operand")? { + let py = dialect.py(); + if !dialect.hasattr("render_cursor_operand")? { return Ok(column.to_owned()); } method( - adapter, + dialect, "render_cursor_operand", &[ ("expression", string(py, column)), diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/future_safety.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/future_safety.rs index 7de00a6118..ef47c62055 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/future_safety.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/future_safety.rs @@ -1,9 +1,16 @@ //! Future cursor safety evidence and exception boundary. -use crate::planner_resolution::_helpers::cursor::cursor_safety::{ - horizon, replace, safety_duration, safety_error, timestamp_bound, utc_clock, -}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use crate::planner_resolution::_helpers::cursor::cursor_safety::horizon; +use crate::planner_resolution::_helpers::cursor::cursor_safety::replace; +use crate::planner_resolution::_helpers::cursor::cursor_safety::safety_duration; +use crate::planner_resolution::_helpers::cursor::cursor_safety::safety_error; +use crate::planner_resolution::_helpers::cursor::cursor_safety::timestamp_bound; +use crate::planner_resolution::_helpers::cursor::cursor_safety::utc_clock; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; use pyo3::types::{PyDict, PyTuple}; use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_cursor::algebra::main::compare::compare; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/maximum_safety.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/maximum_safety.rs index ff3fd29285..8b6a8e8f0c 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/maximum_safety.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/maximum_safety.rs @@ -1,14 +1,22 @@ //! Maximum automatic-start recovery and evidence boundary. use crate::planner_resolution::_helpers::cursor::cursor_bounds::apply_typed_cursor_replay_policy; -use crate::planner_resolution::_helpers::cursor::cursor_bounds::{ - optional_scalar, required_scalar, scalars, -}; -use crate::planner_resolution::_helpers::cursor::cursor_safety::{ - horizon, replace, safety_duration, safety_error, timestamp_bound, utc_clock, -}; +use crate::planner_resolution::_helpers::cursor::cursor_bounds::optional_scalar; +use crate::planner_resolution::_helpers::cursor::cursor_bounds::required_scalar; +use crate::planner_resolution::_helpers::cursor::cursor_bounds::scalars; +use crate::planner_resolution::_helpers::cursor::cursor_safety::horizon; +use crate::planner_resolution::_helpers::cursor::cursor_safety::replace; +use crate::planner_resolution::_helpers::cursor::cursor_safety::safety_duration; +use crate::planner_resolution::_helpers::cursor::cursor_safety::safety_error; +use crate::planner_resolution::_helpers::cursor::cursor_safety::timestamp_bound; +use crate::planner_resolution::_helpers::cursor::cursor_safety::utc_clock; use crate::planner_resolution::_helpers::references::values::string; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyErr; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; use pyo3::types::PyDict; use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_cursor::algebra::main::compare::compare; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_function_sql.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_function_sql.rs index 24046a923e..bf60cd55a5 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_function_sql.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_function_sql.rs @@ -5,6 +5,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyString}; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::models::Dialect; use sqlbuild_python_project::resource_views::main::function_reads::body_sql::body_sql; #[pyfunction] @@ -19,7 +20,7 @@ pub fn resolve_function_sql<'py>( kwargs, ( &[ - "adapter", + "dialect", "function", "model_locations", "seed_locations", @@ -32,7 +33,7 @@ pub fn resolve_function_sql<'py>( ), )?; let resolution = Resolution { - adapter: args.value("adapter")?, + dialect: Dialect::passed(args.value("dialect")?)?, external: py.None().into_bound(py), source_map: args.value("source_map")?, source_warehouse_columns: args.value("source_warehouse_columns")?, diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/models.rs b/crates/sqlbuild-python-planner/src/planner_resolution/models.rs index 7db929d5a7..f9835adfd0 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/models.rs @@ -15,6 +15,7 @@ use pyo3::types::PyDictMethods; use pyo3::types::PyList; use pyo3::types::PyString; use pyo3::types::PyTuple; +use sqlbuild_python_base::native_host::models::Dialect; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; @@ -59,7 +60,7 @@ pub struct ModelSqlRequest<'py> { /// What resolving one model's, hook's or function's references reads. #[derive(Debug)] pub struct Resolution<'py> { - pub adapter: Bound<'py, PyAny>, + pub(crate) dialect: Bound<'py, Dialect>, pub(crate) external: Bound<'py, PyAny>, pub(crate) source_map: Bound<'py, PyAny>, pub(crate) source_warehouse_columns: Bound<'py, PyAny>, @@ -71,12 +72,12 @@ pub struct Resolution<'py> { impl<'py> Resolution<'py> { /// Resolution against a `ModelPlanContext`'s relations. pub fn of_context( - adapter: Bound<'py, PyAny>, + dialect: Bound<'py, Dialect>, context: &Bound<'py, PyAny>, external: Bound<'py, PyAny>, ) -> PyResult { Ok(Self { - adapter, + dialect, external, source_map: class_field(context, |c: &ModelPlanContext| &c.source_map)?, source_warehouse_columns: class_field(context, |c: &ModelPlanContext| { @@ -89,17 +90,17 @@ impl<'py> Resolution<'py> { } pub(crate) fn resolve(&self, sql: &str, filtering: &CursorFiltering<'py>) -> PyResult { - let py = self.adapter.py(); + let py = self.dialect.py(); let resolved: String = resolve_sources( sql, (&self.source_map, &self.source_warehouse_columns), - &self.adapter, + &self.dialect, filtering, )?; let resolved: String = resolve_refs( &resolved, (&self.model_locations, &self.seed_locations), - &self.adapter, + &self.dialect, filtering, )?; let resolved: String = resolve_dbt_ref_references(&resolved, Some(&self.external))?; @@ -108,12 +109,12 @@ impl<'py> Resolution<'py> { } else { PyDict::new(py).into_any() }; - let resolved: String = resolve_udf_references(&resolved, &locations, &self.adapter)?; - resolve_table_function_references(&resolved, &locations, &self.adapter) + let resolved: String = resolve_udf_references(&resolved, &locations, &self.dialect)?; + resolve_table_function_references(&resolved, &locations, &self.dialect) } pub(crate) fn assert_resolved(&self, sql: &str, label: &str) -> PyResult<()> { - let py = self.adapter.py(); + let py = self.dialect.py(); assert_no_unresolved_sql_markers( py, &PyString::new(py, sql), @@ -122,7 +123,7 @@ impl<'py> Resolution<'py> { } pub(crate) fn unbounded(&self, sql: Bound<'py, PyAny>, label: &str) -> PyResult { - let py = self.adapter.py(); + let py = self.dialect.py(); let resolved = self.resolve( &sql.extract::()?, &CursorFiltering { diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/artifacts.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/artifacts.rs index ed30375b64..f47193ce8e 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/artifacts.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/artifacts.rs @@ -1,10 +1,10 @@ //! Scenario artifact dataclasses and adapter identifier callbacks. -use crate::planner_scenario::_helpers::values::{error, string}; -use crate::planner_scenario::constants::{ - SCENARIO_PLAN_HASH_COLLISION, SCENARIO_PLAN_INVALID_HASH_PREFIX, - SCENARIO_PLAN_RELATION_COLLISION, -}; +use crate::planner_scenario::_helpers::values::error; +use crate::planner_scenario::_helpers::values::string; +use crate::planner_scenario::constants::SCENARIO_PLAN_HASH_COLLISION; +use crate::planner_scenario::constants::SCENARIO_PLAN_INVALID_HASH_PREFIX; +use crate::planner_scenario::constants::SCENARIO_PLAN_RELATION_COLLISION; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::planner_scenario::main::{hash_prefix, parse_artifact_name}; diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/cli.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/cli.rs index 372837a2b1..de0d2005c3 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/cli.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/cli.rs @@ -13,6 +13,8 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::models::Dialect; +use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -37,7 +39,7 @@ pub(crate) fn build_cli_scenario_plan<'py>( &[ "scenario", "pipeline_result", - "adapter", + "dialect", "project_name", "source_lexical_syntax", ], @@ -46,7 +48,7 @@ pub(crate) fn build_cli_scenario_plan<'py>( )?; let scenario = args.value("scenario")?; let project = native_field(&args.value("pipeline_result")?, "project")?; - let adapter = args.value("adapter")?; + let dialect = Dialect::passed(args.value("dialect")?)?; let (graph, warnings) = plan_scenario_graph(&scenario, &project, true, None)?; raise_for_warnings(warnings.as_any())?; let hashes = object( @@ -101,7 +103,7 @@ pub(crate) fn build_cli_scenario_plan<'py>( ("artifacts", PyTuple::new(py, identities)?.into_any()), ( "identifier_limit", - adapter.call_method0("maximum_identifier_length")?, + dialect.syntax("maximum_identifier_length")?.call0()?, ), ], )?), @@ -111,7 +113,7 @@ pub(crate) fn build_cli_scenario_plan<'py>( project: project.clone(), graph_plan: graph.clone(), relation_map: map, - render_qualified_name: native_field(&adapter, "render_qualified_name")?, + render_qualified_name: dialect.syntax("render_qualified_name")?, database, schema, })?; @@ -122,7 +124,7 @@ pub(crate) fn build_cli_scenario_plan<'py>( &[ ("scenario", scenario), ("project", project), - ("adapter", adapter), + ("dialect", dialect.into_any()), ("graph_plan", graph), ("relation_plan", relations), ( diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/entries.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/entries.rs index 44e9e48e1b..60a765269f 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/entries.rs @@ -11,6 +11,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyString; use pyo3::types::{PyDictMethods, PyTuple}; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::by_attribute::by_attribute; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -86,7 +87,7 @@ pub(crate) fn function_entry<'py>( Some(keywords( py, &[ - ("adapter", context.adapter.clone()), + ("dialect", context.dialect.clone().into_any()), ("function", function.clone()), ( "model_locations", @@ -104,7 +105,7 @@ pub(crate) fn function_entry<'py>( ("source_warehouse_columns", warehouse_columns.clone()), ( "star_exclude_keyword", - context.adapter.call_method0("star_exclude_keyword")?, + context.dialect.syntax("star_exclude_keyword")?.call0()?, ), ], )?), diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/execution.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/execution.rs index 46c3b16a06..b072621ad9 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/execution.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/execution.rs @@ -14,11 +14,14 @@ use crate::planner_scenario::_helpers::sql::resolve_sql; use crate::planner_scenario::_helpers::sql::source_label; use crate::planner_scenario::_helpers::values::error; use crate::planner_scenario::_helpers::values::string; -use crate::planner_scenario::constants::{ - CURSOR_INTRINSIC_NAMES, CURSOR_TYPE_KEY, INCREMENTAL_MODE_KEY, INTEGER_CURSOR_TYPE, - MICROBATCH_INCREMENTAL_MODE, SCENARIO_EXPECTED_COLUMNS_HELP, SCENARIO_PLAN_EXPECTED_COLUMNS, - SCENARIO_PLAN_INVALID_FIXTURE, -}; +use crate::planner_scenario::constants::CURSOR_INTRINSIC_NAMES; +use crate::planner_scenario::constants::CURSOR_TYPE_KEY; +use crate::planner_scenario::constants::INCREMENTAL_MODE_KEY; +use crate::planner_scenario::constants::INTEGER_CURSOR_TYPE; +use crate::planner_scenario::constants::MICROBATCH_INCREMENTAL_MODE; +use crate::planner_scenario::constants::SCENARIO_EXPECTED_COLUMNS_HELP; +use crate::planner_scenario::constants::SCENARIO_PLAN_EXPECTED_COLUMNS; +use crate::planner_scenario::constants::SCENARIO_PLAN_INVALID_FIXTURE; use crate::planner_scenario::models::SqlContext; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyTuple}; @@ -28,6 +31,8 @@ use sqlbuild_analysis::compiler::main::scenario_reader_sql::scenario_reader_sql; use sqlbuild_discovery::constants::{SCENARIO_EXPECTED_COLUMNS_ALL, SCENARIO_EXPECTED_COLUMNS_KEY}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::models::Dialect; +use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -49,9 +54,10 @@ use sqlbuild_python_project::planner_identity::main::execution_order::execution_ use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_strategy::main::complete_fixtures::complete_fixtures as complete_relation_fixtures; use sqlbuild_python_project::planner_strategy::main::planning_context::planning_context; -use sqlbuild_python_project::sql_test_glue::constants::{ - DEFAULT_INTEGER_END, DEFAULT_INTEGER_START, DEFAULT_TIMESTAMP_END, DEFAULT_TIMESTAMP_START, -}; +use sqlbuild_python_project::sql_test_glue::constants::DEFAULT_INTEGER_END; +use sqlbuild_python_project::sql_test_glue::constants::DEFAULT_INTEGER_START; +use sqlbuild_python_project::sql_test_glue::constants::DEFAULT_TIMESTAMP_END; +use sqlbuild_python_project::sql_test_glue::constants::DEFAULT_TIMESTAMP_START; #[pyfunction] #[pyo3(signature = (**kwargs))] @@ -67,7 +73,7 @@ pub(crate) fn build_scenario_execution_plan<'py>( &[ "scenario", "project", - "adapter", + "dialect", "graph_plan", "relation_plan", "source_lexical_syntax", @@ -83,7 +89,7 @@ pub(crate) fn build_scenario_execution_plan<'py>( let project = args.value("project")?; let graph = args.value("graph_plan")?; let context = SqlContext { - adapter: args.value("adapter")?, + dialect: Dialect::passed(args.value("dialect")?)?, relations: args.value("relation_plan")?, syntax: args.value("source_lexical_syntax")?, label: source_label(&scenario)?, @@ -171,7 +177,7 @@ pub(crate) fn build_scenario_execution_plan<'py>( &[ ("model", model), ("snapshot", snapshot.clone()), - ("adapter", context.adapter.clone()), + ("dialect", context.dialect.clone().into_any()), ("context", model_context), ( "sql_analysis_enabled", @@ -246,7 +252,7 @@ fn complete_fixtures<'py>( let planning = planning_context(project, None)?; let completion = complete_relation_fixtures( &planning, - &context.adapter, + &context.dialect, (&names, PyTuple::new(py, groups)?.as_any()), )?; if !completion.diagnostics.is_empty() { @@ -298,8 +304,9 @@ fn model_context<'py>( source_map: native_field(&context.relations, "source_map")?.unbind(), source_warehouse_columns: columns.clone().unbind(), star_exclude_keyword: context - .adapter - .call_method0("star_exclude_keyword")? + .dialect + .syntax("star_exclude_keyword")? + .call0()? .unbind(), runtime_cursor_producer_names: PyFrozenSet::empty(py)?.into_any().unbind(), future_cursor_config: none(), @@ -330,8 +337,10 @@ fn compared_columns( Some(model) => inferred_names(&model)?, None => None, }; - let dialect: Option = - native_field(&context.adapter, "sql_analysis_dialect_name")?.extract()?; + let dialect: Option = context + .dialect + .syntax("sql_analysis_dialect_name")? + .extract()?; let columns = scenario_expected_columns(authored_sql, dialect.as_deref(), model_columns.as_deref()); let scenario_name: String = native_field(scenario, "name")?.extract()?; diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/fixtures.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/fixtures.rs index 32509765e2..b845471e52 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/fixtures.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/fixtures.rs @@ -14,6 +14,7 @@ use sqlbuild_analysis::compiler::main::scenario_reader_ctes::scenario_reader_cte use sqlbuild_analysis::compiler::main::scenario_reader_sql::scenario_reader_sql; use sqlbuild_analysis::compiler::models::ScenarioHelperScope; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::models::Dialect; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -171,7 +172,7 @@ pub(crate) fn build_scenario_fixture_plans<'py>( "scenario", "graph_plan", "relation_plan", - "adapter", + "dialect", "source_lexical_syntax", ], &["fixture_sql_overrides"], @@ -179,7 +180,7 @@ pub(crate) fn build_scenario_fixture_plans<'py>( )?; let scenario = args.value("scenario")?; let context = SqlContext { - adapter: args.value("adapter")?, + dialect: Dialect::passed(args.value("dialect")?)?, relations: args.value("relation_plan")?, syntax: args.value("source_lexical_syntax")?, label: source_label(&scenario)?, diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/helper_scope.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/helper_scope.rs index 42a7e3491f..039f579c34 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/helper_scope.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/helper_scope.rs @@ -3,6 +3,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::compiler::main::scenario_helper_scope::scenario_helper_scope as helper_scope; use sqlbuild_analysis::compiler::models::{ScenarioHelperScope, ScenarioScopeInput}; +use sqlbuild_python_base::native_host::types::Syntax; use crate::planner_scenario::_helpers::values::error; use crate::planner_scenario::constants::SCENARIO_PLAN_INTERNAL; @@ -33,8 +34,10 @@ pub(crate) fn scenario_helper_scope( ctes.push((name, body)); } let syntax: LexicalSyntaxInput = native_field(&context.syntax, "native_mapping")?.extract()?; - let dialect: Option = - native_field(&context.adapter, "sql_analysis_dialect_name")?.extract()?; + let dialect: Option = context + .dialect + .syntax("sql_analysis_dialect_name")? + .extract()?; helper_scope(&ScenarioScopeInput { ctes: &ctes, checks: &[], diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/relations.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/relations.rs index 035c44aca4..e76a84ef08 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/relations.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/relations.rs @@ -1,10 +1,10 @@ //! Build scenario relation locations through adapter qualification callbacks. use crate::planner_scenario::_helpers::values::error; -use crate::planner_scenario::constants::{REF, SOURCE}; -use crate::planner_scenario::constants::{ - SCENARIO_PLAN_INTERNAL, SCENARIO_PLAN_MISSING_RELATION_TARGET, -}; +use crate::planner_scenario::constants::REF; +use crate::planner_scenario::constants::SCENARIO_PLAN_INTERNAL; +use crate::planner_scenario::constants::SCENARIO_PLAN_MISSING_RELATION_TARGET; +use crate::planner_scenario::constants::SOURCE; use crate::planner_scenario::models::ScenarioRelations; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods}; diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/sql.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/sql.rs index 700030b8e8..2aae522903 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/sql.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/sql.rs @@ -12,6 +12,7 @@ use sqlbuild_analysis::planner_scenario::{ errors::MarkerError, main::replace_markers::replace_markers, }; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::models::Dialect; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; @@ -24,7 +25,7 @@ pub(crate) fn resolve_sql( sql: &str, project_sources: bool, ) -> PyResult { - let py = context.adapter.py(); + let py = context.dialect.py(); let syntax: LexicalSyntaxInput = native_field(&context.syntax, "native_mapping")?.extract()?; let mut result = sql.to_owned(); for (kind, field) in [ @@ -47,7 +48,7 @@ pub(crate) fn resolve_sql( result = replace_markers(&result, kind, Some(syntax.clone().into()), |name, marker| { let target = mapping_get(&locations, name)?; let target = if target.is_none() {target} else if kind == crate::planner_scenario::constants::SOURCE_MARKER { - object(py, render_source_relation(&target, Some(&context.adapter.clone()).filter(|value| !value.is_none()))?)? + object(py, render_source_relation(&target, Some(context.dialect.as_any()))?)? } else {native_field(&target, "qualified_name")?}; if target.is_none() { let marker = string(py, marker).repr()?.to_str()?.to_owned(); @@ -131,7 +132,7 @@ pub(crate) fn resolve_scenario_check_sql( &[ "sql", "relation_plan", - "adapter", + "dialect", "source_label", "source_lexical_syntax", ], @@ -139,7 +140,7 @@ pub(crate) fn resolve_scenario_check_sql( ), )?; let context = SqlContext { - adapter: args.value("adapter")?, + dialect: Dialect::passed(args.value("dialect")?)?, relations: args.value("relation_plan")?, syntax: args.value("source_lexical_syntax")?, label: args.value("source_label")?.extract()?, diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/models.rs b/crates/sqlbuild-python-planner/src/planner_scenario/models.rs index b385cc1b2f..3b30aac6aa 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/models.rs @@ -1,10 +1,11 @@ //! Public records of planner scenario. use pyo3::prelude::{Bound, PyAny}; +use sqlbuild_python_base::native_host::models::Dialect; #[derive(Debug)] pub(crate) struct SqlContext<'py> { - pub adapter: Bound<'py, PyAny>, + pub dialect: Bound<'py, Dialect>, pub relations: Bound<'py, PyAny>, pub syntax: Bound<'py, PyAny>, pub label: String, diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs index f94e6c0176..0e50277e5f 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs @@ -2,6 +2,7 @@ use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; use sqlbuild_analysis::graph::models::GraphIndexes; +use sqlbuild_python_base::native_host::models::Dialect; use crate::planner_strategy::_helpers::audits::{ attachment, outside_target_lineage, resolve_effective_run_scope, @@ -39,9 +40,10 @@ use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::planner_strategy::constants::{ - AUDIT_TYPES_MODULE, COMPILE_TYPES_MODULE, MODEL_ATTACHMENT, SOURCE_ATTACHMENT, -}; +use sqlbuild_python_project::planner_strategy::constants::AUDIT_TYPES_MODULE; +use sqlbuild_python_project::planner_strategy::constants::COMPILE_TYPES_MODULE; +use sqlbuild_python_project::planner_strategy::constants::MODEL_ATTACHMENT; +use sqlbuild_python_project::planner_strategy::constants::SOURCE_ATTACHMENT; use sqlbuild_python_project::planner_strategy::models::AuditPlanning; /// One attribute read of a compiled audit. @@ -63,7 +65,7 @@ fn render_query<'py>( ("model_locations", request.model_locations.clone()), ("seed_locations", request.seed_locations.clone()), ("source_map", request.source_map.clone()), - ("adapter", request.adapter.clone()), + ("adapter", request.dialect.clone().into_any()), ], )?))?, )?; @@ -101,7 +103,7 @@ pub(crate) fn plan_audit_of<'py>( model_locations: request.get("model_locations")?, seed_locations: request.get("seed_locations")?, source_map: request.get("source_map")?, - adapter: request.get("adapter")?, + dialect: Dialect::passed(request.get("dialect")?)?, model_materializations: request.get("model_materializations")?, lineage: Py::new( request.py(), diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_inputs.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_inputs.rs index 24b958f4ef..9ef4b4acb0 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_inputs.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_inputs.rs @@ -6,6 +6,7 @@ use crate::planner_strategy::main::values::text::text; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyFrozenSet, PyString, PyTuple}; +use sqlbuild_python_base::native_host::models::Dialect; use sqlbuild_analysis::planner_strategy::main::cursor_input_column::cursor_input_column_failure; use sqlbuild_analysis::planner_strategy::models::{CursorInputColumnCheck, CursorInputColumnOwner}; @@ -25,10 +26,12 @@ use sqlbuild_python_front::model_config::main::config::config_text_object::confi use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; -use sqlbuild_python_project::planner_strategy::constants::{ - ENFORCED_CONTRACT, INCREMENTAL_MATERIALIZATION, MODEL_REFERENCE, SEED_REFERENCE, - SOURCE_REFERENCE, TIMESTAMP_CURSOR, -}; +use sqlbuild_python_project::planner_strategy::constants::ENFORCED_CONTRACT; +use sqlbuild_python_project::planner_strategy::constants::INCREMENTAL_MATERIALIZATION; +use sqlbuild_python_project::planner_strategy::constants::MODEL_REFERENCE; +use sqlbuild_python_project::planner_strategy::constants::SEED_REFERENCE; +use sqlbuild_python_project::planner_strategy::constants::SOURCE_REFERENCE; +use sqlbuild_python_project::planner_strategy::constants::TIMESTAMP_CURSOR; use sqlbuild_python_project::planner_strategy::models::CursorInputLookups; use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; @@ -171,10 +174,10 @@ pub(crate) fn reject_unexposed_cursor_input_columns<'py>( /// The qualified relation one cursor input reference reads, when it resolves. fn input_relation<'py>( - (lookups, adapter): (&CursorInputLookups<'py>, &Bound<'py, PyAny>), + (lookups, dialect): (&CursorInputLookups<'py>, &Bound<'py, Dialect>), (kind, name): (&str, &str), ) -> PyResult> { - let py: Python<'py> = adapter.py(); + let py: Python<'py> = dialect.py(); let location: Bound<'py, PyAny> = match kind { MODEL_REFERENCE => { let model: Bound<'py, PyAny> = mapping_get(&lookups.model_locations, name)?; @@ -190,7 +193,7 @@ fn input_relation<'py>( if source.is_none() { return Ok(None); } - return render_source_relation(&source, Some(adapter)).map(Some); + return render_source_relation(&source, Some(dialect.as_any())).map(Some); } _ => py.None().into_bound(py), }; @@ -237,7 +240,7 @@ pub(crate) fn cursor_input_relations_of<'py>( ) -> PyResult> { let request = Request::new("build_cursor_input_relations", request)?; for name in [ - "adapter", + "dialect", "model_locations", "models_by_name", "seed_locations", @@ -247,7 +250,10 @@ pub(crate) fn cursor_input_relations_of<'py>( request.get(name)?; } build_cursor_input_relations( - (&request.get("model")?, &request.get("adapter")?), + ( + &request.get("model")?, + &Dialect::passed(request.get("dialect")?)?, + ), &request.get("cursor_column")?, &lookups_of(&request)?, ) @@ -255,7 +261,7 @@ pub(crate) fn cursor_input_relations_of<'py>( /// Cursor-bearing input relations of an incremental model, for runtime range discovery. pub(crate) fn build_cursor_input_relations<'py>( - (model, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (model, dialect): (&Bound<'py, PyAny>, &Bound<'py, Dialect>), cursor_column: &Bound<'py, PyAny>, lookups: &CursorInputLookups<'py>, ) -> PyResult> { @@ -287,7 +293,7 @@ pub(crate) fn build_cursor_input_relations<'py>( let kind: String = text(&native_field(&reference, "ref_kind")?)?; let name: Bound<'py, PyAny> = native_field(&reference, "ref_name")?; let name_text: String = text(&name)?; - let Some(relation) = input_relation((lookups, adapter), (&kind, &name_text))? else { + let Some(relation) = input_relation((lookups, dialect), (&kind, &name_text))? else { continue; }; let producer: Option> = if kind == MODEL_REFERENCE { diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_types.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_types.rs index 3fa759c175..9a7b6249c8 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_types.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_types.rs @@ -10,7 +10,8 @@ use sqlbuild_analysis::planner_strategy::main::cursor_type_warning::cursor_type_ use sqlbuild_analysis::planner_strategy::models::CursorTypeCheck; use sqlbuild_host::entries::models::ExecutorHost; -use crate::planner_strategy::_helpers::values::{optional_string, plan_warning}; +use crate::planner_strategy::_helpers::values::optional_string; +use crate::planner_strategy::_helpers::values::plan_warning; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/lineage.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/lineage.rs index d81b0868fe..a40404cdc8 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/lineage.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/lineage.rs @@ -8,7 +8,8 @@ use sqlbuild_analysis::planner_strategy::errors::LineageFailure; use sqlbuild_analysis::planner_strategy::models::{ProducerFacts, ReferenceIdentity}; use sqlbuild_analysis::planner_strategy::types::ReferenceGraph; -use crate::planner_strategy::_helpers::microbatches::{CAP_ACTIONS, microbatch_limit_config}; +use crate::planner_strategy::_helpers::microbatches::CAP_ACTIONS; +use crate::planner_strategy::_helpers::microbatches::microbatch_limit_config; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatch_limits.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatch_limits.rs index a51ea10ee2..cdd7301f00 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatch_limits.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatch_limits.rs @@ -16,9 +16,9 @@ use sqlbuild_core::plan_display::main::effective_timestamp_grain::effective_time use sqlbuild_core::plan_display::models::TimestampGrainFacts; use crate::plan_output::main::resolve_effective_microbatch_batch_size::resolve_effective_microbatch_batch_size; -use crate::planner_strategy::_helpers::microbatches::{ - CAP_ACTIONS, capped_microbatch_bounds, count_microbatches, -}; +use crate::planner_strategy::_helpers::microbatches::CAP_ACTIONS; +use crate::planner_strategy::_helpers::microbatches::capped_microbatch_bounds; +use crate::planner_strategy::_helpers::microbatches::count_microbatches; use crate::planner_strategy::_helpers::values::optional_string; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/model_entries.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/model_entries.rs index bc5e45b43f..c9593853db 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/model_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/model_entries.rs @@ -1,11 +1,14 @@ //! Assemble model plan dataclasses from resolved plan decisions and authored model config. +use crate::planner_strategy::main::model_entries::column_info::column_info; use crate::planner_strategy::main::values::input_error::input_error; use crate::planner_strategy::main::values::text::text; use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; +use sqlbuild_python_base::native_host::models::Dialect; +use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_analysis::planner_strategy::main::group_replacement_refusal::group_replacement_refusal; use sqlbuild_analysis::planner_strategy::main::materialization_type::materialization_type; @@ -19,15 +22,16 @@ use sqlbuild_model_config::execution_limits::main::parse_execution_limit::parse_ use sqlbuild_model_config::execution_limits::models::ExecutionLimit; use sqlbuild_model_config::model_validation::constants::PHYSICAL_STORAGE_KEYS; -use crate::planner_strategy::_helpers::cursor_types::{ - cursor_type_warning_record, warehouse_column_types, -}; -use crate::planner_strategy::_helpers::microbatch_limits::{replay_grain, unknown_grain}; +use crate::planner_strategy::_helpers::cursor_types::cursor_type_warning_record; +use crate::planner_strategy::_helpers::cursor_types::warehouse_column_types; +use crate::planner_strategy::_helpers::microbatch_limits::replay_grain; +use crate::planner_strategy::_helpers::microbatch_limits::unknown_grain; use crate::planner_strategy::_helpers::microbatches::microbatch_limit_config; -use crate::planner_strategy::_helpers::strategy::{ - model_warning_records, resolve_schema_actions, warning_inputs, -}; -use crate::planner_strategy::_helpers::values::{optional_string, plan_warning}; +use crate::planner_strategy::_helpers::strategy::model_warning_records; +use crate::planner_strategy::_helpers::strategy::resolve_schema_actions; +use crate::planner_strategy::_helpers::strategy::warning_inputs; +use crate::planner_strategy::_helpers::values::optional_string; +use crate::planner_strategy::_helpers::values::plan_warning; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; @@ -52,11 +56,14 @@ use sqlbuild_python_project::compiled_views::main::compiled_models::model_name:: use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; use sqlbuild_python_project::compiled_views::main::compiled_models::relative_path::relative_path; -use sqlbuild_python_project::planner_strategy::constants::{ - CUSTOM_MATERIALIZATION, DDL_SEPARATOR, ENFORCED_CONTRACT, GROUP_REPLACEMENT_UNSUPPORTED_CODE, - MICROBATCH_MODE, PHYSICAL_STORAGE_DIALECT, PHYSICAL_STORAGE_UNSUPPORTED_CODE, - TABLE_RECREATING_ACTIONS, -}; +use sqlbuild_python_project::planner_strategy::constants::CUSTOM_MATERIALIZATION; +use sqlbuild_python_project::planner_strategy::constants::DDL_SEPARATOR; +use sqlbuild_python_project::planner_strategy::constants::ENFORCED_CONTRACT; +use sqlbuild_python_project::planner_strategy::constants::GROUP_REPLACEMENT_UNSUPPORTED_CODE; +use sqlbuild_python_project::planner_strategy::constants::MICROBATCH_MODE; +use sqlbuild_python_project::planner_strategy::constants::PHYSICAL_STORAGE_DIALECT; +use sqlbuild_python_project::planner_strategy::constants::PHYSICAL_STORAGE_UNSUPPORTED_CODE; +use sqlbuild_python_project::planner_strategy::constants::TABLE_RECREATING_ACTIONS; use sqlbuild_python_project::planner_strategy::models::ModelEntryDecisions; /// Model config keys copied to the plan entry as optional strings, with their entry fields. @@ -103,14 +110,6 @@ fn or_none<'py>(py: Python<'py>, value: Option>) -> Bound<'py, value.unwrap_or_else(|| none(py)) } -/// `ColumnInfo(name=..., type=...)`. -fn column_info<'py>( - class: &Bound<'py, PyAny>, - (name, column_type): (Bound<'py, PyAny>, Bound<'py, PyAny>), -) -> PyResult> { - call_with(class, &[("name", name), ("type", column_type)]) -} - /// Declared schema columns as `ColumnInfo`: typed columns only, or every column with an empty default type. pub(crate) fn schema_columns<'py>( model: &Bound<'py, PyAny>, @@ -256,7 +255,7 @@ fn logical_ddl<'py>( (warehouse_columns, cursor_bounds): (&Bound<'py, PyAny>, Bound<'py, PyAny>), ) -> PyResult { let py: Python<'py> = request.model.py(); - let adapter: &Bound<'py, PyAny> = &request.adapter; + let dialect: &Bound<'py, Dialect> = &request.dialect; let model: &Bound<'py, PyAny> = &request.model; let destination: Bound<'py, PyAny> = sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( @@ -314,7 +313,7 @@ fn logical_ddl<'py>( } _ => return Ok(String::new()), }; - let statements: Vec = call_with(&adapter.getattr(method)?, &pairs)? + let statements: Vec = call_with(&dialect.getattr(method)?, &pairs)? .try_iter()? .map(|statement| statement?.extract()) .collect::>()?; @@ -352,8 +351,10 @@ fn storage_fields<'py>( } } if !keys.is_empty() { - let dialect: Option = - native_field(&request.adapter, "sql_analysis_dialect_name")?.extract()?; + let dialect: Option = request + .dialect + .syntax("sql_analysis_dialect_name")? + .extract()?; if dialect.as_deref() != Some(PHYSICAL_STORAGE_DIALECT) { return Err(input_error( py, @@ -362,7 +363,7 @@ fn storage_fields<'py>( "model '{}' sets {}, but adapter '{}' has no physical table layout settings", model_name(&request.model)?, keys.join(", "), - native_field(&request.adapter, "adapter_name")?.str()? + request.dialect.syntax("adapter_name")?.str()? ), Some(PHYSICAL_STORAGE_UNSUPPORTED_CODE), )); @@ -669,10 +670,10 @@ pub(crate) fn assemble_model_plan_entry<'py>( if let Some(refusal) = group_replacement_refusal( &model_name(model)?, ( - &native_field(&request.adapter, "adapter_name")? - .str()? - .to_string(), - native_field(&request.adapter, "sql_analysis_dialect_name")? + &request.dialect.syntax("adapter_name")?.str()?.to_string(), + request + .dialect + .syntax("sql_analysis_dialect_name")? .extract::>()? .as_deref(), ), diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs index fe225e30fe..b620572c5d 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs @@ -14,7 +14,9 @@ use sqlbuild_analysis::planner_strategy::models::{ PlanAction, PlanReason, SchemaActionKind, WarningInputs, }; -use crate::planner_strategy::_helpers::values::{change_facts, finding_rows, plan_warning}; +use crate::planner_strategy::_helpers::values::change_facts; +use crate::planner_strategy::_helpers::values::finding_rows; +use crate::planner_strategy::_helpers::values::plan_warning; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::types::RecordId; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs index 82e778151a..69c28222c5 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs @@ -5,6 +5,7 @@ use crate::planner_strategy::main::audit_entries::audit_is_selected::audit_is_se use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyList, PyListMethods}; use sqlbuild_python_base::boundary::models::Request; +use sqlbuild_python_base::native_host::models::Dialect; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_project::compiled_views::main::project::audits::audits; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; @@ -34,7 +35,7 @@ pub fn build_selected_audit_entries<'py>( model_locations: native_field(&relations, "model_locations")?, seed_locations: native_field(&relations, "seed_locations")?, source_map: native_field(&relations, "source_read_map")?, - adapter: request.get("adapter")?, + dialect: Dialect::passed(request.get("dialect")?)?, model_materializations: request.get("model_materializations")?, lineage: native_field(&project, "lineage_graph")?.extract()?, }; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/build_cursor_input_relations.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/build_cursor_input_relations.rs index ea671898f1..9d0b1005b2 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/build_cursor_input_relations.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/build_cursor_input_relations.rs @@ -3,14 +3,15 @@ use crate::planner_strategy::_helpers::cursor_inputs; use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyTuple; +use sqlbuild_python_base::native_host::models::Dialect; use sqlbuild_python_project::planner_strategy::models::CursorInputLookups; /// Cursor-bearing input relations of an incremental model, for runtime range discovery. #[inline] pub fn build_cursor_input_relations<'py>( - (model, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (model, dialect): (&Bound<'py, PyAny>, &Bound<'py, Dialect>), cursor_column: &Bound<'py, PyAny>, lookups: &CursorInputLookups<'py>, ) -> PyResult> { - cursor_inputs::build_cursor_input_relations((model, adapter), cursor_column, lookups) + cursor_inputs::build_cursor_input_relations((model, dialect), cursor_column, lookups) } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/column_info.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/column_info.rs new file mode 100644 index 0000000000..f4eae54437 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/column_info.rs @@ -0,0 +1,12 @@ +//! One `ColumnInfo` the planner hands the adapter. + +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::boundary::main::call_with::call_with; + +/// `ColumnInfo(name=..., type=...)`. +pub(crate) fn column_info<'py>( + class: &Bound<'py, PyAny>, + (name, column_type): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + call_with(class, &[("name", name), ("type", column_type)]) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/mod.rs index 4725a86594..82e66a1b59 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/mod.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/mod.rs @@ -1,4 +1,5 @@ pub mod assemble_model_plan_entry; +pub(crate) mod column_info; pub mod extract_seed_columns; pub mod resolve_microbatch_lookback; pub mod resolve_microbatch_lookback_function; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs index bf0e2ea08b..877dabe529 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs @@ -5,33 +5,35 @@ use crate::planner_resolution::main::cursor::resolve_effective_timestamp_grain:: use crate::planner_resolution::main::cursor::resolve_start_cursor_config::resolve_start_cursor_config; use crate::planner_resolution::main::safety::maximum_allowed_start::maximum_allowed_start; use crate::planner_strategy::main::lineage::resolve_lineage_reference::resolve_lineage_reference; -use crate::planner_warehouse::_helpers::inspection::snapshot::inspection_concurrency; use crate::planner_warehouse::_helpers::inspection::watermarks::validate_watermark_column; use crate::planner_warehouse::constants::{ CURSOR_ALGEBRA_ERROR, CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, DATE_VALUE, LOG_DEBUG_EVENT, - PLANNER_LOGGER, QUERY_RECORD, RECORD_QUERY, RUN_BOUNDED, RUN_RECORDED_QUERY, + PLANNER_LOGGER, }; -use crate::planner_warehouse::main::inspection::adapter_cursor_sql::adapter_cursor_sql; -use crate::planner_warehouse::main::inspection::bound_task::bound_task; +use crate::planner_warehouse::models::EligibleCursorQuery; +use crate::planner_warehouse::models::Read; use crate::planner_warehouse::models::SnapshotRequest; +use crate::planner_warehouse::types::CursorProgress; +use crate::planner_warehouse::types::WarehouseReads; use pyo3::IntoPyObject; -use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyFloat, PyTuple}; +use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::planner_changes::constants::FORWARD_ONLY_POLICY; use sqlbuild_analysis::planner_warehouse::main::cursor_bounds::cursor_bounds; use sqlbuild_analysis::planner_warehouse::main::cursor_query_groups::cursor_query_groups; use sqlbuild_analysis::planner_warehouse::main::cursor_query_identity::cursor_query_identity; +use sqlbuild_analysis::planner_warehouse::main::cursor_query_sql::cursor_query_sql; use sqlbuild_analysis::planner_warehouse::models::{CursorTagRequest, PhysicalCursorQuery}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::models::Dialect; +use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; use sqlbuild_python_front::cursor::main::intervals::partition::partition; use sqlbuild_python_front::cursor::main::scalars::compare_values::compare_values; use sqlbuild_python_front::cursor::main::scalars::render_value::render_value; @@ -49,12 +51,12 @@ use sqlbuild_python_project::planner_phases::main::model_key::model_key; use sqlbuild_python_project::planner_phases::main::native_member::native_member; use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_phases::main::optional_text::optional_text; -use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; use sqlbuild_python_project::planner_phases::main::planner_model::planner_model; use sqlbuild_python_project::planner_phases::main::text::text; use sqlbuild_python_project::planner_phases::main::tuple::tuple; use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; +use std::cell::RefCell; use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -236,7 +238,7 @@ fn runtime_producer_names( fn upstream_relation<'py>( reference: &Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, + dialect: &Bound<'py, Dialect>, maps: &ModelMaps<'py>, (deferred, selected): (&Bound<'py, PyAny>, Option<&HashSet>), ) -> PyResult> { @@ -271,10 +273,7 @@ fn upstream_relation<'py>( { return object( py, - render_source_relation( - &source_entry(&source)?, - Some(adapter).filter(|value| !value.is_none()), - )?, + render_source_relation(&source_entry(&source)?, Some(dialect.as_any()))?, ); } Ok(py.None().into_bound(py)) @@ -459,7 +458,7 @@ fn collect_models<'py>( } let relation = upstream_relation( &reference, - &request.adapter, + &request.dialect, maps, (&inputs.deferred_locations, selected_names.as_ref()), )?; @@ -577,57 +576,6 @@ fn physical_queries(models: &[CursorModel<'_>]) -> Vec { cursor_query_groups(&requests) } -/// Run one physical cursor statement; a failure leaves only that relation unavailable. -pub(crate) fn read_cursor_query<'py>( - py: Python<'py>, - sql: &str, - (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult> { - let started = Instant::now(); - let outcome = call_with( - execute, - &[("connection", connection.clone()), ("sql", text(py, sql))], - ) - .and_then(|result| result.call_method0("fetchall")); - let elapsed = started.elapsed().as_secs_f64(); - let (row, error, count) = match outcome { - Ok(rows) => { - let count = rows.len()?; - let row = if count > 0 { - rows.get_item(0)? - } else { - py.None().into_bound(py) - }; - ( - row, - py.None().into_bound(py), - count.into_pyobject(py)?.into_any(), - ) - } - Err(error) if error.is_instance_of::(py) => { - let value = error.value(py).clone().into_any(); - (py.None().into_bound(py), value, py.None().into_bound(py)) - } - Err(error) => return Err(error), - }; - let error_text = if error.is_none() { - py.None().into_bound(py) - } else { - error.str()?.into_any() - }; - let record = call_with( - &host_entry(py, QUERY_RECORD)?, - &[ - ("sql", text(py, sql)), - ("elapsed_seconds", PyFloat::new(py, elapsed).into_any()), - ("row_count", count), - ("error", error_text), - ], - )?; - call_with(&host_entry(py, RECORD_QUERY)?, &[("record", record)])?; - Ok(PyTuple::new(py, [row, PyFloat::new(py, elapsed).into_any(), error])?.into_any()) -} - fn report_cursor_query( py: Python<'_>, query: &PhysicalCursorQuery, @@ -721,56 +669,27 @@ fn fan_out<'py>(query: &PhysicalCursorQuery, row: &Bound<'py, PyAny>) -> PyResul Ok(values) } +/// The values the statements read and the failed tags; `None` when inputs are left to the run. fn execute_queries<'py>( request: &SnapshotRequest<'py>, queries: &[PhysicalCursorQuery], - concurrency: i64, -) -> PyResult<(Values<'py>, HashSet)> { +) -> PyResult, HashSet)>> { let py = request.project.py(); - let total = queries.len(); - let on_progress = &request.on_progress; - let handles = (&request.connection, &request.execute); - let outcomes: Vec> = if concurrency <= 1 || total <= 1 { - let mut outcomes: Vec<_> = Vec::new(); - for (position, query) in queries.iter().enumerate() { - let identity = cursor_query_identity(query, (position + 1, total)); - if !on_progress.is_none() { - on_progress.call1((format!("Inspecting cursor bounds {identity}..."),))?; - } - let outcome = - read_cursor_query(py, &adapter_cursor_sql(&request.adapter, query)?, handles)?; - report_cursor_query(py, query, (&outcome, &identity), on_progress)?; - outcomes.push(outcome); - } - outcomes - } else { - if !on_progress.is_none() { - on_progress.call1((format!( - "Inspecting cursor bounds for {total} relations ({} concurrent)...", - (concurrency as usize).min(total) - ),))?; - } - let mut tasks: Vec<_> = Vec::new(); - for query in queries { - tasks.push(bound_task( - py, - adapter_cursor_sql(&request.adapter, query)?, - handles, - )?); - } - call_with( - &host_entry(py, RUN_BOUNDED)?, - &[ - ("tasks", tuple(py, tasks)?), - ("concurrency", concurrency.into_pyobject(py)?.into_any()), - ( - "on_complete", - completion_reporter(py, queries.to_vec(), on_progress)?, - ), - ], - )? - .try_iter()? - .collect::>()? + let statements = || -> PyResult> { + queries + .iter() + .map(|query| dialect_cursor_sql(&request.dialect, query)) + .collect() + }; + let progress = (queries, &request.on_progress); + let Read::Known(outcomes) = + request + .reads + .get() + .reads() + .cursor_bounds(py, &statements, &progress)? + else { + return Ok(None); }; let mut values = Values::new(); let mut failed: HashSet<_> = HashSet::new(); @@ -781,67 +700,8 @@ fn execute_queries<'py>( } values.extend(fan_out(query, &outcome.get_item(0)?)?); } - Ok((values, failed)) -} - -fn eligible_task<'py>( - py: Python<'py>, - (model_name, target, cursor_column): (String, String, String), - sql: String, - (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult> { - let connection: Py = connection.clone().unbind(); - let execute: Py = execute.clone().unbind(); - Ok(PyCFunction::new_closure( - py, - None, - None, - move |args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>| -> PyResult> { - let py = args.py(); - let run_connection = connection.clone_ref(py); - let run_execute = execute.clone_ref(py); - let run_sql = sql.clone(); - let run = PyCFunction::new_closure( - py, - None, - None, - move |args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>| -> PyResult> { - let py = args.py(); - Ok(run_execute - .bind(py) - .call((), Some(&keywords(py, &[ - ("connection", run_connection.bind(py).clone()), - ("sql", text(py, &run_sql)), - ])?))? - .call_method0("fetchall")? - .unbind()) - }, - )?; - let rows = call_with( - &host_entry(py, RUN_RECORDED_QUERY)?, - &[("sql", text(py, &sql)), ("run", run.into_any())], - ); - match rows { - Ok(rows) => Ok(rows.unbind()), - Err(error) if error.is_instance_of::(py) => { - let wrapped: PyErr = planner_error( - py, - &format!( - "model '{model_name}': failed to query highest eligible target cursor for {target}.{cursor_column}: {}", - error.value(py).str()? - ), - (None, None), - ); - wrapped.set_cause(py, Some(error)); - Err(wrapped) - } - Err(error) => Err(error), - } - }, - )? - .into_any()) + Ok(Some((values, failed))) } - fn eligible_target_maxes<'py>( request: &SnapshotRequest<'py>, models: &[CursorModel<'py>], @@ -888,7 +748,7 @@ fn eligible_target_maxes<'py>( continue; } let sql = call_with( - &native_field(&request.adapter, "render_max_cursor_at_or_before")?, + &request.dialect.syntax("render_max_cursor_at_or_before")?, &[ ( "relation", @@ -912,33 +772,24 @@ fn eligible_target_maxes<'py>( )?; requests.push((info, sql)); } - let mut tasks: Vec<_> = Vec::new(); + let mut queries: Vec = Vec::new(); for (info, sql) in &requests { - tasks.push(eligible_task( - py, - ( - info.model_name.clone(), - info.target_relation.clone().unwrap_or_default(), - info.cursor_column.clone(), - ), - sql.extract()?, - (&request.connection, &request.execute), - )?); + queries.push(EligibleCursorQuery { + model_name: info.model_name.clone(), + target: info.target_relation.clone().unwrap_or_default(), + cursor_column: info.cursor_column.clone(), + sql: sql.extract()?, + }); } - let rows_by_request = call_with( - &host_entry(py, RUN_BOUNDED)?, - &[ - ("tasks", tuple(py, tasks)?), - ( - "concurrency", - inspection_concurrency(&request.adapter, &request.connection)? - .into_pyobject(py)? - .into_any(), - ), - ], - )?; - for ((info, _), rows) in requests.iter().zip(rows_by_request.try_iter()?) { - let rows = rows?; + let Read::Known(rows_by_request) = request + .reads + .get() + .reads() + .eligible_cursor_maxes(py, queries)? + else { + return Ok(values); + }; + for ((info, _), rows) in requests.iter().zip(rows_by_request) { if !rows.is_truthy()? { continue; } @@ -1147,44 +998,47 @@ fn assemble<'py>( Ok(snapshots) } -/// Plan previews: target `MAX` values from the synced copy; inputs are read when the run starts. -fn synced_copy_values<'py>( - facade: &SyncedCopyAdapter, +/// Plan previews: target `MAX` values from the synced state; inputs are read when the run starts. +fn synced_values<'py>( + reads: &dyn WarehouseReads, models: &mut [CursorModel<'py>], ) -> PyResult> { let mut values = Values::new(); for info in models { let reads_inputs: bool = !info.upstreams.is_empty(); info.upstreams.clear(); - let observed: Option<&str> = info - .target_tag - .as_ref() - .and_then(|_| facade.target_max(&info.model_name)); - match (&info.target_tag, observed) { + let observed: Option = match &info.target_tag { + Some(_) => match reads.target_max(&info.model_name) { + Read::Known(observed) => Some(observed), + Read::Unknown(_) => None, + }, + None => None, + }; + match (&info.target_tag, observed.as_deref()) { (Some(tag), Some(observed)) => { let value = parse_warehouse_cursor(&text(info.effective_grain.py(), observed))?; let lifted = floor_lifts(info, &value)?; values.insert(tag.clone(), value); if reads_inputs && (info.replays || lifted) { - facade.mark_unknown(format!( + reads.note_unknown(format!( "cursor window of {} (the run starts it away from the last build's cursor)", info.model_name )); } else if reads_inputs { - facade.mark_window(&info.model_name, observed); + reads.note_window(&info.model_name, observed); } if !info.start_cursor_config.is_none() && !native_field(&info.start_cursor_config, "max_ahead")?.is_none() { - facade.mark_unknown(format!("future-cursor safety of {}", info.model_name)); + reads.note_unknown(format!("future-cursor safety of {}", info.model_name)); } } - (Some(_), None) => facade.mark_unknown(format!( + (Some(_), None) => reads.note_unknown(format!( "cursor position of {} (no successful build recorded it)", info.model_name )), (None, _) if reads_inputs => { - facade.mark_unknown(format!("cursor window of {}", info.model_name)); + reads.note_unknown(format!("cursor window of {}", info.model_name)); } (None, _) => {} } @@ -1192,12 +1046,12 @@ fn synced_copy_values<'py>( Ok(values) } -/// Cursor MIN/MAX values of the selected incremental models, by model name. +/// Cursor MIN/MAX values by model name, and whether the reads left cursor inputs to the run. pub(crate) fn cursor_snapshots<'py>( request: &SnapshotRequest<'py>, existing_relations: &Bound<'py, PyDict>, inputs: &CursorInputs<'py>, -) -> PyResult> { +) -> PyResult<(Bound<'py, PyDict>, bool)> { let py = request.project.py(); let maps = ModelMaps { models: by_name(&request.project, "models")?, @@ -1207,16 +1061,14 @@ pub(crate) fn cursor_snapshots<'py>( }; let mut models = collect_models(request, existing_relations, inputs, &maps)?; if models.is_empty() { - return Ok(PyDict::new(py)); - } - if let Ok(facade) = request.adapter.downcast::() { - let values = synced_copy_values(facade.get(), &mut models)?; - return assemble(py, &models, (&values, &HashSet::new())); + return Ok((PyDict::new(py), false)); } let queries = physical_queries(&models); let started = Instant::now(); - let concurrency = inspection_concurrency(&request.adapter, &request.connection)?; - let (values, failed) = execute_queries(request, &queries, concurrency)?; + let Some((values, failed)) = execute_queries(request, &queries)? else { + let values = synced_values(request.reads.get().reads(), &mut models)?; + return Ok((assemble(py, &models, (&values, &HashSet::new()))?, true)); + }; let values = eligible_target_maxes(request, &models, values, &inputs.invocation_time)?; if !request.on_progress.is_none() { let logical: usize = queries @@ -1230,5 +1082,72 @@ pub(crate) fn cursor_snapshots<'py>( started.elapsed().as_secs_f64() ),))?; } - assemble(py, &models, (&values, &failed)) + Ok((assemble(py, &models, (&values, &failed))?, false)) +} + +/// The query's MIN/MAX statement, each bound read as text the way the adapter renders it. +pub(crate) fn dialect_cursor_sql( + dialect: &Bound<'_, Dialect>, + query: &PhysicalCursorQuery, +) -> PyResult { + let py = dialect.py(); + let render = dialect.syntax(CURSOR_TEXT_METHOD)?; + let failure: RefCell> = RefCell::new(None); + let sql: String = cursor_query_sql(query, &|expression: &str| { + call_with( + &render, + &[ + ("expression", text(py, expression)), + ("cursor_type", text(py, &query.cursor_type)), + ], + ) + .and_then(|rendered| rendered.extract::()) + .unwrap_or_else(|error| { + failure.replace(Some(error)); + String::new() + }) + }); + match failure.into_inner() { + Some(error) => Err(error), + None => Ok(sql), + } +} + +/// Progress lines of one plan's cursor bound statements: the statements' queries and `on_progress`. +impl<'py> CursorProgress<'py> for (&[PhysicalCursorQuery], &Bound<'py, PyAny>) { + fn starting(&self, index: usize) -> PyResult<()> { + let (queries, on_progress) = *self; + if !on_progress.is_none() { + let identity = cursor_query_identity(&queries[index], (index + 1, queries.len())); + on_progress.call1((format!("Inspecting cursor bounds {identity}..."),))?; + } + Ok(()) + } + + fn concurrent(&self, concurrency: usize) -> PyResult<()> { + let (queries, on_progress) = *self; + if !on_progress.is_none() { + on_progress.call1((format!( + "Inspecting cursor bounds for {} relations ({concurrency} concurrent)...", + queries.len() + ),))?; + } + Ok(()) + } + + fn finished(&self, index: usize, outcome: &Bound<'py, PyAny>) -> PyResult<()> { + let (queries, on_progress) = *self; + let identity = cursor_query_identity(&queries[index], (index + 1, queries.len())); + report_cursor_query( + outcome.py(), + &queries[index], + (outcome, &identity), + on_progress, + ) + } + + fn completion_callback(&self, py: Python<'py>) -> PyResult> { + let (queries, on_progress) = *self; + completion_reporter(py, queries.to_vec(), on_progress) + } } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs index 58eb73eae7..94a171e4bb 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs @@ -64,11 +64,11 @@ pub(crate) fn sqlite_error(py: Python<'_>, error: &PyErr) -> PyResult { Ok(error.is_instance(py, &imported(py, "sqlite3", "DatabaseError")?)) } -pub(crate) fn close(py: Python<'_>, connection: Option>) -> PyResult<()> { - let Some(connection) = connection else { +pub(crate) fn close(py: Python<'_>, store: Option>) -> PyResult<()> { + let Some(store) = store else { return Ok(()); }; - match connection.bind(py).call_method0("close") { + match store.bind(py).call_method0("close") { Ok(_) => Ok(()), Err(error) if sqlite_error(py, &error)? => Ok(()), Err(error) => Err(error), diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs index 6fec23f4ec..56000c8401 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs @@ -3,15 +3,16 @@ use crate::planner_phases::main::rebuilds::effectively_full_refreshed_model_names::effectively_full_refreshed_model_names; use crate::planner_warehouse::_helpers::inspection::cursors::{CursorInputs, cursor_snapshots}; use crate::planner_warehouse::constants::{ - ACTIVE_CATALOG, FUNCTION_NODE_TYPES, METADATA_NAME_FILTER_LIMIT, MIGRATE_FROM_CONFIG_KEY, - RELATIONS_MAIN_COLUMNS, RELATIONS_MAIN_LIST, + FUNCTION_NODE_TYPES, METADATA_NAME_FILTER_LIMIT, MIGRATE_FROM_CONFIG_KEY, }; +use crate::planner_warehouse::models::FingerprintRead; +use crate::planner_warehouse::models::PlannerReads; use crate::planner_warehouse::models::SnapshotRequest; +use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; -use pyo3::{IntoPyObject, IntoPyObjectExt}; use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_MODEL; use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_SEED; use sqlbuild_analysis::graph::main::reachable_keys::reachable_keys; @@ -20,10 +21,9 @@ use sqlbuild_core::constants::MIGRATION_TABLE_NAME; use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; use sqlbuild_model_config::types::{AuthoredNode, NodeKind}; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::models::Dialect; +use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; @@ -33,7 +33,6 @@ use sqlbuild_python_front::model_config::main::config::model_values::model_value use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; use sqlbuild_python_project::compiled_views::main::compiled_models::references::references; use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; -use sqlbuild_python_project::fingerprints::main::read_latest_fingerprints::read_latest_fingerprints; use sqlbuild_python_project::fingerprints::main::select_latest_fingerprints::select_latest_fingerprints; use sqlbuild_python_project::graph::main::nodes::native_key::native_key; use sqlbuild_python_project::graph::models::KeyCache; @@ -244,31 +243,6 @@ fn metadata_name_filter( Ok(Some(names)) } -fn active_catalog<'py>( - adapter: &Bound<'py, PyAny>, - connection: &Bound<'py, PyAny>, -) -> PyResult> { - call_with( - &host_entry(adapter.py(), ACTIVE_CATALOG)?, - &[ - ("adapter", adapter.clone()), - ("connection", connection.clone()), - ], - ) -} - -/// The concurrency of the active inspection catalog, one without a catalog. -pub(crate) fn inspection_concurrency( - adapter: &Bound<'_, PyAny>, - connection: &Bound<'_, PyAny>, -) -> PyResult { - let catalog = active_catalog(adapter, connection)?; - if catalog.is_none() { - return Ok(1); - } - native_field(&catalog, "concurrency")?.extract() -} - fn prefetch_schemas<'py>( request: &SnapshotRequest<'py>, database: &Bound<'py, PyAny>, @@ -276,10 +250,6 @@ fn prefetch_schemas<'py>( relevant: &Bound<'py, PyAny>, ) -> PyResult<()> { let py = request.project.py(); - let catalog = active_catalog(&request.adapter, &request.connection)?; - if catalog.is_none() { - return Ok(()); - } let mut scopes: Vec> = Vec::new(); for schema in schemas { scopes.push(PyTuple::new(py, [database.clone(), text(py, schema)])?.into_any()); @@ -310,48 +280,32 @@ fn prefetch_schemas<'py>( .into_any(), ); } - call_with( - &native_field(&catalog, "prefetch_schema_listings")?, - &[ - ("scopes", tuple(py, scopes)?), - ("best_effort", true.into_pyobject(py)?.to_owned().into_any()), - ], - )?; - Ok(()) + request + .reads + .get() + .reads() + .prefetch_schema_listings(py, tuple(py, scopes)?) } -/// List relations through the inspection catalog wrappers. +/// The listed relations of `database` narrowed to `schemas` and `names`. pub(crate) fn list_relations<'py>( - (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + reads: &Bound<'py, PlannerReads>, database: &Bound<'py, PyAny>, schemas: Bound<'py, PyAny>, names: Bound<'py, PyAny>, ) -> PyResult> { - call_with( - &host_entry(adapter.py(), RELATIONS_MAIN_LIST)?, - &[ - ("adapter", adapter.clone()), - ("connection", connection.clone()), - ("database", database.clone()), - ("schemas", schemas), - ("names", names), - ], - ) + reads + .get() + .reads() + .relations(reads.py(), database, (schemas, names)) } -/// Columns of the relations through the inspection catalog wrappers. +/// The columns of each listed relation, keyed by identity. pub(crate) fn columns_for<'py>( - (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + reads: &Bound<'py, PlannerReads>, relations: Bound<'py, PyAny>, ) -> PyResult> { - call_with( - &host_entry(adapter.py(), RELATIONS_MAIN_COLUMNS)?, - &[ - ("adapter", adapter.clone()), - ("connection", connection.clone()), - ("relations", relations), - ], - ) + reads.get().reads().columns(reads.py(), relations) } fn optional_names<'py>( @@ -384,12 +338,7 @@ fn gather_relations<'py>( names: Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyDict>, StateSchemas)> { let py = request.project.py(); - let relations = list_relations( - (&request.adapter, &request.connection), - database, - schemas, - names, - )?; + let relations = list_relations(&request.reads, database, schemas, names)?; let logical = PyDict::new(py); for node in destination_nodes(&request.project)? { logical.set_item( @@ -460,7 +409,7 @@ fn gather_columns<'py>( return Ok(result); } let all = columns_for( - (&request.adapter, &request.connection), + &request.reads, tuple( py, needed @@ -543,40 +492,22 @@ fn gather_fingerprints<'py>( )?; for schema in schemas { let schema_lower = lower(&text(py, schema))?; - let mut set = read_latest_fingerprints(Some(&keywords( + let table_exists: bool = state_schemas.contains(&schema_lower); + let read_names: Option> = if read_unfiltered { + None + } else { + node_names_value.extract()? + }; + let mut set = request.reads.get().reads().fingerprints( py, - &[ - ("connection", request.connection.clone()), - ("execute", request.execute.clone()), - ( - "table_exists", - state_schemas - .contains(&schema_lower) - .into_pyobject(py)? - .to_owned() - .into_any(), - ), - ("database", database.clone()), - ("schema", text(py, schema)), - ( - "render_qualified_name", - native_field(&request.adapter, "render_qualified_name")?, - ), - ( - "render_read_latest_sql", - native_field(&request.adapter, "render_read_latest_fingerprints_sql")?, - ), - ( - "node_names", - if read_unfiltered { - py.None().into_bound(py) - } else { - node_names_value.clone() - }, - ), - ("filtered_node_types", filtered_types.clone()), - ], - )?))?; + FingerprintRead { + database: database.extract()?, + schema, + table_exists, + node_names: read_names, + filtered_node_types: filtered_types.extract()?, + }, + )?; if read_unfiltered { let by_identity = native_field(&set, "fingerprints_by_identity")?; let values: Vec> = if by_identity.is_none() { @@ -690,11 +621,11 @@ fn read_node_result_fingerprints<'py>( python_nodes: &Bound<'py, PyDict>, ) -> PyResult<()> { let py = request.project.py(); - let node_database: Bound<'py, PyAny> = database.into_bound_py_any(py)?; + let node_database: Bound<'py, PyAny> = database.clone().into_bound_py_any(py)?; let node_schema: Bound<'py, PyAny> = text(py, &schema); let mut table_exists = false; for relation in list_relations( - (&request.adapter, &request.connection), + &request.reads, &node_database, tuple(py, vec![node_schema.clone()])?, tuple(py, vec![text(py, FINGERPRINT_TABLE_NAME)])?, @@ -704,43 +635,24 @@ fn read_node_result_fingerprints<'py>( let name: String = native_field(&relation?, "name")?.extract()?; table_exists |= name.eq_ignore_ascii_case(FINGERPRINT_TABLE_NAME); } - let set = read_latest_fingerprints(Some(&keywords( + let set = request.reads.get().reads().fingerprints( py, - &[ - ("connection", request.connection.clone()), - ("execute", request.execute.clone()), - ( - "table_exists", - table_exists.into_pyobject(py)?.to_owned().into_any(), - ), - ("database", node_database), - ("schema", node_schema), - ( - "render_qualified_name", - native_field(&request.adapter, "render_qualified_name")?, - ), - ( - "render_read_latest_sql", - native_field(&request.adapter, "render_read_latest_fingerprints_sql")?, - ), - ("node_names", tuple(py, Vec::new())?), - ( - "filtered_node_types", - tuple( - py, - [ - NODE_TYPE_MODEL, - FUNCTION_NODE_TYPES[0], - FUNCTION_NODE_TYPES[1], - NODE_TYPE_SEED, - ] - .iter() - .map(|name| text(py, name)) - .collect(), - )?, - ), - ], - )?))?; + FingerprintRead { + database, + schema: &schema, + table_exists, + node_names: Some(Vec::new()), + filtered_node_types: [ + NODE_TYPE_MODEL, + FUNCTION_NODE_TYPES[0], + FUNCTION_NODE_TYPES[1], + NODE_TYPE_SEED, + ] + .iter() + .map(|name| (*name).to_owned()) + .collect(), + }, + )?; let by_identity = native_field(&set, "fingerprints_by_identity")?; if by_identity.is_none() { return Ok(()); @@ -799,7 +711,7 @@ pub(crate) fn warehouse_snapshot<'py>( } else { native_field(scope, "model_keys")? }; - let cursors = cursor_snapshots( + let (cursors, cursor_inputs_at_run) = cursor_snapshots( request, &relations, &CursorInputs { @@ -833,13 +745,19 @@ pub(crate) fn warehouse_snapshot<'py>( ("existing_columns", columns.into_any()), ("fingerprints", fingerprints), ("cursor_snapshots", cursors.into_any()), + ( + "cursor_inputs_at_run", + pyo3::types::PyBool::new(py, cursor_inputs_at_run) + .to_owned() + .into_any(), + ), ( "source_freshness_state_schemas", names_set(py, &states.freshness)?, ), ( "column_dialect", - request.adapter.call_method0("sql_analysis_dialect")?, + request.dialect.syntax("sql_analysis_dialect")?.call0()?, ), ( "old_name_view_state_schemas", @@ -865,7 +783,7 @@ pub(crate) fn gather_warehouse_snapshot<'py>( "gather_warehouse_snapshot", options, ( - &["project", "adapter", "connection", "execute"], + &["project", "dialect", "reads"], &[ "selected_keys", "full_refresh", @@ -877,9 +795,8 @@ pub(crate) fn gather_warehouse_snapshot<'py>( ), )?; let project = &args.value("project")?; - let adapter = &args.value("adapter")?; - let connection = &args.value("connection")?; - let execute = &args.value("execute")?; + let dialect = Dialect::passed(args.value("dialect")?)?; + let reads = PlannerReads::passed(args.value("reads")?)?; let selected_keys = &args.value("selected_keys")?; let full_refresh: bool = args.flag("full_refresh")?; let full_refresh_model_names = &args.value("full_refresh_model_names")?; @@ -889,9 +806,8 @@ pub(crate) fn gather_warehouse_snapshot<'py>( warehouse_snapshot(&SnapshotRequest { project: project.clone(), - adapter: adapter.clone(), - connection: connection.clone(), - execute: execute.clone(), + dialect, + reads, selected_keys: selected_keys.clone(), full_refresh, full_refresh_model_names: full_refresh_model_names.clone(), @@ -914,8 +830,8 @@ pub(crate) fn gather_redirected_cursor_snapshots<'py>( ( &[ "project", - "adapter", - "connection", + "dialect", + "reads", "existing_relations", "target_relations", "cursor_scope", @@ -925,8 +841,8 @@ pub(crate) fn gather_redirected_cursor_snapshots<'py>( ), )?; let project = &args.value("project")?; - let adapter = &args.value("adapter")?; - let connection = &args.value("connection")?; + let dialect = Dialect::passed(args.value("dialect")?)?; + let reads = PlannerReads::passed(args.value("reads")?)?; let existing_relations = args.value("existing_relations")?; let existing_relations = existing_relations.downcast::()?; let target_relations = args.value("target_relations")?.downcast_into::()?; @@ -944,9 +860,8 @@ pub(crate) fn gather_redirected_cursor_snapshots<'py>( } let request = SnapshotRequest { project: project.clone(), - adapter: adapter.clone(), - connection: connection.clone(), - execute: native_field(adapter, "execute")?, + dialect, + reads, selected_keys: frozenset(py, &keys)?, full_refresh: false, full_refresh_model_names: full_refresh_model_names.clone(), @@ -955,7 +870,7 @@ pub(crate) fn gather_redirected_cursor_snapshots<'py>( cursor_scope: cursor_scope.clone(), node_results_location: None, }; - let snapshots = cursor_snapshots( + let (snapshots, _) = cursor_snapshots( &request, existing_relations, &CursorInputs { diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/watermarks.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/watermarks.rs index c22cfb4a51..99280ad6fc 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/watermarks.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/watermarks.rs @@ -1,9 +1,9 @@ //! Validate reliable watermark column contracts before warehouse inspection. use crate::planner_strategy::main::cursor_types::classify_cursor_sql_type::classify_cursor_sql_type; -use crate::planner_warehouse::constants::{ - CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, WATERMARK_CONTRACT_CODE, -}; +use crate::planner_warehouse::constants::CURSOR_TYPE_INTEGER; +use crate::planner_warehouse::constants::CURSOR_TYPE_TIMESTAMP; +use crate::planner_warehouse::constants::WATERMARK_CONTRACT_CODE; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods}; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/mod.rs index 349ca8f38c..55a2b818a4 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/mod.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod inspection; pub(crate) mod sources; +pub(crate) mod synced; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/semantic_sources.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/semantic_sources.rs index eabd9cf952..28faf17221 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/semantic_sources.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/semantic_sources.rs @@ -1,15 +1,14 @@ //! Complete missing source metadata in bounded, database-grouped batches. -use crate::planner_warehouse::_helpers::inspection::snapshot::{columns_for, list_relations}; -use crate::planner_warehouse::constants::ACTIVE_CATALOG; +use crate::planner_warehouse::_helpers::inspection::snapshot::columns_for; +use crate::planner_warehouse::_helpers::inspection::snapshot::list_relations; +use crate::planner_warehouse::models::PlannerReads; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_model_config::config_values::models::ConfigNode; -use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; @@ -58,30 +57,20 @@ fn sorted_texts<'py>(py: Python<'py>, values: BTreeSet) -> PyResult( - (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + reads: &Bound<'py, PlannerReads>, database: &Bound<'py, PyAny>, entries: &[Bound<'py, PyAny>], ) -> PyResult>> { - let py = adapter.py(); + let py = reads.py(); let mut pending: Vec> = entries.to_vec(); let mut relations: Vec> = Vec::new(); - let catalog = call_with( - &host_entry(py, ACTIVE_CATALOG)?, - &[ - ("adapter", adapter.clone()), - ("connection", connection.clone()), - ], - )?; let mut scoped: Vec<_> = Vec::new(); for entry in entries { if !native_field(entry, "schema")?.is_none() { scoped.push(entry.clone()); } } - if !catalog.is_none() - && native_field(&catalog, "schema_scoped")?.is_truthy()? - && !scoped.is_empty() - { + if !scoped.is_empty() && reads.get().reads().lists_by_schema(py)? { let mut schemas: BTreeSet<_> = BTreeSet::new(); let mut names: BTreeSet<_> = BTreeSet::new(); for entry in &scoped { @@ -93,7 +82,7 @@ fn source_candidates<'py>( names.insert(wanted_name(entry)?.extract::()?); } let listed = list_relations( - (adapter, connection), + reads, database, sorted_texts(py, schemas)?, sorted_texts(py, names)?, @@ -122,7 +111,7 @@ fn source_candidates<'py>( names.insert(wanted_name(entry)?.extract::()?); } let listed = list_relations( - (adapter, connection), + reads, database, py.None().into_bound(py), sorted_texts(py, names)?, @@ -143,18 +132,14 @@ fn source_candidates<'py>( /// Source relation candidates by name; schema listings first answer schema-scoped catalogs. #[pyfunction] -#[pyo3(signature = (*, adapter, connection, database, entries))] +#[pyo3(signature = (*, reads, database, entries))] pub(crate) fn list_source_candidates<'py>( - adapter: &Bound<'py, PyAny>, - connection: &Bound<'py, PyAny>, + reads: &Bound<'py, PlannerReads>, database: &Bound<'py, PyAny>, entries: &Bound<'py, PyAny>, ) -> PyResult> { let entries: Vec> = entries.try_iter()?.collect::>()?; - tuple( - adapter.py(), - source_candidates((adapter, connection), database, &entries)?, - ) + tuple(reads.py(), source_candidates(reads, database, &entries)?) } /// Reuse collected columns and inspect only selected, unresolved table sources. @@ -172,8 +157,7 @@ pub(crate) fn semantic_source_columns<'py>( ( &[ "project", - "adapter", - "connection", + "reads", "source_read_map", "columns", "selected_keys", @@ -182,8 +166,7 @@ pub(crate) fn semantic_source_columns<'py>( ), )?; let project = &args.value("project")?; - let adapter = &args.value("adapter")?; - let connection = &args.value("connection")?; + let reads = args.value("reads")?; let source_read_map = &args.value("source_read_map")?; let columns = args.value("columns")?.downcast_into::()?; let columns = &columns; @@ -191,10 +174,11 @@ pub(crate) fn semantic_source_columns<'py>( let py = project.py(); if !native_field(&native_field(project, "settings")?, "sql_analysis")?.is_truthy()? - || connection.is_none() + || reads.is_none() { return Ok(columns.clone()); } + let reads = &PlannerReads::passed(reads)?; let mut required: BTreeSet = BTreeSet::new(); for model in compiled_models(project)?.try_iter()? { let model = model?; @@ -239,7 +223,7 @@ pub(crate) fn semantic_source_columns<'py>( } let result = columns.copy()?; for (database, entries) in groups { - let relations = source_candidates((adapter, connection), &database, &entries)?; + let relations = source_candidates(reads, &database, &entries)?; let mut chosen: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); for entry in &entries { let mut candidates: Vec<_> = Vec::new(); @@ -266,10 +250,7 @@ pub(crate) fn semantic_source_columns<'py>( for (_, relation) in &chosen { unique.set_item(relation, py.None())?; } - columns_for( - (adapter, connection), - PyTuple::new(py, unique.keys())?.into_any(), - )? + columns_for(reads, PyTuple::new(py, unique.keys())?.into_any())? }; for (name, relation) in chosen { let lower = |value: Bound<'py, PyAny>| -> PyResult> { diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs index accc47be4b..76a89a10d8 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs @@ -1,16 +1,16 @@ //! Warehouse source relation listing and column gathering for planning. -use crate::observations::main::capture_source_columns::capture_source_columns; use crate::planner_warehouse::_helpers::inspection::snapshot::{columns_for, list_relations}; use crate::planner_warehouse::constants::{ COLUMN_INFO, METADATA_NAME_FILTER_LIMIT, RELATION_LOOKUP, }; +use crate::planner_warehouse::models::PlannerReads; +use crate::planner_warehouse::models::Read; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; use sqlbuild_python_project::compiled_views::main::project::sources::sources; use sqlbuild_python_project::planner_phases::main::frozenset::frozenset; use sqlbuild_python_project::planner_phases::main::text::text; @@ -81,7 +81,7 @@ fn names_tuple<'py>(py: Python<'py>, names: &BTreeSet) -> PyResult( project: &Bound<'py, PyAny>, - (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + reads: &Bound<'py, PlannerReads>, source_entries: Option>, ) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyAny>)> { let py = project.py(); @@ -98,32 +98,18 @@ pub(crate) fn source_inspection<'py>( let sql: String = expression.extract()?; let source: Bound<'py, PyAny> = native_field(entry, "name")?; let source_name: String = source.extract()?; - let described: Vec = match adapter.downcast::() { - Ok(facade) => match facade.get().described_columns(&source_name, &sql)? { - Some(described) => described, - None => { - facade.get().mark_unknown(format!( - "query columns of expression source {source_name} (planned as declared)" - )); - native_field(entry, "columns")? - .try_iter()? - .map(|column| native_field(&column?, "name")?.extract()) - .collect::>()? - } - }, - Err(_) => { - let names = call_with( - &native_field(adapter, "query_column_names")?, - &[("connection", connection.clone()), ("sql", expression)], - )?; - let described: Vec = names + let described: Vec = + match reads + .get() + .reads() + .described_columns(py, &source_name, &sql)? + { + Read::Known(described) => described, + Read::Unknown(_) => native_field(entry, "columns")? .try_iter()? - .map(|name| name?.extract()) - .collect::>()?; - capture_source_columns(&source_name, &sql, &described); - described - } - }; + .map(|column| native_field(&column?, "name")?.extract()) + .collect::>()?, + }; let mut columns: Vec<_> = Vec::new(); for name in &described { columns.push(call_with( @@ -183,7 +169,7 @@ pub(crate) fn source_inspection<'py>( } } let relations = list_relations( - (adapter, connection), + reads, &database_value, names_tuple(py, &schemas)?, match name_filter(&entries, database, &schemas)? { @@ -217,7 +203,7 @@ pub(crate) fn source_inspection<'py>( chosen.push(relation.clone()); } } - let all = columns_for((adapter, connection), tuple(py, chosen)?)?; + let all = columns_for(reads, tuple(py, chosen)?)?; for (_, name, identity) in &scoped { if all.contains(identity)? { result.set_item(name, all.get_item(identity)?)?; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_deferral.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_deferral.rs index 30f0337351..b17a99ad39 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_deferral.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_deferral.rs @@ -2,7 +2,8 @@ use sqlbuild_host::entries::models::UserCodeHost; -use crate::planner_warehouse::constants::{MODEL_HOOK_CONFIG_KEYS, SQL_HOOK_ENTRY}; +use crate::planner_warehouse::constants::MODEL_HOOK_CONFIG_KEYS; +use crate::planner_warehouse::constants::SQL_HOOK_ENTRY; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs index 5a3b9c02d1..43cdd09499 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs @@ -2,8 +2,10 @@ use crate::freshness::main::build_direct_source_freshness_planning_result_function::build_direct_source_freshness_planning_result_function; use crate::freshness::main::build_direct_source_freshness_propagation_result_function::build_direct_source_freshness_propagation_result_function; -use crate::planner_warehouse::_helpers::inspection::snapshot::{destination_nodes, lower}; +use crate::planner_warehouse::_helpers::inspection::snapshot::destination_nodes; +use crate::planner_warehouse::_helpers::inspection::snapshot::lower; use crate::planner_warehouse::constants::RUN_SCOPE; +use crate::planner_warehouse::models::PlannerReads; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; use pyo3::types::PyDict as KeywordDict; @@ -14,8 +16,9 @@ use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::models::Dialect; +use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; use sqlbuild_python_project::planner_phases::main::replace::replace; use sqlbuild_python_project::planner_phases::main::text::text; use sqlbuild_python_project::planner_phases::main::tuple::tuple; @@ -72,13 +75,16 @@ pub(crate) fn build_planner_source_freshness_result_of<'py>( "build_planner_source_freshness_result", options, ( - &["project", "adapter", "connection", "scope", "relations"], + &["project", "dialect", "reads", "scope", "relations"], &["freshness_state_schemas"], ), )?; build_planner_source_freshness_result( (&args.value("project")?, &args.value("scope")?), - (&args.value("adapter")?, &args.value("connection")?), + ( + &Dialect::passed(args.value("dialect")?)?, + &PlannerReads::passed(args.value("reads")?)?, + ), &args.value("relations")?, &args.value("freshness_state_schemas")?, ) @@ -87,7 +93,7 @@ pub(crate) fn build_planner_source_freshness_result_of<'py>( /// The source freshness evaluation of one inspection scope against the warehouse. pub(crate) fn build_planner_source_freshness_result<'py>( (project, scope): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (dialect, reads): (&Bound<'py, Dialect>, &Bound<'py, PlannerReads>), relations: &Bound<'py, PyAny>, freshness_state_schemas: &Bound<'py, PyAny>, ) -> PyResult> { @@ -100,15 +106,11 @@ pub(crate) fn build_planner_source_freshness_result<'py>( lowered_state.insert(schema?.extract()?); } } - let synced_copy = adapter.is_instance_of::(); let exists = PyDict::new(py); for schema in &schemas { exists.set_item( schema, - PyBool::new( - py, - !synced_copy && lowered_state.contains(&lower(&text(py, schema))?), - ), + PyBool::new(py, lowered_state.contains(&lower(&text(py, schema))?)), )?; } let sources = tuple( @@ -126,8 +128,8 @@ pub(crate) fn build_planner_source_freshness_result<'py>( call_with( build_direct_source_freshness_planning_result_function(py)?.as_any(), &[ - ("adapter", adapter.clone()), - ("connection", connection.clone()), + ("dialect", dialect.clone().into_any()), + ("reads", reads.clone().into_any()), ("sources", sources), ("state_database", state_database(project)?), ( @@ -138,7 +140,7 @@ pub(crate) fn build_planner_source_freshness_result<'py>( ("run_id", text(py, "planning")), ( "render_qualified_name", - native_field(adapter, "render_qualified_name")?, + dialect.syntax("render_qualified_name")?, ), ("state_table_exists_by_schema", exists.into_any()), ( diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs index 1e743fe847..be90865386 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs @@ -1,9 +1,9 @@ //! Selected-scope validation that external source tables exist in the warehouse. -use crate::planner_warehouse::constants::{ - MISSING_SOURCE_TABLE_CODE, RUN_BOUNDED, WILDCARD_TABLE_CHARACTER, -}; -use pyo3::IntoPyObject; +use crate::planner_warehouse::constants::MISSING_SOURCE_TABLE_CODE; +use crate::planner_warehouse::constants::WILDCARD_TABLE_CHARACTER; +use crate::planner_warehouse::models::PlannerReads; +use crate::planner_warehouse::models::Read; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; use pyo3::types::PyDict as KeywordDict; @@ -13,17 +13,12 @@ use sqlbuild_analysis::planner_scope::constants::SOURCE; use sqlbuild_analysis::planner_warehouse::main::missing_source_tables_help::missing_source_tables_help; use sqlbuild_analysis::planner_warehouse::main::missing_source_tables_message::missing_source_tables_message; use sqlbuild_analysis::planner_warehouse::models::MissingSourceTable; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::models::Dialect; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; -use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; use sqlbuild_python_front::discovery::main::project::yaml_entry_line::yaml_entry_line; use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; -use sqlbuild_python_project::planner_phases::main::tuple::tuple; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; @@ -73,10 +68,14 @@ fn external_table(entry: &Bound<'_, PyAny>) -> PyResult { } fn read_relation<'py>( - adapter: &Bound<'py, PyAny>, + dialect: &Bound<'py, Dialect>, entry: &Bound<'py, PyAny>, ) -> PyResult> { - Ok(PyString::new(adapter.py(), &render_source_relation(entry, Some(adapter))?).into_any()) + Ok(PyString::new( + dialect.py(), + &render_source_relation(entry, Some(dialect.as_any()))?, + ) + .into_any()) } fn declaration_location(source: &Bound<'_, PyAny>) -> PyResult { @@ -98,7 +97,7 @@ fn declaration_location(source: &Bound<'_, PyAny>) -> PyResult { /// Raise S405 when a selected resource reads an external source table that does not exist. pub(crate) fn check_selected_source_tables_exist<'py>( project: &Bound<'py, PyAny>, - (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (dialect, reads): (&Bound<'py, Dialect>, &Bound<'py, PlannerReads>), scope: &Bound<'py, PyAny>, relations: &Bound<'py, PyAny>, ) -> PyResult<()> { @@ -129,38 +128,21 @@ pub(crate) fn check_selected_source_tables_exist<'py>( sorted.sort(); unlisted.push((source, entry, sorted)); } - if unlisted.is_empty() || adapter.is_instance_of::() { + if unlisted.is_empty() { return Ok(()); } - let probe = native_field(adapter, "probe_relation_read")?; - let mut tasks: Vec<_> = Vec::new(); - for (_, entry, _) in &unlisted { - let fields = keywords( - py, - &[ - ("connection", connection.clone()), - ("relation", read_relation(adapter, entry)?), - ], - )?; - tasks.push(imported(py, "functools", "partial")?.call((&probe,), Some(&fields))?); - } - let concurrency: i64 = imported(py, "builtins", "int")? - .call1((native_field(adapter, "metadata_inspection_concurrency")?,))? - .extract()?; - let probes = call_with( - &host_entry(py, RUN_BOUNDED)?, - &[ - ("tasks", tuple(py, tasks)?), - ( - "concurrency", - concurrency.max(1).into_pyobject(py)?.into_any(), - ), - ], - )?; + let relations = || -> PyResult>> { + unlisted + .iter() + .map(|(_, entry, _)| read_relation(dialect, entry)) + .collect() + }; + let Read::Known(probes) = reads.get().reads().read_access(py, &relations)? else { + return Ok(()); + }; let mut missing: Vec<_> = Vec::new(); let mut entries: Vec<_> = Vec::new(); - for ((source, entry, read_by), probe) in unlisted.into_iter().zip(probes.try_iter()?) { - let probe = probe?; + for ((source, entry, read_by), probe) in unlisted.into_iter().zip(probes) { let status = native_field(&probe, "status")?; if status.eq("readable")? { continue; @@ -187,7 +169,7 @@ pub(crate) fn check_selected_source_tables_exist<'py>( return Ok(()); } for (item, entry) in missing.iter_mut().zip(&entries).take(10) { - item.relation = read_relation(adapter, entry)?.str()?.extract()?; + item.relation = read_relation(dialect, entry)?.str()?.extract()?; } let message = missing_source_tables_message(&missing); let sources = PyDict::new(py); @@ -224,16 +206,13 @@ pub(crate) fn check_selected_source_tables_exist_binding<'py>( py, "check_selected_source_tables_exist", options, - ( - &["project", "adapter", "connection", "scope", "relations"], - &[], - ), + (&["project", "dialect", "reads", "scope", "relations"], &[]), )?; let project = &args.value("project")?; - let adapter = &args.value("adapter")?; - let connection = &args.value("connection")?; + let dialect = &Dialect::passed(args.value("dialect")?)?; + let reads = &PlannerReads::passed(args.value("reads")?)?; let scope = &args.value("scope")?; let relations = &args.value("relations")?; - check_selected_source_tables_exist(project, (adapter, connection), scope, relations) + check_selected_source_tables_exist(project, (dialect, reads), scope, relations) } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/synced/exception_types.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/synced/exception_types.rs new file mode 100644 index 0000000000..bd0a2c3e48 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/synced/exception_types.rs @@ -0,0 +1,11 @@ +//! The error a plan preview raises for a warehouse read it refuses. + +use pyo3::create_exception; +use pyo3::exceptions::PyRuntimeError; + +create_exception!( + sqlbuild._native, + WarehouseAccessRefusedError, + PyRuntimeError, + "Raised when a plan preview reaches past the synced warehouse state for the warehouse." +); diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/synced/mod.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/synced/mod.rs new file mode 100644 index 0000000000..b75b726f39 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/synced/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod exception_types; +pub(crate) mod synced_reads; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/synced/synced_reads.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/synced/synced_reads.rs new file mode 100644 index 0000000000..4db9775282 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/synced/synced_reads.rs @@ -0,0 +1,90 @@ +//! Warehouse reads answered from synced state, refusing what it cannot answer. + +use crate::planner_strategy::main::model_entries::column_info::column_info; +use crate::planner_warehouse::_helpers::synced::exception_types::WarehouseAccessRefusedError; +use chrono::SecondsFormat; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use sqlbuild_host::entries::models::AdapterHost; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_state::warehouse_state::models::{StateFingerprint, StateRelation}; + +/// The error a preview raises for a read only a live plan makes. +pub(crate) fn refused(access: &str) -> pyo3::PyErr { + WarehouseAccessRefusedError::new_err(format!( + "plan previews read only the synced warehouse state; refused warehouse access: {access}" + )) +} + +pub(crate) fn relation_info<'py>( + py: Python<'py>, + relation: &StateRelation, +) -> PyResult> { + call_with( + &host_entry(py, AdapterHost::RelationInfo)?, + &[ + ( + "database", + relation.database.as_deref().into_pyobject(py)?.into_any(), + ), + ("schema", PyString::new(py, &relation.schema).into_any()), + ("name", PyString::new(py, &relation.name).into_any()), + ( + "relation_type", + PyString::new(py, &relation.relation_type).into_any(), + ), + ], + ) +} + +pub(crate) fn relation_columns<'py>( + py: Python<'py>, + relation: &StateRelation, +) -> PyResult> { + let class: Bound<'py, PyAny> = host_entry(py, AdapterHost::ColumnInfo)?; + let columns: Vec> = relation + .columns + .iter() + .map(|column| { + column_info( + &class, + ( + PyString::new(py, &column.name).into_any(), + PyString::new(py, &column.data_type).into_any(), + ), + ) + }) + .collect::>()?; + PyTuple::new(py, columns) +} + +pub(crate) fn fingerprint<'py>( + datetime: &Bound<'py, PyAny>, + fingerprint: &StateFingerprint, +) -> PyResult> { + let py: Python<'py> = datetime.py(); + let kwargs: Bound<'py, PyDict> = PyDict::new(py); + kwargs.set_item("node_type", &fingerprint.node_type)?; + kwargs.set_item("node_name", &fingerprint.node_name)?; + kwargs.set_item("target_database", &fingerprint.target_database)?; + kwargs.set_item("target_schema", &fingerprint.target_schema)?; + kwargs.set_item("target_name", &fingerprint.target_name)?; + kwargs.set_item("run_id", &fingerprint.run_id)?; + kwargs.set_item("definition_hash", &fingerprint.definition_hash)?; + kwargs.set_item("version_hash", &fingerprint.version_hash)?; + kwargs.set_item("schema_fingerprint", &fingerprint.schema_fingerprint)?; + kwargs.set_item("definition", &fingerprint.definition)?; + kwargs.set_item("metadata_json", &fingerprint.metadata_json)?; + kwargs.set_item( + "ts", + datetime.call_method1( + "fromisoformat", + (fingerprint.ts.to_rfc3339_opts(SecondsFormat::AutoSi, true),), + )?, + )?; + new_record(py, RecordId::Fingerprint, (), Some(&kwargs)) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs index e60f0d0305..7f31dc14a4 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs @@ -3,20 +3,8 @@ use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; use sqlbuild_python_base::records::types::RecordId; -pub(crate) const RELATIONS_MAIN_LIST: AdapterHost = AdapterHost::ListRelationsForInspection; - -pub(crate) const RELATIONS_MAIN_COLUMNS: AdapterHost = AdapterHost::GetColumnsForInspection; - -pub(crate) const ACTIVE_CATALOG: AdapterHost = AdapterHost::ActiveInspectionCatalog; - pub const RUN_BOUNDED: AdapterHost = AdapterHost::RunBoundedInspections; -pub(crate) const RECORD_QUERY: AdapterHost = AdapterHost::RecordInspectionQuery; - -pub(crate) const RUN_RECORDED_QUERY: AdapterHost = AdapterHost::RunRecordedInspectionQuery; - -pub(crate) const QUERY_RECORD: AdapterHost = AdapterHost::InspectionQueryRecord; - pub(crate) const COLUMN_INFO: AdapterHost = AdapterHost::ColumnInfo; pub(crate) const RELATION_LOOKUP: AdapterHost = AdapterHost::RelationLookup; @@ -51,3 +39,18 @@ pub(crate) const CURSOR_TYPE_INTEGER: &str = "integer"; pub(crate) const MISSING_SOURCE_TABLE_CODE: &str = "S405"; pub(crate) const WATERMARK_CONTRACT_CODE: &str = "S302"; + +/// Why a plan preview leaves a source's freshness unknown; its plan lists freshness as unknown. +pub const PREVIEW_MESSAGE: &str = "a plan preview does not observe source freshness"; + +/// Items a preview always reports as known only once the run plans live. +pub const PREVIEW_UNKNOWNS: [&str; 5] = [ + "source freshness", + "retention", + "column rename support", + "sources outside mirrored schemas", + "recorded migrations", +]; + +/// What a preview's cursor reads leave to the run. +pub const CURSOR_INPUTS: &str = "cursor inputs"; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/adapter_cursor_sql.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/adapter_cursor_sql.rs deleted file mode 100644 index e079335ca7..0000000000 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/adapter_cursor_sql.rs +++ /dev/null @@ -1,38 +0,0 @@ -//! The query's MIN/MAX statement, each bound read as text the way the adapter renders it. - -use crate::planner_warehouse::_helpers::inspection::cursors::CURSOR_TEXT_METHOD; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; -use sqlbuild_analysis::planner_warehouse::main::cursor_query_sql::cursor_query_sql; -use sqlbuild_analysis::planner_warehouse::models::PhysicalCursorQuery; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_project::planner_phases::main::text::text; -use std::cell::RefCell; - -/// The query's MIN/MAX statement, each bound read as text the way the adapter renders it. -pub fn adapter_cursor_sql( - adapter: &Bound<'_, PyAny>, - query: &PhysicalCursorQuery, -) -> PyResult { - let py = adapter.py(); - let render = native_field(adapter, CURSOR_TEXT_METHOD)?; - let failure: RefCell> = RefCell::new(None); - let sql: String = cursor_query_sql(query, &|expression: &str| { - call_with( - &render, - &[ - ("expression", text(py, expression)), - ("cursor_type", text(py, &query.cursor_type)), - ], - ) - .and_then(|rendered| rendered.extract::()) - .unwrap_or_else(|error| { - failure.replace(Some(error)); - String::new() - }) - }); - match failure.into_inner() { - Some(error) => Err(error), - None => Ok(sql), - } -} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/bound_task.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/bound_task.rs deleted file mode 100644 index 30c54090f5..0000000000 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/bound_task.rs +++ /dev/null @@ -1,27 +0,0 @@ -//! Bound task. - -use crate::planner_warehouse::_helpers::inspection::cursors::read_cursor_query; -use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; -use pyo3::types::{PyCFunction, PyDict, PyTuple}; - -/// Bound task. -pub fn bound_task<'py>( - py: Python<'py>, - sql: String, - (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult> { - let connection: Py = connection.clone().unbind(); - let execute: Py = execute.clone().unbind(); - Ok(PyCFunction::new_closure( - py, - None, - None, - move |args: &Bound<'_, PyTuple>, - _kwargs: Option<&Bound<'_, PyDict>>| - -> PyResult> { - let py = args.py(); - Ok(read_cursor_query(py, &sql, (connection.bind(py), execute.bind(py)))?.unbind()) - }, - )? - .into_any()) -} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/columns_for.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/columns_for.rs new file mode 100644 index 0000000000..6cc54caf39 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/columns_for.rs @@ -0,0 +1,14 @@ +//! The columns of listed relations. + +use crate::planner_warehouse::_helpers::inspection::snapshot; +use crate::planner_warehouse::models::PlannerReads; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// The columns of each listed relation, keyed by identity. +#[inline] +pub fn columns_for<'py>( + reads: &Bound<'py, PlannerReads>, + relations: Bound<'py, PyAny>, +) -> PyResult> { + snapshot::columns_for(reads, relations) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/dialect_cursor_sql.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/dialect_cursor_sql.rs new file mode 100644 index 0000000000..73e40877c4 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/dialect_cursor_sql.rs @@ -0,0 +1,15 @@ +//! A physical cursor query rendered in a plan's SQL dialect. + +use crate::planner_warehouse::_helpers::inspection::cursors; +use pyo3::prelude::{Bound, PyResult}; +use sqlbuild_analysis::planner_warehouse::models::PhysicalCursorQuery; +use sqlbuild_python_base::native_host::models::Dialect; + +/// A physical cursor query rendered in a plan's SQL dialect. +#[inline] +pub fn dialect_cursor_sql( + dialect: &Bound<'_, Dialect>, + query: &PhysicalCursorQuery, +) -> PyResult { + cursors::dialect_cursor_sql(dialect, query) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/list_relations.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/list_relations.rs new file mode 100644 index 0000000000..356f7912bc --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/list_relations.rs @@ -0,0 +1,16 @@ +//! The listed relations of a database, narrowed to schemas and names. + +use crate::planner_warehouse::_helpers::inspection::snapshot; +use crate::planner_warehouse::models::PlannerReads; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// The listed relations of `database` narrowed to `schemas` and `names`. +#[inline] +pub fn list_relations<'py>( + reads: &Bound<'py, PlannerReads>, + database: &Bound<'py, PyAny>, + schemas: Bound<'py, PyAny>, + names: Bound<'py, PyAny>, +) -> PyResult> { + snapshot::list_relations(reads, database, schemas, names) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/mod.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/mod.rs index b6ff3dc93c..0e33931cab 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/mod.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/mod.rs @@ -1,7 +1,8 @@ -pub mod adapter_cursor_sql; -pub mod bound_task; +pub mod columns_for; +pub mod dialect_cursor_sql; pub mod gather_redirected_cursor_snapshots; pub mod gather_warehouse_snapshot_function; +pub mod list_relations; pub mod migration_fingerprint_work_info_function; pub mod planner_metadata_name_filter_function; pub mod warehouse_snapshot; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/mod.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/mod.rs index 87c68376c1..6e71f1975a 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/mod.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/mod.rs @@ -1,2 +1,3 @@ pub mod inspection; +pub mod reads; pub mod sources; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/reads/_unknown.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/reads/_unknown.rs new file mode 100644 index 0000000000..9152f40285 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/reads/_unknown.rs @@ -0,0 +1,15 @@ +//! A read whose answer is not known. + +use crate::planner_warehouse::models::Read; +use crate::planner_warehouse::types::WarehouseReads; +use std::borrow::Cow; + +/// Answer `Unknown(item)` and note the item, so the plan reports it. +pub(in crate::planner_warehouse) fn unknown( + reads: &dyn WarehouseReads, + item: impl Into>, +) -> Read { + let item: Cow<'static, str> = item.into(); + reads.note_unknown(item.clone().into_owned()); + Read::Unknown(item) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/reads/is_refused_access.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/reads/is_refused_access.rs new file mode 100644 index 0000000000..e9343e73bb --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/reads/is_refused_access.rs @@ -0,0 +1,10 @@ +//! Whether an error is a plan preview's refusal of a warehouse read. + +use crate::planner_warehouse::_helpers::synced::exception_types::WarehouseAccessRefusedError; +use pyo3::{PyErr, Python}; + +/// Whether `error` is a plan preview's refusal of a warehouse read. +#[inline] +pub fn is_refused_access(py: Python<'_>, error: &PyErr) -> bool { + error.is_instance_of::(py) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/reads/mod.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/reads/mod.rs new file mode 100644 index 0000000000..d5604f6a20 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/reads/mod.rs @@ -0,0 +1,4 @@ +pub(in crate::planner_warehouse) mod _unknown; +pub mod is_refused_access; +pub mod refused_access; +pub mod register_exception_types; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/reads/refused_access.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/reads/refused_access.rs new file mode 100644 index 0000000000..d1db741a55 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/reads/refused_access.rs @@ -0,0 +1,10 @@ +//! The error a plan preview raises for a read only a live plan makes. + +use crate::planner_warehouse::_helpers::synced::synced_reads::refused; +use pyo3::PyErr; + +/// The error a plan preview raises for a read only a live plan makes. +#[inline] +pub fn refused_access(access: &str) -> PyErr { + refused(access) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/reads/register_exception_types.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/reads/register_exception_types.rs new file mode 100644 index 0000000000..bfffd446f3 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/reads/register_exception_types.rs @@ -0,0 +1,12 @@ +//! Register the warehouse read exception types. + +use crate::planner_warehouse::_helpers::synced::exception_types::WarehouseAccessRefusedError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the warehouse read exception types. +pub fn register_exception_types(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add( + "WarehouseAccessRefusedError", + module.py().get_type::(), + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result.rs index a5f1839f35..14402f304e 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result.rs @@ -1,19 +1,21 @@ //! The source freshness evaluation of one inspection scope against the warehouse. use crate::planner_warehouse::_helpers::sources::source_freshness; +use crate::planner_warehouse::models::PlannerReads; use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::native_host::models::Dialect; /// The source freshness evaluation of one inspection scope against the warehouse. #[inline] pub fn build_planner_source_freshness_result<'py>( (project, scope): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (dialect, reads): (&Bound<'py, Dialect>, &Bound<'py, PlannerReads>), relations: &Bound<'py, PyAny>, freshness_state_schemas: &Bound<'py, PyAny>, ) -> PyResult> { source_freshness::build_planner_source_freshness_result( (project, scope), - (adapter, connection), + (dialect, reads), relations, freshness_state_schemas, ) diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist.rs index 6ad31a35c8..b223fbbacd 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist.rs @@ -1,20 +1,17 @@ //! Raise S405 when a selected resource reads an external source table that does not exist. use crate::planner_warehouse::_helpers::sources::source_tables; +use crate::planner_warehouse::models::PlannerReads; use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::native_host::models::Dialect; /// Raise S405 when a selected resource reads an external source table that does not exist. #[inline] pub fn check_selected_source_tables_exist<'py>( project: &Bound<'py, PyAny>, - (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (dialect, reads): (&Bound<'py, Dialect>, &Bound<'py, PlannerReads>), scope: &Bound<'py, PyAny>, relations: &Bound<'py, PyAny>, ) -> PyResult<()> { - source_tables::check_selected_source_tables_exist( - project, - (adapter, connection), - scope, - relations, - ) + source_tables::check_selected_source_tables_exist(project, (dialect, reads), scope, relations) } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns.rs index d3d36b5bb4..3c837638a7 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns.rs @@ -1,17 +1,17 @@ //! Gather source columns. use crate::planner_warehouse::_helpers::sources::source_columns::source_inspection; +use crate::planner_warehouse::models::PlannerReads; use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::pyfunction; use pyo3::types::PyDict; #[pyfunction] -#[pyo3(signature = (*, project, adapter, connection, source_entries=None))] +#[pyo3(signature = (*, project, reads, source_entries=None))] pub fn gather_source_columns<'py>( project: &Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, - connection: &Bound<'py, PyAny>, + reads: &Bound<'py, PlannerReads>, source_entries: Option>, ) -> PyResult> { - Ok(source_inspection(project, (adapter, connection), source_entries)?.0) + Ok(source_inspection(project, reads, source_entries)?.0) } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_inspection.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_inspection.rs index 62fe3bdf29..ca19258d2e 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_inspection.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_inspection.rs @@ -1,16 +1,15 @@ //! Gather source inspection. use crate::planner_warehouse::_helpers::sources::source_columns::source_inspection; +use crate::planner_warehouse::models::PlannerReads; use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyTuple; -/// Gather source inspection. pub fn gather_source_inspection<'py>( project: &Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, - connection: &Bound<'py, PyAny>, + reads: &Bound<'py, PlannerReads>, source_entries: Option>, ) -> PyResult> { - let (columns, listed) = source_inspection(project, (adapter, connection), source_entries)?; + let (columns, listed) = source_inspection(project, reads, source_entries)?; Ok(PyTuple::new(project.py(), [columns.into_any(), listed])?.into_any()) } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/mod.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/mod.rs index 106e3b1428..15cccaede4 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/mod.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/mod.rs @@ -4,3 +4,4 @@ pub(crate) mod _helpers; pub mod constants; pub mod main; pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs index 76f09c8206..3f669b09b4 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs @@ -14,7 +14,19 @@ use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::connect; use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::count_computation; use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::package_version; use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::sqlite_error; +use crate::planner_warehouse::_helpers::synced::synced_reads; +use crate::planner_warehouse::_helpers::synced::synced_reads::refused; +use crate::planner_warehouse::_helpers::synced::synced_reads::relation_columns; +use crate::planner_warehouse::_helpers::synced::synced_reads::relation_info; +use crate::planner_warehouse::constants::CURSOR_INPUTS; +use crate::planner_warehouse::constants::PREVIEW_MESSAGE; +use crate::planner_warehouse::constants::PREVIEW_UNKNOWNS; +use crate::planner_warehouse::main::reads::_unknown::unknown; +use crate::planner_warehouse::types::CursorProgress; +use crate::planner_warehouse::types::WarehouseReads; +use chrono::SecondsFormat; use pyo3::exceptions::PyOSError; +use pyo3::exceptions::PyRuntimeError; use pyo3::exceptions::PyValueError; use pyo3::prelude::Bound; use pyo3::prelude::Py; @@ -29,40 +41,59 @@ use pyo3::types::PyDict; use pyo3::types::PyList; use pyo3::types::PyListMethods; use pyo3::types::PyTuple; +use pyo3::types::{PyDictMethods, PyString}; use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_entry::fingerprint_cache_entry; use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_key::fingerprint_cache_key; use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_lookup::fingerprint_cache_lookup; use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_shared_digest::fingerprint_cache_shared_digest; use sqlbuild_analysis::planner_warehouse::models::FingerprintCacheKey; +use sqlbuild_host::entries::models::AdapterHost; use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::models::Dialect; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::native_host::models::Location; +use sqlbuild_python_front::native_host::types::ConsumptionReads; +use sqlbuild_python_front::state_rows::models::EventTableLayout; +use sqlbuild_python_project::fingerprints::main::selected_set::selected_set; use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_state::consumption::constants::RECENT_EXECUTIONS; +use sqlbuild_state::consumption::main::recent_executions::recent_executions; +use sqlbuild_state::consumption::models::ConsumptionRow; +use sqlbuild_state::warehouse_state::models::{ + InvalidObservation, RelationFilter, StateLocation, StateRelation, WarehouseState, +}; +use std::borrow::Cow; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::HashMap; +use std::sync::Arc; +use std::sync::{Mutex, PoisonError}; /// Reuse migration fingerprints across rename maps, planning phases and invocations. #[pyclass(module = "sqlbuild._native")] #[derive(Debug)] pub struct MigrationFingerprintCache { - pub(crate) ref_names: HashMap<(String, String), Vec>, - pub(crate) fingerprints: HashMap>, - pub(crate) database_path: Option>, - pub(crate) connection: Option>, - pub(crate) connection_opened: bool, - pub(crate) pending: Vec<(String, String)>, - pub(crate) shared_digest: String, + ref_names: HashMap<(String, String), Vec>, + fingerprints: HashMap>, + database_path: Option>, + store: Option>, + store_opened: bool, + pending: Vec<(String, String)>, + shared_digest: String, } impl MigrationFingerprintCache { /// A cache persisting under `root`, or an in-memory cache without one. - pub fn open(py: Python<'_>, root: Option>) -> PyResult { + pub(crate) fn open(py: Python<'_>, root: Option>) -> PyResult { let root = root.filter(|root| !root.is_none()); let database_path = match &root { Some(root) => Some( - root.call_method1("__truediv__", (CACHE_DIRECTORY_NAME,))? - .call_method1("__truediv__", (CACHE_DATABASE_NAME,))? + root.call_method1("joinpath", (CACHE_DIRECTORY_NAME, CACHE_DATABASE_NAME))? .unbind(), ), None => None, @@ -80,14 +111,14 @@ impl MigrationFingerprintCache { ref_names: HashMap::new(), fingerprints: HashMap::new(), database_path, - connection: None, - connection_opened: false, + store: None, + store_opened: false, pending: Vec::new(), shared_digest, }) } - pub(crate) fn cache_key(&self, key: &MemoKey) -> PyResult { + fn cache_key(&self, key: &MemoKey) -> PyResult { if self.database_path.is_none() { return Ok(String::new()); } @@ -101,14 +132,11 @@ impl MigrationFingerprintCache { .map_err(|error| PyValueError::new_err(format!("{error:?}"))) } - pub(crate) fn open_connection(&mut self, py: Python<'_>) -> PyResult>> { - if self.connection_opened { - return Ok(self - .connection - .as_ref() - .map(|connection| connection.clone_ref(py))); + fn open_store(&mut self, py: Python<'_>) -> PyResult>> { + if self.store_opened { + return Ok(self.store.as_ref().map(|store| store.clone_ref(py))); } - self.connection_opened = true; + self.store_opened = true; let Some(path) = &self.database_path else { return Ok(None); }; @@ -116,36 +144,29 @@ impl MigrationFingerprintCache { if !path.call_method0("is_file")?.is_truthy()? { return Ok(None); } - self.connection = match connect(path) { - Ok(connection) => Some(connection.unbind()), + self.store = match connect(path) { + Ok(store) => Some(store.unbind()), Err(error) if error.is_instance_of::(py) || sqlite_error(py, &error)? => { None } Err(error) => return Err(error), }; - Ok(self - .connection - .as_ref() - .map(|connection| connection.clone_ref(py))) + Ok(self.store.as_ref().map(|store| store.clone_ref(py))) } - pub(crate) fn stored( - &mut self, - py: Python<'_>, - cache_key: &str, - ) -> PyResult>> { - let Some(connection) = self.open_connection(py)? else { + fn stored(&mut self, py: Python<'_>, cache_key: &str) -> PyResult>> { + let Some(store) = self.open_store(py)? else { return Ok(None); }; - let row = connection + let row = store .bind(py) .call_method1("execute", (SELECT_SQL, PyTuple::new(py, [cache_key])?)) .and_then(|cursor| cursor.call_method0("fetchone")); let row = match row { Ok(row) => row, Err(error) if sqlite_error(py, &error)? => { - self.connection = None; - close(py, Some(connection))?; + self.store = None; + close(py, Some(store))?; return Ok(None); } Err(error) => return Err(error), @@ -160,7 +181,7 @@ impl MigrationFingerprintCache { Ok(fingerprint_cache_lookup(&contents, cache_key)) } - pub(crate) fn remember(&mut self, cache_key: &str, fingerprint: Option<&str>) -> PyResult<()> { + fn remember(&mut self, cache_key: &str, fingerprint: Option<&str>) -> PyResult<()> { if self.database_path.is_none() { return Ok(()); } @@ -173,21 +194,21 @@ impl MigrationFingerprintCache { Ok(()) } - pub(crate) fn write_pending( + fn write_pending( &self, py: Python<'_>, pending: &[(String, String)], - connection: Option>, + store: Option>, ) -> (Option>, PyResult<()>) { let Some(path) = &self.database_path else { - return (connection, Ok(())); + return (store, Ok(())); }; if pending.is_empty() { - return (connection, Ok(())); + return (store, Ok(())); } let path = path.bind(py); - let connection = match connection { - Some(connection) => connection, + let store = match store { + Some(store) => store, None => { let created = call_with( &match native_field(path, "parent") @@ -209,7 +230,7 @@ impl MigrationFingerprintCache { } }; let written = (|| -> PyResult<()> { - let open = connection.bind(py); + let open = store.bind(py); open.call_method1("execute", (CREATE_CACHE_TABLE_SQL,))?; let rows = PyList::empty(py); for (key, entry) in pending { @@ -219,7 +240,7 @@ impl MigrationFingerprintCache { open.call_method0("commit")?; Ok(()) })(); - (Some(connection), written) + (Some(store), written) } } @@ -227,13 +248,13 @@ impl MigrationFingerprintCache { impl MigrationFingerprintCache { #[new] #[pyo3(signature = (*, root=None))] - pub(crate) fn new(py: Python<'_>, root: Option>) -> PyResult { + fn new(py: Python<'_>, root: Option>) -> PyResult { Self::open(py, root) } /// The migration fingerprint, keyed only by the renames this model can see. #[pyo3(signature = (**options))] - pub(crate) fn fingerprint( + fn fingerprint( &mut self, py: Python<'_>, options: Option>, @@ -307,12 +328,12 @@ impl MigrationFingerprintCache { } /// Write fingerprints computed since the last persist; failures only skip reuse. - pub(crate) fn persist(&mut self, py: Python<'_>) -> PyResult<()> { + fn persist(&mut self, py: Python<'_>) -> PyResult<()> { let pending = std::mem::take(&mut self.pending); - let connection = self.connection.take(); - self.connection_opened = false; - let (connection, written) = self.write_pending(py, &pending, connection); - let closed = close(py, connection); + let store = self.store.take(); + self.store_opened = false; + let (store, written) = self.write_pending(py, &pending, store); + let closed = close(py, store); match written { Err(error) if error.is_instance_of::(py) || sqlite_error(py, &error)? => {} Err(error) => return Err(error), @@ -333,16 +354,452 @@ pub(crate) struct FingerprintWork { /// Everything one snapshot gather needs. #[derive(Debug)] pub struct SnapshotRequest<'py> { - pub project: Bound<'py, PyAny>, - pub adapter: Bound<'py, PyAny>, - pub connection: Bound<'py, PyAny>, - pub execute: Bound<'py, PyAny>, - pub selected_keys: Bound<'py, PyAny>, - pub full_refresh: bool, - pub full_refresh_model_names: Bound<'py, PyAny>, - pub on_progress: Bound<'py, PyAny>, - pub deferred_locations: Bound<'py, PyAny>, - pub cursor_scope: Bound<'py, PyAny>, + pub(crate) project: Bound<'py, PyAny>, + pub(crate) dialect: Bound<'py, Dialect>, + pub(crate) reads: Bound<'py, PlannerReads>, + pub(crate) selected_keys: Bound<'py, PyAny>, + pub(crate) full_refresh: bool, + pub(crate) full_refresh_model_names: Bound<'py, PyAny>, + pub(crate) on_progress: Bound<'py, PyAny>, + pub(crate) deferred_locations: Bound<'py, PyAny>, + pub(crate) cursor_scope: Bound<'py, PyAny>, /// Python nodes' `(database, schema)`; their fingerprints are read there too. `None` when unknown. - pub node_results_location: Option<(Option, Option)>, + pub(crate) node_results_location: Option<(Option, Option)>, +} + +/// One warehouse read's answer: what the warehouse holds, or the item a plan reports as unknown. +pub enum Read { + Known(T), + Unknown(Cow<'static, str>), +} + +/// A read the warehouse adapter answers in one call. +pub enum AdapterRead<'py> { + /// Each table freshness request's metadata, keyed by request. + TableFreshness(Bound<'py, PyAny>), + /// One table's `[database, schema, name]` freshness metadata. + SingleTableFreshness([Bound<'py, PyAny>; 3]), + /// The rows of one source freshness query. + FreshnessQuery(Bound<'py, PyAny>), + /// Why a destination cannot rename columns, `None` when it can. + ColumnRenameUnavailable(Bound<'py, PyAny>), +} + +/// A read of what the warehouse state tables recorded. +pub enum StateHistoryRead<'a, 'py> { + /// The latest recorded source freshness of `(database, schema)`, as a latest-records read. + SourceFreshness { + location: (&'a Bound<'py, PyAny>, &'a Bound<'py, PyAny>), + table_exists: bool, + source_names: Option<&'a [String]>, + }, + /// The migration state tables of `schemas` in `database`, narrowed to `names`. + MigrationTables { + database: &'a Bound<'py, PyAny>, + filter: (Bound<'py, PyAny>, Bound<'py, PyAny>), + }, + /// The recorded migration events of one state table in `(database, schema)`, as a tuple. + MigrationEvents { + layout: &'a EventTableLayout, + location: (Bound<'py, PyAny>, Bound<'py, PyAny>), + stored: Option<&'a [String]>, + }, +} + +/// Where to read the newest fingerprints of one state location, narrowed as a filtered read is. +#[derive(Debug)] +pub struct FingerprintRead<'a> { + pub(crate) database: Option, + pub(crate) schema: &'a str, + pub(crate) table_exists: bool, + /// Node names to keep, or every node when `None`. + pub(crate) node_names: Option>, + /// Node types the name filter applies to; other types are kept whole. + pub(crate) filtered_node_types: Vec, +} + +/// One target cursor `MAX` at or before the latest allowed start, for one model. +#[derive(Debug)] +pub struct EligibleCursorQuery { + pub(crate) model_name: String, + pub(crate) target: String, + pub(crate) cursor_column: String, + pub(crate) sql: String, +} + +/// The warehouse reads one plan runs through. +#[pyclass(frozen, module = "sqlbuild._native", name = "WarehouseReads")] +#[derive(Debug)] +pub struct PlannerReads { + reads: Arc, +} + +impl PlannerReads { + pub fn new(reads: Arc) -> Self { + Self { reads } + } + + /// The reads behind this handle. + pub fn reads(&self) -> &dyn WarehouseReads { + self.reads.as_ref() + } + + /// The reads a keyword argument passed; anything else is refused. + pub fn passed(value: Bound<'_, PyAny>) -> PyResult> { + Ok(value.cast_into::()?) + } +} + +impl ConsumptionReads for dyn WarehouseReads + '_ { + fn read_recent(&self, py: Python<'_>, location: &Location) -> PyResult> { + self.recent_consumption(py, location) + } + + fn read_executions( + &self, + py: Python<'_>, + location: &Location, + execution_ids: &[&str], + ) -> PyResult> { + self.consumption_executions(py, location, execution_ids) + } +} + +/// The reads of one plan preview over a synced state, and what the plan found unknown. +#[derive(Debug)] +pub struct SyncedReads { + state: Arc, + unknowns: Mutex>, + windows: Mutex>, +} + +impl SyncedReads { + /// Reads over `state`, which several plans may share. + pub fn new(state: Arc) -> Self { + Self { + state, + unknowns: Mutex::new(BTreeSet::new()), + windows: Mutex::new(BTreeMap::new()), + } + } + + /// When the state was synced, as ISO 8601 text; `None` before every required table synced. + pub fn synced_at(&self) -> Option { + self.state + .synced_at() + .map(|at| at.to_rfc3339_opts(SecondsFormat::AutoSi, true)) + } + + /// What the preview could not know, in a stable order. + pub fn unknowns(&self) -> Vec { + let mut unknowns: Vec = PREVIEW_UNKNOWNS + .iter() + .map(|item| (*item).to_owned()) + .collect(); + unknowns.extend( + self.unknowns + .lock() + .unwrap_or_else(PoisonError::into_inner) + .iter() + .filter(|item| !PREVIEW_UNKNOWNS.contains(&item.as_str())) + .cloned(), + ); + unknowns + } + + /// Each model's window start, read up to the latest data when the run starts. + pub fn windows(&self) -> BTreeMap { + self.windows + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + + /// Every synced consumption row of the table in `(database, schema)`. + pub(crate) fn consumption( + &self, + (database, schema): (Option<&str>, &str), + ) -> Vec { + self.state.consumption_rows(&StateLocation { + database: database.map(str::to_owned), + schema: schema.to_owned(), + }) + } + + fn listed<'py>( + &self, + py: Python<'py>, + database: &Bound<'py, PyAny>, + (schemas, names): (Bound<'py, PyAny>, Bound<'py, PyAny>), + ) -> PyResult> { + let database: Option = database.extract()?; + let schemas: Option> = schemas.extract()?; + let names: Option> = names.extract()?; + let relations: Vec> = self + .state + .relations(&RelationFilter { + database: database.as_deref(), + schemas: schemas.as_deref(), + names: names.as_deref(), + }) + .into_iter() + .map(|relation| relation_info(py, relation)) + .collect::>()?; + PyTuple::new(py, relations) + } +} + +impl WarehouseReads for SyncedReads { + fn relations<'py>( + &self, + py: Python<'py>, + database: &Bound<'py, PyAny>, + filter: (Bound<'py, PyAny>, Bound<'py, PyAny>), + ) -> PyResult> { + Ok(self.listed(py, database, filter)?.into_any()) + } + + fn columns<'py>( + &self, + py: Python<'py>, + relations: Bound<'py, PyAny>, + ) -> PyResult> { + let found: Bound<'py, PyDict> = PyDict::new(py); + for relation in relations.try_iter()? { + let relation: Bound<'py, PyAny> = relation?; + let database: Option = native_field(&relation, "database")?.extract()?; + let schema: Option = native_field(&relation, "schema")?.extract()?; + let name: String = native_field(&relation, "name")?.extract()?; + let schemas: Option> = schema.clone().map(|schema| vec![schema]); + let names: [String; 1] = [name.clone()]; + let listed: Vec<&StateRelation> = self.state.relations(&RelationFilter { + database: database.as_deref(), + schemas: schemas.as_deref(), + names: Some(&names), + }); + if let Some(listed) = listed.first() { + found.set_item((database, schema, name), relation_columns(py, listed)?)?; + } + } + Ok(found.into_any()) + } + + fn fingerprints<'py>( + &self, + py: Python<'py>, + read: FingerprintRead<'_>, + ) -> PyResult> { + if !read.table_exists { + return new_record_with( + py, + RecordId::FingerprintSet, + &[ + ("schema", PyString::new(py, read.schema).into_any()), + ("fingerprints", PyDict::new(py).into_any()), + ], + ); + } + let datetime: Bound<'py, PyAny> = imported(py, "datetime", "datetime")?; + let location = StateLocation { + database: read.database, + schema: read.schema.to_owned(), + }; + let latest: Vec> = self + .state + .latest_fingerprints(&location) + .into_iter() + .map(|stored| synced_reads::fingerprint(&datetime, stored)) + .collect::>()?; + selected_set( + py, + &location.schema, + latest, + (read.node_names, read.filtered_node_types), + ) + } + + fn within_inspection<'py>( + &self, + _py: Python<'py>, + plan: &mut dyn FnMut() -> PyResult>, + ) -> PyResult> { + plan() + } + + fn inspection_concurrency(&self, _py: Python<'_>) -> PyResult { + Ok(1) + } + + fn prefetch_schema_listings<'py>( + &self, + _py: Python<'py>, + _scopes: Bound<'py, PyAny>, + ) -> PyResult<()> { + Ok(()) + } + + fn lists_by_schema(&self, _py: Python<'_>) -> PyResult { + Ok(false) + } + + fn cursor_bounds<'py>( + &self, + _py: Python<'py>, + _statements: &dyn Fn() -> PyResult>, + _progress: &dyn CursorProgress<'py>, + ) -> PyResult>>> { + Ok(Read::Unknown(CURSOR_INPUTS.into())) + } + + fn eligible_cursor_maxes<'py>( + &self, + _py: Python<'py>, + _queries: Vec, + ) -> PyResult>>> { + Ok(Read::Unknown(CURSOR_INPUTS.into())) + } + + fn target_max(&self, model: &str) -> Read { + match self.state.target_max(model) { + Some(observed) => Read::Known(observed.to_owned()), + None => Read::Unknown(format!("cursor position of {model}").into()), + } + } + + fn note_window(&self, model: &str, start: &str) { + self.windows + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(model.to_owned(), start.to_owned()); + } + + fn note_unknown(&self, item: String) { + self.unknowns + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(item); + } + + fn described_columns( + &self, + _py: Python<'_>, + source: &str, + sql: &str, + ) -> PyResult>> { + let described = self.state.described_columns(source, sql).map_err( + |InvalidObservation { source, reason }| { + PyRuntimeError::new_err(format!( + "invalid source_columns observation for {source}: {reason}" + )) + }, + )?; + Ok(match described { + Some(described) => Read::Known(described), + None => unknown( + self, + format!("query columns of expression source {source} (planned as declared)"), + ), + }) + } + + fn read_access<'py>( + &self, + _py: Python<'py>, + _relations: &dyn Fn() -> PyResult>>, + ) -> PyResult>>> { + Ok(unknown(self, "read access to these sources")) + } + + fn state_history<'py>( + &self, + _py: Python<'py>, + read: StateHistoryRead<'_, 'py>, + ) -> PyResult>> { + Ok(unknown( + self, + match read { + StateHistoryRead::SourceFreshness { .. } => "source freshness", + StateHistoryRead::MigrationTables { .. } + | StateHistoryRead::MigrationEvents { .. } => "recorded migrations", + }, + )) + } + + fn adapter_read<'py>( + &self, + py: Python<'py>, + read: AdapterRead<'py>, + ) -> PyResult>> { + let requests: Bound<'py, PyAny> = match read { + AdapterRead::TableFreshness(requests) => requests, + AdapterRead::SingleTableFreshness(_) | AdapterRead::FreshnessQuery(_) => { + return Ok(unknown(self, "source freshness")); + } + AdapterRead::ColumnRenameUnavailable(_) => { + return Ok(unknown(self, "column rename support")); + } + }; + let unavailable = + host_entry(py, AdapterHost::TableFreshnessMetadata)?.getattr("unavailable")?; + let outcome = call_with( + &unavailable, + &[("message", PyString::new(py, PREVIEW_MESSAGE).into_any())], + )?; + let metadata: Bound<'py, PyDict> = PyDict::new(py); + for request in requests.try_iter()? { + metadata.set_item(request?, &outcome)?; + } + Ok(Read::Known(metadata.into_any())) + } + + fn table_types(&self) -> Read<()> { + unknown(self, "table types") + } + + fn retention<'py>(&self, _py: Python<'py>) -> PyResult>> { + Ok(unknown(self, "retention")) + } + + fn old_name_view_columns<'py>( + &self, + _py: Python<'py>, + _location: (Bound<'py, PyAny>, Bound<'py, PyAny>), + ) -> PyResult>> { + Ok(unknown(self, "recorded migrations")) + } + + fn deferred_relations<'py>( + &self, + _py: Python<'py>, + _location: (Bound<'py, PyAny>, Bound<'py, PyAny>), + ) -> PyResult> { + Err(refused("deferred target relations")) + } + + fn recent_consumption( + &self, + _py: Python<'_>, + location: &Location, + ) -> PyResult> { + Ok(recent_executions( + &self.consumption((location.database.as_deref(), &location.schema)), + RECENT_EXECUTIONS, + )) + } + + fn consumption_executions( + &self, + _py: Python<'_>, + location: &Location, + execution_ids: &[&str], + ) -> PyResult> { + let mut rows: Vec = + self.consumption((location.database.as_deref(), &location.schema)); + rows.retain(|row| execution_ids.contains(&row.row.execution_id.as_str())); + Ok(rows) + } + + fn close_consumption(&self, _py: Python<'_>) -> PyResult<()> { + Ok(()) + } } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs new file mode 100644 index 0000000000..a036a75f7f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs @@ -0,0 +1,158 @@ +//! The warehouse reads planning consults, as traits. + +use crate::planner_warehouse::models::AdapterRead; +use crate::planner_warehouse::models::EligibleCursorQuery; +use crate::planner_warehouse::models::FingerprintRead; +use crate::planner_warehouse::models::Read; +use crate::planner_warehouse::models::StateHistoryRead; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_front::native_host::models::Location; +use sqlbuild_state::consumption::models::ConsumptionRow; + +/// Progress of the cursor bound statements a plan runs. +pub trait CursorProgress<'py> { + /// Before statement `index` runs, when statements run one at a time. + fn starting(&self, index: usize) -> PyResult<()>; + /// Before statements run `concurrency` at a time. + fn concurrent(&self, concurrency: usize) -> PyResult<()>; + /// After statement `index` ran, with its `(row, seconds, error)` outcome. + fn finished(&self, index: usize, outcome: &Bound<'py, PyAny>) -> PyResult<()>; + /// The `on_complete(index=, result=)` callback concurrent statements report through. + fn completion_callback(&self, py: Python<'py>) -> PyResult>; +} + +/// Everything a plan reads from the warehouse, live or from a synced state that may answer unknown. +pub trait WarehouseReads: Send + Sync + std::fmt::Debug { + /// The listed relations of `database` narrowed to `schemas` and `names`, as `RelationInfo`s. + fn relations<'py>( + &self, + py: Python<'py>, + database: &Bound<'py, PyAny>, + filter: (Bound<'py, PyAny>, Bound<'py, PyAny>), + ) -> PyResult>; + + /// The columns of each listed relation, keyed by `(database, schema, name)`. + fn columns<'py>( + &self, + py: Python<'py>, + relations: Bound<'py, PyAny>, + ) -> PyResult>; + + /// The newest fingerprint per node of one state location, as a `FingerprintSet`. + fn fingerprints<'py>( + &self, + py: Python<'py>, + read: FingerprintRead<'_>, + ) -> PyResult>; + + /// Run `plan` with one inspection catalog serving its listings. + fn within_inspection<'py>( + &self, + py: Python<'py>, + plan: &mut dyn FnMut() -> PyResult>, + ) -> PyResult>; + + /// How many metadata reads may run at once. + fn inspection_concurrency(&self, py: Python<'_>) -> PyResult; + + /// Warm the listings of `(database, schema)` scopes the plan will read, where that helps. + fn prefetch_schema_listings<'py>( + &self, + py: Python<'py>, + scopes: Bound<'py, PyAny>, + ) -> PyResult<()>; + + /// Whether listing by schema and name finds relations a name-only listing would. + fn lists_by_schema(&self, py: Python<'_>) -> PyResult; + + /// Each statement's `(row, seconds, error)`; `statements` renders SQL only for reads that run it. + fn cursor_bounds<'py>( + &self, + py: Python<'py>, + statements: &dyn Fn() -> PyResult>, + progress: &dyn CursorProgress<'py>, + ) -> PyResult>>>; + + /// Each eligible target cursor query's rows. + fn eligible_cursor_maxes<'py>( + &self, + py: Python<'py>, + queries: Vec, + ) -> PyResult>>>; + + /// The model's cursor `MAX` text as its newest successful build observed it. + fn target_max(&self, model: &str) -> Read; + + /// Note that `model` reads from `start` up to the latest data when the run starts. + fn note_window(&self, model: &str, start: &str); + + /// Note one item the plan reports as unknown until the run plans live. + fn note_unknown(&self, item: String); + + /// The columns an expression source's `sql` yields. + fn described_columns( + &self, + py: Python<'_>, + source: &str, + sql: &str, + ) -> PyResult>>; + + /// Whether each relation `relations` renders can be read, as `RelationReadProbe`s in order. + fn read_access<'py>( + &self, + py: Python<'py>, + relations: &dyn Fn() -> PyResult>>, + ) -> PyResult>>>; + + /// What the warehouse state tables recorded for `read`. + fn state_history<'py>( + &self, + py: Python<'py>, + read: StateHistoryRead<'_, 'py>, + ) -> PyResult>>; + + /// What the warehouse adapter answers to `read` in one call. + fn adapter_read<'py>( + &self, + py: Python<'py>, + read: AdapterRead<'py>, + ) -> PyResult>>; + + /// Whether listed relations carry their table type. + fn table_types(&self) -> Read<()>; + + /// The callable that reads one retention request's state. + fn retention<'py>(&self, py: Python<'py>) -> PyResult>>; + + /// The stored columns of the old-name view table in `(database, schema)`. + fn old_name_view_columns<'py>( + &self, + py: Python<'py>, + location: (Bound<'py, PyAny>, Bound<'py, PyAny>), + ) -> PyResult>>; + + /// Relations of the deferred target's `(database, schema)`, by name; deferral plans live only. + fn deferred_relations<'py>( + &self, + py: Python<'py>, + location: (Bound<'py, PyAny>, Bound<'py, PyAny>), + ) -> PyResult>; + + /// The newest executions per relation recorded in `location`'s consumption table. + fn recent_consumption( + &self, + py: Python<'_>, + location: &Location, + ) -> PyResult>; + + /// The named executions recorded in `location`'s consumption table. + fn consumption_executions( + &self, + py: Python<'_>, + location: &Location, + execution_ids: &[&str], + ) -> PyResult>; + + /// Close what reading consumption history opened. + fn close_consumption(&self, py: Python<'_>) -> PyResult<()>; +} diff --git a/crates/sqlbuild-python-project/src/analysis/_helpers/catalog.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/catalog.rs index a7abc19801..3ea5e12ed0 100644 --- a/crates/sqlbuild-python-project/src/analysis/_helpers/catalog.rs +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/catalog.rs @@ -6,8 +6,9 @@ use crate::analysis::_helpers::conversions::columns; use crate::analysis::_helpers::conversions::relations; use crate::analysis::_helpers::native_classes::CatalogInput; use crate::analysis::_helpers::normalization_results::normalization_results; +use crate::analysis::models::CompactAnalysisJob; use crate::analysis::models::Positional; -use crate::analysis::models::{CompactAnalysisJob, ProjectCatalog}; +use crate::analysis::models::ProjectCatalog; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::pymethods; diff --git a/crates/sqlbuild-python-project/src/analysis/_helpers/relation_names.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/relation_names.rs index 42d5c7533f..0ccc85a694 100644 --- a/crates/sqlbuild-python-project/src/analysis/_helpers/relation_names.rs +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/relation_names.rs @@ -1,14 +1,60 @@ //! Relations named in SQL and temporary tables it creates, for hard-coded name checks. -use crate::analysis::types::RelationRow; +use std::collections::HashMap; +use std::sync::{Mutex, PoisonError}; + +use crate::analysis::types::{ExtractedRows, RelationRow}; use pyo3::PyErr; use sqlbuild_analysis::relation_names::errors::RelationNamesFailure; -use sqlbuild_analysis::relation_names::models::RelationName; +use sqlbuild_analysis::relation_names::models::{ExtractedRelations, RelationName}; use sqlbuild_core::panics::main::native_failure::native_failure; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; const RELATION_NAMES_CONTEXT: &str = "native relation name extraction"; +/// Distinct SQL texts the memo keeps before starting over; a 10K-model project holds about 10K. +const EXTRACTED_LIMIT: usize = 100_000; + +/// Extractions by dialect and SQL text, reused by a location's previews of one publication. +pub(crate) static EXTRACTED: Mutex> = Mutex::new(None); + +/// Extracted rows by `(dialect, SQL text)`. +pub(crate) type ExtractedMemo = HashMap<(Option, String), ExtractedRows>; + +pub(crate) fn remembered( + memo: &Mutex>, + dialect: Option<&str>, + sql: &str, +) -> Option { + let memo = memo.lock().unwrap_or_else(PoisonError::into_inner); + memo.as_ref()? + .get(&(dialect.map(str::to_owned), sql.to_owned())) + .cloned() +} + +pub(crate) fn remember( + memo: &Mutex>, + dialect: Option<&str>, + extracted: impl IntoIterator, +) { + let mut memo = memo.lock().unwrap_or_else(PoisonError::into_inner); + let entries = memo.get_or_insert_with(HashMap::new); + if entries.len() > EXTRACTED_LIMIT { + entries.clear(); + } + for (sql, rows) in extracted { + entries.insert((dialect.map(str::to_owned), sql), rows); + } +} + +pub(crate) fn rows_of(extracted: Option) -> ExtractedRows { + extracted.map( + |ExtractedRelations { + relations, + temporary, + }| { (relations.into_iter().map(relation_row).collect(), temporary) }, + ) +} pub(crate) fn relation_row(relation: RelationName) -> RelationRow { (relation.name, relation.schema, relation.database) diff --git a/crates/sqlbuild-python-project/src/analysis/main/extract_sql_relation_names.rs b/crates/sqlbuild-python-project/src/analysis/main/extract_sql_relation_names.rs index 9b2e06b121..ad0ef1d4e4 100644 --- a/crates/sqlbuild-python-project/src/analysis/main/extract_sql_relation_names.rs +++ b/crates/sqlbuild-python-project/src/analysis/main/extract_sql_relation_names.rs @@ -1,11 +1,11 @@ //! `(relations, temporary names)` named in `sql`, or None if unparseable. -use crate::analysis::_helpers::relation_names::relation_names_error; -use crate::analysis::_helpers::relation_names::relation_row; -use crate::analysis::types::RelationRow; +use crate::analysis::_helpers::relation_names::{ + EXTRACTED, relation_names_error, remember, remembered, rows_of, +}; +use crate::analysis::types::ExtractedRows; use pyo3::prelude::{PyResult, Python}; use sqlbuild_analysis::relation_names::main::extract_relation_names::extract_relation_names; -use sqlbuild_analysis::relation_names::models::ExtractedRelations; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::types::CompilerDetach; @@ -14,15 +14,15 @@ pub fn extract_sql_relation_names( py: Python<'_>, sql: &str, dialect: Option<&str>, -) -> PyResult, Vec)>> { +) -> PyResult { + if let Some(rows) = remembered(&EXTRACTED, dialect, sql) { + return Ok(rows); + } let extracted = py .compiler_detach(|| Ok(extract_relation_names(sql, dialect))) .map_err(compiler_error)? .map_err(relation_names_error)?; - Ok(extracted.map( - |ExtractedRelations { - relations, - temporary, - }| { (relations.into_iter().map(relation_row).collect(), temporary) }, - )) + let rows: ExtractedRows = rows_of(extracted); + remember(&EXTRACTED, dialect, [(sql.to_owned(), rows.clone())]); + Ok(rows) } diff --git a/crates/sqlbuild-python-project/src/analysis/main/mod.rs b/crates/sqlbuild-python-project/src/analysis/main/mod.rs index bb9d5d1db2..b7a9f89e5a 100644 --- a/crates/sqlbuild-python-project/src/analysis/main/mod.rs +++ b/crates/sqlbuild-python-project/src/analysis/main/mod.rs @@ -1,6 +1,7 @@ pub mod extract_sql_relation_names; pub mod ignores_quoted_case; pub mod normalize_analysis_sql; +pub mod prefetch_sql_relation_names; pub mod register_binding_catalog; pub mod register_normalization; pub mod schema_validation_request; diff --git a/crates/sqlbuild-python-project/src/analysis/main/prefetch_sql_relation_names.rs b/crates/sqlbuild-python-project/src/analysis/main/prefetch_sql_relation_names.rs new file mode 100644 index 0000000000..5a7a0f29d2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/main/prefetch_sql_relation_names.rs @@ -0,0 +1,34 @@ +//! Extract every SQL text not extracted yet, off the GIL, so later reads find them. + +use crate::analysis::_helpers::relation_names::{ + EXTRACTED, relation_names_error, remember, remembered, rows_of, +}; +use crate::analysis::types::ExtractedRows; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_analysis::relation_names::errors::RelationNamesFailure; +use sqlbuild_analysis::relation_names::models::ExtractedRelations; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::types::CompilerDetach; + +/// Extract every SQL text not extracted yet, in parallel off the GIL, so later reads find them. +pub fn prefetch_sql_relation_names( + py: Python<'_>, + sqls: Vec, + dialect: Option<&str>, + extract_all: impl FnOnce(&[String]) -> Vec, RelationNamesFailure>> + + Send, +) -> PyResult<()> { + let missing: Vec = sqls + .into_iter() + .filter(|sql| remembered(&EXTRACTED, dialect, sql).is_none()) + .collect(); + let extracted: Vec, RelationNamesFailure>> = py + .compiler_detach(|| Ok(extract_all(&missing))) + .map_err(compiler_error)?; + let mut rows: Vec<(String, ExtractedRows)> = Vec::with_capacity(missing.len()); + for (sql, extracted) in missing.into_iter().zip(extracted) { + rows.push((sql, rows_of(extracted.map_err(relation_names_error)?))); + } + remember(&EXTRACTED, dialect, rows); + Ok(()) +} diff --git a/crates/sqlbuild-python-project/src/analysis/types.rs b/crates/sqlbuild-python-project/src/analysis/types.rs index dca7b1ab92..7f107e0827 100644 --- a/crates/sqlbuild-python-project/src/analysis/types.rs +++ b/crates/sqlbuild-python-project/src/analysis/types.rs @@ -2,3 +2,6 @@ /// `(name, schema, database)`. pub type RelationRow = (String, Option, Option); + +/// `(relations, temporary names)` named in a SQL text, or None if unparseable. +pub type ExtractedRows = Option<(Vec, Vec)>; diff --git a/crates/sqlbuild-python-project/src/analysis_session/_helpers/query_columns.rs b/crates/sqlbuild-python-project/src/analysis_session/_helpers/query_columns.rs index 2ed7350994..fe7b47728b 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/_helpers/query_columns.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/_helpers/query_columns.rs @@ -16,9 +16,9 @@ use sqlbuild_core::panics::main::native_failure::native_failure; use sqlbuild_python_base::positional_record; use crate::analysis::models::ProjectCatalog; -use crate::analysis_session::_helpers::nullability_rules::{ - RuleFailure, nullability_callback, raised, -}; +use crate::analysis_session::_helpers::nullability_rules::RuleFailure; +use crate::analysis_session::_helpers::nullability_rules::nullability_callback; +use crate::analysis_session::_helpers::nullability_rules::raised; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::types::CompilerDetach; diff --git a/crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs b/crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs index 927e17c6b8..71817ad314 100644 --- a/crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs @@ -14,14 +14,18 @@ use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple}; use sqlbuild_analysis::assembly::analysis_session::types::Pairs; use crate::analysis::models::ProjectCatalog; -use crate::analysis_session::_helpers::profile_inputs::{aliased_dialect, profile_catalog}; -use crate::analysis_session::_helpers::query_columns::{ - QueryRequestRow, QueryRow, infer_query_columns, -}; -use crate::analysis_session::_helpers::session::{ - ContractRow, ExpressionShapesRow, PivotModelRow, infer_expression_source_shapes, -}; -use crate::analysis_session::models::{FamilyRow, NativeModelAnalysisSession, PivotRequestRow}; +use crate::analysis_session::_helpers::profile_inputs::aliased_dialect; +use crate::analysis_session::_helpers::profile_inputs::profile_catalog; +use crate::analysis_session::_helpers::query_columns::QueryRequestRow; +use crate::analysis_session::_helpers::query_columns::QueryRow; +use crate::analysis_session::_helpers::query_columns::infer_query_columns; +use crate::analysis_session::_helpers::session::ContractRow; +use crate::analysis_session::_helpers::session::ExpressionShapesRow; +use crate::analysis_session::_helpers::session::PivotModelRow; +use crate::analysis_session::_helpers::session::infer_expression_source_shapes; +use crate::analysis_session::models::FamilyRow; +use crate::analysis_session::models::NativeModelAnalysisSession; +use crate::analysis_session::models::PivotRequestRow; use crate::analysis_session::types::ColumnsRow; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::compile_driver::constants::EXPRESSION_SHAPE_BATCHES; diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs index ca64cee07a..53d25310cf 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs @@ -12,9 +12,9 @@ use sqlbuild_analysis::graph::models::{JobSelection, ParsedSelector}; use sqlbuild_cli::native_commands::main::effective_target::effective_target; use sqlbuild_discovery::orchestration_files::models::{OrchestrationInventory, RunFile}; -use crate::compiled_project::constants::{ - JOB_SELECTOR_KINDS, SELECTOR_INTERSECTION, SENSOR_RUN_ARGUMENT, -}; +use crate::compiled_project::constants::JOB_SELECTOR_KINDS; +use crate::compiled_project::constants::SELECTOR_INTERSECTION; +use crate::compiled_project::constants::SENSOR_RUN_ARGUMENT; use sqlbuild_python_front::discovery::main::sources::orchestration_declarations::orchestration_declarations; /// Each job and group's selection, and the names of those a schedule or sensor starts. diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/getters.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/getters.rs index 51ed97700f..769e1eebee 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/getters.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/getters.rs @@ -7,10 +7,14 @@ use crate::compiled_views::models::NativeCompiledModel; use crate::compiled_views::models::NativeInputsView; use crate::compiled_views::models::NativeModelInput; use crate::compiled_views::models::NativeProjectView; -use crate::resource_views::models::{NativeAuditInput, NativeCompiledAudit}; -use crate::resource_views::models::{NativeCompiledFunction, NativeFunctionInput}; -use crate::resource_views::models::{NativeCompiledSeed, NativeSeedInput}; -use crate::resource_views::models::{NativeCompiledSource, NativeSourceInput}; +use crate::resource_views::models::NativeAuditInput; +use crate::resource_views::models::NativeCompiledAudit; +use crate::resource_views::models::NativeCompiledFunction; +use crate::resource_views::models::NativeCompiledSeed; +use crate::resource_views::models::NativeCompiledSource; +use crate::resource_views::models::NativeFunctionInput; +use crate::resource_views::models::NativeSeedInput; +use crate::resource_views::models::NativeSourceInput; pub(crate) fn read_native_audit_input<'py>( value: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs b/crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs index 861ac32076..b6bf3c6d15 100644 --- a/crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs +++ b/crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs @@ -19,9 +19,8 @@ use sqlbuild_sqltext::compiler::main::model_output_columns::model_output_columns use crate::compiled_project::models::NativeCompiledProject; use crate::compiled_views::main::compiled_models::enum_columns::enum_columns; use crate::compiled_views::main::project::compiled_project_facts::compiled_project_facts; -use crate::contracts::_helpers::model_contracts::{ - DiagnosticRow, evaluate_project_model_contracts, -}; +use crate::contracts::_helpers::model_contracts::DiagnosticRow; +use crate::contracts::_helpers::model_contracts::evaluate_project_model_contracts; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_io.rs b/crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_io.rs index cbf9f38911..271dcf245e 100644 --- a/crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_io.rs +++ b/crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_io.rs @@ -12,12 +12,10 @@ use sqlbuild_analysis::fingerprint_storage::models::RelevantRead; use sqlbuild_host::entries::models::AdapterHost; -use crate::fingerprints::_helpers::latest_rows::{ - fingerprint_input_error, fingerprint_set_from_rows, -}; -use crate::fingerprints::_helpers::storage_sql::{ - fingerprint_insert_batches, fingerprint_insert_sql, -}; +use crate::fingerprints::_helpers::latest_rows::fingerprint_input_error; +use crate::fingerprints::_helpers::latest_rows::fingerprint_set_from_rows; +use crate::fingerprints::_helpers::storage_sql::fingerprint_insert_batches; +use crate::fingerprints::_helpers::storage_sql::fingerprint_insert_sql; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::new_record::new_record; diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/mod.rs b/crates/sqlbuild-python-project/src/fingerprints/main/mod.rs index ea51d13f10..a97c63b92b 100644 --- a/crates/sqlbuild-python-project/src/fingerprints/main/mod.rs +++ b/crates/sqlbuild-python-project/src/fingerprints/main/mod.rs @@ -5,3 +5,4 @@ pub mod register_query_hashes; pub mod register_storage_io; pub mod row_fingerprint; pub mod select_latest_fingerprints; +pub mod selected_set; diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/selected_set.rs b/crates/sqlbuild-python-project/src/fingerprints/main/selected_set.rs new file mode 100644 index 0000000000..998331e669 --- /dev/null +++ b/crates/sqlbuild-python-project/src/fingerprints/main/selected_set.rs @@ -0,0 +1,36 @@ +//! The `FingerprintSet` of `latest` fingerprints, narrowed to `node_names` plus unfiltered node types + +use crate::fingerprints::_helpers::latest_rows::borrowed; +use crate::fingerprints::_helpers::latest_rows::identity_of; +use crate::fingerprints::_helpers::latest_rows::indexed_set; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_analysis::fingerprint_storage::main::selected_positions::selected_positions; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::types::RecordId; + +/// `latest` narrowed to `node_names` as a filtered latest read would be; `None` keeps every fingerprint. +pub fn selected_set<'py>( + py: Python<'py>, + schema: &str, + latest: Vec>, + (node_names, filtered_node_types): (Option>, Vec), +) -> PyResult> { + let Some(node_names) = node_names else { + return indexed_set(py, schema, &latest); + }; + if node_names.is_empty() && filtered_node_types.is_empty() { + let kwargs: Bound<'py, PyDict> = PyDict::new(py); + kwargs.set_item("schema", schema)?; + kwargs.set_item("fingerprints", PyDict::new(py))?; + return new_record(py, RecordId::FingerprintSet, (), Some(&kwargs)); + } + let identities: Vec<(String, String)> = + latest.iter().map(identity_of).collect::>()?; + let selected: Vec> = + selected_positions(&borrowed(&identities), &node_names, &filtered_node_types) + .into_iter() + .filter_map(|position| latest.get(position).cloned()) + .collect(); + indexed_set(py, schema, &selected) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs b/crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs index 6dd300ebc1..88460a1e35 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs @@ -2,6 +2,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyString; +use std::collections::HashMap; +use std::sync::Mutex; +use std::sync::PoisonError; use sqlbuild_analysis::planner_identity::constants::{FINGERPRINT_HELP, QUERY_FINGERPRINT_FAILED}; use sqlbuild_analysis::sql_tokens::main::baseline_fingerprint::baseline_fingerprint; @@ -15,13 +18,46 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::types::CompilerDetach; +/// Query fingerprints by dialect and SQL text, reused across a plan and a location's later previews. +pub(crate) static QUERY_FINGERPRINTS: Mutex> = Mutex::new(None); +/// Distinct SQL texts the memo keeps before starting over; a 10K-model project holds about 10K. +const QUERY_FINGERPRINTS_LIMIT: usize = 100_000; + +/// Fingerprint results by `(dialect, SQL text)`. +pub(crate) type QueryFingerprints = HashMap<(Option, String), Result>; + +/// `compiler_query_fingerprint`, remembered by dialect and SQL text. +pub(crate) fn query_fingerprint( + memo: &Mutex>, + sql: &str, + dialect: Option<&str>, +) -> Result { + let key: (Option, String) = (dialect.map(str::to_owned), sql.to_owned()); + if let Some(known) = memo + .lock() + .unwrap_or_else(PoisonError::into_inner) + .as_ref() + .and_then(|memo| memo.get(&key)) + { + return known.clone(); + } + let fingerprint: Result = compiler_query_fingerprint(sql, dialect); + let mut memo = memo.lock().unwrap_or_else(PoisonError::into_inner); + let entries: &mut QueryFingerprints = memo.get_or_insert_with(HashMap::new); + if entries.len() > QUERY_FINGERPRINTS_LIMIT { + entries.clear(); + } + entries.insert(key, fingerprint.clone()); + fingerprint +} + /// The query fingerprint of a model, or the planner input error naming the model. pub(crate) fn model_query_hash( py: Python<'_>, model_name: &str, (query_sql, dialect): (&str, Option<&str>), ) -> PyResult { - py.compiler_detach(|| compiler_query_fingerprint(query_sql, dialect)) + py.compiler_detach(|| query_fingerprint(&QUERY_FINGERPRINTS, query_sql, dialect)) .map_err(|error| fingerprint_error(py, &format!("Model '{model_name}' SQL"), error)) } diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/baselined_model_query_hash.rs b/crates/sqlbuild-python-project/src/planner_identity/main/baselined_model_query_hash.rs index 4327a91cd3..026257d5c4 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/main/baselined_model_query_hash.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/main/baselined_model_query_hash.rs @@ -2,8 +2,8 @@ use crate::planner_identity::_helpers::hashing::model_query_hash; use crate::planner_identity::_helpers::hashing::recorded_baseline; +use crate::planner_identity::_helpers::hashing::{QUERY_FINGERPRINTS, query_fingerprint}; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use sqlbuild_analysis::sql_tokens::main::compiler_query_fingerprint::compiler_query_fingerprint; /// The model's query fingerprint, or its recorded one while the recorded SQL still fingerprints the same. pub fn baselined_model_query_hash( @@ -14,6 +14,6 @@ pub fn baselined_model_query_hash( ) -> PyResult { let current: String = model_query_hash(py, model_name, (query_sql, dialect))?; recorded_baseline(py, current, recorded, |sql| { - compiler_query_fingerprint(sql, dialect) + query_fingerprint(&QUERY_FINGERPRINTS, sql, dialect) }) } diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_completion.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_completion.rs index 6f5ff146a1..99635b66ee 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_completion.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_completion.rs @@ -2,6 +2,7 @@ use crate::planner_strategy::main::complete_fixtures::complete_fixtures; use crate::planner_strategy::models::FixtureCompletion; +use sqlbuild_python_base::native_host::models::Dialect; use std::collections::HashSet; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -39,9 +40,9 @@ pub(crate) fn complete_prepared<'py, 'a>( &sqlbuild_analysis::planner_fixtures::models::FixtureReadRequirements, ), expected: ExpectedTypes<'a>, - adapter: &Bound<'py, PyAny>, + dialect: &Bound<'py, Dialect>, ) -> PyResult> { - let py = adapter.py(); + let py = dialect.py(); let python_keys: Vec<(FixtureKey, Bound<'py, PyAny>)> = prepared.keys.clone(); let python_key = |key: &FixtureKey| -> Option<&Bound<'py, PyAny>> { python_keys @@ -116,7 +117,7 @@ pub(crate) fn complete_prepared<'py, 'a>( quoted_names: quoted_names(&prepared, original)?, completed, }, - adapter, + dialect, )?; prepared.sql.set_item(original, sql)?; completed_keys.insert(key.clone()); @@ -140,7 +141,7 @@ pub(crate) fn complete_prepared<'py, 'a>( quoted_names: quoted_names(&prepared, original)?, completed: Vec::new(), }, - adapter, + dialect, )?; prepared.sql.set_item(original, sql)?; } @@ -172,10 +173,10 @@ fn quoted_names<'py>( /// Complete one chain's fixture groups, building the planning context when none is given. #[pyfunction] -#[pyo3(signature = (*, project, adapter, request, planning_context = None))] +#[pyo3(signature = (*, project, dialect, request, planning_context = None))] pub(crate) fn build_relation_fixture_completion<'py>( project: &Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, + dialect: &Bound<'py, Dialect>, request: (Vec, Bound<'py, PyAny>), planning_context: Option>, ) -> PyResult> { @@ -188,7 +189,7 @@ pub(crate) fn build_relation_fixture_completion<'py>( &built } }; - let completion = complete_fixtures(context, adapter, (&request.0, &request.1))?; + let completion = complete_fixtures(context, dialect, (&request.0, &request.1))?; let expected = PyDict::new(py); for (key, types) in &completion.expected_types.overlay { let columns = PyDict::new(py); diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_context.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_context.rs index e4772f64ab..7df823914f 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_context.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_context.rs @@ -19,14 +19,17 @@ use crate::native_host::models::ModelHandle; use crate::planner_strategy::_helpers::fixtures::fixture_metadata::model_fixture_relation; use crate::planner_strategy::_helpers::fixtures::fixture_reads::lineage_reads; use crate::planner_strategy::_helpers::fixtures::fixture_types::projections; -use crate::planner_strategy::_helpers::fixtures::fixture_values::{columns, folded_name}; -use crate::planner_strategy::constants::{ - MODEL_REFERENCE_KIND, SEED_REFERENCE_KIND, SOURCE_REFERENCE_KIND, -}; -use crate::planner_strategy::models::{ - FixtureReadCacheHost, FixtureRelation, ModelFixtureFacts, RelationFixturePlanningContext, -}; -use crate::planner_strategy::types::{FixtureKey, FixtureRelationInput}; +use crate::planner_strategy::_helpers::fixtures::fixture_values::columns; +use crate::planner_strategy::_helpers::fixtures::fixture_values::folded_name; +use crate::planner_strategy::constants::MODEL_REFERENCE_KIND; +use crate::planner_strategy::constants::SEED_REFERENCE_KIND; +use crate::planner_strategy::constants::SOURCE_REFERENCE_KIND; +use crate::planner_strategy::models::FixtureReadCacheHost; +use crate::planner_strategy::models::FixtureRelation; +use crate::planner_strategy::models::ModelFixtureFacts; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use crate::planner_strategy::types::FixtureKey; +use crate::planner_strategy::types::FixtureRelationInput; use crate::resource_views::main::resource_reads::name::name; use crate::resource_views::main::seed_reads::schema_entry::schema_entry; use crate::resource_views::main::source_reads::source_entry::source_entry; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_metadata.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_metadata.rs index fd0d0e684c..e874a437e8 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_metadata.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_metadata.rs @@ -7,7 +7,8 @@ use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; use crate::compiled_views::main::compiled_models::inferred_rows::inferred_rows; use crate::compiled_views::main::compiled_models::schema_entry::schema_entry; use crate::compiled_views::main::compiled_models::star_flags::star_flags; -use crate::planner_strategy::_helpers::fixtures::fixture_values::{columns, folded_name}; +use crate::planner_strategy::_helpers::fixtures::fixture_values::columns; +use crate::planner_strategy::_helpers::fixtures::fixture_values::folded_name; use crate::planner_strategy::constants::ENFORCED_CONTRACT; use crate::planner_strategy::models::FixtureRelation; use sqlbuild_project::types::InferredRow; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_preparation.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_preparation.rs index 9e313d23bc..13d12561a9 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_preparation.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_preparation.rs @@ -4,8 +4,11 @@ use crate::planner_strategy::main::complete_empty_fixture_sql::complete_empty_fi use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::planner_fixtures::main::empty_fixture_marker::empty_fixture_marker; +use sqlbuild_python_base::native_host::models::Dialect; +use sqlbuild_python_base::native_host::types::Syntax; -use crate::planner_strategy::_helpers::fixtures::fixture_values::{diagnostic, label}; +use crate::planner_strategy::_helpers::fixtures::fixture_values::diagnostic; +use crate::planner_strategy::_helpers::fixtures::fixture_values::label; use crate::planner_strategy::models::RelationFixturePlanningContext; use crate::planner_strategy::types::FixtureKey; use sqlbuild_python_base::boundary::main::value_error::value_error; @@ -24,7 +27,7 @@ pub(crate) struct PreparedFixtures<'py> { pub(crate) fn prepare<'py>( groups: &Bound<'py, PyAny>, context: &RelationFixturePlanningContext, - adapter: &Bound<'py, PyAny>, + dialect: &Bound<'py, Dialect>, ) -> PyResult> { let py = groups.py(); let inferences = context.fixture_inferences.bind(py); @@ -47,7 +50,7 @@ pub(crate) fn prepare<'py>( let mut sql: String = sql.extract()?; if empty_fixture_marker(&sql).map_err(value_error)? { let (empty_sql, error) = - complete_empty_fixture_sql(&sql, context.relations.get(&fixture_key), adapter)?; + complete_empty_fixture_sql(&sql, context.relations.get(&fixture_key), dialect)?; if let Some(error) = error { prepared .diagnostics @@ -62,7 +65,7 @@ pub(crate) fn prepare<'py>( kwargs.set_item("query_sql", &sql)?; kwargs.set_item( "inference_profile", - adapter.call_method0("expression_inference_profile")?, + dialect.syntax("expression_inference_profile")?.call0()?, )?; let inference = inferences.call_method("infer", (), Some(&kwargs))?; if !inference.is_none() { diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_reads.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_reads.rs index 8949fddebc..c21dc7ccb2 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_reads.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_reads.rs @@ -11,6 +11,8 @@ use sqlbuild_analysis::planner_fixtures::models::{ }; use sqlbuild_analysis::required_columns::models::ColumnRead; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; +use sqlbuild_python_base::native_host::models::Dialect; +use sqlbuild_python_base::native_host::types::Syntax; use std::collections::{HashMap, HashSet}; use crate::compiled_views::main::compiled_models::dep_rows::dep_rows; @@ -32,9 +34,9 @@ pub(crate) fn required_fixture_columns( context: &RelationFixturePlanningContext, ordered_model_names: &[String], fixture_keys: &HashSet, - adapter: &Bound<'_, PyAny>, + dialect: &Bound<'_, Dialect>, ) -> PyResult { - let py = adapter.py(); + let py = dialect.py(); let mut schema: Vec = Vec::new(); let mut authoritative: HashMap> = HashMap::new(); for key in fixture_keys { @@ -50,8 +52,9 @@ pub(crate) fn required_fixture_columns( authoritative.insert(key.clone(), names.iter().cloned().collect()); } } - let dialect = adapter - .call_method0("expression_inference_profile")? + let analysis_dialect = dialect + .syntax("expression_inference_profile")? + .call0()? .getattr("sql_analysis_dialect")? .extract::>()?; let mut models: Vec = Vec::new(); @@ -81,10 +84,10 @@ pub(crate) fn required_fixture_columns( key: FixtureReadKey { sql: query_sql(&model)?.extract()?, references, - dialect: dialect.clone(), + dialect: analysis_dialect.clone(), }, high_confidence: facts.reads.clone(), - stars: model_stars(&model, &schema, adapter)?, + stars: model_stars(&model, &schema, dialect)?, }); } let owner = work_counts(py)?; @@ -131,7 +134,7 @@ pub(crate) fn lineage_reads(model: &Bound<'_, PyAny>) -> PyResult, schema: &[FixtureStarRelation], - adapter: &Bound<'_, PyAny>, + dialect: &Bound<'_, Dialect>, ) -> PyResult> { if !star_flags(model)?.0 { return Ok(Vec::new()); @@ -157,8 +160,9 @@ fn model_stars( let inputs: Option> = proven_inputs(model)?; let dialect: Option = if inputs.is_none() && selected.iter().any(|relation| !relation.columns.is_empty()) { - adapter - .call_method0("expression_inference_profile")? + dialect + .syntax("expression_inference_profile")? + .call0()? .getattr("sql_analysis_dialect")? .extract()? } else { diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_selection.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_selection.rs index 7ab6d934ca..c62bee7701 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_selection.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_selection.rs @@ -10,7 +10,9 @@ use sqlbuild_analysis::planner_fixtures::main::{ }; use crate::planner_strategy::_helpers::fixtures::fixture_preparation::PreparedFixtures; -use crate::planner_strategy::_helpers::fixtures::fixture_values::{columns, diagnostic, label}; +use crate::planner_strategy::_helpers::fixtures::fixture_values::columns; +use crate::planner_strategy::_helpers::fixtures::fixture_values::diagnostic; +use crate::planner_strategy::_helpers::fixtures::fixture_values::label; use crate::planner_strategy::models::RelationFixturePlanningContext; use crate::planner_strategy::types::FixtureKey; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_sql.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_sql.rs index f2ff9d0dba..ebd97af2a6 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_sql.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_sql.rs @@ -9,6 +9,7 @@ use sqlbuild_analysis::planner_fixtures::main::{ use sqlbuild_analysis::planner_fixtures::models::{ FixtureColumn, FixtureSqlInput, SuppliedProjection, }; +use sqlbuild_python_base::native_host::models::Dialect; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -27,7 +28,7 @@ pub(crate) struct CompletionSql<'py, 'a> { pub(crate) fn render_completed( input: CompletionSql<'_, '_>, - adapter: &Bound<'_, PyAny>, + dialect: &Bound<'_, Dialect>, ) -> PyResult { let mut supplied: Vec = Vec::new(); if !input.typed_nulls.is_empty() { @@ -54,7 +55,7 @@ pub(crate) fn render_completed( completed: &input.completed, }, |name, exact| { - adapter + dialect .call_method1( if exact { "render_exact_identifier" diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_types.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_types.rs index 4222c87804..7a4ffe29ce 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_types.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_types.rs @@ -11,8 +11,10 @@ use sqlbuild_core::text::main::python_casefold::python_casefold; use crate::compiled_views::main::compiled_models::lineage_facts::lineage_facts; use crate::compiled_views::models::LineageFactRow; -use crate::planner_strategy::constants::{DIRECT_TRANSFORM, MODEL_REFERENCE_KIND}; -use crate::planner_strategy::models::{ExpectedTypes, RelationFixturePlanningContext}; +use crate::planner_strategy::constants::DIRECT_TRANSFORM; +use crate::planner_strategy::constants::MODEL_REFERENCE_KIND; +use crate::planner_strategy::models::ExpectedTypes; +use crate::planner_strategy::models::RelationFixturePlanningContext; use crate::planner_strategy::types::FixtureKey; /// Each chain model's known column types with direct lineage types propagated from its inputs. diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_empty_fixture_sql.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_empty_fixture_sql.rs index fe7dffadd8..5822633131 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_empty_fixture_sql.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_empty_fixture_sql.rs @@ -2,17 +2,19 @@ use crate::planner_strategy::_helpers::fixtures::fixture_sql::fixture_marker; use crate::planner_strategy::models::FixtureRelation; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult}; use sqlbuild_analysis::planner_fixtures::main::{ empty_fixture_error::empty_fixture_error, render_empty::render_empty, }; use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; +use sqlbuild_python_base::native_host::models::Dialect; +use sqlbuild_python_base::native_host::types::Syntax; /// An empty-fixture marker rendered as the relation's typed empty projection, or why it cannot be. pub fn complete_empty_fixture_sql( sql: &str, relation: Option<&FixtureRelation>, - adapter: &Bound<'_, PyAny>, + dialect: &Bound<'_, Dialect>, ) -> PyResult<(String, Option)> { if !fixture_marker(sql)? { return Ok((sql.into(), None)); @@ -27,8 +29,9 @@ pub fn complete_empty_fixture_sql( } Ok(( render_empty(metadata, |name| { - adapter - .call_method1("render_identifier", (name,))? + dialect + .syntax("render_identifier")? + .call1((name,))? .extract::() })?, None, diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_fixtures.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_fixtures.rs index 2edf3b0b59..6ddfd488b5 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_fixtures.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_fixtures.rs @@ -8,17 +8,18 @@ use crate::planner_strategy::_helpers::fixtures::fixture_types::expected_fixture use crate::planner_strategy::models::FixtureCompletion; use crate::planner_strategy::models::RelationFixturePlanningContext; use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::native_host::models::Dialect; /// Complete the fixture groups of one chain against the planning context. pub fn complete_fixtures<'py, 'a>( context: &'a RelationFixturePlanningContext, - adapter: &Bound<'py, PyAny>, + dialect: &Bound<'py, Dialect>, (ordered_model_names, fixture_groups): (&[String], &Bound<'py, PyAny>), ) -> PyResult> { - let py = adapter.py(); - let prepared = prepare(fixture_groups, context, adapter)?; + let py = dialect.py(); + let prepared = prepare(fixture_groups, context, dialect)?; let fixture_keys = fixture_keys_requiring_analysis(&prepared, context)?; - let required = required_fixture_columns(context, ordered_model_names, &fixture_keys, adapter)?; + let required = required_fixture_columns(context, ordered_model_names, &fixture_keys, dialect)?; let expected = expected_fixture_types(context, py, ordered_model_names)?; - complete_prepared(prepared, (context, &required), expected, adapter) + complete_prepared(prepared, (context, &required), expected, dialect) } diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_typed_null_columns.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_typed_null_columns.rs index 99860782a3..9333c39dd2 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_typed_null_columns.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_typed_null_columns.rs @@ -4,6 +4,7 @@ use crate::planner_strategy::_helpers::fixtures::fixture_sql::CompletionSql; use crate::planner_strategy::_helpers::fixtures::fixture_sql::render_completed; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::native_host::models::Dialect; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Fixture SQL with its untyped null columns cast to the expected types. @@ -11,7 +12,7 @@ pub fn complete_typed_null_columns( sql: &str, inference: &Bound<'_, PyAny>, expected_types: &[(String, String)], - adapter: &Bound<'_, PyAny>, + dialect: &Bound<'_, Dialect>, ) -> PyResult { let py = inference.py(); let typed_nulls = PyDict::new(py); @@ -33,6 +34,6 @@ pub fn complete_typed_null_columns( quoted_names: native_field(inference, "quoted_names")?, completed: Vec::new(), }, - adapter, + dialect, ) } diff --git a/crates/sqlbuild-python-project/src/planner_strategy/models.rs b/crates/sqlbuild-python-project/src/planner_strategy/models.rs index 7895145136..3b2b6e4e49 100644 --- a/crates/sqlbuild-python-project/src/planner_strategy/models.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/models.rs @@ -11,6 +11,7 @@ use sqlbuild_analysis::planner_fixtures::models::{ FixtureColumn, FixtureReadCache, FixtureTypeProjection, }; use sqlbuild_analysis::required_columns::models::ColumnRead; +use sqlbuild_python_base::native_host::models::Dialect; use std::collections::{BTreeSet, HashMap}; use std::sync::{Arc, Mutex}; @@ -71,7 +72,7 @@ pub(crate) struct ExpectedTypes<'a> { #[derive(Debug)] pub struct ModelEntryDecisions<'py> { pub model: Bound<'py, pyo3::types::PyAny>, - pub adapter: Bound<'py, pyo3::types::PyAny>, + pub dialect: Bound<'py, Dialect>, pub snapshot: Bound<'py, pyo3::types::PyAny>, pub change_result: Bound<'py, pyo3::types::PyAny>, pub action: Bound<'py, pyo3::types::PyAny>, @@ -120,7 +121,7 @@ pub struct AuditPlanning<'py> { pub model_locations: Bound<'py, pyo3::types::PyAny>, pub seed_locations: Bound<'py, pyo3::types::PyAny>, pub source_map: Bound<'py, pyo3::types::PyAny>, - pub adapter: Bound<'py, pyo3::types::PyAny>, + pub dialect: Bound<'py, Dialect>, pub model_materializations: Bound<'py, pyo3::types::PyAny>, pub lineage: Py, } diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_boundary.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_boundary.rs index 38373ed435..9d869ca651 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_boundary.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_boundary.rs @@ -16,7 +16,8 @@ use sqlbuild_orchestration::snapshots::models::SnapshotNode; use crate::compiled_project::main::plain_value::plain_value; use crate::compiled_project::models::NativeCompiledProject; use crate::compiled_views::models::NativeProjectView; -use crate::project_snapshot::_helpers::snapshot::snapshot_project::{destination, insert_node}; +use crate::project_snapshot::_helpers::snapshot::snapshot_project::destination; +use crate::project_snapshot::_helpers::snapshot::snapshot_project::insert_node; use crate::resource_views::models::NativeCompiledSeed; use crate::resource_views::types::SeedViews; use sqlbuild_python_base::records::main::fields::field::field; diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs index b4babc2ea7..d03fc007b2 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs @@ -1,6 +1,7 @@ //! Compile TOML jobs and groups and Python calendars and sensors against retained graph data before publication. use std::collections::{BTreeMap, BTreeSet}; +use std::num::NonZeroU32; use std::path::Path; use pyo3::exceptions::PyValueError; @@ -8,14 +9,18 @@ use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use serde_json::Value; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; +use sqlbuild_config::project::main::load_pool_limits::load_pool_limits; +use sqlbuild_config::project::main::project_config_paths::project_config_paths; +use sqlbuild_config::project::models::PoolLimits; use sqlbuild_discovery::orchestration_files::models::{ FilePosition, InitialStatus, NamedModule, OrchestrationInventory, RunFile, ScheduleSource, }; use sqlbuild_orchestration::declarations::main::group_dependencies::group_dependencies; +use sqlbuild_orchestration::declarations::main::group_wait_reasons::group_wait_reasons; use sqlbuild_orchestration::declarations::main::snapshot_definitions::snapshot_definitions; use sqlbuild_orchestration::declarations::models::{ - DeclaredAt, Definitions, GroupDefinition, JobDefinition, Schedule, TriggerDeclaration, - TriggerDefinition, TriggerStatus, + ConcurrencySource, DeclaredAt, Definitions, GroupDefinition, JobDefinition, PoolSlots, + ResolvedConcurrency, Schedule, TriggerDeclaration, TriggerDefinition, TriggerStatus, }; use sqlbuild_orchestration::snapshots::models::SnapshotInput; @@ -23,6 +28,7 @@ use crate::graph::models::ProjectSelection; use crate::native_host::main::compile_calendars::compile_calendars; use crate::native_host::main::compile_sensors::compile_sensors; use crate::native_host::models::CalendarRequest; +use sqlbuild_python_base::orchestration_reads::main::dependency_failures::dependency_failures; use sqlbuild_python_base::orchestration_reads::main::imported_declarations::imported_declarations; use sqlbuild_python_base::orchestration_reads::main::run_controls::run_controls; use sqlbuild_python_base::orchestration_reads::main::run_resources::run_resources; @@ -121,10 +127,50 @@ fn matching_imports( Ok(()) } +/// Each pool the run takes, with its amount and the pool's limit on the target. +fn run_pools( + run: &RunFile, + limits: &BTreeMap, +) -> PyResult> { + let mut pools: BTreeMap = BTreeMap::new(); + for (pool, amount) in &run.pools { + let slots = limits + .get(pool) + .copied() + .and_then(NonZeroU32::new) + .zip(NonZeroU32::new(*amount)); + let Some((limit, amount)) = slots.filter(|(limit, amount)| amount <= limit) else { + return Err(PyValueError::new_err(format!( + "{} takes pool {pool}, which the project does not declare with room for its slots", + run.name.value + ))); + }; + pools.insert(pool.clone(), PoolSlots { amount, limit }); + } + Ok(pools) +} + +/// The job's own `concurrency` setting when it has one, otherwise the target's resolved value. +fn job_concurrency( + settings: &BTreeMap, + target: ResolvedConcurrency, +) -> ResolvedConcurrency { + match settings.get("concurrency").and_then(Value::as_u64) { + Some(limit) => ResolvedConcurrency { + limit, + source: ConcurrencySource::Job, + }, + None => target, + } +} + pub(crate) fn compile_declarations( py: Python<'_>, mut input: SnapshotInput, - (project_dir, graph, selected): (&Path, &ProjectGraph, &ProjectSelection), + ((project_dir, graph, selected), concurrency): ( + (&Path, &ProjectGraph, &ProjectSelection), + ResolvedConcurrency, + ), (inventory, declarations): (&OrchestrationInventory, Vec>), ) -> PyResult { if inventory.is_empty() && declarations.is_empty() { @@ -191,6 +237,9 @@ pub(crate) fn compile_declarations( .map(str::to_owned) .collect::>() }; + let pool_limits: PoolLimits = load_pool_limits(&project_config_paths(project_dir)) + .map_err(|_| PyValueError::new_err("cannot read [pools] of sqlbuild_project.toml"))?; + let limits: &BTreeMap = pool_limits.on(&input.target); let mut jobs: BTreeMap = BTreeMap::new(); let mut triggers: BTreeMap = BTreeMap::new(); let mut calendar_requests: Vec = Vec::new(); @@ -229,7 +278,8 @@ pub(crate) fn compile_declarations( description: run.description.clone(), selection, settings: job.settings.clone(), - concurrency_groups: run.concurrency_groups.iter().cloned().collect(), + pools: run_pools(run, limits)?, + resolved_concurrency: Some(job_concurrency(&job.settings, concurrency)), select: Some(job.select.value.clone()), exclude, targets: run.targets.iter().cloned().collect(), @@ -278,7 +328,8 @@ pub(crate) fn compile_declarations( description: run.description.clone(), selection, settings: BTreeMap::new(), - concurrency_groups: run.concurrency_groups.iter().cloned().collect(), + pools: run_pools(run, limits)?, + resolved_concurrency: Some(concurrency), select: None, exclude: None, targets: run.targets.iter().cloned().collect(), @@ -288,7 +339,9 @@ pub(crate) fn compile_declarations( on_failure: controls.on_failure, resources: None, }, + wait_reasons: group_wait_reasons(&input, &jobs, &dependencies), jobs: dependencies, + dependency_failure: dependency_failures(&group.dependencies, &members), }, ); } diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_models.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_models.rs index 44aeaa81e2..032381d580 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_models.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_models.rs @@ -17,9 +17,9 @@ use crate::compiled_project::models::NativeCompiledProject; use crate::compiled_views::models::NativeCompiledModel; use crate::compiled_views::models::NativeProjectView; use crate::compiled_views::types::ModelInputRecord; -use crate::project_snapshot::_helpers::snapshot::snapshot_project::{ - dependency_keys, destination, insert_node, -}; +use crate::project_snapshot::_helpers::snapshot::snapshot_project::dependency_keys; +use crate::project_snapshot::_helpers::snapshot::snapshot_project::destination; +use crate::project_snapshot::_helpers::snapshot::snapshot_project::insert_node; use sqlbuild_python_front::attachments::main::analysis_cursor_splices::analysis_cursor_splices; /// Each model's snapshot node, and the `columns` document of ranges kept out of node identity. diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_resources.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_resources.rs index c7791c304a..86770ccb32 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_resources.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_resources.rs @@ -11,12 +11,12 @@ use sqlbuild_orchestration::snapshots::models::SnapshotNode; use crate::compiled_project::main::plain_value::plain_value; use crate::compiled_project::models::NativeCompiledProject; use crate::compiled_views::models::NativeProjectView; -use crate::project_snapshot::_helpers::snapshot::snapshot_boundary::{ - boundary_nodes, columns, metadata, -}; -use crate::project_snapshot::_helpers::snapshot::snapshot_project::{ - dependency_keys, destination, insert_node, -}; +use crate::project_snapshot::_helpers::snapshot::snapshot_boundary::boundary_nodes; +use crate::project_snapshot::_helpers::snapshot::snapshot_boundary::columns; +use crate::project_snapshot::_helpers::snapshot::snapshot_boundary::metadata; +use crate::project_snapshot::_helpers::snapshot::snapshot_project::dependency_keys; +use crate::project_snapshot::_helpers::snapshot::snapshot_project::destination; +use crate::project_snapshot::_helpers::snapshot::snapshot_project::insert_node; use crate::resource_views::models::NativeCompiledFunction; pub(crate) fn resource_nodes( diff --git a/crates/sqlbuild-python-project/src/project_snapshot/main/compile_declarations.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/compile_declarations.rs index de4e6710ad..c20b2bdec1 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/main/compile_declarations.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/compile_declarations.rs @@ -5,6 +5,7 @@ use crate::project_snapshot::_helpers::snapshot::snapshot_declarations; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; +use sqlbuild_orchestration::declarations::models::ResolvedConcurrency; use sqlbuild_orchestration::snapshots::models::SnapshotInput; use std::path::Path; @@ -13,13 +14,16 @@ use std::path::Path; pub fn compile_declarations( py: Python<'_>, input: SnapshotInput, - (project_dir, graph, selected): (&Path, &ProjectGraph, &ProjectSelection), + ((project_dir, graph, selected), concurrency): ( + (&Path, &ProjectGraph, &ProjectSelection), + ResolvedConcurrency, + ), (inventory, declarations): (&OrchestrationInventory, Vec>), ) -> PyResult { snapshot_declarations::compile_declarations( py, input, - (project_dir, graph, selected), + ((project_dir, graph, selected), concurrency), (inventory, declarations), ) } diff --git a/crates/sqlbuild-python-project/src/project_snapshot/models.rs b/crates/sqlbuild-python-project/src/project_snapshot/models.rs index d4dc746edd..7ec951f026 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/models.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/models.rs @@ -35,7 +35,7 @@ use sqlbuild_orchestration::runs::models::RunChange; use sqlbuild_orchestration::runs::models::RunCommand; use sqlbuild_orchestration::runs::models::RunExecution; use sqlbuild_orchestration::runs::models::RunOutcome; -use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; +use sqlbuild_orchestration::snapshots::main::publication::publish_remote_snapshot::publish_remote_snapshot; use sqlbuild_orchestration::snapshots::models::SnapshotInput; use sqlbuild_orchestration::snapshots::models::SnapshotRecord; use sqlbuild_orchestration::worker::errors::WorkerError; @@ -48,6 +48,7 @@ use std::collections::VecDeque; use std::sync::Arc; use std::sync::Mutex; use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicU32; use std::sync::atomic::Ordering; use uuid::Uuid; @@ -118,27 +119,29 @@ impl NativeOrchestrationSnapshot { #[derive(Debug)] pub struct WorkerBuild { pub worker: Py, - pub(crate) execution: RunExecution, - pub(crate) completed: Mutex>, - pub(crate) checks: Mutex>>, + execution: RunExecution, + completed: Mutex>, + checks: Mutex>>, /// The snapshot's audits and SQL tests, each with the node keys it checks. - pub(crate) attached: BTreeMap>, + attached: BTreeMap>, /// Each SQL test with every node a failure of it blocks in a build: its expected, assertion and reference models and the functions it checks. - pub(crate) blocked: BTreeMap>, + blocked: BTreeMap>, /// How many of each function's planned tests have still to report; a function is built before its tests run. - pub(crate) untested: Mutex>, + untested: Mutex>, /// A built function's result, held until its tests report so a failing one can fail it instead. - pub(crate) held: Mutex>, + held: Mutex>, /// Each snapshot node's direct upstream node keys, for naming what a skipped node waited on. - pub(crate) upstream: BTreeMap>, + upstream: BTreeMap>, /// The failed node each finished node traces back to: itself when it failed, its cause when skipped. - pub(crate) causes: Mutex>, + causes: Mutex>, /// The worst attached outcome of each node, for commands that only run tests or audits. - pub(crate) checked: Mutex>, + checked: Mutex>, /// Whether the command returned, having run every check it planned, rather than being refused or raising. - pub(crate) returned: AtomicBool, + returned: AtomicBool, /// Why the worker refused the run before executing anything, reported as the run's status reason. - pub(crate) refusal: Mutex>, + refusal: Mutex>, + /// Error-severity audit failures the build let through, reported before the run finishes. + ignored_audit_failures: AtomicU32, } impl WorkerBuild { @@ -159,6 +162,7 @@ impl WorkerBuild { causes: Mutex::default(), checked: Mutex::default(), returned: AtomicBool::new(false), + ignored_audit_failures: AtomicU32::new(0), refusal: Mutex::default(), }); for key in state.execution.skipped_nodes.keys() { @@ -176,8 +180,16 @@ impl WorkerBuild { self.execution.claim.request.run_id } + /// The build let `count` error-severity audit failures through. + pub fn ignored_audit_failures(&self, count: i64) { + self.ignored_audit_failures.store( + u32::try_from(count.max(0)).unwrap_or(u32::MAX), + Ordering::SeqCst, + ); + } + /// Test and audit runs execute only attached checks, so their nodes take the checks' outcome. - pub(crate) fn checks_only(&self) -> bool { + fn checks_only(&self) -> bool { matches!(self.command(), RunCommand::Test | RunCommand::Audit) } @@ -262,7 +274,7 @@ impl WorkerBuild { } /// Count each function's tests this build plans, so a built function waits only for tests that will report. - pub(crate) fn count_planned_tests(&self, plan: &Bound<'_, PyAny>) -> PyResult<()> { + fn count_planned_tests(&self, plan: &Bound<'_, PyAny>) -> PyResult<()> { let mut untested = self.untested.lock().map_err(|_| poisoned())?; for entry in field(plan, "test_entries")?.try_iter()? { let test = format!("{SQL_TEST_KEY_PREFIX}{}", field(&entry?, "name")?.str()?); @@ -300,7 +312,7 @@ impl WorkerBuild { } /// A skip in a build names the failed node upstream of it; a test or audit run that raised skips what it never checked, with no cause. - pub(crate) fn complete( + fn complete( &self, py: Python<'_>, key: &str, @@ -473,7 +485,7 @@ impl WorkerBuild { } /// The failed node nearest upstream of `key` in this run, or the cause a skipped one carries. - pub(crate) fn failed_upstream(&self, key: &str) -> PyResult> { + fn failed_upstream(&self, key: &str) -> PyResult> { let causes = self.causes.lock().map_err(|_| poisoned())?; let mut seen: BTreeSet<&str> = BTreeSet::from([key]); let mut pending: VecDeque<&str> = VecDeque::from([key]); @@ -491,7 +503,7 @@ impl WorkerBuild { } /// A selected SQL test completes itself; a failing one, found by its compiled name such as `parent [case]`, fails the nodes it blocks; each test that reports releases a function waiting on its tests. - pub(crate) fn finished_test(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { + fn finished_test(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { let py = result.py(); let outcome: String = field(result, "outcome")?.str()?.extract()?; let passed = SqlTestOutcome::from_value(&outcome) == Some(SqlTestOutcome::Pass); @@ -525,7 +537,7 @@ impl WorkerBuild { } /// Count `test` as reported for each function it checks, and publish a held function once none of its tests is left. - pub(crate) fn release_tested(&self, py: Python<'_>, test: &str) -> PyResult<()> { + fn release_tested(&self, py: Python<'_>, test: &str) -> PyResult<()> { let mut released: Vec<(String, Completion)> = Vec::new(); { let mut untested = self.untested.lock().map_err(|_| poisoned())?; @@ -549,12 +561,7 @@ impl WorkerBuild { } /// Publish a node's result, or hold a built function's while its tests have still to report. - pub(crate) fn complete_or_hold( - &self, - py: Python<'_>, - key: &str, - completion: Completion, - ) -> PyResult<()> { + fn complete_or_hold(&self, py: Python<'_>, key: &str, completion: Completion) -> PyResult<()> { let waiting = !self.checks_only() && matches!(completion.0, NodeOutcome::Success | NodeOutcome::Warn) && self @@ -671,12 +678,7 @@ impl WorkerBuild { self.finish_check(result.py(), &key, outcome) } - pub(crate) fn finish_check( - &self, - py: Python<'_>, - key: &str, - outcome: NodeOutcome, - ) -> PyResult<()> { + fn finish_check(&self, py: Python<'_>, key: &str, outcome: NodeOutcome) -> PyResult<()> { self.worker.bind(py).get().change( py, &format!("check-finished/{key}"), @@ -789,6 +791,14 @@ impl WorkerBuild { .map_err(|_| poisoned())? .clone() .unwrap_or_else(|| "worker executor completed and closed its connections".into()); + let ignored: u32 = self.ignored_audit_failures.load(Ordering::SeqCst); + if ignored > 0 { + self.worker.bind(py).get().change( + py, + "worker-audit-failures-ignored", + RunChange::AuditFailuresIgnored { count: ignored }, + )?; + } self.worker.bind(py).get().change( py, "worker-finished", diff --git a/crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs b/crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs index d289897091..a5ebabaafb 100644 --- a/crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs +++ b/crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs @@ -21,7 +21,8 @@ use sqlbuild_pysource::literal_sql::main::literal_sql_calls::literal_sql_calls; use sqlbuild_pysource::literal_sql::models::FunctionSource; use crate::graph::main::nodes::model_python_hook_names::model_python_hook_names; -use crate::references::_helpers::reference_facts::{ProjectFacts, project_facts}; +use crate::references::_helpers::reference_facts::ProjectFacts; +use crate::references::_helpers::reference_facts::project_facts; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::native_host::main::host_entry::host_entry; diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs index 4d3c222ee1..e57df87031 100644 --- a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs @@ -8,7 +8,8 @@ use sqlbuild_host::entries::models::AdapterHost; use crate::analysis::constants::CASE_SENSITIVE_BINDING_DIALECTS; use crate::analysis::main::normalize_analysis_sql::normalize_analysis_sql; -use crate::analysis::models::{BindingCatalog, CatalogKey}; +use crate::analysis::models::BindingCatalog; +use crate::analysis::models::CatalogKey; use crate::analysis_session::main::infer_native_expression_source_shapes::infer_native_expression_source_shapes; use crate::compiled_project::models::ReferenceTuple; use crate::compiled_views::main::compiled_models::declared_rows::declared_rows; diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs index 1b4a796e94..3efb430ca5 100644 --- a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs @@ -8,13 +8,16 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple}; -use crate::analysis::constants::{CASE_SENSITIVE_BINDING_DIALECTS, NATIVE_DIALECT_ALIASES}; +use crate::analysis::constants::CASE_SENSITIVE_BINDING_DIALECTS; +use crate::analysis::constants::NATIVE_DIALECT_ALIASES; use crate::analysis::main::schema_validation_request::schema_validation_request; use crate::analysis::main::schema_validations::schema_validations; use crate::analysis::models::BindingPositions; use crate::analysis::models::SchemaRequestParts; use crate::analysis_session::main::infer_native_query_columns::infer_native_query_columns; -use crate::semantic_shapes::_helpers::shapes::{model_inputs, project_shapes, published_shape}; +use crate::semantic_shapes::_helpers::shapes::model_inputs; +use crate::semantic_shapes::_helpers::shapes::project_shapes; +use crate::semantic_shapes::_helpers::shapes::published_shape; use crate::semantic_shapes::constants::SOURCE_REBINDING_ANALYSES; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::Request; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs index 067d5c84bc..60d0a5ae42 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs @@ -10,11 +10,15 @@ use sqlbuild_cursor::algebra::models::{Grain, Scalar}; use crate::compiled_views::main::compiled_models::name_object::name_object; use crate::compiled_views::main::compiled_models::query_sql::query_sql; -use crate::sql_test_glue::constants::{ - CURSOR_ALGEBRA_EXCEPTIONS, CURSOR_END_KEY, CURSOR_START_KEY, DEFAULT_INTEGER_END, - DEFAULT_INTEGER_START, DEFAULT_TIMESTAMP_END, DEFAULT_TIMESTAMP_START, INTEGER_CURSOR_TYPE, - TIMESTAMP_CURSOR_TYPE, -}; +use crate::sql_test_glue::constants::CURSOR_ALGEBRA_EXCEPTIONS; +use crate::sql_test_glue::constants::CURSOR_END_KEY; +use crate::sql_test_glue::constants::CURSOR_START_KEY; +use crate::sql_test_glue::constants::DEFAULT_INTEGER_END; +use crate::sql_test_glue::constants::DEFAULT_INTEGER_START; +use crate::sql_test_glue::constants::DEFAULT_TIMESTAMP_END; +use crate::sql_test_glue::constants::DEFAULT_TIMESTAMP_START; +use crate::sql_test_glue::constants::INTEGER_CURSOR_TYPE; +use crate::sql_test_glue::constants::TIMESTAMP_CURSOR_TYPE; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/fixture_validation.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/fixture_validation.rs index d32a0a8fd2..2f59202a0b 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/fixture_validation.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/fixture_validation.rs @@ -1,5 +1,6 @@ //! Static SQL-test fixture shape validation over native fixture completion. +use sqlbuild_python_base::native_host::models::Dialect; use std::collections::BTreeSet; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -31,7 +32,8 @@ pub(crate) fn validated_fixtures<'py>( ) -> PyResult; 4], Vec>> { let py = test.py(); let (chain, [refs, sources, seeds, expected]) = request; - let adapter = inputs.1; + let dialect = Dialect::of(inputs.1)?; + let dialect = &dialect; let resource_types = record_class(py, RecordId::CompiledResourceType)?; let model = native_field(&resource_types, "MODEL")?; let kinds = [ @@ -48,7 +50,7 @@ pub(crate) fn validated_fixtures<'py>( ], )?; let chain: Vec = chain.extract()?; - let completion = complete_fixtures(context, adapter, (&chain, groups.as_any()))?; + let completion = complete_fixtures(context, dialect, (&chain, groups.as_any()))?; let mut errors: BTreeSet = BTreeSet::new(); for diagnostic in &completion.diagnostics { let key = native_field(diagnostic, "key")?; @@ -81,7 +83,7 @@ pub(crate) fn validated_fixtures<'py>( let (completed_sql, empty_error) = complete_empty_fixture_sql( &sql.extract::()?, context.relations.get(&model_key), - adapter, + dialect, )?; if let Some(error) = empty_error { errors.insert(format!("{}: expected output '{name}' {error}", location()?)); @@ -91,7 +93,7 @@ pub(crate) fn validated_fixtures<'py>( kwargs.set_item("query_sql", &completed_sql)?; kwargs.set_item( "inference_profile", - adapter.call_method0("expression_inference_profile")?, + dialect.call_method0("expression_inference_profile")?, )?; let inference = context @@ -107,7 +109,7 @@ pub(crate) fn validated_fixtures<'py>( .unwrap_or_default(); completed_expected.set_item( &name, - complete_typed_null_columns(&completed_sql, &inference, types, adapter)?, + complete_typed_null_columns(&completed_sql, &inference, types, dialect)?, )?; Some(native_field(&inference, "columns")?) } diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_assembly.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_assembly.rs index 554d9c1d1c..64b8aed0e5 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_assembly.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_assembly.rs @@ -7,7 +7,8 @@ use crate::compiled_views::main::compiled_models::name_object::name_object; use crate::planner_strategy::main::planning_context::planning_context; use crate::planner_strategy::models::RelationFixturePlanningContext; use crate::sql_test_glue::_helpers::plan::plan_driver::planned as planned_tests; -use crate::sql_test_glue::models::{SqlTestAssembly, SqlTestPlanOptions}; +use crate::sql_test_glue::models::SqlTestAssembly; +use crate::sql_test_glue::models::SqlTestPlanOptions; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_driver.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_driver.rs index f935a7abb8..2f4b969e0f 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_driver.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_driver.rs @@ -7,9 +7,8 @@ use pyo3::types::{PyDict, PyTuple}; use crate::planner_phases::main::object::object; use crate::planner_strategy::main::count_chain_resolution::count_chain_resolution; use crate::sql_test_glue::_helpers::plan::plan_values::tuple; -use crate::sql_test_glue::_helpers::plan::planning::{ - ChainRequest, resolve_compiled_sql_test_chains, -}; +use crate::sql_test_glue::_helpers::plan::planning::ChainRequest; +use crate::sql_test_glue::_helpers::plan::planning::resolve_compiled_sql_test_chains; use crate::sql_test_glue::models::PlanRow; use crate::sql_test_glue::models::SqlTestPlanOptions; use sqlbuild_python_base::boundary::models::KeywordArguments; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_entries.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_entries.rs index 8e2028dd65..f22b0d047b 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_entries.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_entries.rs @@ -9,7 +9,8 @@ use sqlbuild_model_config::execution_limits::models::ExecutionLimit; use crate::planner_strategy::constants::MODEL_REFERENCE_KIND; use crate::planner_strategy::models::RelationFixturePlanningContext; use crate::sql_test_glue::_helpers::plan::plan_values::model_payload; -use crate::sql_test_glue::constants::{TABLE_FN, UDF}; +use crate::sql_test_glue::constants::TABLE_FN; +use crate::sql_test_glue::constants::UDF; use sqlbuild_analysis::compiler::main::missing_expected_columns::missing_expected_columns; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::record_class::record_class; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_rendering.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_rendering.rs index 5201e859d9..e02fee468d 100644 --- a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_rendering.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_rendering.rs @@ -2,14 +2,15 @@ use crate::compiled_views::main::compiled_models::name_object::name_object; use crate::compiled_views::main::compiled_models::query_sql::query_sql; -use crate::sql_test_glue::_helpers::plan::plan_values::{model_payload, string}; +use crate::sql_test_glue::_helpers::plan::plan_values::model_payload; +use crate::sql_test_glue::_helpers::plan::plan_values::string; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; -use crate::sql_test_glue::_helpers::inputs::cursor_window::{ - declared_window_cursor_models, declares_cursor_window, render_test_cursor_intrinsics, -}; +use crate::sql_test_glue::_helpers::inputs::cursor_window::declared_window_cursor_models; +use crate::sql_test_glue::_helpers::inputs::cursor_window::declares_cursor_window; +use crate::sql_test_glue::_helpers::inputs::cursor_window::render_test_cursor_intrinsics; use crate::sql_test_glue::constants::SENTINEL; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index 49686bc982..d10755d5a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -1,5 +1,6 @@ //! `sqb audit`: compile the selected audits, run them with nested progress, and report. +use sqlbuild_python_base::native_host::models::Dialect; use std::collections::BTreeSet; use std::sync::Arc; @@ -182,7 +183,7 @@ fn compile_audits<'py>( )?))?; let plan_output: Bound<'py, PyAny> = build_audit_command_plan( &native_field(&context, "project")?, - &adapter, + &Dialect::of(&adapter)?, &native_field(&context, "scope")?, &native_field(&context, "relations")?, )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs index 9d3546c385..60837a78cc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs @@ -12,14 +12,13 @@ use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_host::models::BuildStatus; -use crate::bindings::_helpers::build_command::execution::{ - Outcome, Preparation, write_run_context, -}; +use crate::bindings::_helpers::build_command::execution::Outcome; +use crate::bindings::_helpers::build_command::execution::Preparation; +use crate::bindings::_helpers::build_command::execution::write_run_context; use crate::bindings::_helpers::build_command::invocation::BuildContext; use crate::bindings::_helpers::check::check_results::check_result_text; -use crate::bindings::_helpers::cost_report::degradation::{ - finalize_build_cost as finalize_cost, render_build_cost, -}; +use crate::bindings::_helpers::cost_report::degradation::finalize_build_cost as finalize_cost; +use crate::bindings::_helpers::cost_report::degradation::render_build_cost; use crate::bindings::_helpers::cost_report::functions::cost_output_payload; use sqlbuild_core::build_progress::models::PythonNodeText; use sqlbuild_core::check_command::models::{CheckResultText, CheckStatus}; @@ -378,6 +377,9 @@ pub(crate) fn complete_direct_build<'py>( native_field(result, "failed")?.is_truthy() })?; let exit_code: i32 = build_exit_code(succeeded, (python_failed, checks_failed)); + if let Some(worker) = &context.worker { + worker.ignored_audit_failures(result.ignored_audit_failure_count); + } let target_dir: Bound<'py, PyAny> = joined(&context.get("effective_project_dir")?, &["target"])?; write_runtime_target(&target_dir, &plan, result)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index a95586b7aa..7186236739 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -12,17 +12,18 @@ use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_host::models::{BuildExecutionResult, BuildStatus, ExecutionStatus}; use crate::bindings::_helpers::build_command::invocation::BuildContext; -use crate::bindings::_helpers::build_command::lifecycle::{ - DirectPythonLifecycle, LifecycleInputs, prepare_direct_python_lifecycle, -}; -use crate::bindings::_helpers::check::check_core::{ - build_check_relation_targets, load_results_by_loader_name, record_python_run_state_results, - relevant_check_functions, -}; +use crate::bindings::_helpers::build_command::lifecycle::DirectPythonLifecycle; +use crate::bindings::_helpers::build_command::lifecycle::LifecycleInputs; +use crate::bindings::_helpers::build_command::lifecycle::prepare_direct_python_lifecycle; +use crate::bindings::_helpers::check::check_core::build_check_relation_targets; +use crate::bindings::_helpers::check::check_core::load_results_by_loader_name; +use crate::bindings::_helpers::check::check_core::record_python_run_state_results; +use crate::bindings::_helpers::check::check_core::relevant_check_functions; use crate::bindings::_helpers::check::check_results::write_check_results; use crate::bindings::_helpers::command_runtime::warehouses::describe_connection_warehouse; use crate::bindings::_helpers::sql_test_command::test_command::timeout_record; -use crate::bindings::_helpers::sql_test_command::timeouts::{TimeoutKind, resolve_run_timeout}; +use crate::bindings::_helpers::sql_test_command::timeouts::TimeoutKind; +use crate::bindings::_helpers::sql_test_command::timeouts::resolve_run_timeout; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; @@ -59,7 +60,8 @@ use sqlbuild_python_planner::consumption::main::held_back_text::held_back_text; use sqlbuild_python_planner::consumption::main::policy_line::policy_line; use sqlbuild_python_planner::consumption::main::prepare::prepare as prepare_consumption; use sqlbuild_python_planner::consumption::main::run_reason_text::run_reason_text; -use sqlbuild_python_planner::consumption::models::{ConsumptionRecorder, Prepared}; +use sqlbuild_python_planner::consumption::models::ConsumptionRecorder; +use sqlbuild_python_planner::consumption::models::Prepared; use sqlbuild_python_planner::freshness::main::append_freshness_records::append_freshness_records; use sqlbuild_python_planner::observations::main::append_build_observations::append_build_observations; use sqlbuild_python_planner::planner_resolution::main::cursor::type_cursor_overrides::type_cursor_overrides; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index e843ca0896..eb8395976e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -10,7 +10,9 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::build_command::defer_clone::run_defer_clone_boundary_prephase; use crate::bindings::_helpers::build_command::invocation::BuildContext; -use crate::bindings::_helpers::build_command::policies::{enforce_plan_policies, model_limit}; +use crate::bindings::_helpers::build_command::policies::enforce_plan_policies; +use crate::bindings::_helpers::build_command::policies::model_limit; +use crate::bindings::_helpers::command_runtime::audit_failures::resolve_ignore_audit_failures; use crate::bindings::_helpers::compile_driver::plan_pipeline::run_compile_pipeline; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -77,6 +79,17 @@ fn request_options<'py>(context: &BuildContext<'py>) -> PyResult) -> PyResult<()> { defer_clone::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_cache.rs index af9ef841bc..b051c45169 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_cache.rs @@ -8,9 +8,9 @@ use sqlbuild_analysis::definition_changes::main::definition_digest::definition_d use sqlbuild_rules::build_identity::main::native_build_identity::native_build_identity; use crate::bindings::_helpers::change_selection::base_compile::NOTES_EXTENSION; -use crate::bindings::_helpers::change_selection::fingerprints::{ - environment_digest_with, process_environment, referenced_environment, -}; +use crate::bindings::_helpers::change_selection::fingerprints::environment_digest_with; +use crate::bindings::_helpers::change_selection::fingerprints::process_environment; +use crate::bindings::_helpers::change_selection::fingerprints::referenced_environment; /// Cached base snapshots kept per project; older ones are removed after each compile. const KEPT_SNAPSHOTS: usize = 8; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs index f8d5922e53..c4983c66af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs @@ -18,9 +18,10 @@ use sqlbuild_analysis::definition_changes::main::sql_definition_digest::sql_defi use sqlbuild_analysis::definition_changes::main::sql_macro_definition_digest::sql_macro_definition_digest; use sqlbuild_analysis::definition_changes::models::{DefinitionSnapshot, NodeDefinition}; -use crate::bindings::_helpers::change_selection::fingerprints::{ - Fingerprinting, directory_digest, file_digest, text_of, -}; +use crate::bindings::_helpers::change_selection::fingerprints::Fingerprinting; +use crate::bindings::_helpers::change_selection::fingerprints::directory_digest; +use crate::bindings::_helpers::change_selection::fingerprints::file_digest; +use crate::bindings::_helpers::change_selection::fingerprints::text_of; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_planner::planner_identity::main::function_local_hashes::function_local_hashes; use sqlbuild_python_planner::planner_identity::main::seed_identity::seed_identity; @@ -62,7 +63,7 @@ const PROJECT_CONFIG_ELSEWHERE: [&str; 11] = [ ]; const LOCAL_CONFIG_ELSEWHERE: [&str; 4] = ["settings", "setting_overrides", "vars", "scenario"]; /// Settings that tune how a run executes or validates, not what a node computes. -const RUNTIME_SETTINGS: [&str; 10] = [ +const RUNTIME_SETTINGS: [&str; 11] = [ "sql_analysis", "require_sql_analysis", "allow_warehouse_round_trips", @@ -73,6 +74,7 @@ const RUNTIME_SETTINGS: [&str; 10] = [ "scenario_timeout", "scenario_timeout_seconds", "delta_compile", + "ignore_audit_failures", ]; const PYTHON_SUFFIX: &str = ".py"; /// Discovered Python node collections, in `PYTHON_NODE_KINDS` order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 74337e8a27..987b9f61b3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -40,13 +40,13 @@ use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_selection::main::parse_project_selector::parse_project_selector; use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; -use crate::bindings::_helpers::check::check_core::{ - ReadSideInputs, load_results_by_loader_name, record_python_run_state_results, - run_read_side_dependencies, -}; -use crate::bindings::_helpers::check::check_results::{ - check_result_text, results_json, write_results, -}; +use crate::bindings::_helpers::check::check_core::ReadSideInputs; +use crate::bindings::_helpers::check::check_core::load_results_by_loader_name; +use crate::bindings::_helpers::check::check_core::record_python_run_state_results; +use crate::bindings::_helpers::check::check_core::run_read_side_dependencies; +use crate::bindings::_helpers::check::check_results::check_result_text; +use crate::bindings::_helpers::check::check_results::results_json; +use crate::bindings::_helpers::check::check_results::write_results; use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/registration.rs index f1df493414..7ffc271e36 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/registration.rs @@ -2,7 +2,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::check::{check_command, check_core, check_results}; +use crate::bindings::_helpers::check::check_command; +use crate::bindings::_helpers::check::check_core; +use crate::bindings::_helpers::check::check_results; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { check_command::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index e5db142c99..7627073e23 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -35,9 +35,8 @@ use crate::bindings::_helpers::scenario_command::test_command::run_scenario_test use crate::bindings::_helpers::scopes::scope_command::run_scope_command; use crate::bindings::_helpers::seed_command::command::run_seed_command; use crate::bindings::_helpers::sql_test_command::test_command::run_test_command; -use crate::bindings::_helpers::workspace_commands::commands::{ - run_playground_command, run_skills_command, -}; +use crate::bindings::_helpers::workspace_commands::commands::run_playground_command; +use crate::bindings::_helpers::workspace_commands::commands::run_skills_command; use pyo3::IntoPyObjectExt; use pyo3::exceptions::{PyAttributeError, PyOSError, PyTypeError, PyUnicodeError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; @@ -334,27 +333,27 @@ fn selector_file_error(py: Python<'_>, path: &Bound<'_, PyAny>, error: &PyErr) - )) } -/// Call the route's command implementation, or the handler a test injected for its slot. -fn call_route<'py>( +/// The positional and keyword arguments a route passes its implementation, or the handler a test injected. +type RouteArguments<'py> = (Vec>, Bound<'py, PyDict>); + +fn route_arguments<'py>( invocation: &Invocation<'py>, route: &Route, - handler: Option>, - recall: &InvocationRecall, -) -> PyResult> { + injected: bool, +) -> PyResult> { let py: Python<'py> = invocation.py(); let mut positional: Vec> = Vec::new(); let keywords: Bound<'py, PyDict> = PyDict::new(py); - let positional_count: usize = match &handler { - Some(_) => route.shape.slot_positional, - None => { - if let Some(extra) = route.implementation_extra { - keywords.set_item(extra.name, invocation.value(extra.source)?)?; - } - if route.shape.implementation_keywords { - 0 - } else { - route.shape.slot_positional - } + let positional_count: usize = if injected { + route.shape.slot_positional + } else { + if let Some(extra) = route.implementation_extra { + keywords.set_item(extra.name, invocation.value(extra.source)?)?; + } + if route.shape.implementation_keywords { + 0 + } else { + route.shape.slot_positional } }; for (index, field) in route.arguments.iter().enumerate() { @@ -365,6 +364,18 @@ fn call_route<'py>( keywords.set_item(field.name, value)?; } } + Ok((positional, keywords)) +} + +/// Call the route's command implementation, or the handler a test injected for its slot. +fn call_route<'py>( + invocation: &Invocation<'py>, + route: &Route, + handler: Option>, + recall: &InvocationRecall, +) -> PyResult> { + let py: Python<'py> = invocation.py(); + let (positional, keywords) = route_arguments(invocation, route, handler.is_some())?; match handler { Some(handler) => handler.call(PyTuple::new(py, positional)?, Some(&keywords)), None => run_implementation(route.implementation, (positional, keywords), recall), @@ -444,6 +455,42 @@ fn run_implementation<'py>( code.into_bound_py_any(py) } +/// Run `body` with the request record a parsed command's route passes, in the scopes `sqb` runs it in. +pub(crate) fn with_routed_request<'py, T>( + args: &Bound<'py, PyAny>, + body: impl FnOnce(&Bound<'py, PyAny>) -> PyResult, +) -> PyResult { + let py: Python<'py> = args.py(); + within(&warehouse_scope(args)?, |_| { + let invocation = Invocation::new(args)?; + let _secrets = MissingSecretsScope::enter( + args.getattr_opt(ALLOW_MISSING_SECRETS_DEST)? + .is_some_and(|allowed| allowed.is_truthy().unwrap_or(false)), + ); + let route = match command_dispatch(&command_facts(args)?) { + Dispatch::Run(route, None) => route, + Dispatch::Run(_, Some(failure)) | Dispatch::Fail(failure) => { + return Err(user_error(py, failure)); + } + Dispatch::Nothing => { + return Err(PyTypeError::new_err("the command line names no command")); + } + }; + let project_dir: PathBuf = invocation.value(Source::EffectiveProjectDir)?.extract()?; + let selected_target: Option = match invocation.value(Source::Attr("target")) { + Ok(target) => optional_text(&target)?, + Err(_) => None, + }; + with_job_scope((project_dir, selected_target), || { + let arguments = route_arguments(&invocation, route, false)?; + let request = argument(&arguments, "request")?.ok_or_else(|| { + PyTypeError::new_err(format!("{}() takes no request", route.implementation)) + })?; + body(&request) + }) + }) +} + fn routed_command<'py>( args: &Bound<'py, PyAny>, handlers: Option<&Bound<'py, PyAny>>, @@ -555,5 +602,7 @@ pub(crate) fn command_facts(args: &Bound<'_, PyAny>) -> PyResult { scenario_retain: flag("scenario_retain")?, scenario_snapshot_sync: flag("scenario_sync_snapshots")? || flag("scenario_refresh")?, raw_query_diff, + ignores_failures_of_no_audits: attr("ignore_audit_failures")?.is_truthy()? + && !flag("run_audits")?, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs index 68a3017237..5973502dd8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs @@ -3,6 +3,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_python_base::native_host::models::Dialect; +use sqlbuild_python_planner::live_reads::models::LiveReads; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::Request; @@ -158,8 +160,17 @@ pub(crate) fn timed_clone_pipeline<'py>( ("project", destination_project.clone()), ("select", native_field(&pipeline.options, "select")?), ("exclude", native_field(&pipeline.options, "exclude")?), - ("adapter", pipeline.adapter.clone()), - ("connection", native_field(&pipeline.connection, "handle")?), + ("dialect", Dialect::of(&pipeline.adapter)?.into_any()), + ( + "reads", + LiveReads::new( + &pipeline.adapter, + &native_field(&pipeline.connection, "handle")?, + None, + ) + .handle(pipeline.adapter.py())? + .into_any(), + ), ("origin_project", origin_project.clone()), ("origin_source_project", origin_source_project), ("destination_source_project", destination_source_project), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/registration.rs index bca0112872..58dfc5e687 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/registration.rs @@ -2,7 +2,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::clone::{clone_command, clone_pipeline}; +use crate::bindings::_helpers::clone::clone_command; +use crate::bindings::_helpers::clone::clone_pipeline; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { clone_command::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/audit_failures.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/audit_failures.rs new file mode 100644 index 0000000000..0a97a9a39e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/audit_failures.rs @@ -0,0 +1,42 @@ +//! Whether one build lets failing audits hold nothing back: the flag, the target, then `[settings]`. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_config::effective::main::effective_ignore_audit_failures::effective_ignore_audit_failures; +use sqlbuild_config::effective::models::AuditFailureLayers; + +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::compile_attachment::main::build_effective_settings::build_effective_settings; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; +use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; + +const IGNORE_AUDIT_FAILURES: &str = "ignore_audit_failures"; + +/// `request`'s flag (a job run's settings arrive as it), else the selected target's, else `[settings]`. +pub(crate) fn resolve_ignore_audit_failures( + request: &Bound<'_, PyAny>, + discovered: &Bound<'_, PyAny>, + selected_target: &Bound<'_, PyAny>, +) -> PyResult { + let py = request.py(); + let project_config: Bound<'_, PyAny> = native_field(discovered, "project_config")?; + let local_config: Bound<'_, PyAny> = native_field(discovered, "local_config")?; + let target: Option = + match resolve_target_name(py, &project_config, &local_config, selected_target)? { + Some(name) => native_field( + &resolve_target_config(py, &project_config, &local_config, name.as_any())?, + IGNORE_AUDIT_FAILURES, + )? + .extract()?, + None => None, + }; + let settings: bool = native_field( + &build_effective_settings(&project_config, &local_config)?, + IGNORE_AUDIT_FAILURES, + )? + .extract()?; + Ok(effective_ignore_audit_failures(&AuditFailureLayers { + requested: native_field(request, IGNORE_AUDIT_FAILURES)?.extract()?, + target, + settings, + })) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs index 4d947aefd7..9c24ebb2e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs @@ -1,7 +1,15 @@ pub(crate) mod connection; + pub(crate) mod invocation; + pub(crate) mod preview_target; + pub(crate) mod registration; + pub(crate) mod runtime_errors; + pub(crate) mod warehouse_scope; + pub(crate) mod warehouses; + +pub(crate) mod audit_failures; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index 57c5e43654..4a1a0e691a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -1,4 +1,9 @@ pub(crate) mod plan_pipeline; + pub(crate) mod plan_result; + pub(crate) mod registration; + pub(crate) mod static_context; + +pub(crate) mod rules_at_load; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs index d2b3895b16..f4a1a822b6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs @@ -4,13 +4,19 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyTuple}; use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_planner::live_reads::models::LiveReads; +use sqlbuild_python_planner::planner_phases::constants::COMPILING; +use sqlbuild_python_planner::planner_phases::main::timing::timed::timed; +use sqlbuild_python_planner::planner_warehouse::models::PlannerReads; use sqlbuild_rules::lint_text::main::counts_toward_exit::counts_toward_exit; use sqlbuild_rules::rule_results::main::default_rules_notice::default_rules_notice; use sqlbuild_rules::rule_results::main::format_skipped_models::format_skipped_models; -use crate::bindings::_helpers::compile_driver::plan_result::{ - PlanningInputs, connection_with_hooks, pipeline_result, -}; +use crate::bindings::_helpers::compile_driver::plan_result::PlanningInputs; +use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hooks; +use crate::bindings::_helpers::compile_driver::plan_result::pipeline_result; +use crate::bindings::_helpers::compile_driver::rules_at_load::rules_ran_at_load; +use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -53,37 +59,77 @@ pub(crate) fn run_compile_pipeline<'py>( options, (&["discovered_inputs", "adapter"], &["options", "hooks"]), )?; - let discovered: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; - let adapter: Bound<'py, PyAny> = arguments.value("adapter")?; - let options: Bound<'py, PyAny> = - apply_unit_test_selectors(&arguments.value("options")?, &discovered)?; - let hooks: Bound<'py, PyAny> = arguments.value("hooks")?; + planned_project( + ( + &arguments.value("discovered_inputs")?, + &arguments.value("adapter")?, + ), + (&arguments.value("options")?, arguments.value("hooks")?), + (None, None), + ) +} + +/// Compile unless `held` is compiled, gate on rules unless they ran at load, and plan. +pub(crate) fn planned_project<'py>( + (discovered, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (options, hooks): (&Bound<'py, PyAny>, Bound<'py, PyAny>), + (held, planned_through): (Option<&Bound<'py, PyAny>>, Option<&Bound<'py, PyAny>>), +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let options: Bound<'py, PyAny> = apply_unit_test_selectors(options, discovered)?; let hooks: Bound<'py, PyAny> = if hooks.is_none() { - host_entry(py, ExecutorHost::ConnectionHooks)?.call0()? + call_with(&host_entry(py, ExecutorHost::ConnectionHooks)?, &[])? } else { hooks }; let on_progress: Bound<'py, PyAny> = native_field(&hooks, "on_progress")?; let expansion_reuse: Bound<'py, EarlySqlLint> = Bound::new(py, EarlySqlLint::new(false, false))?; - let on_inputs_ready: Bound<'py, PyAny> = - EarlySqlLint::inputs_ready_hook(&expansion_reuse, GENERIC_DIALECT, &adapter); - let compiled: Bound<'py, PyAny> = compiled_phase( - (&discovered, &adapter), - &options, - &on_progress, - Some(&on_inputs_ready), - )?; - let project: Bound<'py, PyAny> = native_field(&compiled, "project")?; - let graph: Bound<'py, PyAny> = project_graph(&project)?; - run_rules_gate( - (&discovered, &adapter, expansion_reuse.as_any()), - (&options, &on_progress), - (&project, &graph), - )?; - planned((&discovered, &adapter), (&options, &hooks), &compiled) + let compiled: Bound<'py, PyAny> = match held { + Some(held) => { + progress( + &on_progress, + "Planning from the published compiled project.", + )?; + let configured: Bound<'py, PyAny> = native_field(&options, "connection_config")?; + if configured.is_none() { + held.clone() + } else { + replaced(held, &[("connection_config", configured)])? + } + } + None => { + let on_inputs_ready: Bound<'py, PyAny> = + EarlySqlLint::inputs_ready_hook(&expansion_reuse, GENERIC_DIALECT, adapter); + timed(COMPILING, || { + compiled_phase( + (discovered, adapter), + &options, + &on_progress, + Some(&on_inputs_ready), + ) + })? + } + }; + if !rules_ran_at_load() { + let project: Bound<'py, PyAny> = native_field(&compiled, "project")?; + let graph: Bound<'py, PyAny> = project_graph(&project)?; + run_rules_gate( + (discovered, adapter, expansion_reuse.as_any()), + (&options, &on_progress), + (&project, &graph), + )?; + } + planned( + (discovered, adapter), + (&options, &hooks), + (&compiled, planned_through), + ) } +/// The failing findings' `RulesResult`, their rendered text and the default-rules notice. +pub(crate) type RuleFailures<'py> = (Bound<'py, PyAny>, String, Option); + /// Run the configured rules and stop the command when any of them finds a fault. fn run_rules_gate<'py>( (discovered, adapter, expansion_reuse): ( @@ -94,6 +140,36 @@ fn run_rules_gate<'py>( (options, on_progress): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), (project, graph): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult<()> { + let Some((_, rendered, notice)) = configured_rule_failures( + (discovered, adapter, expansion_reuse), + (options, on_progress), + (project, graph), + )? + else { + return Ok(()); + }; + let notice: Vec<(&str, String)> = notice + .map(|notice| (DEFAULT_RULES_NOTICE_ATTR, notice)) + .into_iter() + .collect(); + Err(new_annotated_error( + discovered.py(), + ErrorId::Rules, + format!("configured compiler rules failed:\n{rendered}"), + ¬ice, + )) +} + +/// Evaluate the configured rules; the failing findings, or `None` when none fails. +pub(crate) fn configured_rule_failures<'py>( + (discovered, adapter, expansion_reuse): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), + (options, on_progress): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (project, graph): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult>> { let py: Python<'py> = discovered.py(); let rules_error: Bound<'py, PyAny> = error_class(py, ErrorId::Rules)?; let project_dir: Bound<'py, PyAny> = native_field(discovered, "project_dir")?; @@ -156,7 +232,7 @@ fn run_rules_gate<'py>( } } if failing.is_empty() { - return Ok(()); + return Ok(None); } let findings: Bound<'py, PyAny> = PyTuple::new(py, failing)?.into_any(); let summary: Bound<'py, PyAny> = new_record_with( @@ -182,24 +258,16 @@ fn run_rules_gate<'py>( .map(|finding| native_field(&finding?, "code")?.extract()) .collect::>()?; let select_omitted: bool = native_field(&config, "select")?.is_none(); - let notice: Vec<(&str, String)> = - default_rules_notice(select_omitted, codes.iter().map(String::as_str)) - .map(|notice| (DEFAULT_RULES_NOTICE_ATTR, notice)) - .into_iter() - .collect(); - Err(new_annotated_error( - py, - ErrorId::Rules, - format!("configured compiler rules failed:\n{rendered}"), - ¬ice, - )) + let notice: Option = + default_rules_notice(select_omitted, codes.iter().map(String::as_str)); + Ok(Some((summary, rendered, notice))) } /// Open the planning connection, plan, and record the compile and planning times. fn planned<'py>( (discovered, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), (options, hooks): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - compiled: &Bound<'py, PyAny>, + (compiled, planned_through): (&Bound<'py, PyAny>, Option<&Bound<'py, PyAny>>), ) -> PyResult> { let py: Python<'py> = discovered.py(); let tracker: Bound<'py, PyAny> = timing_tracker(py)?; @@ -217,13 +285,31 @@ fn planned<'py>( .node_locations(adapter)?; Some((database.extract()?, schema.extract()?)) }; - let opened: Bound<'py, PyAny> = connection_with_hooks(adapter, &connection_config, hooks)?; - connection = Some(opened.clone()); + let (opened, reads): (Bound<'py, PyAny>, Bound<'py, PlannerReads>) = match planned_through { + Some(through) => { + let on_start: Bound<'py, PyAny> = native_field(hooks, "on_connection_start")?; + if !on_start.is_none() { + on_start.call1((1,))?; + } + ( + py.None().into_bound(py), + PlannerReads::passed(through.clone())?, + ) + } + None => { + let opened: Bound<'py, PyAny> = + connection_with_hooks(adapter, &connection_config, hooks)?; + connection = Some(opened.clone()); + let reads = LiveReads::new(adapter, &opened, None).handle(adapter.py())?; + (opened, reads) + } + }; pipeline_result( &PlanningInputs { discovered_inputs: discovered, adapter, connection: &opened, + reads: &reads, node_results_location, }, &project, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 60f4c4bec3..3f1c5c11c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -1,6 +1,8 @@ //! Plan a compiled project over an open warehouse connection, and open that connection. use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_python_base::native_host::models::Dialect; +use sqlbuild_python_planner::planner_warehouse::models::PlannerReads; use sqlbuild_python_compile::compile_driver::main::stages::build_deferred_locations::build_deferred_locations; use sqlbuild_python_compile::compile_driver::main::stages::gather_deferred_relations::gather_deferred_relations; @@ -40,7 +42,9 @@ type Deferral<'py> = (Bound<'py, PyAny>, Bound<'py, PyAny>); pub(crate) struct PlanningInputs<'a, 'py> { pub(crate) discovered_inputs: &'a Bound<'py, PyAny>, pub(crate) adapter: &'a Bound<'py, PyAny>, + /// The live planning connection, `None` when the plan reads a synced state. pub(crate) connection: &'a Bound<'py, PyAny>, + pub(crate) reads: &'a Bound<'py, PlannerReads>, /// Python nodes' `(database, schema)` under the active target. pub(crate) node_results_location: Option<(Option, Option)>, } @@ -185,6 +189,7 @@ pub(crate) fn pipeline_result<'py>( "selection_diagnostics", "plan_sql_tests", "record_migration_fingerprints", + "ignore_audit_failures", ], )?, )?; @@ -194,8 +199,8 @@ pub(crate) fn pipeline_result<'py>( py, &[ ("project", project.clone()), - ("adapter", inputs.adapter.clone()), - ("connection", inputs.connection.clone()), + ("dialect", Dialect::of(inputs.adapter)?.into_any()), + ("reads", inputs.reads.clone().into_any()), ("selection", selection), ("overrides", overrides), ("deferral", deferral), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/rules_at_load.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/rules_at_load.rs new file mode 100644 index 0000000000..5c8b44d767 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/rules_at_load.rs @@ -0,0 +1,32 @@ +//! Work on a location's published snapshot skips the rules gate: the location ran them at load. + +use std::cell::Cell; + +thread_local! { + /// Whether the running work plans a snapshot whose rules passed when the location loaded it. + static RULES_RAN_AT_LOAD: Cell = const { Cell::new(false) }; +} + +/// Marks plans and runs on a published snapshot for one scope, restored when the scope ends. +pub(crate) struct RulesAtLoadScope { + previous: bool, +} + +impl RulesAtLoadScope { + pub(crate) fn enter() -> Self { + Self { + previous: RULES_RAN_AT_LOAD.replace(true), + } + } +} + +impl Drop for RulesAtLoadScope { + fn drop(&mut self) { + RULES_RAN_AT_LOAD.set(self.previous); + } +} + +/// Whether the running work plans a published snapshot, whose rules passed at load. +pub(crate) fn rules_ran_at_load() -> bool { + RULES_RAN_AT_LOAD.get() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs index 6def3694f9..0e973d9f4a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs @@ -3,6 +3,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyFrozenSet}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_python_base::native_host::models::Dialect; use sqlbuild_python_compile::compile_driver::main::stages::build_deferred_locations::build_deferred_locations; use sqlbuild_python_compile::compile_driver::main::stages::resolve_deferred_target_config::resolve_deferred_target_config; @@ -149,7 +150,7 @@ pub(crate) fn compile_static_command_context<'py>( py, &[ ("project", project.clone()), - ("adapter", adapter), + ("dialect", Dialect::of(&adapter)?.into_any()), ("scope", scope.clone()), ( "deferral", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/registration.rs index fb8534ed64..bca58ce5f5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/registration.rs @@ -2,7 +2,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::cost_command::{bounds, command, refresh}; +use crate::bindings::_helpers::cost_command::bounds; +use crate::bindings::_helpers::cost_command::command; +use crate::bindings::_helpers::cost_command::refresh; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { command::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/registration.rs index 3b2f71866c..23e2cbdb65 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/registration.rs @@ -2,7 +2,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::cost_report::{degradation, exception_types, functions}; +use crate::bindings::_helpers::cost_report::degradation; +use crate::bindings::_helpers::cost_report::exception_types; +use crate::bindings::_helpers::cost_report::functions; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { degradation::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/registration.rs index 0700489b8f..d6a6f1f9a3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/registration.rs @@ -3,7 +3,10 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; -use crate::bindings::_helpers::dagster_invocation::{arguments, failures, results, translator}; +use crate::bindings::_helpers::dagster_invocation::arguments; +use crate::bindings::_helpers::dagster_invocation::failures; +use crate::bindings::_helpers::dagster_invocation::results; +use crate::bindings::_helpers::dagster_invocation::translator; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(arguments::dagster_cli_arguments, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/debug.rs index 2608350b05..81d03e04d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/debug.rs @@ -4,7 +4,10 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_core::dbt_command::constants::NO_CONNECTION_OPTION; use crate::bindings::_helpers::dbt_command::auto_init::args_tuple; -use crate::bindings::_helpers::dbt_command::support::{color_enabled, flag, status, stream}; +use crate::bindings::_helpers::dbt_command::support::color_enabled; +use crate::bindings::_helpers::dbt_command::support::flag; +use crate::bindings::_helpers::dbt_command::support::status; +use crate::bindings::_helpers::dbt_command::support::stream; use crate::bindings::_helpers::dbt_interop::pipeline::debug_dbt_from_project; use crate::bindings::_helpers::debug_command::run_debug::run_debug_command; use sqlbuild_python_base::boundary::main::keywords::keywords; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs index 11d70fdfe2..1488bae594 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs @@ -17,9 +17,8 @@ use sqlbuild_host::entries::models::ThirdPartyHost; use crate::bindings::_helpers::dbt_interop::profile::build_dbt_init_project_toml; use crate::bindings::_helpers::dbt_interop::profile::normalize_dbt_profile_connection; -use crate::bindings::_helpers::dbt_interop::profile_load::{ - load_dbt_project_metadata, load_selected_dbt_profile_output, -}; +use crate::bindings::_helpers::dbt_interop::profile_load::load_dbt_project_metadata; +use crate::bindings::_helpers::dbt_interop::profile_load::load_selected_dbt_profile_output; use crate::bindings::_helpers::dbt_interop::runner::launch_os_error; use sqlbuild_config::dbt_profile::models::DbtProjectMetadata; use sqlbuild_python_base::boundary::main::call_with::call_with; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/registration.rs index 50dcc0c8ee..d0ab17ec03 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/registration.rs @@ -2,7 +2,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::dbt_command::{command, init, init_project}; +use crate::bindings::_helpers::dbt_command::command; +use crate::bindings::_helpers::dbt_command::init; +use crate::bindings::_helpers::dbt_command::init_project; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { command::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs index 653db671ba..6eb8bda7e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs @@ -8,7 +8,8 @@ use crate::bindings::_helpers::dbt_interop::execution::render_dbt_execution_summ use crate::bindings::_helpers::dbt_interop::pipeline_prologue::progress; use crate::bindings::_helpers::dbt_interop::plan::render_dbt_interop_plan; use crate::bindings::_helpers::dbt_interop::plan_output::build_sqlbuild_plan_output; -use crate::bindings::_helpers::dbt_interop::plan_output::{connection_config, connection_hooks}; +use crate::bindings::_helpers::dbt_interop::plan_output::connection_config; +use crate::bindings::_helpers::dbt_interop::plan_output::connection_hooks; use crate::bindings::_helpers::dbt_interop::runtime_execution::execute_dbt_commands; use crate::bindings::_helpers::dbt_interop::sqlbuild_work::execute_dbt_sqlbuild_work; use sqlbuild_python_base::boundary::main::keywords::keywords; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs index 53cf8e9db5..284894aded 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs @@ -4,10 +4,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::native_host::models::Dialect; +use sqlbuild_python_planner::live_reads::models::LiveReads; -use crate::bindings::_helpers::command_runtime::connection::{ - anchor_local_database_path, with_dbt_profile, -}; +use crate::bindings::_helpers::command_runtime::connection::anchor_local_database_path; +use crate::bindings::_helpers::command_runtime::connection::with_dbt_profile; use crate::bindings::_helpers::command_runtime::warehouses::apply_command_warehouse; use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hooks; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -204,14 +205,20 @@ pub(crate) fn build_sqlbuild_plan_output<'py>( ], )?; let discovered = native_field(&environment, "discovered_inputs")?; + let adapter = native_field(&environment, "adapter")?; let result = build_execution_plan( py, Some(keywords( py, &[ ("project", native_field(&environment, "project")?), - ("adapter", native_field(&environment, "adapter")?), - ("connection", connection.clone()), + ("dialect", Dialect::of(&adapter)?.into_any()), + ( + "reads", + LiveReads::new(&adapter, &connection, None) + .handle(adapter.py())? + .into_any(), + ), ("selection", selection), ("overrides", overrides), ("deferral", deferral), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs index f188970794..cb99f3e04a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs @@ -2,9 +2,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::dbt_interop::{ - cli, execution, graph, manifest, orchestration, plan, profile, -}; +use crate::bindings::_helpers::dbt_interop::cli; +use crate::bindings::_helpers::dbt_interop::execution; +use crate::bindings::_helpers::dbt_interop::graph; +use crate::bindings::_helpers::dbt_interop::manifest; +use crate::bindings::_helpers::dbt_interop::orchestration; +use crate::bindings::_helpers::dbt_interop::plan; +use crate::bindings::_helpers::dbt_interop::profile; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { cli::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index 5f753700c8..2ff1f9b431 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -22,17 +22,19 @@ use sqlbuild_core::debug_report::models::{ DebugReport, DebugRuntime, WarehouseCheck, }; -use crate::bindings::_helpers::cli::failure_cause::{FailureCause, report_failure_cause}; -use crate::bindings::_helpers::command_runtime::connection::{ - local_database_files, project_connection_config, -}; -use crate::bindings::_helpers::command_runtime::runtime_errors::{ - expected_resolution_error, resolution_message, -}; +use crate::bindings::_helpers::cli::failure_cause::FailureCause; +use crate::bindings::_helpers::cli::failure_cause::report_failure_cause; +use crate::bindings::_helpers::command_runtime::connection::local_database_files; +use crate::bindings::_helpers::command_runtime::connection::project_connection_config; +use crate::bindings::_helpers::command_runtime::runtime_errors::expected_resolution_error; +use crate::bindings::_helpers::command_runtime::runtime_errors::resolution_message; use crate::bindings::_helpers::command_runtime::warehouse_scope::command_warehouse_scope; -use crate::bindings::_helpers::command_runtime::warehouses::{ - CommandScope, WarehouseInputs, apply_warehouse, command_scope, find_authored, resolve_warehouse, -}; +use crate::bindings::_helpers::command_runtime::warehouses::CommandScope; +use crate::bindings::_helpers::command_runtime::warehouses::WarehouseInputs; +use crate::bindings::_helpers::command_runtime::warehouses::apply_warehouse; +use crate::bindings::_helpers::command_runtime::warehouses::command_scope; +use crate::bindings::_helpers::command_runtime::warehouses::find_authored; +use crate::bindings::_helpers::command_runtime::warehouses::resolve_warehouse; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs index d20d5cc532..48e2805581 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -16,9 +16,11 @@ use sqlbuild_cli::diff_report::types::PhaseSeconds; use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; -use crate::bindings::_helpers::diff_report::support::{ - call, cli_error, error_text, object, progress_line, -}; +use crate::bindings::_helpers::diff_report::support::call; +use crate::bindings::_helpers::diff_report::support::cli_error; +use crate::bindings::_helpers::diff_report::support::error_text; +use crate::bindings::_helpers::diff_report::support::object; +use crate::bindings::_helpers::diff_report::support::progress_line; use crate::bindings::_helpers::diff_report::tolerances::cli_tolerance_overrides; use sqlbuild_analysis::diff_selection::main::raw_query_shape::raw_query_shape; use sqlbuild_analysis::diff_selection::models::RawQueryShape; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs index 4357437f57..81c7cea66c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs @@ -3,10 +3,10 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; -use crate::bindings::_helpers::diff_report::command::{ - execute_prepared_query_diff, run_diff_command, -}; -use crate::bindings::_helpers::diff_report::output::{render_diff_error_json, render_diff_output}; +use crate::bindings::_helpers::diff_report::command::execute_prepared_query_diff; +use crate::bindings::_helpers::diff_report::command::run_diff_command; +use crate::bindings::_helpers::diff_report::output::render_diff_error_json; +use crate::bindings::_helpers::diff_report::output::render_diff_output; use crate::bindings::_helpers::diff_report::tolerances; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/registration.rs index 28114d91b5..af67faa3ca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/registration.rs @@ -3,7 +3,8 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use crate::bindings::_helpers::format_command::cache::FORMAT_RESULT_STORE_FILE_NAME; -use crate::bindings::_helpers::format_command::{command, files}; +use crate::bindings::_helpers::format_command::command; +use crate::bindings::_helpers::format_command::files; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { command::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index 78feedae01..cf27c4ae40 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -1,6 +1,8 @@ //! `sqb freshness`: observe selected source freshness without writing state. +use sqlbuild_python_base::native_host::models::Dialect; use sqlbuild_python_base::records::main::construct::new_error::new_error; +use sqlbuild_python_planner::live_reads::models::LiveReads; use std::collections::BTreeSet; use pyo3::exceptions::PyValueError; @@ -233,8 +235,8 @@ fn observe( let observed_at: Bound<'_, PyAny> = imported(py, "datetime", "datetime")? .call_method1("now", (imported(py, "datetime", "UTC")?,))?; let observer: Observer<'_> = Observer { - adapter: adapter.clone(), - connection: connection.clone(), + dialect: Dialect::of(adapter)?, + reads: LiveReads::new(adapter, &connection, None).handle(adapter.py())?, observed_at, }; let state: PreviousState<'_> = PreviousState { by_identity }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index 19a279f42a..114cba5874 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -27,16 +27,19 @@ use sqlbuild_core::janitor_text::models::{JanitorOutcome, JanitorPlanText, Janit use sqlbuild_python_base::records::main::construct::new_error::new_error; use std::time::Instant; -use crate::bindings::_helpers::cli::machine_output::{ - report_machine_output_written, take_machine_output_written, -}; +use crate::bindings::_helpers::cli::machine_output::report_machine_output_written; +use crate::bindings::_helpers::cli::machine_output::take_machine_output_written; use crate::bindings::_helpers::command_runtime::preview_target::validate_preview_target; use crate::bindings::_helpers::diff_report::support::cli_error; -use crate::bindings::_helpers::janitor::janitor_phases::{ - JanitorCompiled, build_janitor_execution_plan, close_janitor_warehouse, - compile_janitor_project, connect_janitor_warehouse, execute_janitor_cleanup, output_stream, -}; -use crate::bindings::_helpers::janitor::janitor_plan::{janitor_plan_text, janitor_result_counts}; +use crate::bindings::_helpers::janitor::janitor_phases::JanitorCompiled; +use crate::bindings::_helpers::janitor::janitor_phases::build_janitor_execution_plan; +use crate::bindings::_helpers::janitor::janitor_phases::close_janitor_warehouse; +use crate::bindings::_helpers::janitor::janitor_phases::compile_janitor_project; +use crate::bindings::_helpers::janitor::janitor_phases::connect_janitor_warehouse; +use crate::bindings::_helpers::janitor::janitor_phases::execute_janitor_cleanup; +use crate::bindings::_helpers::janitor::janitor_phases::output_stream; +use crate::bindings::_helpers::janitor::janitor_plan::janitor_plan_text; +use crate::bindings::_helpers::janitor::janitor_plan::janitor_result_counts; use crate::bindings::_helpers::janitor::janitor_progress::JanitorProgressPrinter; use crate::bindings::_helpers::scenario_command::setup::stream_isatty; use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs index c8b64ad572..9acfa79849 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs @@ -14,20 +14,34 @@ use sqlbuild_analysis::janitor_planning::main::query_diff_artifact_parts::query_ use sqlbuild_analysis::review_command::constants::LEFTOVER_REVIEW_SCHEMA_AGE_DAYS; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::janitor::janitor_classification::{ - Candidate, Policy, Protection, SchemaClassification, classify_target_schemas, plan_archives, -}; -use crate::bindings::_helpers::janitor::janitor_facts::{ - Warehouse, WarehouseFacts, collect_target_schemas, direct_state_prune_candidates, - gather_warehouse_facts, query_diff_artifact_candidates, -}; -use crate::bindings::_helpers::janitor::janitor_keys::{ - Folded, Key, SchemaKey, archive_timestamp, days, dedupe_schema_keys, integer, now_utc, record, - sorted_schemas, timestamp_fields, tuple, union_exact, -}; -use crate::bindings::_helpers::janitor::janitor_old_names::{ - OldNamePlanning, pending_migration_origins, plan_old_name_views, -}; +use crate::bindings::_helpers::janitor::janitor_classification::Candidate; +use crate::bindings::_helpers::janitor::janitor_classification::Policy; +use crate::bindings::_helpers::janitor::janitor_classification::Protection; +use crate::bindings::_helpers::janitor::janitor_classification::SchemaClassification; +use crate::bindings::_helpers::janitor::janitor_classification::classify_target_schemas; +use crate::bindings::_helpers::janitor::janitor_classification::plan_archives; +use crate::bindings::_helpers::janitor::janitor_facts::Warehouse; +use crate::bindings::_helpers::janitor::janitor_facts::WarehouseFacts; +use crate::bindings::_helpers::janitor::janitor_facts::collect_target_schemas; +use crate::bindings::_helpers::janitor::janitor_facts::direct_state_prune_candidates; +use crate::bindings::_helpers::janitor::janitor_facts::gather_warehouse_facts; +use crate::bindings::_helpers::janitor::janitor_facts::query_diff_artifact_candidates; +use crate::bindings::_helpers::janitor::janitor_keys::Folded; +use crate::bindings::_helpers::janitor::janitor_keys::Key; +use crate::bindings::_helpers::janitor::janitor_keys::SchemaKey; +use crate::bindings::_helpers::janitor::janitor_keys::archive_timestamp; +use crate::bindings::_helpers::janitor::janitor_keys::days; +use crate::bindings::_helpers::janitor::janitor_keys::dedupe_schema_keys; +use crate::bindings::_helpers::janitor::janitor_keys::integer; +use crate::bindings::_helpers::janitor::janitor_keys::now_utc; +use crate::bindings::_helpers::janitor::janitor_keys::record; +use crate::bindings::_helpers::janitor::janitor_keys::sorted_schemas; +use crate::bindings::_helpers::janitor::janitor_keys::timestamp_fields; +use crate::bindings::_helpers::janitor::janitor_keys::tuple; +use crate::bindings::_helpers::janitor::janitor_keys::union_exact; +use crate::bindings::_helpers::janitor::janitor_old_names::OldNamePlanning; +use crate::bindings::_helpers::janitor::janitor_old_names::pending_migration_origins; +use crate::bindings::_helpers::janitor::janitor_old_names::plan_old_name_views; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs index 075ffb283c..95b469fdb9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs @@ -11,7 +11,10 @@ use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_model_config::model_validation::main::fixed_duration_seconds::fixed_duration_seconds; use crate::bindings::_helpers::diff_report::support::cli_error; -use crate::bindings::_helpers::janitor::janitor_keys::{Folded, Key, record, tuple}; +use crate::bindings::_helpers::janitor::janitor_keys::Folded; +use crate::bindings::_helpers::janitor::janitor_keys::Key; +use crate::bindings::_helpers::janitor::janitor_keys::record; +use crate::bindings::_helpers::janitor::janitor_keys::tuple; use crate::bindings::_helpers::janitor::janitor_planning::completed; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/registration.rs index ae838bfb40..dc7e2f754b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/registration.rs @@ -3,11 +3,12 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; -use crate::bindings::_helpers::janitor::janitor_planning::{ - build_janitor_archive_name, build_janitor_plan_binding, build_query_diff_artifact_name, - parse_query_diff_artifact_name, -}; -use crate::bindings::_helpers::janitor::{janitor_command, janitor_plan}; +use crate::bindings::_helpers::janitor::janitor_command; +use crate::bindings::_helpers::janitor::janitor_plan; +use crate::bindings::_helpers::janitor::janitor_planning::build_janitor_archive_name; +use crate::bindings::_helpers::janitor::janitor_planning::build_janitor_plan_binding; +use crate::bindings::_helpers::janitor::janitor_planning::build_query_diff_artifact_name; +use crate::bindings::_helpers::janitor::janitor_planning::parse_query_diff_artifact_name; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { janitor_command::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index 5bdc985fbd..9985b75133 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -16,18 +16,23 @@ use sqlbuild_orchestration::declarations::models::{ use sqlbuild_orchestration::dev_events::main::compile_summary::compile_summary; use sqlbuild_orchestration::dev_events::main::emit_dev_steps::emit_dev_steps; use sqlbuild_orchestration::dev_events::models::{DevPhase, DevStepState}; +use sqlbuild_orchestration::platform::models::LocationRuleFailure; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; use sqlbuild_orchestration::warehouse_metadata::models::WarehouseStateLocation; -use crate::bindings::_helpers::location_command::constants::{EXECUTE_ACTION, PUBLISH_ACTION}; +use crate::bindings::_helpers::location_command::constants::EXECUTE_ACTION; +use crate::bindings::_helpers::location_command::constants::PUBLISH_ACTION; +use crate::bindings::_helpers::location_command::load_rules::load_rules; use crate::bindings::_helpers::location_command::runtime::run_runtime; -use crate::bindings::_helpers::location_command::snapshot::{ - build_orchestration_snapshot, orchestration_snapshot, -}; -use crate::bindings::_helpers::location_command::worker::{blocked_nodes, execute_worker}; -use crate::bindings::_helpers::query_command::project::{ - ProjectAdapter, ProjectInputs, project_adapter, -}; +use crate::bindings::_helpers::location_command::snapshot::build_orchestration_snapshot; +use crate::bindings::_helpers::location_command::snapshot::orchestration_snapshot; +use crate::bindings::_helpers::location_command::snapshot_connection::add_connection; +use crate::bindings::_helpers::location_command::types::HeldProject; +use crate::bindings::_helpers::location_command::worker::blocked_nodes; +use crate::bindings::_helpers::location_command::worker::execute_worker; +use crate::bindings::_helpers::query_command::project::ProjectAdapter; +use crate::bindings::_helpers::query_command::project::ProjectInputs; +use crate::bindings::_helpers::query_command::project::project_adapter; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::write_console::write_console; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -122,11 +127,14 @@ fn metadata( }) } +/// A compiled snapshot, its blocked nodes and the compile's `CompiledProjectPhaseResult`. +type CompiledSnapshot<'py> = (NativeOrchestrationSnapshot, NodeKeys, Bound<'py, PyAny>); + fn compile_location<'py>( py: Python<'py>, arguments: &KeywordArguments<'py>, input: &SnapshotInput, -) -> PyResult<(NativeOrchestrationSnapshot, ProjectAdapter<'py>, NodeKeys)> { +) -> PyResult<(CompiledSnapshot<'py>, ProjectAdapter<'py>)> { status(py, "Discovering location project...")?; steps(py, &[(DevPhase::Discover, DevStepState::Started)], None)?; let project = project_adapter( @@ -140,11 +148,11 @@ fn compile_location<'py>( (DevPhase::Compile, DevStepState::Started), ]; steps(py, &discovered, None)?; - let (snapshot, blocked) = compile_snapshot(py, arguments, (input, &project), None)?; + let compiled = compile_snapshot(py, arguments, (input, &project), None)?; status(py, "Location snapshot compiled.")?; - let compiled = [(DevPhase::Compile, DevStepState::Finished)]; - steps(py, &compiled, Some(compile_summary(&snapshot.input)))?; - Ok((snapshot, project, blocked)) + let finished = [(DevPhase::Compile, DevStepState::Finished)]; + steps(py, &finished, Some(compile_summary(&compiled.0.input)))?; + Ok((compiled, project)) } /// Compile the discovered project into a snapshot, with a run's variables when it carries them. @@ -153,7 +161,7 @@ pub(super) fn compile_snapshot<'py>( arguments: &KeywordArguments<'py>, (input, project): (&SnapshotInput, &ProjectAdapter<'py>), vars: Option<&Bound<'py, PyAny>>, -) -> PyResult<(NativeOrchestrationSnapshot, NodeKeys)> { +) -> PyResult> { let ProjectAdapter { project_dir, discovered, @@ -185,7 +193,8 @@ pub(super) fn compile_snapshot<'py>( .is_some_and(|action| action == EXECUTE_ACTION), ), )?; - Ok((snapshot, blocked_nodes(&project)?)) + let blocked = blocked_nodes(&project)?; + Ok((snapshot, blocked, compiled)) } pub(super) fn location_arguments<'py>( @@ -209,14 +218,17 @@ pub(super) fn location_arguments<'py>( ) } -pub(super) fn publish_location( - py: Python<'_>, - arguments: &KeywordArguments<'_>, -) -> PyResult<( +/// A published record, its earliest calendar refresh and the state locations its nodes write. +pub(super) type Published = ( SnapshotRecord, Option>, Vec, -)> { +); + +pub(super) fn publish_location<'py>( + py: Python<'py>, + arguments: &KeywordArguments<'py>, +) -> PyResult), LocationRuleFailure>> { if arguments .value("action")? .extract::>()? @@ -238,7 +250,14 @@ pub(super) fn publish_location( .value("project_dir")? .extract::()?; let input = metadata(py, arguments, &directory)?; - let (snapshot, _, _) = compile_location(py, arguments, &input)?; + let ((mut snapshot, _, compiled), project) = compile_location(py, arguments, &input)?; + if let Some(failure) = load_rules(py, arguments, (&project, &compiled))? { + return Ok(Err(failure)); + } + snapshot.input = add_connection( + snapshot.input, + (&project, &arguments.value("selected_target")?), + )?; let locations: BTreeSet<_> = snapshot .input .nodes @@ -277,8 +296,9 @@ pub(super) fn publish_location( &arguments.value("server_url")?.extract::()?, &bearer, )?.map_err(|error| { - let message = if matches!(error, sqlbuild_orchestration::api::errors::ClientError::Transport(_)) { - format!("Publication outcome is unknown; generation {} may already be published. Check the current snapshot before retrying: {error}", input.generation) + let message = if let sqlbuild_orchestration::api::errors::ClientError::Transport(transport) = &error { + let cause = if transport.is_timeout() { " (the server did not answer in time)" } else { "" }; + format!("Publication outcome is unknown; generation {} may already be published. Check the current snapshot before retrying: {error}{cause}", input.generation) } else { error.to_string() }; @@ -286,7 +306,10 @@ pub(super) fn publish_location( })?; status(py, "Location snapshot published.")?; steps(py, &[(DevPhase::Publish, DevStepState::Finished)], None)?; - Ok((record, refresh_at, locations.into_iter().collect())) + Ok(Ok(( + (record, refresh_at, locations.into_iter().collect()), + (project.discovered, compiled), + ))) } /// One-shot publication and the supervisor's private stdio runtime share the same compiler path. @@ -322,7 +345,7 @@ pub(crate) fn run_location<'py>( arguments.value("run_id")?.extract::()?, arguments.value("claim_id")?.extract::()?, ); - let (snapshot, project, blocked) = compile_location(py, &arguments, &input)?; + let ((snapshot, blocked, _), project) = compile_location(py, &arguments, &input)?; return execute_worker( py, (&snapshot, project, blocked), @@ -339,7 +362,12 @@ pub(crate) fn run_location<'py>( return run_runtime(py, options.unwrap_or_else(|| PyDict::new(py))); } let publication = (|| { - let (record, _, _) = publish_location(py, &arguments)?; + let ((record, _, _), _) = publish_location(py, &arguments)?.map_err(|failure| { + PyValueError::new_err(format!( + "Compile failed: {} rule findings", + failure.finding_count + )) + })?; let output = serde_json::to_string(&record) .map_err(|error| PyValueError::new_err(error.to_string()))?; write_console(py, "stdout", &format!("{output}\n")).map_err(|error| { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs index 27ea6aed44..ef9d973459 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -34,3 +34,7 @@ pub(crate) const JOB_RUN_COUNTS: [&str; 2] = ["definition", "source_data"]; pub(crate) const SQL_TEST_NODE_KIND: &str = "sql_test"; /// The setting-row field the published providers document leaves out: secret matching reads it, the UI does not. pub(crate) const SNAPSHOT_HIDDEN_SETTING: &str = "alias"; +/// The runtime request key of a plan preview: `{"plan": {"argv", "copy"}}`. +pub(crate) const PLAN_REQUEST: &str = "plan"; +/// The code of a failed plan preview whose error names none. +pub(crate) const PLAN_FAILED_CODE: &str = "plan_failed"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/load_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/load_rules.rs new file mode 100644 index 0000000000..f60c44857a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/load_rules.rs @@ -0,0 +1,81 @@ +//! A location load is a full compile: the configured rules run after it, as `sqb compile` runs them. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_orchestration::platform::constants::{ + MAX_REPORTED_RULE_FINDINGS, MAX_RULE_FINDING_CHARS, +}; +use sqlbuild_orchestration::platform::models::{LocationRuleFailure, LocationRuleFinding}; + +use crate::bindings::_helpers::compile_driver::plan_pipeline::configured_rule_failures; +use crate::bindings::_helpers::query_command::project::ProjectAdapter; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::setup::project_graph::project_graph; +use sqlbuild_python_compile::rules::models::EarlySqlLint; +use sqlbuild_python_output::recall::main::write_stderr::write_stderr; + +/// The configured rules' findings when they fail the compiled project, printed as `sqb compile` would. +pub(super) fn load_rules<'py>( + py: Python<'py>, + arguments: &KeywordArguments<'py>, + (project, compiled): (&ProjectAdapter<'py>, &Bound<'py, PyAny>), +) -> PyResult> { + let options = new_record_with( + py, + RecordId::CompilePipelineOptions, + &[("selected_target", arguments.value("selected_target")?)], + )?; + let expansion = Bound::new(py, EarlySqlLint::new(false, false))?; + let compiled_project = field(compiled, "project")?; + let graph = project_graph(&compiled_project)?; + write_stderr(py, "Evaluating configured rules...\n")?; + let Some((summary, rendered, notice)) = configured_rule_failures( + (&project.discovered, &project.adapter, expansion.as_any()), + (&options, &py.None().into_bound(py)), + (&compiled_project, &graph), + )? + else { + write_stderr(py, "Configured rules passed.\n")?; + return Ok(None); + }; + let mut findings: Vec = Vec::new(); + let mut finding_count: u64 = 0; + for finding in field(&summary, "findings")?.try_iter()? { + let finding = finding?; + finding_count += 1; + if findings.len() < MAX_REPORTED_RULE_FINDINGS { + let path = field(&finding, "path")?; + findings.push(LocationRuleFinding { + code: bounded(field(&finding, "code")?.extract()?), + message: bounded(field(&finding, "message")?.extract()?), + path: (!path.is_none()) + .then(|| path.str().map(|path| bounded(path.to_string()))) + .transpose()?, + line: field(&finding, "line")?.extract()?, + }); + } + } + let noun = if finding_count == 1 { + "rule finding" + } else { + "rule findings" + }; + let notice = notice + .map(|notice| format!("\n{notice}")) + .unwrap_or_default(); + write_stderr( + py, + &format!("Compile failed: {finding_count} {noun}\n{rendered}{notice}\n"), + )?; + Ok(Some(LocationRuleFailure { + finding_count, + findings, + })) +} + +/// `text` cut to the reported bound. +fn bounded(text: String) -> String { + text.chars().take(MAX_RULE_FINDING_CHARS).collect() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs index f4a9eb49c7..914ed0ee35 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs @@ -1,7 +1,11 @@ pub(crate) mod command; mod constants; +mod load_rules; mod runtime; +mod runtime_plan; mod snapshot; +mod snapshot_connection; mod snapshot_lineage; mod snapshot_providers; +pub(crate) mod types; mod worker; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs index e48f589c3d..3fdb5dbfcd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs @@ -14,12 +14,16 @@ use sqlbuild_orchestration::warehouse_metadata::models::{ WarehouseHistoryFailure, WarehouseHistoryKind, WarehouseHistoryRead, }; -use crate::bindings::_helpers::location_command::command::{location_arguments, publish_location}; +use crate::bindings::_helpers::location_command::command::location_arguments; +use crate::bindings::_helpers::location_command::command::publish_location; use crate::bindings::_helpers::location_command::constants::DUCKDB_LOCK_CONFLICT; +use crate::bindings::_helpers::location_command::constants::PLAN_REQUEST; +use crate::bindings::_helpers::location_command::runtime_plan::runtime_plan; +use crate::bindings::_helpers::location_command::types::HeldProject; use crate::bindings::_helpers::native_host::orchestration_sensors::evaluate_sensor; -use crate::bindings::_helpers::native_host::warehouse_metadata::{ - close_history_session, open_history_session, read_history, -}; +use crate::bindings::_helpers::native_host::warehouse_metadata::close_history_session; +use crate::bindings::_helpers::native_host::warehouse_metadata::open_history_session; +use crate::bindings::_helpers::native_host::warehouse_metadata::read_history; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::native_host::main::redirect_stdout_to_stderr::redirect_stdout_to_stderr; use sqlbuild_python_output::recall::main::write_stderr::write_stderr; @@ -30,6 +34,7 @@ pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResul within(&redirect, |_| { let mut session = None; let mut stream = None; + let mut held: Option> = None; let mut reported_busy = false; let mut serve = || -> PyResult { loop { @@ -48,8 +53,13 @@ pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResul let output = if let Some(generation) = request.as_i64() { options.set_item("generation", generation)?; let arguments = location_arguments(py, Some(options.clone()))?; - let record = match publish_location(py, &arguments) { - Ok(record) => Some(record), + held = None; + let (published, rules) = match publish_location(py, &arguments) { + Ok(Ok((published, project))) => { + held = Some(project); + (Some(published), None) + } + Ok(Err(rules)) => (None, Some(rules)), Err(error) => { write_stderr( py, @@ -57,10 +67,15 @@ pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResul )?; let failed = generation_failed(generation, error.to_string()); py.detach(|| emit_dev_event(&failed))?; - None + (None, None) } }; - serde_json::to_string(&record) + serde_json::to_string(&serde_json::json!({ + "published": published, + "rules": rules, + })) + } else if let Some(plan) = request.get(PLAN_REQUEST) { + serde_json::to_string(&runtime_plan(py, held.as_ref(), plan)?) } else if let Some(kind) = WarehouseHistoryKind::ALL .iter() .find(|kind| request.get(kind.as_str()).is_some()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime_plan.rs new file mode 100644 index 0000000000..ae8134f59d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime_plan.rs @@ -0,0 +1,120 @@ +//! A location's plan previews, run in its runtime from its last publication over its synced state. + +use std::sync::{Arc, Mutex, PoisonError}; + +use pyo3::PyErr; +use pyo3::exceptions::{PyException, PyValueError}; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyList, PyString}; +use serde_json::{Value, json}; +use sqlbuild_orchestration::warehouse_metadata::main::location_reads::preview_state_parts::preview_state_parts; +use sqlbuild_orchestration::warehouse_metadata::models::PreviewCopy; +use sqlbuild_state::warehouse_state::main::warehouse_state::warehouse_state; +use sqlbuild_state::warehouse_state::models::WarehouseState; + +use crate::bindings::_helpers::cli::argv::parse_cli_invocation; +use crate::bindings::_helpers::cli::dispatch::with_routed_request; +use crate::bindings::_helpers::location_command::constants::PLAN_FAILED_CODE; +use crate::bindings::_helpers::location_command::types::HeldProject; +use crate::bindings::_helpers::plan_command::plan::PlanProject; +use crate::bindings::_helpers::plan_command::plan::location_plan; +use crate::bindings::_helpers::review_command::support::attribute; +use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_python_output::recall::main::write_stderr::write_stderr; + +/// The synced state the runtime plans previews from, and the mirror version it was read at. +struct HeldState { + version: String, + state: Arc, +} + +/// One state for the runtime's life; a newer mirror version replaces it. +static HELD_STATE: Mutex> = Mutex::new(None); + +/// The state a request plans from: the held state at its mirror version, else built from its copy file. +fn requested_state( + held_state: &Mutex>, + request: &Value, +) -> PyResult> { + let mut held = held_state.lock().unwrap_or_else(PoisonError::into_inner); + let version: Option<&str> = request["version"].as_str(); + if let Some(current) = held + .as_ref() + .filter(|held| Some(held.version.as_str()) == version) + { + return Ok(Arc::clone(¤t.state)); + } + let path: &str = request["copy"].as_str().unwrap_or_default(); + let text: String = std::fs::read_to_string(path)?; + let copy: PreviewCopy = serde_json::from_str(&text) + .map_err(|error| PyValueError::new_err(format!("invalid synced copy: {error}")))?; + let version: Option = copy.version.clone(); + let state: Arc = Arc::new(warehouse_state(preview_state_parts( + copy, + request["consumption"].as_bool().unwrap_or(false), + ))); + *held = version.map(|version| HeldState { + version, + state: Arc::clone(&state), + }); + Ok(state) +} + +/// One plan request's reply: `sqb plan`'s exit code and JSON output, or why it failed. +pub(super) fn runtime_plan<'py>( + py: Python<'py>, + held: Option<&HeldProject<'py>>, + request: &Value, +) -> PyResult { + let argv: Vec = serde_json::from_value(request["argv"].clone()) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + let select: Vec = serde_json::from_value(request["select"].clone()).unwrap_or_default(); + let state: Arc = requested_state(&HELD_STATE, request)?; + let planned = (|| -> PyResult<(i32, Option)> { + let argv: Vec> = + argv.iter().map(|part| PyString::new(py, part)).collect(); + let args = field(&parse_cli_invocation(py, argv, false)?, "args")?; + if args.is_none() { + return Err(PyException::new_err("the plan command line is invalid")); + } + if !select.is_empty() { + args.setattr("select", PyList::new(py, &select)?)?; + } + let project = match held { + Some((discovered, compiled)) => PlanProject::Held { + discovered, + compiled, + }, + None => PlanProject::Compiled, + }; + with_routed_request(&args, |request| location_plan(request, (project, &state))) + })(); + Ok(match planned { + Ok((exit_code, Some(output))) => json!({"exit_code": exit_code, "output": output}), + Ok((_, None)) => { + json!({"failed": {"code": PLAN_FAILED_CODE, "message": "the plan printed no JSON", "help": null}}) + } + Err(error) if error.is_instance_of::(py) => { + let failure = failure(py, &error)?; + write_stderr( + py, + &format!( + "Location plan failed: {}\n", + failure["message"].as_str().unwrap_or_default() + ), + )?; + json!({"failed": failure}) + } + Err(error) => return Err(error), + }) +} + +/// A failed plan's `code`, `message` and `help`, as `sqb plan` prints a coded error. +fn failure(py: Python<'_>, error: &PyErr) -> PyResult { + let value = error.value(py); + Ok(json!({ + "code": attribute(value.as_any(), "code")?.unwrap_or_else(|| PLAN_FAILED_CODE.to_owned()), + "message": attribute(value.as_any(), "message")?.unwrap_or(value.str()?.to_string()), + "help": attribute(value.as_any(), "help")?, + })) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index 7b5b0c5c4a..f556259c9f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -1,8 +1,7 @@ //! Location publication assembles catalog data and planner-owned local identities. -use std::collections::BTreeMap; -use std::path::PathBuf; - +use crate::bindings::_helpers::location_command::snapshot_lineage::column_lineage_document; +use crate::bindings::_helpers::location_command::snapshot_providers::providers_document; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; @@ -13,12 +12,10 @@ use sqlbuild_core::models::DiagnosticSeverity; use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; use sqlbuild_discovery::project_validation::main::validate_orchestration_names::validate_orchestration_names; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; - -use crate::bindings::_helpers::location_command::snapshot_lineage::column_lineage_document; -use crate::bindings::_helpers::location_command::snapshot_providers::providers_document; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::orchestration_reads::main::environment_hash::environment_hash; use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_python_front::command_runtime::main::published_concurrency::published_concurrency; use sqlbuild_python_front::discovery::main::project::named_node_rows::named_node_rows; use sqlbuild_python_front::discovery::main::project::project_facts::project_facts; use sqlbuild_python_front::discovery::main::project::validation_error::validation_error; @@ -35,6 +32,8 @@ use sqlbuild_python_project::project_snapshot::main::model_nodes::model_nodes; use sqlbuild_python_project::project_snapshot::main::python_declarations::python_declarations; use sqlbuild_python_project::project_snapshot::main::resource_nodes::resource_nodes; use sqlbuild_python_project::project_snapshot::models::NativeOrchestrationSnapshot; +use std::collections::BTreeMap; +use std::path::PathBuf; /// Compile catalog nodes and identities in the location, never in the server. #[pyfunction] @@ -120,10 +119,18 @@ pub(crate) fn orchestration_snapshot<'py>( facts.orchestration = inventory.names(); validate_orchestration_names(&facts).map_err(|failure| validation_error(py, failure))?; let declarations = python_declarations(py, &project_dir, &inventory)?; + let concurrency = published_concurrency( + &project_dir, + &discovered_inputs, + project.effective_target_name().as_deref(), + )?; input = compile_declarations( py, input, - (&project_dir, graph.bind(py).get().graph(), &selected), + ( + (&project_dir, graph.bind(py).get().graph(), &selected), + concurrency, + ), (&inventory, declarations), )?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_connection.rs new file mode 100644 index 0000000000..c2a10b83dd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_connection.rs @@ -0,0 +1,64 @@ +//! The target's non-secret connection settings, published with each snapshot for the Connections page. + +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use serde_json::{Map, Value}; +use sqlbuild_orchestration::snapshots::constants::CONNECTION_DOCUMENT; +use sqlbuild_orchestration::snapshots::main::reads::connection_details::connection_details; +use sqlbuild_orchestration::snapshots::models::SnapshotInput; + +use crate::bindings::_helpers::command_runtime::connection::project_connection_config; +use crate::bindings::_helpers::command_runtime::warehouse_scope::command_warehouse_scope; +use crate::bindings::_helpers::query_command::project::ProjectAdapter; +use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::records::main::fields::field::field; + +/// The resolved connection's text settings; only `connection_details` decides which are published. +fn text_settings( + project: &ProjectAdapter<'_>, + selected_target: &Bound<'_, PyAny>, +) -> PyResult> { + let py: Python<'_> = selected_target.py(); + let none: Bound<'_, PyAny> = py.None().into_bound(py); + let unscoped: Bound<'_, PyAny> = + Bound::new(py, command_warehouse_scope(&none, &none, None)?)?.into_any(); + let config = within(&unscoped, |_| { + project_connection_config( + &project.discovered, + (&project.project_dir, selected_target), + (&none, false), + false, + ) + })?; + let mut settings: Map = Map::new(); + for (key, value) in config.cast_into::()?.iter() { + let shown = if value.is_none() { + Value::Null + } else if value.is_instance_of::() { + Value::String(value.extract()?) + } else { + Value::Bool(true) + }; + settings.insert(key.extract()?, shown); + } + Ok(settings) +} + +/// Add the `connection` document; a connection that cannot be resolved here, such as one naming an unset secret, publishes none. +pub(super) fn add_connection( + mut input: SnapshotInput, + (project, selected_target): (&ProjectAdapter<'_>, &Bound<'_, PyAny>), +) -> PyResult { + let py: Python<'_> = selected_target.py(); + let settings = match text_settings(project, selected_target) { + Ok(settings) => settings, + Err(error) if error.is_instance_of::(py) => return Ok(input), + Err(error) => return Err(error), + }; + let adapter: String = field(&project.adapter, "adapter_name")?.str()?.extract()?; + let details = serde_json::to_value(connection_details(&adapter, &settings)) + .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?; + input.documents.insert(CONNECTION_DOCUMENT.into(), details); + Ok(input) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/types.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/types.rs new file mode 100644 index 0000000000..e04b3a0868 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/types.rs @@ -0,0 +1,6 @@ +//! Values a location's runtime keeps between requests. + +use pyo3::prelude::{Bound, PyAny}; + +/// The discovered inputs and `CompiledProjectPhaseResult` of the project a location last published. +pub(crate) type HeldProject<'py> = (Bound<'py, PyAny>, Bound<'py, PyAny>); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index e1ad555707..7a88a487a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -17,22 +17,26 @@ use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_orchestration::runs::models::{ RunChange, RunCommand, RunKind, RunOutcome, RunRequest, RunWindow, }; -use sqlbuild_orchestration::snapshots::main::node_hashes::node_hashes; +use sqlbuild_orchestration::snapshots::main::reads::node_hashes::node_hashes; use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::audit_command::command::run_audit; use crate::bindings::_helpers::build_command::command::run_build; use crate::bindings::_helpers::cli::argv::parse_cli_invocation; -use crate::bindings::_helpers::cli::dispatch::{Invocation, command_facts, warehouse_scope}; -use crate::bindings::_helpers::command_runtime::warehouses::{ - WarehouseInputs, configured_warehouses, -}; +use crate::bindings::_helpers::cli::dispatch::Invocation; +use crate::bindings::_helpers::cli::dispatch::command_facts; +use crate::bindings::_helpers::cli::dispatch::warehouse_scope; +use crate::bindings::_helpers::command_runtime::warehouses::WarehouseInputs; +use crate::bindings::_helpers::command_runtime::warehouses::configured_warehouses; +use crate::bindings::_helpers::compile_driver::rules_at_load::RulesAtLoadScope; use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command; use crate::bindings::_helpers::location_command::command::compile_snapshot; -use crate::bindings::_helpers::location_command::constants::{ - CHANGED_SETTING, COMMAND_IMPLEMENTATIONS, JOB_RUN_COUNTS, JOB_RUN_REACH, REACH_SETTING, - SQL_TEST_NODE_KIND, -}; +use crate::bindings::_helpers::location_command::constants::CHANGED_SETTING; +use crate::bindings::_helpers::location_command::constants::COMMAND_IMPLEMENTATIONS; +use crate::bindings::_helpers::location_command::constants::JOB_RUN_COUNTS; +use crate::bindings::_helpers::location_command::constants::JOB_RUN_REACH; +use crate::bindings::_helpers::location_command::constants::REACH_SETTING; +use crate::bindings::_helpers::location_command::constants::SQL_TEST_NODE_KIND; use crate::bindings::_helpers::query_command::project::ProjectAdapter; use crate::bindings::_helpers::seed_command::command::run_seed; use crate::bindings::_helpers::sql_test_command::test_command::run_test; @@ -326,7 +330,7 @@ fn record_compiled_vars<'py>( }; write_stderr(py, "Compiling with the run's variables...\n")?; let vars = json_value_to_python(py, vars)?; - let (compiled, _) = compile_snapshot(py, arguments, (input, project), Some(&vars))?; + let (compiled, _, _) = compile_snapshot(py, arguments, (input, project), Some(&vars))?; let node_hashes: BTreeMap = node_hashes(&compiled.input) .map_err(|error| PyValueError::new_err(error.to_string()))? .into_iter() @@ -383,6 +387,7 @@ pub(super) fn execute_worker<'py>( record_compiled_vars(py, (&owner, &frozen), (arguments, input, &project))?; let (request, scope) = command_request(py, &discovered, arguments, &owner)?; write_stderr(py, "Run inputs verified. Executing frozen selection...\n")?; + let _rules = RulesAtLoadScope::enter(); within(&scope, |_| { run_cancellable(py, &worker, (&request, &discovered)) }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs index df8641ddac..3779722f71 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs @@ -4,13 +4,17 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::types::PyTuple; use sqlbuild_host::entries::main::host_entry_labels::host_entry_labels; -use sqlbuild_python_base::warehouse_access::main::register_synced_copy::register_synced_copy; +use sqlbuild_python_base::native_host::models::Dialect; +use sqlbuild_python_planner::live_reads::main::register_live_reads::register_live_reads; +use sqlbuild_python_planner::planner_warehouse::main::reads::register_exception_types::register_exception_types; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add( "HOST_ENTRIES", PyTuple::new(module.py(), host_entry_labels())?, )?; - register_synced_copy(module)?; + register_exception_types(module)?; + module.add_class::()?; + register_live_reads(module)?; sqlbuild_python_base::orchestration_reads::main::register_orchestration_declarations::register_orchestration_declarations(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index 29d7f8f6dd..f3324e48a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -15,7 +15,9 @@ use sqlbuild_core::constants::{ RESOURCE_COSTS_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, }; use sqlbuild_core::models::SqlValue; -use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use sqlbuild_orchestration::warehouse_metadata::constants::{ + HISTORY_PAGE_BYTES, MAX_HISTORY_PAGE_RECORDS, +}; use sqlbuild_orchestration::warehouse_metadata::models::{ ConsumptionRead, NodeResultsRead, WarehouseHistoryPage, WarehouseHistoryRead, WarehouseHistoryRecord, WarehouseStateLocation, @@ -31,12 +33,12 @@ use sqlbuild_state::observations::constants::OBSERVATION_COLUMN_TYPES; use sqlbuild_state::resource_costs::constants::RESOURCE_COST_COLUMN_TYPES; use sqlbuild_state::state_tables::main::event_id_index::event_id_index; use sqlbuild_state::state_tables::main::read_history_sql::read_history_sql; -use sqlbuild_state::state_tables::models::HistoryRead; +use sqlbuild_state::state_tables::models::{HistoryRead, NewestVersions}; use crate::bindings::_helpers::command_runtime::connection::project_connection_config; -use crate::bindings::_helpers::query_command::project::{ - ProjectAdapter, ProjectInputs, project_adapter, -}; +use crate::bindings::_helpers::query_command::project::ProjectAdapter; +use crate::bindings::_helpers::query_command::project::ProjectInputs; +use crate::bindings::_helpers::query_command::project::project_adapter; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::constants::HISTORY_STREAM_PAGES; @@ -52,6 +54,15 @@ use sqlbuild_python_planner::freshness::main::row_record::row_record; use sqlbuild_python_project::compiled_project::main::plain_value::plain_value; use sqlbuild_python_project::fingerprints::main::row_fingerprint::row_fingerprint; +/// A node's builds share these columns, as the control store groups them. +const FINGERPRINT_IDENTITY_COLUMNS: [&str; 5] = [ + "node_type", + "node_name", + "target_database", + "target_schema", + "target_name", +]; + /// One read-only adapter session serves every page of one synchronization. pub(crate) struct HistorySession<'py> { adapter: Bound<'py, PyAny>, @@ -70,6 +81,10 @@ pub(crate) struct HistoryStream<'py> { next_after: Vec>, /// Rows the bounded SELECT can still return; at zero the next page reads from its cursor. remaining: usize, + /// Rows fetched but not delivered, because the last page reached its byte budget. + carried: std::collections::VecDeque>, + /// The cursor returned fewer rows than asked for, so the table has no more rows. + drained: bool, qualified: String, columns: Vec, timestamp: String, @@ -166,7 +181,13 @@ pub(crate) fn read_history<'py>( "metadata state schema must not be blank", )); } - read_page(py, (session, stream), request) + let (mut page, stream) = read_page(py, (session, stream), request)?; + page.wakes_compute = session + .adapter + .getattr("history_reads_wake_compute") + .and_then(|value| value.extract::()) + .unwrap_or(false); + Ok((page, stream)) } /// Any request but the open scan's next page discards that result before running other SQL. @@ -324,8 +345,8 @@ fn read_page<'py>( OBSERVED_AT_COLUMN, ), }; - let limit = limit.unwrap_or(MAX_HISTORY_RECORDS); - if limit == 0 || limit > MAX_HISTORY_RECORDS { + let limit = limit.unwrap_or(MAX_HISTORY_PAGE_RECORDS); + if limit == 0 || limit > MAX_HISTORY_PAGE_RECORDS { return Err(PyValueError::new_err( "metadata page limit is outside the supported bounds", )); @@ -355,6 +376,7 @@ fn read_page<'py>( next_after: None, change_token: None, unchanged: false, + wakes_compute: false, }; return Ok((page, None)); } @@ -369,6 +391,7 @@ fn read_page<'py>( next_after: None, change_token, unchanged: true, + wakes_compute: false, }; return Ok((page, None)); } @@ -504,6 +527,15 @@ fn read_page<'py>( }) .collect(); let projection: Vec<&str> = projection.iter().map(String::as_str).collect(); + let newest = newest_versions(request).map(|(partition, versions)| NewestVersions { + partition, + observation: match request { + WarehouseHistoryRead::AuditResults(_) => &["result_id"], + _ if has_event_id => &["event_id", "run_id"], + _ => &["run_id"], + }, + versions, + }); let window = if session.local { limit } else { @@ -520,6 +552,7 @@ fn read_page<'py>( adapter: &adapter_name, has_event_id, limit: window, + newest, }, ) .map_err(PyValueError::new_err)?; @@ -535,6 +568,8 @@ fn read_page<'py>( cursor, next_after: Vec::new(), remaining: window, + carried: std::collections::VecDeque::new(), + drained: false, qualified, columns: columns.iter().map(|column| (*column).to_owned()).collect(), timestamp: timestamp.to_owned(), @@ -546,31 +581,51 @@ fn read_page<'py>( Ok((page, stream)) } -/// Fetch one page from the scan's result; a full page keeps the stream open for the next. +/// Fetch one page from the scan's result; rows past the byte budget wait in the stream. fn stream_page<'py>( py: Python<'py>, (session, request): (&HistorySession<'py>, &WarehouseHistoryRead), mut stream: HistoryStream<'py>, limit: usize, ) -> PyResult<(WarehouseHistoryPage, Option>)> { - let columns: Vec<&str> = stream.columns.iter().map(String::as_str).collect(); + let columns: Vec = stream.columns.clone(); + let columns: Vec<&str> = columns.iter().map(String::as_str).collect(); let tuple = py.import("builtins")?.getattr("tuple")?; let mut records: Vec = Vec::new(); let mut next_after = None; let mut read_count = 0; - let fetch = limit.min(stream.remaining); - for row in stream - .cursor - .call_method1("fetchmany", (fetch,))? - .try_iter()? - { - read_count += 1; - let row = tuple.call1((row?,))?; + let mut bytes = 0; + while read_count < limit { + let Some(row) = stream.carried.pop_front() else { + if stream.drained || stream.remaining == 0 { + break; + } + let fetch = (limit - read_count).min(stream.remaining); + let rows: Vec> = stream + .cursor + .call_method1("fetchmany", (fetch,))? + .try_iter()? + .map(|row| tuple.call1((row?,))) + .collect::>()?; + stream.remaining -= rows.len(); + stream.drained = rows.len() < fetch; + stream.carried.extend(rows); + continue; + }; let record = decode_record( py, (request, &row, &stream.qualified), (&columns, &stream.timestamp), )?; + let size = serde_json::to_vec(&record) + .map_err(|error| PyValueError::new_err(error.to_string()))? + .len(); + if read_count > 0 && bytes + size > HISTORY_PAGE_BYTES { + stream.carried.push_front(row); + break; + } + bytes += size; + read_count += 1; let mut cursor = vec![None; stream.cursor_indexes.len()]; for (index, value) in cursor.iter_mut().enumerate() { let raw = row.get_item(stream.cursor_indexes[index])?; @@ -604,13 +659,18 @@ fn stream_page<'py>( } next_after = Some(cursor); } - stream.remaining -= read_count; + let more_rows = !stream.carried.is_empty() || !stream.drained; let open = match &next_after { - Some(after) if read_count == fetch && stream.remaining > 0 => { - stream.next_after.clone_from(after); - Some(stream) + Some(after) if more_rows && (!stream.carried.is_empty() || stream.remaining > 0) => { + if session.local { + discard_stream(&session.connection, Some(stream))?; + None + } else { + stream.next_after.clone_from(after); + Some(stream) + } } - Some(_) if read_count == fetch => { + Some(_) if more_rows => { discard_stream(&session.connection, Some(stream))?; None } @@ -625,6 +685,7 @@ fn stream_page<'py>( next_after, change_token: None, unchanged: false, + wakes_compute: false, }; Ok((page, open)) } @@ -659,6 +720,19 @@ fn discard_stream( Ok(()) } +/// A full read of builds or check results keeps each one's newest only: identity columns and count. +fn newest_versions(request: &WarehouseHistoryRead) -> Option<(&'static [&'static str], u16)> { + match request { + WarehouseHistoryRead::Fingerprints(read) if read.observed_since.is_none() => read + .newest_versions + .map(|versions| (FINGERPRINT_IDENTITY_COLUMNS.as_slice(), versions)), + WarehouseHistoryRead::AuditResults(read) if read.observed_since.is_none() => read + .newest_versions + .map(|versions| (["binding_key"].as_slice(), versions)), + _ => None, + } +} + /// The token a complete earlier read recorded; only state-table scans carry one. fn unchanged_token(request: &WarehouseHistoryRead) -> Option<&str> { match request { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index d9345949ba..d2e0d8eeb5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -1,22 +1,34 @@ //! `sqb plan`: compile the plan pipeline and print the plan as text or JSON. +use crate::bindings::_helpers::command_runtime::audit_failures::resolve_ignore_audit_failures; +use sqlbuild_python_planner::planner_phases::constants::BUILD_HISTORY; +use sqlbuild_python_planner::planner_phases::constants::WRITING; +use sqlbuild_python_planner::planner_phases::main::timing::timed::timed; +use sqlbuild_python_planner::planner_phases::models::PlanTimingScope; +use sqlbuild_python_planner::planner_phases::models::PlanTimings; +use std::sync::Arc; + use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, pyfunction, + Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, pyfunction, }; -use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use pyo3::types::{PyBool, PyCFunction, PyDict, PyDictMethods}; use pyo3::wrap_pyfunction; use sqlbuild_config::command_connection::errors::CommandConnectionError; use sqlbuild_core::plan_display::constants::{ - PLAN_SYNCED_COPY_ENV, SYNCED_COPY_DEFER_HELP, SYNCED_COPY_DEFER_REFUSAL, SYNCED_COPY_KEY, + SYNCED_COPY_DEFER_HELP, SYNCED_COPY_DEFER_REFUSAL, SYNCED_COPY_KEY, SYNCED_COPY_READING, }; use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_state::consumption::models::ChangedPolicy; +use sqlbuild_state::warehouse_state::models::WarehouseState; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::command_runtime::preview_target::validate_preview_target; use crate::bindings::_helpers::command_runtime::runtime_errors::raised; -use crate::bindings::_helpers::compile_driver::plan_pipeline::run_compile_pipeline; +use crate::bindings::_helpers::compile_driver::plan_pipeline::planned_project; +use crate::bindings::_helpers::compile_driver::rules_at_load::RulesAtLoadScope; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::plan_command::changed::{changed_json, changed_text}; +use crate::bindings::_helpers::plan_command::changed::changed_json; +use crate::bindings::_helpers::plan_command::changed::changed_text; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -26,7 +38,6 @@ use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use sqlbuild_python_front::compile_attachment::main::build_effective_settings_config::build_effective_settings_config; use sqlbuild_python_output::build_progress::main::with_verbose_inspection::with_verbose_inspection; @@ -36,11 +47,20 @@ use sqlbuild_python_planner::consumption::main::changed_policy::changed_policy; use sqlbuild_python_planner::consumption::main::policy_line::policy_line; use sqlbuild_python_planner::consumption::main::preview_changed::preview_changed; use sqlbuild_python_planner::consumption::models::ChangedDecision; +use sqlbuild_python_planner::live_reads::models::LiveReads; use sqlbuild_python_planner::plan_output::constants::CHANGED_KEY; use sqlbuild_python_planner::plan_output::main::format_plan::format_plan; use sqlbuild_python_planner::plan_output::main::format_plan_json::format_plan_json; +use sqlbuild_python_planner::planner_warehouse::models::PlannerReads; +use sqlbuild_python_planner::planner_warehouse::models::SyncedReads; const COMMAND_NAME: &str = "plan"; +/// The plan document's field holding how long each planning step took. +const TIMINGS_KEY: &str = "timings"; +/// `timings.planned_from` for a location preview over the synced warehouse state. +const SYNCED_STATE: &str = "synced_state"; +/// `timings.planned_from` for a plan that read the warehouse. +const LIVE: &str = "live"; const DEFAULT_SECTION_ENTRIES: i64 = 50; /// The `severity` value of a plan finding that blocks a build. const ERROR_SEVERITY: &str = "error"; @@ -87,17 +107,33 @@ struct PlanInvocation<'py> { connection_progress: Bound<'py, PyAny>, planning_progress: Bound<'py, PyAny>, load_sources: Bound<'py, PyAny>, - /// Set for a location's plan preview: the adapter that answers from the synced copy. - synced_copy: Option>, + /// Set for a location's plan preview: the reads that answer from the synced state. + preview: Option>, +} + +/// A plan preview's reads over the synced state, and the handle the planner reads through. +struct SyncedPlan<'py> { + synced: Arc, + reads: Bound<'py, PlannerReads>, +} + +/// A plan's project: compiled for the request, or the discovered inputs and compiled phase a location holds. +#[derive(Clone, Copy)] +pub(crate) enum PlanProject<'a, 'py> { + Compiled, + Held { + discovered: &'a Bound<'py, PyAny>, + compiled: &'a Bound<'py, PyAny>, + }, } -/// The synced-copy adapter when a location asked for a preview; a preview never defers. -fn synced_copy<'py>( +/// A location preview's reads over the synced `state`, never deferring; other plans read live. +fn preview<'py>( request: &Bound<'py, PyAny>, - adapter: Bound<'py, PyAny>, -) -> PyResult>> { + state: Option<&Arc>, +) -> PyResult>> { let py: Python<'py> = request.py(); - let Ok(path) = std::env::var(PLAN_SYNCED_COPY_ENV) else { + let Some(state) = state else { return Ok(None); }; if native_field(request, "defer_to")?.is_truthy()? { @@ -110,13 +146,15 @@ fn synced_copy<'py>( }, )); } - Ok(Some(Bound::new( - py, - SyncedCopyAdapter::load(py, adapter, &path)?, - )?)) + let synced: Arc = Arc::new(SyncedReads::new(Arc::clone(state))); + let reads: Bound<'py, PlannerReads> = Bound::new(py, PlannerReads::new(synced.clone()))?; + Ok(Some(SyncedPlan { synced, reads })) } -fn resolve_invocation<'py>(request: &Bound<'py, PyAny>) -> PyResult> { +fn resolve_invocation<'py>( + request: &Bound<'py, PyAny>, + (project, state): (PlanProject<'_, 'py>, Option<&Arc>), +) -> PyResult> { let py: Python<'py> = request.py(); let requested_dir: Bound<'py, PyAny> = native_field(request, "project_dir")?; let project_dir: Bound<'py, PyAny> = if requested_dir.is_none() { @@ -124,16 +162,19 @@ fn resolve_invocation<'py>(request: &Bound<'py, PyAny>) -> PyResult = discover_project_inputs( - py, - Some(keywords( + let discovered: Bound<'py, PyAny> = match project { + PlanProject::Held { discovered, .. } => discovered.clone(), + PlanProject::Compiled => discover_project_inputs( py, - &[ - ("project_dir", project_dir.clone()), - ("extract_output_column_locations", flag(py, false)), - ], - )?), - )?; + Some(keywords( + py, + &[ + ("project_dir", project_dir.clone()), + ("extract_output_column_locations", flag(py, false)), + ], + )?), + )?, + }; validate_preview_target( py, discovered.clone(), @@ -182,8 +223,7 @@ fn resolve_invocation<'py>(request: &Bound<'py, PyAny>) -> PyResult> = - synced_copy(request, native_field(&context, "adapter")?)?; + let preview: Option> = preview(request, state)?; Ok(PlanInvocation { project_dir, discovered, @@ -193,21 +233,39 @@ fn resolve_invocation<'py>(request: &Bound<'py, PyAny>) -> PyResult(invocation: &PlanInvocation<'py>) -> PyResult> { let py: Python<'py> = invocation.stream.py(); + let on_progress: Bound<'py, PyAny> = + native_field(&invocation.planning_progress, "on_progress")?; + if invocation.preview.is_some() { + let progress: Py = invocation + .planning_progress + .downcast::()? + .clone() + .unbind(); + let reading = PyCFunction::new_closure(py, None, None, move |arguments, _keywords| { + progress + .get() + .on_progress(arguments.py(), SYNCED_COPY_READING) + })?; + return call_with( + &host_entry(py, ExecutorHost::ConnectionHooks)?, + &[ + ("on_progress", on_progress), + ("on_connection_start", reading.into_any()), + ], + ); + } let connection: &Bound<'py, PyAny> = &invocation.connection_progress; call_with( &host_entry(py, ExecutorHost::ConnectionHooks)?, &[ - ( - "on_progress", - native_field(&invocation.planning_progress, "on_progress")?, - ), + ("on_progress", on_progress), ( "on_connection_start", native_field(connection, "on_connection_start")?, @@ -233,6 +291,7 @@ fn connection_hooks<'py>(invocation: &PlanInvocation<'py>) -> PyResult( request: &Bound<'py, PyAny>, invocation: &PlanInvocation<'py>, + project: PlanProject<'_, 'py>, ) -> PyResult> { let py: Python<'py> = request.py(); let resolver: Bound<'py, PyAny> = resolve_external_sql_reference_resolver( @@ -272,25 +331,30 @@ fn compile_pipeline<'py>( native_field(request, "include_python")?, )?; options.set_item("record_migration_fingerprints", false)?; + let planned_target: Bound<'py, PyAny> = options + .get_item("selected_target")? + .unwrap_or_else(|| py.None().into_bound(py)); + options.set_item( + "ignore_audit_failures", + resolve_ignore_audit_failures(request, &invocation.discovered, &planned_target)?, + )?; let pipeline_options: Bound<'py, PyAny> = new_record(py, RecordId::CompilePipelineOptions, (), Some(&options))?; - run_compile_pipeline( - py, - Some(keywords( - py, - &[ - ("discovered_inputs", invocation.discovered.clone()), - ( - "adapter", - match &invocation.synced_copy { - Some(facade) => facade.clone().into_any(), - None => native_field(&invocation.context, "adapter")?, - }, - ), - ("options", pipeline_options), - ("hooks", connection_hooks(invocation)?), - ], - )?), + let adapter: Bound<'py, PyAny> = native_field(&invocation.context, "adapter")?; + let preview: Option> = invocation + .preview + .as_ref() + .map(|preview| preview.reads.clone().into_any()); + planned_project( + (&invocation.discovered, &adapter), + (&pipeline_options, connection_hooks(invocation)?), + ( + match project { + PlanProject::Held { compiled, .. } => Some(compiled), + PlanProject::Compiled => None, + }, + preview.as_ref(), + ), ) } @@ -304,48 +368,68 @@ fn changed_preview( let Some(policy) = changed_policy(&invocation.project_dir, request, &project)? else { return Ok(None); }; - let adapter: Bound<'_, PyAny> = match &invocation.synced_copy { - Some(facade) => facade.clone().into_any(), - None => native_field(&invocation.context, "adapter")?, + timed(BUILD_HISTORY, || { + changed_decision(request, invocation, (pipeline, &project), &policy) + }) +} + +/// The `--changed` decision under `policy`, read from recorded consumption. +fn changed_decision( + request: &Bound<'_, PyAny>, + invocation: &PlanInvocation<'_>, + (pipeline, project): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + policy: &ChangedPolicy, +) -> PyResult> { + let plan: (&Bound<'_, PyAny>, &Bound<'_, PyAny>) = + (&native_field(pipeline, "plan_output")?, project); + let full_refresh: bool = native_field(request, "full_refresh")?.is_truthy()?; + let py = request.py(); + let reads: Bound<'_, PlannerReads> = match &invocation.preview { + Some(preview) => preview.reads.clone(), + None => LiveReads::new( + &native_field(&invocation.context, "adapter")?, + &py.None().into_bound(py), + Some(&native_field(&invocation.context, "connection_config")?), + ) + .handle(py)?, }; - let (decision, warnings) = preview_changed( - (&native_field(pipeline, "plan_output")?, &project), - ( - &adapter, - &native_field(&invocation.context, "connection_config")?, - ), - (&policy, native_field(request, "full_refresh")?.is_truthy()?), - )?; + let changed = preview_changed(plan, reads.get().reads(), (policy, full_refresh)); + reads.get().reads().close_consumption(py)?; + let (decision, warnings) = changed?; for warning in warnings { write_progress(&invocation.stream, &format!("warning: {warning}\n"))?; } Ok(Some(decision)) } +/// Print the plan; the JSON plan is returned instead when `captured`. fn write_output( request: &Bound<'_, PyAny>, - invocation: &PlanInvocation<'_>, + (invocation, captured): (&PlanInvocation<'_>, bool), pipeline: &Bound<'_, PyAny>, changed: Option<&ChangedDecision>, -) -> PyResult<()> { +) -> PyResult> { let py: Python<'_> = request.py(); let plan: Bound<'_, PyAny> = native_field(pipeline, "plan_output")?; let python_entries: Bound<'_, PyAny> = native_field(pipeline, "python_plan_entries")?; if native_field(request, "json_output")?.is_truthy()? { let mut options: Vec<(&str, Bound<'_, PyAny>)> = vec![("plan", plan), ("python_plan_entries", python_entries)]; - if let Some(facade) = &invocation.synced_copy { + if let Some(facade) = &invocation.preview { options.push(( SYNCED_COPY_KEY, - synced_copy_json(py, facade.get())?.into_any(), + synced_copy_json(py, &facade.synced)?.into_any(), )); } if let Some(decision) = changed { options.push((CHANGED_KEY, changed_json(py, decision)?.into_any())); } let text: String = format_plan_json(py, Some(keywords(py, &options)?))?; + if captured { + return Ok(Some(text)); + } imported(py, "builtins", "print")?.call1((text,))?; - return Ok(()); + return Ok(None); } let display: Bound<'_, PyDict> = PyDict::new(py); display.set_item( @@ -378,17 +462,14 @@ fn write_output( } let text: String = format_plan(py, Some(keywords(py, &options)?))?; print_line(py, &format!("\n{text}"))?; - if let Some(facade) = &invocation.synced_copy { - print_line(py, &synced_copy_text(facade.get()))?; + if let Some(facade) = &invocation.preview { + print_line(py, &synced_copy_text(&facade.synced))?; } - Ok(()) + Ok(None) } /// The plan JSON's account of the synced copy: when it synced, window starts and unknowns. -fn synced_copy_json<'py>( - py: Python<'py>, - facade: &SyncedCopyAdapter, -) -> PyResult> { +fn synced_copy_json<'py>(py: Python<'py>, facade: &SyncedReads) -> PyResult> { let synced: Bound<'py, PyDict> = PyDict::new(py); synced.set_item("synced_at", facade.synced_at())?; synced.set_item("cursor_window_starts", facade.windows())?; @@ -397,7 +478,7 @@ fn synced_copy_json<'py>( } /// The preview footer: when the copy was synced and what stays unknown until the run. -fn synced_copy_text(facade: &SyncedCopyAdapter) -> String { +fn synced_copy_text(facade: &SyncedReads) -> String { let mut lines: Vec = vec![format!( "Planned from the synced copy ({}); the run re-plans live.", facade.synced_at().map_or_else( @@ -428,20 +509,81 @@ fn synced_copy_text(facade: &SyncedCopyAdapter) -> String { #[pyfunction] #[pyo3(signature = (*, request))] pub(crate) fn run_plan_command(request: &Bound<'_, PyAny>) -> PyResult { - let py: Python<'_> = request.py(); - let invocation: PlanInvocation<'_> = resolve_invocation(request)?; + Ok(plan(request, (PlanProject::Compiled, None))?.0) +} + +/// A location's preview from the synced `state`, without rules, which ran at load; vars or a dbt manifest recompile. +pub(crate) fn location_plan<'py>( + request: &Bound<'py, PyAny>, + (held, state): (PlanProject<'_, 'py>, &Arc), +) -> PyResult<(i32, Option)> { + let external_manifest: bool = match held { + PlanProject::Held { discovered, .. } => !native_field( + &native_field(&native_field(discovered, "project_config")?, "dbt")?, + "target_path", + )? + .is_none(), + PlanProject::Compiled => false, + }; + let _rules = RulesAtLoadScope::enter(); + let project: PlanProject<'_, 'py> = + if external_manifest || native_field(request, "cli_vars")?.is_truthy()? { + PlanProject::Compiled + } else { + held + }; + plan(request, (project, Some(state))) +} + +/// One plan: printed, or from a synced state, the JSON plan returned. +fn plan<'py>( + request: &Bound<'py, PyAny>, + (project, state): (PlanProject<'_, 'py>, Option<&Arc>), +) -> PyResult<(i32, Option)> { + let py: Python<'py> = request.py(); + let timing: PlanTimingScope = PlanTimingScope::enter(); + let invocation: PlanInvocation<'py> = resolve_invocation(request, (project, state))?; if !native_field(request, "json_output")?.is_truthy()? { write_progress(&invocation.stream, "\n")?; } let verbose: bool = native_field(request, "verbose")?.is_truthy()?; - let pipeline: Bound<'_, PyAny> = - with_verbose_inspection(py, verbose, || compile_pipeline(request, &invocation))?; + let pipeline: Bound<'py, PyAny> = with_verbose_inspection(py, verbose, || { + compile_pipeline(request, &invocation, project) + })?; let changed: Option = changed_preview(request, &invocation, &pipeline)?; - write_output(request, &invocation, &pipeline, changed.as_ref())?; - Ok(i32::from(plan_has_errors(&native_field( - &pipeline, - "plan_output", - )?)?)) + let output: Option = timed(WRITING, || { + write_output( + request, + (&invocation, state.is_some()), + &pipeline, + changed.as_ref(), + ) + })?; + let timings: PlanTimings = timing.timings(); + let output: Option = output.map(|document| { + with_timings( + document, + &timings.json(if state.is_some() { SYNCED_STATE } else { LIVE }), + ) + }); + if native_field(request, "timings")?.is_truthy()? { + write_progress(&invocation.stream, &format!("\n{}\n", timings.table()))?; + } + Ok(( + i32::from(plan_has_errors(&native_field(&pipeline, "plan_output")?)?), + output, + )) +} + +/// `document` with its `timings` as the first field; the plan is written before its own timings are known. +fn with_timings(document: String, timings: &serde_json::Value) -> String { + match document.strip_prefix('{') { + Some(fields) if fields.trim_start().starts_with('}') => { + format!("{{\"{TIMINGS_KEY}\": {timings}}}") + } + Some(fields) => format!("{{\"{TIMINGS_KEY}\": {timings}, {fields}"), + None => document, + } } /// Whether any plan finding has error severity, which `sqb build` refuses. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/registration.rs index f9933699b4..5365871ca7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/registration.rs @@ -2,7 +2,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::plan_command::{plan, rules}; +use crate::bindings::_helpers::plan_command::plan; +use crate::bindings::_helpers::plan_command::rules; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { plan::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs index a4c60b4b08..e0f55be6bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs @@ -2,11 +2,18 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::{ - planner_changes, planner_entries, planner_identity, planner_migrations, planner_phases, - planner_resolution, planner_scenario, planner_scope, planner_selection, planner_strategy, - planner_warehouse, relation_naming, -}; +use crate::bindings::_helpers::planner_changes; +use crate::bindings::_helpers::planner_entries; +use crate::bindings::_helpers::planner_identity; +use crate::bindings::_helpers::planner_migrations; +use crate::bindings::_helpers::planner_phases; +use crate::bindings::_helpers::planner_resolution; +use crate::bindings::_helpers::planner_scenario; +use crate::bindings::_helpers::planner_scope; +use crate::bindings::_helpers::planner_selection; +use crate::bindings::_helpers::planner_strategy; +use crate::bindings::_helpers::planner_warehouse; +use crate::bindings::_helpers::relation_naming; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { planner_changes::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/registration.rs index e42f61f8aa..95a7ef5169 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/registration.rs @@ -2,7 +2,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::query_command::{dag, query}; +use crate::bindings::_helpers::query_command::dag; +use crate::bindings::_helpers::query_command::query; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { query::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs index ecce1d6d96..1f9d3d666f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs @@ -16,7 +16,8 @@ use crate::bindings::_helpers::build_command::command::run_build_command; use crate::bindings::_helpers::clone::clone_command::run_clone_command; use crate::bindings::_helpers::dagster_invocation::access::string_tuple as texts; use crate::bindings::_helpers::review_command::planning::ReviewSettings; -use crate::bindings::_helpers::review_command::support::{quietly, review_error}; +use crate::bindings::_helpers::review_command::support::quietly; +use crate::bindings::_helpers::review_command::support::review_error; use crate::bindings::_helpers::sql_test_command::test_command::run_test_command; use sqlbuild_python_base::build_command::main::boolean::boolean; use sqlbuild_python_base::build_command::main::text::text; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/support.rs index c37c30d1fa..5424091e4f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/support.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/support.rs @@ -53,7 +53,7 @@ pub(crate) fn plan_error(py: Python<'_>, error: &ReviewPlanError) -> PyErr { } /// `name` of `value` as text, or `None` when it is absent or `None`. -fn attribute(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { +pub(crate) fn attribute(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { match value.getattr_opt(name)? { Some(item) if !item.is_none() => Ok(Some(item.str()?.extract()?)), Some(_) | None => Ok(None), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/registration.rs index 30bbbf8e11..77e34f7e88 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/registration.rs @@ -2,7 +2,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::scenario_command::{capture_command, test_command}; +use crate::bindings::_helpers::scenario_command::capture_command; +use crate::bindings::_helpers::scenario_command::test_command; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { test_command::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs index 98998b7fa8..5e35577dc3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs @@ -9,13 +9,23 @@ use sqlbuild_core::scenario_command::models::ScenarioNamespace; use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::scenario_command::conversions::succeeded; -use crate::bindings::_helpers::scenario_command::presentation::{ - CompletionKind, RelayContext, SectionStatus, begin_section, connection_hooks, progress_relay, -}; -use crate::bindings::_helpers::scenario_command::setup::{ - CompileInputs, boolean, capture_limits, capture_settings, compile, flush, optional_text, - project_connection_config, selected_scenarios, text, write, -}; +use crate::bindings::_helpers::scenario_command::presentation::CompletionKind; +use crate::bindings::_helpers::scenario_command::presentation::RelayContext; +use crate::bindings::_helpers::scenario_command::presentation::SectionStatus; +use crate::bindings::_helpers::scenario_command::presentation::begin_section; +use crate::bindings::_helpers::scenario_command::presentation::connection_hooks; +use crate::bindings::_helpers::scenario_command::presentation::progress_relay; +use crate::bindings::_helpers::scenario_command::setup::CompileInputs; +use crate::bindings::_helpers::scenario_command::setup::boolean; +use crate::bindings::_helpers::scenario_command::setup::capture_limits; +use crate::bindings::_helpers::scenario_command::setup::capture_settings; +use crate::bindings::_helpers::scenario_command::setup::compile; +use crate::bindings::_helpers::scenario_command::setup::flush; +use crate::bindings::_helpers::scenario_command::setup::optional_text; +use crate::bindings::_helpers::scenario_command::setup::project_connection_config; +use crate::bindings::_helpers::scenario_command::setup::selected_scenarios; +use crate::bindings::_helpers::scenario_command::setup::text; +use crate::bindings::_helpers::scenario_command::setup::write; use crate::bindings::_helpers::scenario_command::test_command::TestRun; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 588f117eb9..1f231727c1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -1,5 +1,6 @@ //! `sqb seed`: compile the selected seeds, load them, and report. +use sqlbuild_python_planner::live_reads::models::LiveReads; use std::collections::HashMap; use std::sync::Arc; use std::time::Instant; @@ -269,12 +270,12 @@ fn compile_seeds<'py>( )?; let connection: Bound<'py, PyAny> = connection_with_hooks(&adapter, &connection_config, &hooks)?; - let read: PyResult> = read_selected_seed_fingerprints( - &adapter, - &connection, - &PyTuple::new(py, &selected)?.into_any(), - ) - .map(Bound::into_any); + let read: PyResult> = LiveReads::new(&adapter, &connection, None) + .handle(adapter.py()) + .and_then(|reads| { + read_selected_seed_fingerprints(&reads, &PyTuple::new(py, &selected)?.into_any()) + }) + .map(Bound::into_any); let closed: PyResult> = adapter.call_method1("close", (connection,)); let read: Bound<'py, PyAny> = read?; closed?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs index 0b42953e34..5b2ab9f168 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs @@ -32,7 +32,7 @@ const TEST_MODULE_PREFIX: &str = "test_"; const PYTEST_MODULE: &str = "pytest"; /// The diagnostic code of a Python test run that cannot start. pub(crate) const PYTHON_TESTS_INPUT_CODE: &str = "S000"; -/// The project folder whose `python/` and `providers/` folders mirror the project's. +/// The project folder whose `python/`, `providers/` and `orchestration/` folders mirror the project's. const TESTS_FOLDER: &str = "tests"; /// The discovered Python node collections whose folders a selection's tests mirror. const NODE_COLLECTIONS: [&str; 4] = [ @@ -54,7 +54,7 @@ pub(crate) fn selective(request: &Bound<'_, PyAny>) -> PyResult { || native_field(request, "exclude")?.is_truthy()?) } -/// Every `test_*.py` module under the project's `tests/python/` and `tests/providers/`. +/// Every `test_*.py` module under the project's `tests/python/`, `tests/providers/` and `tests/orchestration/`. pub(crate) fn python_test_modules(invocation: &Bound<'_, PyAny>) -> PyResult> { let project_dir: PathBuf = native_field(invocation, "effective_project_dir")?.extract()?; Ok(python_test_paths(&project_dir) @@ -118,7 +118,7 @@ pub(crate) fn selected_python_tests( ( Some(PYTHON_TESTS_INPUT_CODE), Some( - "add test_*.py modules under tests/python/ or tests/providers/, mirroring the code they test", + "add test_*.py modules under tests/python/, tests/providers/ or tests/orchestration/, mirroring the code they test", ), ), )); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/registration.rs index 0d79653596..2ef7c1e16a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/registration.rs @@ -2,7 +2,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::sql_test_command::{test_command, timeouts}; +use crate::bindings::_helpers::sql_test_command::test_command; +use crate::bindings::_helpers::sql_test_command::timeouts; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { test_command::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index a36b6e5b36..6d91e5f9bc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -1,5 +1,6 @@ //! `sqb test`: compile the selected SQL unit tests, inspect or run them, and report; locally, then run the Python tests. +use sqlbuild_python_base::native_host::models::Dialect; use std::collections::{BTreeSet, HashMap}; use std::path::PathBuf; use std::sync::Arc; @@ -25,23 +26,27 @@ use sqlbuild_host::models::{SqlTestExecutionResult, SqlTestOutcome}; use sqlbuild_host::types::ResultEnum; use sqlbuild_orchestration::runs::models::NodeOutcome; -use crate::bindings::_helpers::change_selection::base_snapshot::{ - BaseComparison, base_comparison, base_variant, -}; -use crate::bindings::_helpers::change_selection::selection::{ - ChangedProject, changed_project, tests_and_models, with_since_section, -}; +use crate::bindings::_helpers::change_selection::base_snapshot::BaseComparison; +use crate::bindings::_helpers::change_selection::base_snapshot::base_comparison; +use crate::bindings::_helpers::change_selection::base_snapshot::base_variant; +use crate::bindings::_helpers::change_selection::selection::ChangedProject; +use crate::bindings::_helpers::change_selection::selection::changed_project; +use crate::bindings::_helpers::change_selection::selection::tests_and_models; +use crate::bindings::_helpers::change_selection::selection::with_since_section; use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::native_host::statement_report::StatementReport; -use crate::bindings::_helpers::sql_test_command::python_tests::{ - PYTHON_TESTS_INPUT_CODE, names_python_node, python_test_modules, run_python_tests, - selected_python_tests, selective, with_python_section, -}; -use crate::bindings::_helpers::sql_test_command::timeouts::{ - RunTimeout, TimeoutKind, resolve_run_timeout, -}; +use crate::bindings::_helpers::sql_test_command::python_tests::PYTHON_TESTS_INPUT_CODE; +use crate::bindings::_helpers::sql_test_command::python_tests::names_python_node; +use crate::bindings::_helpers::sql_test_command::python_tests::python_test_modules; +use crate::bindings::_helpers::sql_test_command::python_tests::run_python_tests; +use crate::bindings::_helpers::sql_test_command::python_tests::selected_python_tests; +use crate::bindings::_helpers::sql_test_command::python_tests::selective; +use crate::bindings::_helpers::sql_test_command::python_tests::with_python_section; +use crate::bindings::_helpers::sql_test_command::timeouts::RunTimeout; +use crate::bindings::_helpers::sql_test_command::timeouts::TimeoutKind; +use crate::bindings::_helpers::sql_test_command::timeouts::resolve_run_timeout; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -372,7 +377,7 @@ fn compile_tests<'py>( py, &[ ("project", context.getattr("project")?), - ("adapter", adapter), + ("dialect", Dialect::of(&adapter)?.into_any()), ("scope", context.getattr("scope")?), ("relations", context.getattr("relations")?), ("case_name", case_name.clone()), diff --git a/crates/sqlbuild-python/src/bindings/tests/test_failure_cause.rs b/crates/sqlbuild-python/src/bindings/tests/test_failure_cause.rs index 6133d4820d..672f3ca1e7 100644 --- a/crates/sqlbuild-python/src/bindings/tests/test_failure_cause.rs +++ b/crates/sqlbuild-python/src/bindings/tests/test_failure_cause.rs @@ -1,5 +1,6 @@ use crate::bindings::_helpers::cli::failure_cause::take_failure_cause; -use crate::bindings::tests::helpers::{report_elsewhere, report_here}; +use crate::bindings::tests::helpers::report_elsewhere; +use crate::bindings::tests::helpers::report_here; use crate::bindings::tests::test_types::FailureCauseTestCase; #[test] diff --git a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs index 8c6f7fbe29..b7cb007599 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs @@ -479,7 +479,8 @@ fn audit_results(options: &DdlOptions) -> String { JSONExtract(payload, 'evidence_count', 'Nullable(Int64)') AS evidence_count, if(JSONHas(payload, 'evidence_truncated'), toUInt8(JSONExtractBool(payload, 'evidence_truncated')), NULL) AS evidence_truncated, - JSONExtract(payload, 'sample_digest', 'Nullable(String)') AS sample_digest + JSONExtract(payload, 'sample_digest', 'Nullable(String)') AS sample_digest, + toUInt8(JSONExtractBool(payload, 'ignored')) AS ignored FROM {events} WHERE event_type = 'audit_completed'" ) diff --git a/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs index 26cfb9721a..6fe9dadf3f 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs @@ -186,7 +186,8 @@ fn audit_results(postgres: bool) -> String { sql_digest, {evidence_count} AS evidence_count, {evidence_truncated} AS evidence_truncated, - {sample_digest} AS sample_digest + {sample_digest} AS sample_digest, + COALESCE({ignored}, 0) AS ignored FROM recall_events WHERE event_type = 'audit_completed'", audit_name = field("audit_name", ""), @@ -204,7 +205,8 @@ WHERE event_type = 'audit_completed'", execution_error = field("execution_error", ""), evidence_count = field("evidence_count", "::BIGINT"), evidence_truncated = field("evidence_truncated", "::BOOLEAN::INTEGER"), - sample_digest = field("sample_digest", "") + sample_digest = field("sample_digest", ""), + ignored = field("ignored", "::BOOLEAN::INTEGER") ) } diff --git a/crates/sqlbuild-recall/src/schema/constants.rs b/crates/sqlbuild-recall/src/schema/constants.rs index 367fe349d5..70a2580dfd 100644 --- a/crates/sqlbuild-recall/src/schema/constants.rs +++ b/crates/sqlbuild-recall/src/schema/constants.rs @@ -3,7 +3,7 @@ use crate::schema::models::{ColumnSpec, ColumnType, Table, TableSpec}; /// The Recall schema version this build writes. Raise it when a column or view is added. -pub const RECALL_SCHEMA_VERSION: i64 = 6; +pub const RECALL_SCHEMA_VERSION: i64 = 7; /// Text that only older schema versions wrote into ClickHouse view definitions, with the last version that wrote it. Version 3 changed only view bodies, so view columns cannot tell version 2 views, which set `join_use_nulls`, from current ones. pub const SUPERSEDED_CLICKHOUSE_VIEW_MARKERS: [(&str, i64); 1] = [("join_use_nulls", 2)]; /// The oldest SQLite library the store's views run on (ordered `group_concat`). diff --git a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs index 32c8e20587..9bd8ada2f8 100644 --- a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs +++ b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs @@ -154,8 +154,8 @@ fn given_clickhouse_statement_sets_when_rendered_then_every_statement_has_the_sh .collect(), required_fragments: &[ "INSERT INTO `observability`.`recall_meta`", - "SELECT 'schema', 'schema_version', '6'", - "6, '9.8.7', now64(6)", + "SELECT 'schema', 'schema_version', '7'", + "7, '9.8.7', now64(6)", ], expected_count: 1, }, diff --git a/crates/sqlbuild-recall/src/store/tests/test_views.rs b/crates/sqlbuild-recall/src/store/tests/test_views.rs index 3e08baa596..05de39325e 100644 --- a/crates/sqlbuild-recall/src/store/tests/test_views.rs +++ b/crates/sqlbuild-recall/src/store/tests/test_views.rs @@ -420,21 +420,40 @@ fn given_audit_completed_facts_when_projected_then_audit_results_carry_outcome_a let env = envelope("audits", NOW); let test_cases = [ProjectionTestCase { description: "audit outcomes name the audited relation and column", - batches: vec![vec![event( - &env, - ("a0", "audit_completed", 1), - r#","run_id":"run""#, - r#"{"audit_name":"revenue_positive","audit_definition_name":"positive","evaluation_mode":"violations","outcome":"warn","severity":"warn","run_scope_phase":"model","attachment_kind":"column","target_schema":"analytics","target_name":"daily_revenue","attached_column_name":"total_revenue_cents","violation_count":3,"sql_digest":"ab"}"#, - )]], - sql: "SELECT audit_name, target_relation, \"column\", outcome, violation_count \ - FROM audit_results", - expected_rows: vec![vec![ - text("revenue_positive"), - text("analytics.daily_revenue"), - text("total_revenue_cents"), - text("warn"), - integer(3), + batches: vec![vec![ + event( + &env, + ("a0", "audit_completed", 1), + r#","run_id":"run""#, + r#"{"audit_name":"revenue_positive","audit_definition_name":"positive","evaluation_mode":"violations","outcome":"warn","severity":"warn","run_scope_phase":"model","attachment_kind":"column","target_schema":"analytics","target_name":"daily_revenue","attached_column_name":"total_revenue_cents","violation_count":3,"sql_digest":"ab"}"#, + ), + event( + &env, + ("a1", "audit_completed", 2), + r#","run_id":"run""#, + r#"{"audit_name":"revenue_present","audit_definition_name":"present","evaluation_mode":"violations","outcome":"error","severity":"error","run_scope_phase":"model","attachment_kind":"model","target_schema":"analytics","target_name":"daily_revenue","violation_count":1,"sql_digest":"cd","ignored":true}"#, + ), ]], + sql: "SELECT audit_name, target_relation, \"column\", outcome, violation_count, ignored \ + FROM audit_results ORDER BY audit_name", + expected_rows: vec![ + vec![ + text("revenue_positive"), + text("analytics.daily_revenue"), + text("total_revenue_cents"), + text("warn"), + integer(3), + integer(0), + ], + vec![ + text("revenue_present"), + text("analytics.daily_revenue"), + RecallValue::Null, + text("error"), + integer(1), + integer(1), + ], + ], }]; for test_case in test_cases { diff --git a/crates/sqlbuild-rules/src/python_rules/constants.rs b/crates/sqlbuild-rules/src/python_rules/constants.rs index a2225bbbf4..0fd11007f0 100644 --- a/crates/sqlbuild-rules/src/python_rules/constants.rs +++ b/crates/sqlbuild-rules/src/python_rules/constants.rs @@ -255,8 +255,8 @@ pub(crate) const PYTHON_RULES: [PythonRuleSpec; 108] = [ applies: PythonFiles::Tests, enabled_by_default: true, options: &[], - message: "Python tests must mirror an existing module under python/ or providers/", - remediation: "Move the test to tests/python/ or tests/providers/ at the path of the module it tests: tests/python/orders/storefront_import/test_main.py tests python/orders/storefront_import/main.py.", + message: "Python tests must mirror an existing module under python/, providers/ or orchestration/", + remediation: "Move the test to tests/python/, tests/providers/ or tests/orchestration/ at the path of the module it tests: tests/python/orders/storefront_import/test_main.py tests python/orders/storefront_import/main.py.", }, PythonRuleSpec { code: "SQBRPYTHON022", diff --git a/crates/sqlbuild-rules/src/python_rules/models.rs b/crates/sqlbuild-rules/src/python_rules/models.rs index d228ee9ad6..9cf765e959 100644 --- a/crates/sqlbuild-rules/src/python_rules/models.rs +++ b/crates/sqlbuild-rules/src/python_rules/models.rs @@ -7,7 +7,7 @@ pub enum PythonFiles { Package, /// Every supported project Python file, including hooks, UDFs and orchestration modules. All, - /// Only the project's Python tests under `tests/python/` and `tests/providers/`. + /// Only the project's Python tests under `tests/python/`, `tests/providers/` and `tests/orchestration/`. Tests, } diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_python_test_rules.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_python_test_rules.rs index 3357c58220..e45d9ae02d 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/test_python_test_rules.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_python_test_rules.rs @@ -185,6 +185,30 @@ fn given_project_python_tests_when_evaluating_test_rules_then_reports_misplaced_ ("SQBRPYTHON021", "tests/python/test_everything.py", 1, 1), ], }, + PythonRunTestCase { + description: "an orchestration test mirrors its sensor code; one mirroring nothing is reported", + files: &[ + ( + "orchestration/sensors/orders/_helpers/restock_window.py", + "def restock_due(hours):\n return hours >= 6\n", + ), + ( + "tests/orchestration/sensors/orders/_helpers/test_restock_window.py", + ROW_MAPPING_TEST, + ), + ( + "tests/orchestration/sensors/inventory/test_stock.py", + ROW_MAPPING_TEST, + ), + ], + config: json!({"select": ["SQBRPYTHON021"]}), + expected_faults: vec![( + "SQBRPYTHON021", + "tests/orchestration/sensors/inventory/test_stock.py", + 1, + 1, + )], + }, PythonRunTestCase { description: "a relative import, a top-level helper and a branch in a test break the test conventions", files: &[ diff --git a/crates/sqlbuild-state/Cargo.toml b/crates/sqlbuild-state/Cargo.toml index 17f6e3272b..e4bf9ae968 100644 --- a/crates/sqlbuild-state/Cargo.toml +++ b/crates/sqlbuild-state/Cargo.toml @@ -8,6 +8,8 @@ publish.workspace = true rust-version.workspace = true [dependencies] +chrono.workspace = true +serde.workspace = true serde_json.workspace = true sha2.workspace = true sqlbuild-core.workspace = true diff --git a/crates/sqlbuild-state/src/lib.rs b/crates/sqlbuild-state/src/lib.rs index e637523fb6..2f5c84c43d 100644 --- a/crates/sqlbuild-state/src/lib.rs +++ b/crates/sqlbuild-state/src/lib.rs @@ -1,4 +1,4 @@ -//! Warehouse state rows SQLBuild appends: event IDs, event SQL and source freshness versions. +//! Warehouse state rows SQLBuild appends, and the warehouse state a plan reads. #![forbid(unsafe_code)] @@ -11,3 +11,4 @@ pub mod microbatches; pub mod observations; pub mod resource_costs; pub mod state_tables; +pub mod warehouse_state; diff --git a/crates/sqlbuild-state/src/state_tables/_helpers/history_sql.rs b/crates/sqlbuild-state/src/state_tables/_helpers/history_sql.rs new file mode 100644 index 0000000000..eb072d18f0 --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/_helpers/history_sql.rs @@ -0,0 +1,89 @@ +//! The SQL pieces a keyset history page is built from: byte-wise comparisons, the newest versions and the cursor filter. + +use crate::state_tables::constants::{NEWEST_FIRST_COLUMN, NEWEST_VERSION_COLUMN}; +use crate::state_tables::models::NewestVersions; + +/// Session and column collations must not merge case, accents or trailing spaces at page boundaries. +pub(crate) fn binary(value: &str, adapter: &str) -> Result { + Ok(match adapter { + "duckdb" | "motherduck" => format!("encode({value})"), + "sqlite" => format!("CAST({value} AS BLOB)"), + "postgres" => format!("convert_to({value}, 'UTF8')"), + "snowflake" => format!("TO_BINARY({value}, 'UTF-8')"), + "bigquery" => format!("CAST({value} AS BYTES)"), + "databricks" => format!("CAST({value} AS BINARY)"), + "sqlserver" => format!("CONVERT(VARBINARY(MAX), CONVERT(NVARCHAR(MAX), {value}))"), + "clickhouse" => value.to_owned(), + _ => { + return Err(format!( + "warehouse history does not support adapter '{adapter}'" + )); + } + }) +} + +/// The table a read pages through, or each partition's newest observations and their filter. +pub(crate) fn versioned_source( + qualified_name: &str, + timestamp_column: &str, + newest: Option>, +) -> (String, Option) { + let Some(newest) = newest else { + return (qualified_name.to_owned(), None); + }; + let partition = newest.partition.join(", "); + let mut order = vec![format!("{NEWEST_FIRST_COLUMN} DESC")]; + order.extend( + newest + .observation + .iter() + .map(|column| format!("{column} DESC")), + ); + ( + format!( + "(SELECT *, DENSE_RANK() OVER (PARTITION BY {partition} ORDER BY {}) AS {NEWEST_VERSION_COLUMN} \ + FROM (SELECT *, MIN({timestamp_column}) OVER (PARTITION BY {partition}, {}) AS {NEWEST_FIRST_COLUMN} \ + FROM {qualified_name}) AS sqb_observations) AS sqb_versions", + order.join(", "), + newest.observation.join(", ") + ), + Some(format!("{NEWEST_VERSION_COLUMN} <= {}", newest.versions)), + ) +} + +/// The rows after a page's last cursor values, in key order; `None` for an empty cursor. +pub(crate) fn after_filter( + (columns, ordered): (&[&str], &[String]), + after: &[Option], + (timestamp_column, adapter): (&str, &str), +) -> Result, String> { + let mut greater: Vec = Vec::new(); + let mut equal: Vec = Vec::new(); + for ((column, ordered), value) in columns.iter().zip(ordered).zip(after) { + let literal = value + .as_ref() + .map(|value| { + if *column == timestamp_column { + Ok(value.clone()) + } else { + binary(value, adapter) + } + }) + .transpose()?; + let comparison = literal.as_ref().map_or_else( + || format!("{column} IS NOT NULL"), + |literal| format!("{ordered} > {literal}"), + ); + let prefix = if equal.is_empty() { + String::new() + } else { + format!("{} AND ", equal.join(" AND ")) + }; + greater.push(format!("({prefix}{comparison})")); + equal.push(literal.as_ref().map_or_else( + || format!("{column} IS NULL"), + |literal| format!("{ordered} = {literal}"), + )); + } + Ok((!greater.is_empty()).then(|| format!("({})", greater.join(" OR ")))) +} diff --git a/crates/sqlbuild-state/src/state_tables/_helpers/mod.rs b/crates/sqlbuild-state/src/state_tables/_helpers/mod.rs new file mode 100644 index 0000000000..5cc07b565b --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod history_sql; diff --git a/crates/sqlbuild-state/src/state_tables/constants.rs b/crates/sqlbuild-state/src/state_tables/constants.rs index 7ebe8b34ce..26a7d2469f 100644 --- a/crates/sqlbuild-state/src/state_tables/constants.rs +++ b/crates/sqlbuild-state/src/state_tables/constants.rs @@ -1,7 +1,15 @@ //! Adapter decisions shared by append-only state history reads. pub(crate) const SQLSERVER_ADAPTER: &str = "sqlserver"; + /// Cursor key of a table without event IDs: always NULL, so it never orders or binds a value. pub(crate) const MISSING_EVENT_ID_KEY: &str = "NULL"; + /// The column every event table identifies its rows by. pub(crate) const EVENT_ID_COLUMN: &str = "event_id"; + +/// The rank a newest-versions read filters on; never projected, so it never reaches a record. +pub(crate) const NEWEST_VERSION_COLUMN: &str = "sqb_version"; + +/// When an observation's first copy was written, so its copies share one rank. +pub(crate) const NEWEST_FIRST_COLUMN: &str = "sqb_first_at"; diff --git a/crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs b/crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs index ab2b046bfb..3aea535a61 100644 --- a/crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs +++ b/crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs @@ -1,5 +1,6 @@ //! Keyset pages preserve distinct observations without rereading or ranking all preceding rows. +use crate::state_tables::_helpers::history_sql::{after_filter, binary, versioned_source}; use crate::state_tables::constants::{EVENT_ID_COLUMN, MISSING_EVENT_ID_KEY, SQLSERVER_ADAPTER}; use crate::state_tables::models::HistoryRead; @@ -17,6 +18,7 @@ pub fn read_history_sql( adapter, has_event_id, limit, + newest, } = read; let mut projection = columns.join(", "); if !columns.contains(&EVENT_ID_COLUMN) { @@ -44,43 +46,18 @@ pub fn read_history_sql( } }) .collect::, _>>()?; - let mut greater: Vec = Vec::new(); - let mut equal: Vec = Vec::new(); - if let Some(after) = after { - for ((column, ordered), value) in columns.iter().zip(&ordered).zip(after) { - let literal = value - .as_ref() - .map(|value| { - if *column == timestamp_column { - Ok(value.clone()) - } else { - binary(value, adapter) - } - }) - .transpose()?; - let comparison = literal.as_ref().map_or_else( - || format!("{column} IS NOT NULL"), - |literal| format!("{ordered} > {literal}"), - ); - let prefix = if equal.is_empty() { - String::new() - } else { - format!("{} AND ", equal.join(" AND ")) - }; - greater.push(format!("({prefix}{comparison})")); - equal.push(literal.as_ref().map_or_else( - || format!("{column} IS NULL"), - |literal| format!("{ordered} = {literal}"), - )); - } - } let mut filters: Vec = Vec::new(); - if !greater.is_empty() { - filters.push(format!("({})", greater.join(" OR "))); - } + filters.extend( + after + .map(|after| after_filter((&columns, &ordered), after, (timestamp_column, adapter))) + .transpose()? + .flatten(), + ); if let Some(since) = observed_since { filters.push(format!("{timestamp_column} >= {since}")); } + let (source, versions) = versioned_source(qualified_name, timestamp_column, newest); + filters.extend(versions); let filter = if filters.is_empty() { String::new() } else { @@ -104,26 +81,7 @@ pub fn read_history_sql( format!(" LIMIT {limit}") }; Ok(format!( - "SELECT {head}{projection} FROM {qualified_name}{filter} ORDER BY {}{tail}", + "SELECT {head}{projection} FROM {source}{filter} ORDER BY {}{tail}", order.join(", ") )) } - -/// Session and column collations must not merge case, accents or trailing spaces at page boundaries. -fn binary(value: &str, adapter: &str) -> Result { - Ok(match adapter { - "duckdb" | "motherduck" => format!("encode({value})"), - "sqlite" => format!("CAST({value} AS BLOB)"), - "postgres" => format!("convert_to({value}, 'UTF8')"), - "snowflake" => format!("TO_BINARY({value}, 'UTF-8')"), - "bigquery" => format!("CAST({value} AS BYTES)"), - "databricks" => format!("CAST({value} AS BINARY)"), - "sqlserver" => format!("CONVERT(VARBINARY(MAX), CONVERT(NVARCHAR(MAX), {value}))"), - "clickhouse" => value.to_owned(), - _ => { - return Err(format!( - "warehouse history does not support adapter '{adapter}'" - )); - } - }) -} diff --git a/crates/sqlbuild-state/src/state_tables/mod.rs b/crates/sqlbuild-state/src/state_tables/mod.rs index 042268f15e..f01181f853 100644 --- a/crates/sqlbuild-state/src/state_tables/mod.rs +++ b/crates/sqlbuild-state/src/state_tables/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod _helpers; pub mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-state/src/state_tables/models.rs b/crates/sqlbuild-state/src/state_tables/models.rs index 5c41f277a2..e7d65b4e86 100644 --- a/crates/sqlbuild-state/src/state_tables/models.rs +++ b/crates/sqlbuild-state/src/state_tables/models.rs @@ -19,4 +19,15 @@ pub struct HistoryRead<'a> { pub adapter: &'a str, pub has_event_id: bool, pub limit: usize, + /// Deliver only each identity's newest rows, so a capped mirror never copies older ones. + pub newest: Option>, +} + +/// The identity a version count applies to and how many newest observations a read keeps. +#[derive(Clone, Copy, Debug)] +pub struct NewestVersions<'a> { + pub partition: &'a [&'a str], + /// Columns naming one observation within the partition; they also order ties, newest first. + pub observation: &'a [&'a str], + pub versions: u16, } diff --git a/crates/sqlbuild-state/src/state_tables/tests/test_sql.rs b/crates/sqlbuild-state/src/state_tables/tests/test_sql.rs index eab9c3779d..d99492efd2 100644 --- a/crates/sqlbuild-state/src/state_tables/tests/test_sql.rs +++ b/crates/sqlbuild-state/src/state_tables/tests/test_sql.rs @@ -5,9 +5,11 @@ use crate::state_tables::main::create_table_sql::create_table_sql; use crate::state_tables::main::read_history_sql::read_history_sql; use crate::state_tables::main::values_insert_sql::values_insert_sql; use crate::state_tables::models::HistoryRead; +use crate::state_tables::models::NewestVersions; use crate::state_tables::tests::helpers::column; use crate::state_tables::tests::test_types::{ - CreateTableTestCase, EventKeyTestCase, HistoryReadTestCase, ValuesInsertTestCase, + CreateTableTestCase, EventKeyTestCase, HistoryReadTestCase, NewestVersionsTestCase, + ValuesInsertTestCase, }; #[test] @@ -104,6 +106,7 @@ fn given_a_table_without_event_ids_when_paging_history_then_the_missing_key_neve adapter: test_case.adapter, has_event_id: test_case.has_event_id, limit: 5, + newest: None, }, ) .expect("supported adapter"), @@ -205,6 +208,7 @@ fn given_event_tables_keyed_by_event_id_when_paging_history_then_each_column_and adapter, has_event_id: true, limit: 5, + newest: None, }, ) .expect("supported adapter"); @@ -236,3 +240,59 @@ fn given_event_tables_keyed_by_event_id_when_paging_history_then_each_column_and } } } + +#[test] +fn given_a_version_limit_when_reading_history_then_only_each_identitys_newest_rows_are_selected() { + let test_cases = [ + NewestVersionsTestCase { + description: "copies of one build share a rank, so the newest three builds of each node arrive whole", + versions: 3, + expected_sql: "SELECT node_name, run_id, ts, event_id FROM (SELECT *, DENSE_RANK() OVER \ + (PARTITION BY node_name ORDER BY sqb_first_at DESC, event_id DESC, run_id DESC) AS sqb_version \ + FROM (SELECT *, MIN(ts) OVER (PARTITION BY node_name, event_id, run_id) AS sqb_first_at \ + FROM s.t) AS sqb_observations) AS sqb_versions WHERE sqb_version <= 3 ORDER BY \ + CASE WHEN node_name IS NULL THEN 0 ELSE 1 END, encode(node_name), \ + CASE WHEN run_id IS NULL THEN 0 ELSE 1 END, encode(run_id), \ + CASE WHEN ts IS NULL THEN 0 ELSE 1 END, ts, \ + CASE WHEN event_id IS NULL THEN 0 ELSE 1 END, encode(event_id) LIMIT 5", + }, + NewestVersionsTestCase { + description: "a limit of one keeps only each node's newest build", + versions: 1, + expected_sql: "SELECT node_name, run_id, ts, event_id FROM (SELECT *, DENSE_RANK() OVER \ + (PARTITION BY node_name ORDER BY sqb_first_at DESC, event_id DESC, run_id DESC) AS sqb_version \ + FROM (SELECT *, MIN(ts) OVER (PARTITION BY node_name, event_id, run_id) AS sqb_first_at \ + FROM s.t) AS sqb_observations) AS sqb_versions WHERE sqb_version <= 1 ORDER BY \ + CASE WHEN node_name IS NULL THEN 0 ELSE 1 END, encode(node_name), \ + CASE WHEN run_id IS NULL THEN 0 ELSE 1 END, encode(run_id), \ + CASE WHEN ts IS NULL THEN 0 ELSE 1 END, ts, \ + CASE WHEN event_id IS NULL THEN 0 ELSE 1 END, encode(event_id) LIMIT 5", + }, + ]; + for test_case in test_cases { + let sql: Result = read_history_sql( + "s.t", + None, + HistoryRead { + columns: &["node_name", "run_id", "ts"], + key_columns: None, + timestamp_column: "ts", + observed_since: None, + adapter: "duckdb", + has_event_id: true, + limit: 5, + newest: Some(NewestVersions { + partition: &["node_name"], + observation: &["event_id", "run_id"], + versions: test_case.versions, + }), + }, + ); + assert_eq!( + sql, + Ok(test_case.expected_sql.to_owned()), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/state_tables/tests/test_types.rs b/crates/sqlbuild-state/src/state_tables/tests/test_types.rs index d33808bbba..30ca9a4683 100644 --- a/crates/sqlbuild-state/src/state_tables/tests/test_types.rs +++ b/crates/sqlbuild-state/src/state_tables/tests/test_types.rs @@ -27,3 +27,9 @@ pub(crate) struct EventKeyTestCase { pub(crate) timestamp_column: &'static str, pub(crate) expected_ordering_keys: &'static [&'static str], } + +pub(crate) struct NewestVersionsTestCase { + pub(crate) description: &'static str, + pub(crate) versions: u16, + pub(crate) expected_sql: &'static str, +} diff --git a/crates/sqlbuild-state/src/warehouse_state/_helpers/indexes.rs b/crates/sqlbuild-state/src/warehouse_state/_helpers/indexes.rs new file mode 100644 index 0000000000..fb1853dae6 --- /dev/null +++ b/crates/sqlbuild-state/src/warehouse_state/_helpers/indexes.rs @@ -0,0 +1,46 @@ +//! Position indexes over the received facts, keyed as the warehouse folds case. + +use std::collections::HashMap; +use std::hash::Hash; + +use crate::warehouse_state::constants::MODEL_NODE_TYPE; +use crate::warehouse_state::models::{WarehouseState, WarehouseStateParts}; + +/// An identifier as the warehouse compares it. +pub(crate) fn folded(value: &str) -> String { + value.to_lowercase() +} + +/// Positions of `items` grouped by `key`, each group in source order. +fn positions( + items: &[T], + key: impl Fn(&T) -> Option, +) -> HashMap> { + let mut grouped: HashMap> = HashMap::new(); + for (position, item) in items.iter().enumerate() { + if let Some(key) = key(item) { + grouped.entry(key).or_default().push(position); + } + } + grouped +} + +/// The state over `parts`, with every index built once. +pub(crate) fn indexed(parts: WarehouseStateParts) -> WarehouseState { + WarehouseState { + relations_by_name: positions(&parts.relations, |relation| Some(folded(&relation.name))), + fingerprints_by_schema: positions(&parts.fingerprints, |fingerprint| { + Some(folded(&fingerprint.location.schema)) + }), + latest_model_runs: positions(&parts.latest_runs, |run| { + (run.node_type == MODEL_NODE_TYPE).then(|| run.node_name.clone()) + }), + observations_by_node: positions(&parts.observations, |observation| { + Some((observation.kind.clone(), observation.node.clone())) + }), + consumption_by_schema: positions(&parts.consumption, |row| { + Some(folded(&row.location.schema)) + }), + parts, + } +} diff --git a/crates/sqlbuild-state/src/warehouse_state/_helpers/mod.rs b/crates/sqlbuild-state/src/warehouse_state/_helpers/mod.rs new file mode 100644 index 0000000000..8d17aece7b --- /dev/null +++ b/crates/sqlbuild-state/src/warehouse_state/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod indexes; +pub(crate) mod queries; diff --git a/crates/sqlbuild-state/src/warehouse_state/_helpers/queries.rs b/crates/sqlbuild-state/src/warehouse_state/_helpers/queries.rs new file mode 100644 index 0000000000..beaf325dfa --- /dev/null +++ b/crates/sqlbuild-state/src/warehouse_state/_helpers/queries.rs @@ -0,0 +1,218 @@ +//! What a plan asks of the warehouse state, each answered from an index. + +use std::collections::BTreeMap; + +use chrono::{DateTime, Utc}; +use sha2::{Digest, Sha256}; +use sqlbuild_core::constants::{FINGERPRINT_TABLE_NAME, STATE_TABLE_NAMES}; + +use crate::consumption::models::{ + ConsumedInput, ConsumptionAppend, ConsumptionRow, NodeKey, RelationKey, +}; +use crate::observations::constants::{CURSOR_MAX_KIND, SOURCE_COLUMNS_KIND}; +use crate::warehouse_state::_helpers::indexes::folded; +use crate::warehouse_state::models::{ + DescribedColumns, InvalidObservation, RelationFilter, StateConsumption, StateFingerprint, + StateLocation, StateRelation, WarehouseState, WarehouseStateParts, +}; + +fn same(left: &str, right: &str) -> bool { + folded(left) == folded(right) +} + +/// Whether a fact stored at `stored` answers a read of `wanted`; an unset database matches any. +fn database_matches(stored: Option<&str>, wanted: Option<&str>) -> bool { + match (stored, wanted) { + (Some(stored), Some(wanted)) => same(stored, wanted), + _ => true, + } +} + +/// Whether `wanted` is unset or holds `value`, compared as the warehouse folds case. +fn any_same(wanted: Option<&[String]>, value: &str) -> bool { + wanted.is_none_or(|wanted| wanted.iter().any(|item| same(value, item))) +} + +/// Whether a listing hides the relation: state tables are planner inputs, not relations, except fingerprints. +fn hidden(relation: &StateRelation) -> bool { + STATE_TABLE_NAMES.contains(&relation.name.as_str()) && relation.name != FINGERPRINT_TABLE_NAME +} + +/// A stored consumption row as `--changed` reads it; a missing text column reads as empty. +fn consumption_row(record: &StateConsumption) -> ConsumptionRow { + let text = |value: &Option| value.clone().unwrap_or_default(); + ConsumptionRow { + row: ConsumptionAppend { + execution_id: text(&record.execution_id), + phase: text(&record.phase), + run_id: text(&record.run_id), + node: NodeKey { + kind: text(&record.node_type), + name: text(&record.node_name), + }, + relation: RelationKey { + database: record.target_database.clone(), + schema: text(&record.target_schema), + name: text(&record.target_name), + }, + version_hash: text(&record.version_hash), + input_count: record.input_count.unwrap_or(-1), + input: consumed_input(record), + }, + recorded_at: record.recorded_at.timestamp_micros(), + } +} + +fn consumed_input(record: &StateConsumption) -> Option { + let kind: String = record.input_kind.clone()?; + Some(ConsumedInput { + kind, + name: record.input_name.clone().unwrap_or_default(), + version: record.input_version.clone().unwrap_or_default(), + }) +} + +impl WarehouseState { + /// The facts as received, for fixtures and parity checks. + pub fn parts(&self) -> &WarehouseStateParts { + &self.parts + } + + /// When every required state table last synced cleanly; `None` before that. + pub fn synced_at(&self) -> Option> { + self.parts.provenance.synced_at + } + + /// The mirror version the facts were read at, when the server versions its mirror. + pub fn version(&self) -> Option<&str> { + self.parts.provenance.version.as_deref() + } + + /// Whether the facts include consumption history. + pub fn has_consumption(&self) -> bool { + self.parts.has_consumption + } + + /// The relations `filter` names, in source order; state tables other than fingerprints are hidden. + pub fn relations(&self, filter: &RelationFilter<'_>) -> Vec<&StateRelation> { + let candidates: Vec<&StateRelation> = match filter.names { + Some(names) => { + let mut positions: Vec = names + .iter() + .filter_map(|name| self.relations_by_name.get(&folded(name))) + .flatten() + .copied() + .collect(); + positions.sort_unstable(); + positions.dedup(); + positions + .into_iter() + .filter_map(|position| self.parts.relations.get(position)) + .collect() + } + None => self.parts.relations.iter().collect(), + }; + candidates + .into_iter() + .filter(|relation| { + !hidden(relation) + && filter.database.is_none_or(|database| { + database_matches(relation.database.as_deref(), Some(database)) + }) + && any_same(filter.schemas, &relation.schema) + }) + .collect() + } + + /// The newest fingerprint of each node in a state location: newest `ts`, then the greatest run ID. + pub fn latest_fingerprints(&self, location: &StateLocation) -> Vec<&StateFingerprint> { + let mut newest: BTreeMap<(&str, &str), &StateFingerprint> = BTreeMap::new(); + let positions = self + .fingerprints_by_schema + .get(&folded(&location.schema)) + .into_iter() + .flatten(); + for fingerprint in positions.filter_map(|position| self.parts.fingerprints.get(*position)) { + if !database_matches( + fingerprint.location.database.as_deref(), + location.database.as_deref(), + ) { + continue; + } + let key = ( + fingerprint.node_type.as_str(), + fingerprint.node_name.as_str(), + ); + let replaces = newest.get(&key).is_none_or(|current| { + (fingerprint.ts, &fingerprint.run_id) > (current.ts, ¤t.run_id) + }); + if replaces { + newest.insert(key, fingerprint); + } + } + newest.into_values().collect() + } + + /// The model's cursor `MAX` text, only when observed by its newest successful build. + pub fn target_max(&self, model: &str) -> Option<&str> { + let observations: &[usize] = self + .observations_by_node + .get(&(CURSOR_MAX_KIND.to_owned(), model.to_owned())) + .map_or(&[], Vec::as_slice); + let runs = self.latest_model_runs.get(model).into_iter().flatten(); + for run in runs.filter_map(|position| self.parts.latest_runs.get(*position)) { + let matching = observations + .iter() + .filter_map(|position| self.parts.observations.get(*position)) + .find(|observation| { + observation.run_id == run.run_id && observation.location == run.location + }); + if let Some(observation) = matching { + return Some(observation.value.as_str()); + } + } + None + } + + /// The source's columns as last described, only when it described exactly `sql`. + pub fn described_columns( + &self, + source: &str, + sql: &str, + ) -> Result>, InvalidObservation> { + let hash: String = format!("{:x}", Sha256::digest(sql.as_bytes())); + let positions = self + .observations_by_node + .get(&(SOURCE_COLUMNS_KIND.to_owned(), source.to_owned())) + .into_iter() + .flatten(); + for observation in positions.filter_map(|position| self.parts.observations.get(*position)) { + let described: DescribedColumns = + serde_json::from_str(&observation.value).map_err(|error| InvalidObservation { + source: source.to_owned(), + reason: error.to_string(), + })?; + if described.sql_hash == hash { + return Ok(Some(described.columns)); + } + } + Ok(None) + } + + /// Every consumption row of the state table at `location`, in source order. + pub fn consumption_rows(&self, location: &StateLocation) -> Vec { + self.consumption_by_schema + .get(&folded(&location.schema)) + .into_iter() + .flatten() + .filter_map(|position| self.parts.consumption.get(*position)) + .filter(|row| { + database_matches( + row.location.database.as_deref(), + location.database.as_deref(), + ) + }) + .map(consumption_row) + .collect() + } +} diff --git a/crates/sqlbuild-state/src/warehouse_state/constants.rs b/crates/sqlbuild-state/src/warehouse_state/constants.rs new file mode 100644 index 0000000000..bf1e872ea6 --- /dev/null +++ b/crates/sqlbuild-state/src/warehouse_state/constants.rs @@ -0,0 +1,4 @@ +//! Kinds and names the warehouse state reads by. + +/// The node type whose latest run carries a cursor `MAX` observation. +pub const MODEL_NODE_TYPE: &str = "model"; diff --git a/crates/sqlbuild-state/src/warehouse_state/main/mod.rs b/crates/sqlbuild-state/src/warehouse_state/main/mod.rs new file mode 100644 index 0000000000..d9c4589c87 --- /dev/null +++ b/crates/sqlbuild-state/src/warehouse_state/main/mod.rs @@ -0,0 +1 @@ +pub mod warehouse_state; diff --git a/crates/sqlbuild-state/src/warehouse_state/main/warehouse_state.rs b/crates/sqlbuild-state/src/warehouse_state/main/warehouse_state.rs new file mode 100644 index 0000000000..e8062ec9d0 --- /dev/null +++ b/crates/sqlbuild-state/src/warehouse_state/main/warehouse_state.rs @@ -0,0 +1,9 @@ +//! The warehouse state a plan reads, indexed once from the facts a builder received. + +use crate::warehouse_state::_helpers::indexes::indexed; +use crate::warehouse_state::models::{WarehouseState, WarehouseStateParts}; + +/// The state over `parts`; lookups after this never scan the facts. +pub fn warehouse_state(parts: WarehouseStateParts) -> WarehouseState { + indexed(parts) +} diff --git a/crates/sqlbuild-state/src/warehouse_state/mod.rs b/crates/sqlbuild-state/src/warehouse_state/mod.rs new file mode 100644 index 0000000000..f01181f853 --- /dev/null +++ b/crates/sqlbuild-state/src/warehouse_state/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-state/src/warehouse_state/models.rs b/crates/sqlbuild-state/src/warehouse_state/models.rs new file mode 100644 index 0000000000..8713d674fb --- /dev/null +++ b/crates/sqlbuild-state/src/warehouse_state/models.rs @@ -0,0 +1,150 @@ +//! The warehouse facts a plan reads, as received and as indexed for lookups. + +use std::collections::HashMap; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +/// The schema holding a set of state tables; a missing database matches any database. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +pub struct StateLocation { + pub database: Option, + pub schema: String, +} + +/// One column of a relation, in warehouse order. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct StateColumn { + pub name: String, + pub data_type: String, +} + +/// One relation of a listed schema, with its columns. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct StateRelation { + pub database: Option, + pub schema: String, + pub name: String, + pub relation_type: String, + pub columns: Vec, +} + +/// The newest recorded fingerprint of one node identity in one state location. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct StateFingerprint { + pub location: StateLocation, + pub node_type: String, + pub node_name: String, + pub target_database: Option, + pub target_schema: Option, + pub target_name: Option, + pub run_id: String, + pub definition_hash: String, + pub version_hash: String, + pub schema_fingerprint: String, + pub definition: String, + pub metadata_json: String, + pub ts: DateTime, +} + +/// The run of a node's newest succeeded build. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct StateLatestRun { + pub location: StateLocation, + pub node_type: String, + pub node_name: String, + pub run_id: String, +} + +/// A node's newest observation of one kind, such as a cursor `MAX` or described source columns. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct StateObservation { + pub location: StateLocation, + pub kind: String, + pub node: String, + pub cursor_type: Option, + pub value: String, + pub run_id: String, + pub observed_at: DateTime, +} + +/// One consumption row as the state table holds it; text columns may be NULL. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct StateConsumption { + pub location: StateLocation, + pub execution_id: Option, + pub phase: Option, + pub run_id: Option, + pub node_type: Option, + pub node_name: Option, + pub target_database: Option, + pub target_schema: Option, + pub target_name: Option, + pub version_hash: Option, + pub input_count: Option, + pub input_kind: Option, + pub input_name: Option, + pub input_version: Option, + pub recorded_at: DateTime, +} + +/// Where the facts come from: a synced copy of the warehouse state. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SyncedProvenance { + /// When every required state table last synced cleanly; `None` before that. + pub synced_at: Option>, + /// The mirror version the copy was read at; `None` from a server without versions. + pub version: Option, +} + +/// The facts as received, in source order: what builders produce and fixtures store. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct WarehouseStateParts { + pub provenance: SyncedProvenance, + pub relations: Vec, + pub fingerprints: Vec, + pub latest_runs: Vec, + pub observations: Vec, + /// Empty unless the copy was read with consumption history. + pub consumption: Vec, + /// Whether `consumption` holds the history, rather than the copy leaving it out. + pub has_consumption: bool, +} + +/// The facts indexed by name; every lookup is a hash lookup over the folded identifier. +#[derive(Clone, Debug)] +pub struct WarehouseState { + pub(crate) parts: WarehouseStateParts, + /// Relation positions by folded name, in source order. + pub(crate) relations_by_name: HashMap>, + /// Fingerprint positions by folded state schema, in source order. + pub(crate) fingerprints_by_schema: HashMap>, + /// Latest-run positions by model name, in source order. + pub(crate) latest_model_runs: HashMap>, + /// Observation positions by kind and node, in source order. + pub(crate) observations_by_node: HashMap<(String, String), Vec>, + /// Consumption positions by folded state schema, in source order. + pub(crate) consumption_by_schema: HashMap>, +} + +/// What a listing names; each unset part matches every relation. +#[derive(Clone, Copy, Debug)] +pub struct RelationFilter<'filter> { + pub database: Option<&'filter str>, + pub schemas: Option<&'filter [String]>, + pub names: Option<&'filter [String]>, +} + +/// A source's described columns, keyed to the SQL that was described. +#[derive(Debug, Deserialize)] +pub(crate) struct DescribedColumns { + pub(crate) sql_hash: String, + pub(crate) columns: Vec, +} + +/// A stored `source_columns` observation that is not valid JSON of the expected shape. +#[derive(Debug, PartialEq, Eq)] +pub struct InvalidObservation { + pub source: String, + pub reason: String, +} diff --git a/crates/sqlbuild-state/src/warehouse_state/tests.rs b/crates/sqlbuild-state/src/warehouse_state/tests.rs new file mode 100644 index 0000000000..584f7d855c --- /dev/null +++ b/crates/sqlbuild-state/src/warehouse_state/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_lookups.rs"] +mod test_lookups; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-state/src/warehouse_state/tests/helpers.rs b/crates/sqlbuild-state/src/warehouse_state/tests/helpers.rs new file mode 100644 index 0000000000..8ab6497cde --- /dev/null +++ b/crates/sqlbuild-state/src/warehouse_state/tests/helpers.rs @@ -0,0 +1,143 @@ +//! Facts of a small shop warehouse: two schemas, fingerprints, runs and observations. + +use chrono::{DateTime, TimeZone, Utc}; + +use crate::warehouse_state::models::{ + StateColumn, StateConsumption, StateFingerprint, StateLatestRun, StateLocation, + StateObservation, StateRelation, SyncedProvenance, WarehouseStateParts, +}; + +pub(crate) fn owned(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +pub(crate) fn at(day: u32) -> DateTime { + Utc.with_ymd_and_hms(2026, 1, day, 12, 0, 0) + .single() + .unwrap_or_default() +} + +pub(crate) fn location(database: Option<&str>, schema: &str) -> StateLocation { + StateLocation { + database: database.map(str::to_owned), + schema: schema.to_owned(), + } +} + +pub(crate) fn relation(database: Option<&str>, schema: &str, name: &str) -> StateRelation { + StateRelation { + database: database.map(str::to_owned), + schema: schema.to_owned(), + name: name.to_owned(), + relation_type: "TABLE".to_owned(), + columns: vec![StateColumn { + name: "ID".to_owned(), + data_type: "NUMBER".to_owned(), + }], + } +} + +pub(crate) fn fingerprint( + at_location: StateLocation, + (node_name, run_id): (&str, &str), + day: u32, +) -> StateFingerprint { + StateFingerprint { + location: at_location.clone(), + node_type: "model".to_owned(), + node_name: node_name.to_owned(), + target_database: at_location.database, + target_schema: Some(at_location.schema), + target_name: Some(node_name.to_owned()), + run_id: run_id.to_owned(), + definition_hash: format!("definition-{run_id}"), + version_hash: format!("version-{run_id}"), + schema_fingerprint: "schema".to_owned(), + definition: "SELECT 1".to_owned(), + metadata_json: "{}".to_owned(), + ts: at(day), + } +} + +pub(crate) fn observation( + kind: &str, + (node, run_id): (&str, &str), + value: &str, +) -> StateObservation { + StateObservation { + location: location(Some("WAREHOUSE"), "SHOP"), + kind: kind.to_owned(), + node: node.to_owned(), + cursor_type: None, + value: value.to_owned(), + run_id: run_id.to_owned(), + observed_at: at(3), + } +} + +pub(crate) fn consumption(at_location: StateLocation, execution_id: &str) -> StateConsumption { + StateConsumption { + location: at_location, + execution_id: Some(execution_id.to_owned()), + phase: Some("succeeded".to_owned()), + run_id: Some("run-2".to_owned()), + node_type: Some("model".to_owned()), + node_name: Some("orders".to_owned()), + target_database: Some("WAREHOUSE".to_owned()), + target_schema: Some("SHOP".to_owned()), + target_name: Some("orders".to_owned()), + version_hash: Some("version".to_owned()), + input_count: Some(0), + input_kind: None, + input_name: None, + input_version: None, + recorded_at: at(2), + } +} + +/// The shop warehouse: `SHOP` holds the models, `archive` an older copy of orders. +pub(crate) fn shop_parts() -> WarehouseStateParts { + let shop = location(Some("WAREHOUSE"), "SHOP"); + WarehouseStateParts { + provenance: SyncedProvenance { + synced_at: Some(at(4)), + version: Some("7".to_owned()), + }, + relations: vec![ + relation(Some("WAREHOUSE"), "SHOP", "ORDERS"), + relation(Some("WAREHOUSE"), "SHOP", "CUSTOMERS"), + relation(None, "archive", "Orders"), + relation(Some("WAREHOUSE"), "SHOP", "_sqlbuild_fingerprints"), + relation(Some("WAREHOUSE"), "SHOP", "_sqlbuild_consumption"), + ], + fingerprints: vec![ + fingerprint(shop.clone(), ("orders", "run-1"), 1), + fingerprint(shop.clone(), ("orders", "run-2"), 2), + fingerprint(shop.clone(), ("customers", "run-b"), 2), + fingerprint(shop.clone(), ("customers", "run-a"), 2), + fingerprint(location(None, "archive"), ("orders", "run-0"), 1), + ], + latest_runs: vec![StateLatestRun { + location: shop.clone(), + node_type: "model".to_owned(), + node_name: "orders".to_owned(), + run_id: "run-2".to_owned(), + }], + observations: vec![ + observation("cursor_max", ("orders", "run-1"), "2026-01-01"), + observation("cursor_max", ("orders", "run-2"), "2026-01-02"), + observation("cursor_max", ("customers", "run-b"), "2026-01-02"), + observation( + "source_columns", + ("raw_refunds", "run-2"), + // SHA-256 of `SELECT 1`. + r#"{"sql_hash": "e004ebd5b5532a4b85984a62f8ad48a81aa3460c1ca07701f386135d72cdecf5", "columns": ["id", "amount"]}"#, + ), + ], + consumption: vec![ + consumption(shop, "execution-1"), + consumption(location(Some("REPORTING"), "SHOP"), "execution-2"), + ], + has_consumption: true, + } +} diff --git a/crates/sqlbuild-state/src/warehouse_state/tests/test_lookups.rs b/crates/sqlbuild-state/src/warehouse_state/tests/test_lookups.rs new file mode 100644 index 0000000000..fa448d8e21 --- /dev/null +++ b/crates/sqlbuild-state/src/warehouse_state/tests/test_lookups.rs @@ -0,0 +1,240 @@ +use crate::warehouse_state::main::warehouse_state::warehouse_state; +use crate::warehouse_state::models::{RelationFilter, WarehouseStateParts}; +use crate::warehouse_state::tests::helpers::{location, owned, shop_parts}; +use crate::warehouse_state::tests::test_types::{ + ConsumptionTestCase, DescribedColumnsTestCase, FingerprintsTestCase, RelationsTestCase, + RoundTripTestCase, TargetMaxTestCase, +}; + +#[test] +fn given_synced_relations_when_listed_then_they_match_as_the_warehouse_folds_case() { + let test_cases = [ + RelationsTestCase { + description: "a name in another case finds the relation in every schema, in source order", + database: None, + schemas: None, + names: Some(&["orders"]), + expected_relations: &["SHOP.ORDERS", "archive.Orders"], + }, + RelationsTestCase { + description: "a schema narrows the named relations", + database: Some("warehouse"), + schemas: Some(&["shop"]), + names: Some(&["ORDERS"]), + expected_relations: &["SHOP.ORDERS"], + }, + RelationsTestCase { + description: "a name asked for twice lists its relation once", + database: None, + schemas: Some(&["SHOP"]), + names: Some(&["orders", "Orders", "customers"]), + expected_relations: &["SHOP.ORDERS", "SHOP.CUSTOMERS"], + }, + RelationsTestCase { + description: "another database hides a relation that names its database", + database: Some("reporting"), + schemas: None, + names: Some(&["orders"]), + expected_relations: &["archive.Orders"], + }, + RelationsTestCase { + description: "an unknown name lists nothing", + database: None, + schemas: None, + names: Some(&["refunds"]), + expected_relations: &[], + }, + RelationsTestCase { + description: "no names lists the schema's relations but state tables other than fingerprints", + database: None, + schemas: Some(&["shop"]), + names: None, + expected_relations: &[ + "SHOP.ORDERS", + "SHOP.CUSTOMERS", + "SHOP._sqlbuild_fingerprints", + ], + }, + RelationsTestCase { + description: "a hidden state table is not found by name either", + database: None, + schemas: None, + names: Some(&["_sqlbuild_consumption"]), + expected_relations: &[], + }, + ]; + let state = warehouse_state(shop_parts()); + for test_case in test_cases { + let schemas: Option> = test_case.schemas.map(owned); + let names: Option> = test_case.names.map(owned); + let listed: Vec = state + .relations(&RelationFilter { + database: test_case.database, + schemas: schemas.as_deref(), + names: names.as_deref(), + }) + .into_iter() + .map(|relation| format!("{}.{}", relation.schema, relation.name)) + .collect(); + assert_eq!( + listed, + owned(test_case.expected_relations), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_recorded_fingerprints_when_a_location_is_read_then_each_node_has_its_newest() { + let test_cases = [ + FingerprintsTestCase { + description: "the newest timestamp wins, then the greatest run ID", + database: Some("warehouse"), + schema: "shop", + expected_fingerprints: &["customers@run-b", "orders@run-2"], + }, + FingerprintsTestCase { + description: "a fingerprint with no database answers any database", + database: Some("warehouse"), + schema: "ARCHIVE", + expected_fingerprints: &["orders@run-0"], + }, + FingerprintsTestCase { + description: "another database has none of the shop's fingerprints", + database: Some("reporting"), + schema: "shop", + expected_fingerprints: &[], + }, + FingerprintsTestCase { + description: "a schema with no fingerprints has none", + database: None, + schema: "staging", + expected_fingerprints: &[], + }, + ]; + let state = warehouse_state(shop_parts()); + for test_case in test_cases { + let read: Vec = state + .latest_fingerprints(&location(test_case.database, test_case.schema)) + .into_iter() + .map(|fingerprint| format!("{}@{}", fingerprint.node_name, fingerprint.run_id)) + .collect(); + assert_eq!( + read, + owned(test_case.expected_fingerprints), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_cursor_observations_when_read_then_only_the_newest_successful_build_counts() { + let test_cases = [ + TargetMaxTestCase { + description: "the observation of the model's newest succeeded run", + model: "orders", + expected_max: Some("2026-01-02"), + }, + TargetMaxTestCase { + description: "an observation from a run that is not the newest succeeded one", + model: "customers", + expected_max: None, + }, + ]; + let state = warehouse_state(shop_parts()); + for test_case in test_cases { + assert_eq!( + state.target_max(test_case.model), + test_case.expected_max, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_described_sources_when_read_then_only_the_described_sql_answers() { + let test_cases = [ + DescribedColumnsTestCase { + description: "the SQL that was described", + source: "raw_refunds", + sql: "SELECT 1", + expected_columns: Some(&["id", "amount"]), + }, + DescribedColumnsTestCase { + description: "an edited expression is not described yet", + source: "raw_refunds", + sql: "SELECT 2", + expected_columns: None, + }, + DescribedColumnsTestCase { + description: "a source never described", + source: "raw_orders", + sql: "SELECT 1", + expected_columns: None, + }, + ]; + let state = warehouse_state(shop_parts()); + for test_case in test_cases { + assert_eq!( + state.described_columns(test_case.source, test_case.sql), + Ok(test_case.expected_columns.map(owned)), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_consumption_in_two_databases_when_one_location_is_read_then_only_its_rows_return() { + let test_cases = [ + ConsumptionTestCase { + description: "the location's database in another case", + database: Some("warehouse"), + schema: "shop", + expected_executions: &["execution-1"], + }, + ConsumptionTestCase { + description: "no database reads the schema in every database", + database: None, + schema: "SHOP", + expected_executions: &["execution-1", "execution-2"], + }, + ]; + let state = warehouse_state(shop_parts()); + for test_case in test_cases { + let executions: Vec = state + .consumption_rows(&location(test_case.database, test_case.schema)) + .into_iter() + .map(|row| row.row.execution_id) + .collect(); + assert_eq!( + executions, + owned(test_case.expected_executions), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_parts_when_stored_as_json_and_read_back_then_the_state_is_unchanged() { + let test_cases = [RoundTripTestCase { + description: "every fact kind of the shop warehouse", + parts: shop_parts, + expected_unchanged: true, + }]; + for test_case in test_cases { + let parts: WarehouseStateParts = (test_case.parts)(); + let stored: String = serde_json::to_string(&parts).unwrap_or_default(); + let read: Option = serde_json::from_str(&stored).ok(); + assert_eq!( + read.map(|read| warehouse_state(read).parts().clone()) == Some(parts), + test_case.expected_unchanged, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/warehouse_state/tests/test_types.rs b/crates/sqlbuild-state/src/warehouse_state/tests/test_types.rs new file mode 100644 index 0000000000..f75ab32c73 --- /dev/null +++ b/crates/sqlbuild-state/src/warehouse_state/tests/test_types.rs @@ -0,0 +1,50 @@ +//! Table cases of warehouse state lookups. + +use crate::warehouse_state::models::WarehouseStateParts; + +/// A listing and the `schema.name` of each relation it returns. +pub(crate) struct RelationsTestCase { + pub(crate) description: &'static str, + pub(crate) database: Option<&'static str>, + pub(crate) schemas: Option<&'static [&'static str]>, + pub(crate) names: Option<&'static [&'static str]>, + pub(crate) expected_relations: &'static [&'static str], +} + +/// A fingerprint read of one state location and the `name@run` of each newest fingerprint. +pub(crate) struct FingerprintsTestCase { + pub(crate) description: &'static str, + pub(crate) database: Option<&'static str>, + pub(crate) schema: &'static str, + pub(crate) expected_fingerprints: &'static [&'static str], +} + +/// A model and the cursor `MAX` its newest successful build observed. +pub(crate) struct TargetMaxTestCase { + pub(crate) description: &'static str, + pub(crate) model: &'static str, + pub(crate) expected_max: Option<&'static str>, +} + +/// A source, the SQL a plan describes it with, and the columns the state knows for it. +pub(crate) struct DescribedColumnsTestCase { + pub(crate) description: &'static str, + pub(crate) source: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_columns: Option<&'static [&'static str]>, +} + +/// A consumption read of one state location and the execution IDs it returns. +pub(crate) struct ConsumptionTestCase { + pub(crate) description: &'static str, + pub(crate) database: Option<&'static str>, + pub(crate) schema: &'static str, + pub(crate) expected_executions: &'static [&'static str], +} + +/// Facts stored as JSON and read back. +pub(crate) struct RoundTripTestCase { + pub(crate) description: &'static str, + pub(crate) parts: fn() -> WarehouseStateParts, + pub(crate) expected_unchanged: bool, +} diff --git a/fensu.toml b/fensu.toml index e19a6994c7..9e9faafec2 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1359,11 +1359,11 @@ paths = ["crates/sqlbuild-rules/src/custom_rule_cache/main/parse_custom_rule_cac reason = "Each reads its own cache format: the SQL rule cache checks its version and the build identity and returns findings by path, the custom-rule cache checks only its version, because its identities already include the build, and returns rules by code. Only the parse-and-filter shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/audit.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/location_health.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/location_work.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_checks.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_location_work.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_maintenance.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_node_sql.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_plans.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/run_reads.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/runs.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/snapshots.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/trigger_reads.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs"] +paths = ["crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/audit.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/changed_builds.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/location_health.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/location_work.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_checks.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_location_work.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_maintenance.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_node_sql.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_plans.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_selection.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/run_reads.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/runs.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/snapshots.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/trigger_reads.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs"] reason = "Each function is one utoipa-annotated axum route: OpenAPI needs one annotated handler per operation, and each body only checks the caller's scope and calls one store operation. Scope checks and store work live in their shared owners." [[dupes.allowlist]] -paths = ["crates/sqlbuild-orchestration/src/authorization/main/list_role_definitions.rs", "crates/sqlbuild-orchestration/src/authorization/main/read_security_audit.rs", "crates/sqlbuild-orchestration/src/credentials/_helpers/active_session.rs", "crates/sqlbuild-orchestration/src/declarations/_helpers/reads.rs", "crates/sqlbuild-orchestration/src/declarations/main/published_job.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_person_jobs.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_person_trigger_changes.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_person_triggers.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_triggers.rs", "crates/sqlbuild-orchestration/src/platform/main/read_location_work.rs", "crates/sqlbuild-orchestration/src/platform/main/read_person_location_health.rs", "crates/sqlbuild-orchestration/src/platform/main/location_health.rs", "crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs", "crates/sqlbuild-orchestration/src/runs/main/list_person_runs.rs", "crates/sqlbuild-orchestration/src/runs/main/list_person_run_nodes.rs", "crates/sqlbuild-orchestration/src/runs/main/list_run_nodes.rs", "crates/sqlbuild-orchestration/src/runs/main/list_runs.rs", "crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs", "crates/sqlbuild-orchestration/src/warehouse_history/_helpers/checks.rs", "crates/sqlbuild-orchestration/src/scheduler/main/list_person_ticks.rs", "crates/sqlbuild-orchestration/src/store/main/read_records.rs", "crates/sqlbuild-orchestration/src/platform/main/location_work.rs", "crates/sqlbuild-orchestration/src/scheduler/main/list_ticks.rs", "crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs", "crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs", "crates/sqlbuild-orchestration/src/snapshots/main/read_current_record.rs", "crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_documents.rs", "crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_nodes.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_catalog.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_check.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_checks.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_maintenance.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_microbatch_coverage.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_statistics.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_node_runs.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_events.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_statements.rs", "crates/sqlbuild-orchestration/src/warehouse_history/_helpers/node_runs.rs"] +paths = ["crates/sqlbuild-orchestration/src/authorization/main/list_role_definitions.rs", "crates/sqlbuild-orchestration/src/authorization/main/read_security_audit.rs", "crates/sqlbuild-orchestration/src/credentials/_helpers/active_session.rs", "crates/sqlbuild-orchestration/src/declarations/_helpers/reads.rs", "crates/sqlbuild-orchestration/src/declarations/main/published_job.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_person_jobs.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_person_trigger_changes.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_person_triggers.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_triggers.rs", "crates/sqlbuild-orchestration/src/platform/main/work/read_location_work.rs", "crates/sqlbuild-orchestration/src/platform/main/health/read_person_location_health.rs", "crates/sqlbuild-orchestration/src/platform/main/health/location_health.rs", "crates/sqlbuild-orchestration/src/snapshots/main/reads/current_snapshot_record.rs", "crates/sqlbuild-orchestration/src/runs/main/listings/list_person_runs.rs", "crates/sqlbuild-orchestration/src/runs/main/listings/list_person_run_nodes.rs", "crates/sqlbuild-orchestration/src/runs/main/listings/list_run_nodes.rs", "crates/sqlbuild-orchestration/src/runs/main/listings/list_runs.rs", "crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs", "crates/sqlbuild-orchestration/src/warehouse_history/_helpers/checks.rs", "crates/sqlbuild-orchestration/src/scheduler/main/list_person_ticks.rs", "crates/sqlbuild-orchestration/src/store/main/read_records.rs", "crates/sqlbuild-orchestration/src/platform/main/work/location_work.rs", "crates/sqlbuild-orchestration/src/scheduler/main/list_ticks.rs", "crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs", "crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs", "crates/sqlbuild-orchestration/src/snapshots/main/reads/read_current_record.rs", "crates/sqlbuild-orchestration/src/snapshots/main/reads/read_snapshot_documents.rs", "crates/sqlbuild-orchestration/src/snapshots/main/reads/read_snapshot_nodes.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_catalog.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_check.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_checks.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_maintenance.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_microbatch_coverage.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_statistics.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_node_runs.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_events.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_statements.rs", "crates/sqlbuild-orchestration/src/warehouse_history/_helpers/node_runs.rs"] reason = "Each is one control-store read with its own SQL and record type. Transactions, person authorization and decoding go through the shared begin_authorized, row_in_target, query_records, read_records and query_bodies helpers; what remains alike is the call shape." [[dupes.allowlist]] @@ -1383,7 +1383,7 @@ paths = ["crates/sqlbuild-orchestration/src/scheduler/main/_next_cron_time.rs", reason = "Forward and backward cron searches differ in search start, inclusivity and stop condition; the fold and gap rules they share live in calendar.rs." [[dupes.allowlist]] -paths = ["crates/sqlbuild-orchestration/src/platform/main/connect_location_work.rs", "crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs"] +paths = ["crates/sqlbuild-orchestration/src/platform/main/work/connect_location_work.rs", "crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs"] reason = "Location and worker clients authenticate different credentials against different endpoint sets; both build on the shared control connection." [[dupes.allowlist]] @@ -1625,14 +1625,25 @@ rule_packs = ["rust"] rule_modules = [ "scripts.fensu_policy.rules.native_boundary", "scripts.fensu_policy.rules.observation_boundary", + "scripts.fensu_policy.rules.planner_reads", "scripts.fensu_policy.rules.unused_state", ] -select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "XSB079"] +select = [ + "FPRS", + "XSB072", + "XSB073", + "XSB075", + "XSB076", + "XSB077", + "XSB078", + "XSB079", + "XSB080", +] [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 50333 } -reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." +thresholds = { max_crate_code_lines = 51200 } +reason = "The Python bindings crate is above the code-line limit; this caps it near its current size until it is split into crates by area. The location and plan bindings of the large-project server work and #1560 grew it to 51,122; moving them out would need new cross-crate imports or a policy exception. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/src/bindings/_helpers"] @@ -1669,23 +1680,32 @@ allowed_paths = [ [targets.rust.rule_options.XSB079] allowed_paths = [ - "crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs=A location's requested plan names the synced copy for its sqb plan child.", - "crates/sqlbuild-core/src/plan_display/constants.rs=Names the synced-copy environment variable.", - "crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs=Preview branch: a refused freshness observation leaves the source unknown.", - "crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs=Registers the synced copy's refusal error with Python.", - "crates/sqlbuild-python-base/src/warehouse_access/_helpers/synced_copy.rs=Defines the synced-copy adapter.", - "crates/sqlbuild-python-base/src/warehouse_access/main/register_synced_copy.rs=Registers the synced copy's refusal error with Python.", - "crates/sqlbuild-python-base/src/warehouse_access/main/mod.rs=Declares the synced-copy entries.", - "crates/sqlbuild-python-base/src/warehouse_access/main/is_warehouse_access_refused.rs=Tells a plan preview's refused warehouse access apart.", - "crates/sqlbuild-python-base/src/warehouse_access/models.rs=Defines the synced-copy adapter the plan preview reads through.", - "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=Only sqb plan activates the synced copy and reports what it could not know.", - "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs=Preview branch: target MAX from the copy, watermark inputs read when the run starts.", - "crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs=Preview branch: table types and retention stay unknown.", - "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs=Preview branch: cursor intrinsics bind to bounds the run resolves.", - "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs=Preview branch: expression columns from the copy instead of DESCRIBE.", - "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs=Preview branch: the copy has no source freshness state to read.", - "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs=Preview branch: sources outside the copy are not probed.", - "crates/sqlbuild-python-front/src/native_host/models.rs=Preview branch: --changed reads consumption history from the copy instead of the warehouse table.", + "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=Only a location's plan preview plans through the synced reads and reports what they left unknown.", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/synced/exception_types.rs=Defines the preview's refusal error.", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/synced/synced_reads.rs=Answers the planner's warehouse reads from the synced state.", + "crates/sqlbuild-python-planner/src/planner_warehouse/main/reads/is_refused_access.rs=Tells a preview's refused read apart, so freshness planning leaves the source unknown.", + "crates/sqlbuild-python-planner/src/planner_warehouse/main/reads/register_exception_types.rs=Registers the preview's refusal error with Python.", + "crates/sqlbuild-python-planner/src/planner_warehouse/models.rs=Defines the synced reads a plan preview plans through.", +] + +[targets.rust.rule_options.XSB080] +planner_paths = [ + "crates/sqlbuild-python-planner/src/planner_changes=Planner phases read the warehouse through WarehouseReads.", + "crates/sqlbuild-python-planner/src/planner_entries=Planner phases read the warehouse through WarehouseReads.", + "crates/sqlbuild-python-planner/src/planner_identity=Planner phases read the warehouse through WarehouseReads.", + "crates/sqlbuild-python-planner/src/planner_migrations=Planner phases read the warehouse through WarehouseReads.", + "crates/sqlbuild-python-planner/src/planner_phases=Planner phases read the warehouse through WarehouseReads.", + "crates/sqlbuild-python-base/src/records/_helpers/planner_records=Planner phases read the warehouse through WarehouseReads.", + "crates/sqlbuild-python/src/bindings/_helpers/planner_registration=Planner phases read the warehouse through WarehouseReads.", + "crates/sqlbuild-python-planner/src/planner_resolution=Planner phases read the warehouse through WarehouseReads.", + "crates/sqlbuild-python-planner/src/planner_scenario=Planner phases read the warehouse through WarehouseReads.", + "crates/sqlbuild-python/src/bindings/_helpers/planner_scope=Planner phases read the warehouse through WarehouseReads.", + "crates/sqlbuild-python/src/bindings/_helpers/planner_selection=Planner phases read the warehouse through WarehouseReads.", + "crates/sqlbuild-python-planner/src/planner_strategy=Planner phases read the warehouse through WarehouseReads.", + "crates/sqlbuild-python-planner/src/planner_warehouse=Planner phases read the warehouse through WarehouseReads.", + "crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs=Source freshness observation reads through WarehouseReads.", + "crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs=Source freshness planning reads through WarehouseReads.", + "crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs=Consumption rows and SQL without the adapter; I/O lives in consumption_io.", ] [targets.rust.rule_options.XSB077] @@ -1940,6 +1960,12 @@ path = "crates/sqlbuild-python-base/src/native_threads/_helpers/exit_signal.rs" symbols = ["signal_interpreter_exit"] reason = "The existing interpreter-exit signal is process-wide and monotonic: every native daemon wait must observe any exit hook before Python finalization. The atomic replaces the same global threading.Event, not per-command lifecycle state." +[[targets.rust.rule_exceptions]] +rule = "FPRSS130" +path = "crates/sqlbuild-cli/src/native_commands/_helpers/command/terminal.rs" +symbols = ["share_terminal"] +reason = "Whether a location's interpreter shares the terminal is process-wide and monotonic: once the supervisor starts it, every later write from any location task must first clear its spinner line." + [targets.rust.rule_options.FPRSR311] allowed_entries = [ "Cargo.toml", @@ -2053,7 +2079,7 @@ paths = ["src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py"] reason = "SQL Server overrides each state table's create SQL to add that table's own key index; the keyed DDL lives in _keyed_state_table_sql." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs", "crates/sqlbuild-python-base/src/warehouse_access/_helpers/synced_copy.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/checking/dsl_headers.rs"] +paths = ["crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/checking/dsl_headers.rs"] reason = "Each turns its own record type into its own Python objects; only the loop that collects them is alike." [[dupes.allowlist]] @@ -2069,7 +2095,7 @@ paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerpr reason = "Typed observation field projections intentionally mirror provenance checks while preserving different required identity fields and timestamp names. Bounds, rejection rules and reconciliation completion share the ingestion owner." [[dupes.allowlist]] -paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_audit_results.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_column_migrations.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_consumption.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_janitor_events.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_microbatches.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_migrations.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_node_results.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_observations.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_old_name_views.rs"] +paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_audit_results.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_column_migrations.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_consumption.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_janitor_events.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_microbatches.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_migrations.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_node_results.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_observations.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_old_name_views.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_resource_costs.rs"] reason = "Typed event projections retain distinct required identities and indexed payload columns. Event persistence, fencing, atomic retries and reconciliation share the events ingestion owner; no per-table lifecycle is copied." [[targets.python.rule_exceptions]] @@ -2483,6 +2509,8 @@ pending_budgets = [ "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs=4", "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/payloads.rs=4", "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/validation.rs=4", + "crates/sqlbuild-python-planner/src/live_reads/_helpers/bounded.rs=3", + "crates/sqlbuild-python-planner/src/live_reads/models.rs=6", "crates/sqlbuild-python-planner/src/microbatches/_helpers/conversion.rs=0", "crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs=1", "crates/sqlbuild-python-planner/src/microbatches/_helpers/statements.rs=5", @@ -2502,7 +2530,7 @@ pending_budgets = [ "crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/relations.rs=1", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/values.rs=1", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs=5", - "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs=3", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs=2", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs=1", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs=2", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs=0", @@ -2530,7 +2558,7 @@ pending_budgets = [ "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_overlay.rs=4", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_warnings.rs=2", - "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs=13", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs=9", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs=4", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs=2", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_references.rs=7", @@ -2568,7 +2596,7 @@ pending_budgets = [ "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs=0", "crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/audit_is_selected.rs=2", "crates/sqlbuild-python-planner/src/planner_strategy/main/values/input_error.rs=1", - "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs=14", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs=11", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs=6", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs=3", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/watermarks.rs=2", @@ -2576,10 +2604,11 @@ pending_budgets = [ "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs=1", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_deferral.rs=1", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs=5", - "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs=5", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs=1", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/synced/synced_reads.rs=1", "crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs=1", "crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/source_freshness_unknown_warnings.rs=1", - "crates/sqlbuild-python-planner/src/planner_warehouse/models.rs=8", + "crates/sqlbuild-python-planner/src/planner_warehouse/models.rs=9", "crates/sqlbuild-python-project/src/analysis/_helpers/binding_catalog.rs=0", "crates/sqlbuild-python-project/src/analysis/main/ignores_quoted_case.rs=1", "crates/sqlbuild-python-project/src/analysis/models.rs=1", @@ -2753,6 +2782,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime_plan.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs=3", @@ -2934,7 +2964,8 @@ pending_budgets = [ "crates/sqlbuild-python-output/src/result_output/_helpers/events/terminal_index.rs=2", "crates/sqlbuild-python-output/src/result_output/_helpers/results/documents.rs=8", "crates/sqlbuild-python-output/src/result_output/models.rs=4", - "crates/sqlbuild-python-planner/src/freshness/models.rs=8", + "crates/sqlbuild-python-planner/src/freshness/models.rs=6", + "crates/sqlbuild-python-planner/src/live_reads/models.rs=3", "crates/sqlbuild-python-planner/src/microbatches/_helpers/conversion.rs=1", "crates/sqlbuild-python-planner/src/plan_output/_helpers/cursor_details.rs=2", "crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs=2", @@ -2944,20 +2975,20 @@ pending_budgets = [ "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs=9", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs=2", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs=2", - "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs=4", - "crates/sqlbuild-python-planner/src/planner_entries/models.rs=56", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs=3", + "crates/sqlbuild-python-planner/src/planner_entries/models.rs=52", "crates/sqlbuild-python-planner/src/planner_identity/_helpers/staleness.rs=4", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/known_input_columns.rs=2", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs=2", - "crates/sqlbuild-python-planner/src/planner_migrations/models.rs=77", + "crates/sqlbuild-python-planner/src/planner_migrations/models.rs=75", "crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs=4", - "crates/sqlbuild-python-planner/src/planner_phases/models.rs=38", + "crates/sqlbuild-python-planner/src/planner_phases/models.rs=36", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs=3", - "crates/sqlbuild-python-planner/src/planner_resolution/models.rs=20", - "crates/sqlbuild-python-planner/src/planner_scenario/models.rs=9", + "crates/sqlbuild-python-planner/src/planner_resolution/models.rs=19", + "crates/sqlbuild-python-planner/src/planner_scenario/models.rs=8", "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/lineage.rs=1", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs=19", - "crates/sqlbuild-python-planner/src/planner_warehouse/models.rs=11", + "crates/sqlbuild-python-planner/src/planner_warehouse/models.rs=8", "crates/sqlbuild-python-project/src/analysis/models.rs=9", "crates/sqlbuild-python-project/src/analysis_session/models.rs=1", "crates/sqlbuild-python-project/src/compiled_project/models.rs=36", @@ -2974,7 +3005,7 @@ pending_budgets = [ "crates/sqlbuild-python-project/src/planner_selection/models.rs=1", "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_preparation.rs=6", "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_sql.rs=3", - "crates/sqlbuild-python-project/src/planner_strategy/models.rs=30", + "crates/sqlbuild-python-project/src/planner_strategy/models.rs=28", "crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs=2", "crates/sqlbuild-python-project/src/references/_helpers/reference_facts.rs=2", "crates/sqlbuild-python-project/src/resource_views/_helpers/audit_views.rs=2", @@ -3111,6 +3142,7 @@ module_layers = [ "microbatches=registry", "model_config=registry", "native_host=front", + "native_host/live_reads=plan", "native_host/orchestration_sensors=commands", "native_host/setup_context=commands", "native_host/warehouse_metadata=commands", @@ -3194,7 +3226,7 @@ paths = ["crates/sqlbuild-analysis/src/warehouse_types/_helpers/type_trees.rs", reason = "Each is a short stack walk over a JSON syntax tree with its own stop and collect rule: cast_targets collects every cast's target type, branch_selects stops at each set-operation branch, and walked lists every object. A shared visitor would need a callback for each rule and save no logic." [[dupes.allowlist]] -paths = ["crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/checking/native_header_parser.rs"] +paths = ["crates/sqlbuild-cli/src/native_commands/_helpers/command/project_facts.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/checking/native_header_parser.rs"] reason = "Field lookups on unrelated value types in different crates: project configuration takes the first matching field of a built object, while rule headers take the last entry of an authored map so a repeated key overrides earlier ones." [[dupes.allowlist]] diff --git a/pyproject.toml b/pyproject.toml index c161935cc4..b8ec3490c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -107,6 +107,7 @@ markers = [ "clickhouse: tests that require a live ClickHouse server", "motherduck: tests that require live MotherDuck credentials", "performance: opt-in performance guard tests", + "preview_performance: the 10K-model plan preview guard, run in the 10K dense compile cell", "cold_compile_performance: fresh-process semantic compile scaling guards", "cache_compile_performance: fresh-process warm and incremental compile cache guards", ] diff --git a/scripts/fensu_policy/_helpers/unused_state.py b/scripts/fensu_policy/_helpers/unused_state.py index 5a9d00018e..1860661234 100644 --- a/scripts/fensu_policy/_helpers/unused_state.py +++ b/scripts/fensu_policy/_helpers/unused_state.py @@ -20,6 +20,8 @@ OBSERVATION_STATE_PATTERN, PARENTHESIS_CLOSE, PARENTHESIS_OPEN, + PLANNER_WAREHOUSE_IDENTIFIER_PATTERN, + PLANNER_WAREHOUSE_LITERAL_PATTERN, POLICY_CONFIG_FILE, REASON_SEPARATOR, REMOVED_CODE, @@ -75,6 +77,8 @@ _OBSERVATION_STATE: re.Pattern[str] = re.compile(OBSERVATION_STATE_PATTERN) _SYNCED_COPY: re.Pattern[str] = re.compile(SYNCED_COPY_PATTERN) +_PLANNER_WAREHOUSE_IDENTIFIER: re.Pattern[str] = re.compile(PLANNER_WAREHOUSE_IDENTIFIER_PATTERN) +_PLANNER_WAREHOUSE_LITERAL: re.Pattern[str] = re.compile(PLANNER_WAREHOUSE_LITERAL_PATTERN) _LEXEMES: re.Pattern[str] = re.compile(RUST_LITERAL_PATTERN, re.DOTALL) _CONTINUATION: re.Pattern[str] = re.compile(RUST_LINE_CONTINUATION) _TEST_MODULE: re.Pattern[str] = re.compile(RUST_TEST_MODULE_PATTERN) @@ -612,6 +616,25 @@ def synced_copy_reference_lines(*, source: str) -> list[int]: return _reference_lines(source=source, pattern=_SYNCED_COPY) +def warehouse_access_lines(*, source: str) -> list[int]: + """One-based code lines naming the adapter, a connection or a warehouse operation.""" + + lines: list[int] = [] + number: int + code: str + raw: str + for number, (code, raw) in enumerate( + zip(code_view(source=source).splitlines(), source.splitlines(), strict=True), start=1 + ): + if REMOVED_CODE in code: + continue + if _PLANNER_WAREHOUSE_IDENTIFIER.search(code) or _PLANNER_WAREHOUSE_LITERAL.search( + raw.partition(RUST_LINE_COMMENT)[0] + ): + lines.append(number) + return lines + + def _reference_lines(*, source: str, pattern: re.Pattern[str]) -> list[int]: lines: list[int] = [] number: int @@ -631,33 +654,49 @@ def confined_references( ) -> list[Fault]: """Faults for each reference outside the allowed paths, and for stale allowed paths.""" + return path_entry_faults( + ctx=ctx, + entries=(option, "allowed_paths", False), + lines=references, + stale=lambda prefix: f"allowed path {prefix} no longer names {subject}; remove the entry", + ) + + +def path_entry_faults( + *, + ctx: RuleContext, + entries: tuple[RuleOption[tuple[str, ...]], str, bool], + lines: Callable[..., list[int]], + stale: Callable[[str], str], +) -> list[Fault]: + """Faults for `lines` on the guarded side of `entries` (option, name, guarded); bad entries.""" + + option: RuleOption[tuple[str, ...]] + name: str + inside: bool + option, name, inside = entries reasons: dict[str, str] malformed: list[str] reasons, malformed = reasoned_entries(entries=ctx.option(option)) prefixes: list[str] = list(reasons) - used: set[str] = set() + matched: set[str] = set() faults: list[Fault] = [] for facts in ctx.rust.files: path: str = facts.file.path.value if facts.test or is_rust_test_path(path=path): continue - lines: list[int] = references(source=facts.source) - if not lines: - continue prefix: str | None = allowed_prefix(path=path, prefixes=prefixes) - if prefix is not None: - used.add(prefix) + if prefix is None and inside: continue - faults.extend( - ctx.fault_for(path=ctx.repo_root / path, line=line, column=0) for line in lines - ) - problems: list[str] = [ - f"allowed_paths entry {entry!r} needs `path=reason`" for entry in malformed - ] + [ - f"allowed path {prefix} no longer names {subject}; remove the entry" - for prefix in prefixes - if prefix not in used - ] + found: list[int] = lines(source=facts.source) + if prefix is not None and (inside or found): + matched.add(prefix) + if (prefix is not None) == inside: + faults.extend( + ctx.fault_for(path=ctx.repo_root / path, line=line, column=0) for line in found + ) + problems: list[str] = [f"{name} entry {entry!r} needs `path=reason`" for entry in malformed] + problems += [stale(prefix) for prefix in prefixes if prefix not in matched] return [ ctx.fault_for(path=ctx.repo_root / POLICY_CONFIG_FILE, line=1, column=0, message=problem) for problem in problems diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index cc388cefcd..be05ba1217 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -501,11 +501,30 @@ default=(), description=( "`path=reason` entries for the Rust files or directories that may name the plan " - "preview's synced-copy adapter." + "preview's synced warehouse reads." ), ) -SYNCED_COPY_PATTERN: str = ( - r"\b(?:SyncedCopyAdapter|SyncedCopyConnection|PLAN_SYNCED_COPY_ENV|" - r"WarehouseAccessRefusedError|is_warehouse_access_refused|register_synced_copy)\b" +SYNCED_COPY_PATTERN: str = r"\b(?:SyncedReads|WarehouseAccessRefusedError)\b" +PLANNER_READ_PATHS: RuleOption[tuple[str, ...]] = RuleOption.string_list( + name="planner_paths", + default=(), + description=( + "`path=reason` entries for the Rust files or directories of planner phases, which read the " + "warehouse only through `WarehouseReads` and write SQL only through `Dialect`." + ), +) +PLANNER_WAREHOUSE_IDENTIFIER_PATTERN: str = ( + r"\b(?:adapter|connection|LiveReads|ConsumptionConnection" + r"|ListRelationsForInspection|GetColumnsForInspection|ActiveInspectionCatalog" + r"|OpenInspectionCatalog|RunRecordedInspectionQuery|RecordInspectionQuery" + r"|RELATIONS_MAIN_LIST|RELATIONS_MAIN_COLUMNS|RUN_RECORDED_QUERY|RECORD_QUERY" + r"|read_latest_fingerprints|read_latest|read_events|stored_old_name_columns)\b" +) +PLANNER_WAREHOUSE_LITERAL_PATTERN: str = ( + r'"connection"|"adapter"\s*\)' + r'|"(?:list_relations|get_columns|get_all_columns|get_columns_for_relations|query_column_names' + r"|probe_relation_read|get_tables?_freshness_metadata|inspect_retention" + r"|column_rename_unavailable_reason|metadata_inspection_concurrency|describe_relation" + r'|relation_exists|schema_exists|render_read_latest_\w+)"' ) RUST_LINE_COMMENT: str = "//" diff --git a/scripts/fensu_policy/rules/observation_boundary.py b/scripts/fensu_policy/rules/observation_boundary.py index 3c52e2e000..9338f9105a 100644 --- a/scripts/fensu_policy/rules/observation_boundary.py +++ b/scripts/fensu_policy/rules/observation_boundary.py @@ -41,8 +41,8 @@ def observation_display_only(*, project: Project, ctx: RuleContext) -> list[Faul slug="synced-copy-preview-only", message="only sqb plan previews may plan from the synced copy", remediation=( - "Plan from the live adapter; only the files in allowed_paths, which define, activate or " - "branch on the plan preview's synced copy, may name it." + "Plan through WarehouseReads; only the files in allowed_paths, which define or choose " + "the plan preview's synced reads, may name them." ), cacheable=False, analyzers=(AnalyzerId.RUST,), diff --git a/scripts/fensu_policy/rules/planner_reads.py b/scripts/fensu_policy/rules/planner_reads.py new file mode 100644 index 0000000000..7bb4aa66c7 --- /dev/null +++ b/scripts/fensu_policy/rules/planner_reads.py @@ -0,0 +1,35 @@ +"""Rule keeping planner phases on `WarehouseReads` and `Dialect`, away from the adapter.""" + +from __future__ import annotations + +from fensu import AnalyzerId, Family, Fault, Project, RuleContext, rule + +from scripts.fensu_policy._helpers.unused_state import ( + path_entry_faults, + warehouse_access_lines, +) +from scripts.fensu_policy.constants import PLANNER_READ_PATHS + + +@rule( + code="XSB080", + family=Family.CUSTOM, + slug="planner-warehouse-reads", + message="planner phases read the warehouse only through WarehouseReads", + remediation=( + "Write SQL through `Dialect` and read warehouse state through `WarehouseReads`; add the " + "read to `WarehouseReads`, with its live and synced answers, instead of reaching the " + "adapter or a connection." + ), + cacheable=False, + analyzers=(AnalyzerId.RUST,), + options=(PLANNER_READ_PATHS,), +) +def planner_warehouse_reads(*, project: Project, ctx: RuleContext) -> list[Fault]: + del project + return path_entry_faults( + ctx=ctx, + entries=(PLANNER_READ_PATHS, "planner_paths", True), + lines=warehouse_access_lines, + stale=lambda prefix: f"planner path {prefix} matches no Rust file; remove the entry", + ) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index c4db851627..d68a377f7b 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -61,6 +61,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [Selection and staleness](concepts/planning/selection-and-staleness.md) (`concepts/planning/selection-and-staleness`) - How selection interacts with change detection, and the stale warnings that stop silent partial rebuilds. - [Column Lineage](concepts/column-lineage.md) (`concepts/column-lineage`) - Trace individual columns through your SQL pipeline - understand where data comes from and where it goes. - [Orchestration](concepts/orchestration.md) (`concepts/orchestration`) - Declare jobs, job groups and schedules in TOML, and calendars and sensors in Python, for a SQLBuild server to run. +- [Concurrency](concepts/concurrency.md) (`concepts/concurrency`) - Choose how many models and runs SQLBuild works on at once: concurrency, max_workers, write claims, overlap and pools. ## Refactor diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index e7b91404d3..68a98b4599 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -53,6 +53,8 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--target` | Build against a configured target instead of the active/default target | | `--no-tests` | Skip SQL unit tests; a build never runs [Python tests](test.md#python-tests) | | `--no-audits` | Skip audits | +| `--ignore-audit-failures` | Run every audit, but let failures through: staged tables are promoted, incremental batches applied and downstream nodes built. Failures are listed as ignored; see [Ignoring audit failures](../concepts/audits.md#ignoring-audit-failures). Not allowed with `--no-audits` | +| `--no-ignore-audit-failures` | Let audit failures block the build even where `ignore_audit_failures` is set | | `--no-python` | Skip read-side Python tasks and assets (loader-side Python still runs for selected sources) | | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | | `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dev.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dev.md index 6920343bf4..3e915c8f14 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dev.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dev.md @@ -35,10 +35,11 @@ sqb dev o open UI · r recompile · s retry warehouse · l logs · q quit ``` -The location compiles the project and publishes it to the server. When you save a model, -orchestration file or other project input, it compiles and publishes again, so the UI and the -next run use the new version. Each change, publication, failure and run gets one line above the -hotkey bar. +The location compiles the project, runs its [Rules](../concepts/rules.md), and publishes it to +the server. When you save a model, orchestration file or other project input, it compiles and +publishes again, so the UI and the next run use the new version. Each change, publication, failure +and run gets one line above the hotkey bar. A compile error or a Rule finding leaves the previous +version published, and its failure line names the error or the first finding. ## Options @@ -120,5 +121,15 @@ timed reads but not the read after each publication or run. for a server other people or machines connect to. - The control store keeps runs, trigger changes and history between sessions. Delete `.sqlbuild/control.sqlite3` to start again from nothing. +- The location is named after the project, the `name` in `sqlbuild_project.toml`. When the + control store already has a location with that name, for example from another target of the + same project, it gets ` (2)`, then ` (3)`, and so on. +- The UI is ready once the project compiles and publishes. The location then reads SQLBuild's + history from the warehouse; plan previews, history and costs show + **Syncing warehouse history** until that first sync completes. See + [Follow a sync](server.md#follow-a-sync). +- The control store keeps the newest 5 builds of each node and results of each check that it copies + from the warehouse; older ones are deleted and the file shrinks. See + [retention](server.md#set-retention). - When `sqb dev` stops, it prints `Development services stopped successfully`. After a failure it prints the last log lines that explain it. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md index f30f489492..a67b97ea21 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md @@ -160,10 +160,17 @@ like a review schema is left alone. ### State history pruning -The janitor prunes old rows from `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness`, keeping -the latest `direct_state_history_versions` rows per identity (default `20`), and from -`_sqlbuild_consumption`, keeping the latest `direct_state_history_versions` builds of each model and -seed. This is separate from relation retention. +The janitor prunes old rows from SQLBuild's state tables, keeping the latest +`direct_state_history_versions` (default `20`) of each: + +| Table | Keeps the latest `direct_state_history_versions` | +| --- | --- | +| `_sqlbuild_fingerprints` | Rows per identity | +| `_sqlbuild_source_freshness` | Rows per identity | +| `_sqlbuild_consumption` | Builds of each model and seed | +| `_sqlbuild_audit_results` | Results of each check | + +This is separate from relation retention. ## Safety diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/location.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/location.md index 448a5c8a42..c6285b0532 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/location.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/location.md @@ -53,5 +53,18 @@ healthcheck: A `falling_back` location still runs scheduled jobs from its last good snapshot. Restarting it does not fix the failed load; fix the project and deploy it again. +Loading a project is a full compile, as [`sqb compile`](compile.md) runs it, including the +configured [Rules](../concepts/rules.md). A Rule finding fails the load like any other compile +error: nothing is published, and the location's output and its page in the UI show the findings: + +```text +Compile failed: 1 rule finding +models/orders.sql:2:8 [SQBRMODEL102] SELECT * is allowed only inside dependency import CTEs +``` + +Runs and plan previews on a published snapshot don't evaluate the Rules again; they passed when it +loaded. After a SQLBuild upgrade that adds Rules to a family you select, a location keeps running +its last good snapshot until the project passes them. + A `missing_secrets` location publishes and runs jobs whose nodes have their secrets. Set the missing variables in its environment and restart it; `sqb debug` in the location lists them. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md index 4885f90263..a68f01bd5f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md @@ -11,6 +11,7 @@ Online: https://sqlbuild.com/docs/cli/plan/ - Usage - Flags - Output +- Timings - Exit codes - Missing upstream dependencies - Missing source tables @@ -35,6 +36,8 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--defer-to` | Resolve unselected model references against another target | | `--as` | Compile for another target and show its plan, including [model migration](../concepts/models/migrations.md) decisions, using the current connection. Inspection only. | | `--json` | Output the plan as JSON | +| `--timings` | After the plan, print how long each planning step took; see [Timings](#timings) | +| `--ignore-audit-failures`, `--no-ignore-audit-failures` | Plan the build `sqb build` would run with the same flag; a plan whose audit failures are ignored starts with `Audit failures won't stop this build`. See [Ignoring audit failures](../concepts/audits.md#ignoring-audit-failures) | | `--selection-diagnostics` | Warn when a selected model will build on changed upstream models that are not selected; see [Selection and staleness](../concepts/planning/selection-and-staleness.md) | | `--full-refresh` | Plan a full rebuild for selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model without this flag | | `--changed` | Show what `sqb build --changed` would run, which nodes it holds back and why; takes `--changed=definition,source_data` and `--reach` like [`sqb build`](build.md#building-only-what-changed). In `--json`, the `changed` object lists `runs`, `held_back` and `unchanged` | @@ -134,6 +137,26 @@ Column migrations (1) When a removed column and an added column look related but are not an exact rename, the schema diff shows a hint such as `similar to amount; if this is a rename, add revenue (migrate_from amount)` until the change is built. +## Timings + +`--timings` prints how long each planning step took, in pipeline order, after the plan: + +``` +Plan timings seconds share + Compiling the project 2.1 20% + Reading warehouse state 3.3 31% + Detecting changes 2.4 23% + Checking source freshness 0.2 2% + Building plan entries 1.6 15% + Preparing test fixtures 0.1 1% + Assembling the plan 0.4 4% + Writing the plan 0.3 3% + Other 0.2 2% + Total 10.6 +``` + +`Reading build history` appears with `--changed`. `Preparing test fixtures` counts only the time the plan waits for them, because they are prepared alongside the other steps. `Other` is the time no step covers. A plan a location makes for the web UI records the same steps in its plan document. + ## Exit codes `sqb plan` exits `0` when the plan is ready, including plans that only have warnings. It exits `1` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index f04ae8e5bb..e199365ff1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -19,6 +19,7 @@ Online: https://sqlbuild.com/docs/cli/server/ - Sync from the warehouse now - Skip overlapping runs and retry failures - Limit worker containers +- Cap pools - Run ad-hoc commands - Plan builds - Define project roles @@ -64,12 +65,11 @@ executable = "/usr/local/bin/sqb" | Section | Keys | | --- | --- | | `[server]` | `host` (default `127.0.0.1`), `port` (`8000`), `database_env` (`SQLBUILD_CONTROL_DATABASE_URL`), `password_origin`, `default_role` (`none`), `scheduler` (`true`) | -| `[retention]` | `statistics_days`, `audit_days`, `snapshot_count`, `run_log_days`, `run_log_count`; see [Set retention](#set-retention) | +| `[retention]` | `statistics_days`, `audit_days`, `snapshot_count`, `run_log_days`, `run_log_count`, `state_history_versions`; see [Set retention](#set-retention) | | `[oidc]` | `origin`, `issuer`, `client_id`, `identities`, and optionally `client_secret_env`, `certificate_authority`, `groups_claim` and `group_roles` | | `[trusted_proxy]` | `origin`, `trusted_peers` and `identities` | -| `[[launchers.docker]]` | `location`, `target`, `server_url`, `max_workers`, `socket`, `location_container`, `executable`, and an optional `run` table; see [Limit worker containers](#limit-worker-containers) | -| `[[launchers.local]]` | `location`, `target`, `server_url`, `max_workers`, and absolute `project_dir` and `executable` paths | -| `[launchers.concurrency_limits]` | The most runs at once for each job `concurrency_groups` name, such as `orders_pool = 2` | +| `[[launchers.docker]]` | `location`, `target`, `server_url`, `max_workers`, `socket`, `location_container`, `executable`, and optional `run` and `pool_caps` tables; see [Limit worker containers](#limit-worker-containers) | +| `[[launchers.local]]` | `location`, `target`, `server_url`, `max_workers`, absolute `project_dir` and `executable` paths, and an optional `pool_caps` table | - A flag overrides the same setting in the file: `sqb server --config server.toml --port 9000` listens on port 9000. @@ -201,6 +201,7 @@ audit_days = 730 snapshot_count = 20 run_log_days = 7 run_log_count = 100 +state_history_versions = 10 ``` | Data | `[retention]` key | Option | Default | Range | @@ -210,6 +211,7 @@ run_log_count = 100 | Published snapshots per location and target | `snapshot_count` | `--snapshot-retention-count` | `10` | `1`–`65535` | | Logs of finished runs | `run_log_days` | `--run-log-retention-days` | `14` | `1`–`65535` | | Finished runs per location and target that keep their logs | `run_log_count` | `--run-log-retention-count` | `200` | `1`–`65535` | +| Builds and check results kept for each node | `state_history_versions` | `--state-history-retention-versions` | `5` on SQLite, `20` on PostgreSQL | `1`–`65535` | | Skipped and covered schedule ticks | None | None | 30 days | Fixed | Statistics samples older than the horizon disappear from statistics reads at once and are then @@ -237,6 +239,23 @@ have finished in its location and target, whichever comes first. The run itself, and the run history the scheduler recorded stay, so run lists, node status and history are unchanged; only the run's log, statements and SQL are no longer shown. A running run keeps its log. +Locations copy the builds and check results recorded in the warehouse's +`_sqlbuild_fingerprints` and `_sqlbuild_audit_results` tables into the control database. The +server keeps the newest `state_history_versions` builds of each node and results of each check, +and always keeps the latest one: + +- A location's first read of a table, and its periodic full re-read, take only the newest + `state_history_versions` rows of each node or check from the warehouse. +- Builds and results that arrive later are added, and older ones are deleted in small batches about + once a minute. +- A SQLite control database gives the freed space back to the disk a little at a time, so the file + shrinks. + +The PostgreSQL default matches the janitor's +[`direct_state_history_versions`](janitor.md#state-history-pruning), so with the janitor +running the server keeps what the warehouse keeps. [`sqb dev`](dev.md) has no +`server.toml` and uses the SQLite default. + The **Server** panel on the **Settings** page shows the control database's size on disk and both run log limits, and each `heartbeat` [platform event](#receive-platform-events) carries the size as `store_bytes`. @@ -364,7 +383,7 @@ The location starts the sync within a few seconds, and the top bar shows when it | --- | --- | --- | --- | | `[control] server` | `SQLBUILD_CONTROL_SERVER` | none | HTTPS origin of the server, as `sqb login --server` takes it | | `[control] location` | `SQLBUILD_CONTROL_LOCATION` | none | Location name a signed-in `sqb build` asks to sync | -| `[control] sync_at_least_every` | `SQLBUILD_CONTROL_SYNC_AT_LEAST_EVERY` | `"30m"` | Longest a location waits between syncs; `"0"` or `"off"` turns it off | +| `[control] sync_at_least_every` | `SQLBUILD_CONTROL_SYNC_AT_LEAST_EVERY` | `"30m"`, or off on Databricks, ClickHouse and MotherDuck | Longest a location waits between syncs; `"0"` or `"off"` turns it off. See [warehouse cost](#warehouse-cost-of-an-always-on-location) | | none | `SQLBUILD_CONTROL_CA_CERT` | none | Extra trusted PEM CA certificate for the server | An environment variable overrides the project file. `sqb location serve --sync-at-least-every` @@ -388,9 +407,9 @@ Details: - When a read fails, the Connections page shows the warehouse error. A read that ran out of memory or time retries a smaller page. Any other error, such as a missing permission or table, retries after 30 seconds, then waits twice as long after each failure in a row. After 5 failures in a row - the sync stops until you request one, the location publishes again, or the next safety sync is - due (`sync_at_least_every`). The safety sync tries once; another failure stops it until the - following one. + the sync stops until a run ends, you request one, the location publishes again, or the next safety + sync is due (`sync_at_least_every`). It then starts again from the first kind of history. A sync + resumed by a run or the safety sync tries once, and another failure stops it again. - `[control]` holds no secrets. The session stays in the login `sqb login` saved for that server. - When a state table holds two rows with the same ID, such as one check result written twice, the server keeps the row with the earlier time. Estimated query costs are the exception: when two @@ -399,6 +418,66 @@ Details: at the location's next full reread, which runs when the location starts and on the first sync at least 5 minutes after the previous full reread. +### Warehouse cost of an always-on location + +A location syncs when it starts, after each run ends, when someone asks, and on the +`sync_at_least_every` interval. Which reads need running compute depends on the adapter: + +| Adapter | Reads that need running compute | Interval sync while idle | +| --- | --- | --- | +| Snowflake | Reading a state table that changed since the last sync. Change checks, the catalog (`SHOW TABLES`, `SHOW VIEWS`, `SHOW COLUMNS`), statistics (one `SHOW OBJECTS` per schema) and existence checks run in cloud services | Every 30 minutes. A sync with nothing new resumes no warehouse; one after another writer changed a state table resumes it once | +| BigQuery | None to resume, but every sync queries each state table and is billed for the bytes it reads | Every 30 minutes | +| Databricks | Every read, including the catalog and existence checks, runs on the SQL warehouse | Off: a stopped warehouse would start each time | +| ClickHouse | Every read is a query, which wakes an idle ClickHouse Cloud service | Off | +| MotherDuck | Every read runs on MotherDuck compute | Off | +| PostgreSQL, SQL Server | The server is always running; a serverless database that pauses when idle wakes on connection | Every 30 minutes; set `sync_at_least_every = "off"` for one that pauses | +| DuckDB, SQLite | None; the location reads the file | Every 30 minutes | + +Where the interval is off, an idle location syncs only after a run ends or when someone asks, with +**Sync now**, `sqb control sync` or a signed-in `sqb build`. A run has just used the warehouse, so +the sync after it rarely resumes anything. Set `sync_at_least_every` to turn the interval back on. + +### Follow a sync + +The location prints a line as it finishes each of its 14 kinds of history, then a summary: + +```text +Reading warehouse history for automatic synchronization... +Warehouse history 1/14: fingerprints, 1,204 rows in 0.8 s +Warehouse history 2/14: source freshness, unchanged in 0.1 s +... +Warehouse history synchronization completed: 14 kinds, 31,204 rows in 12.8 s. +``` + +A page holds up to 2,000 rows and ends early at 8 MiB, so wide rows make smaller pages. A failed +read prints which kind failed, the page size and why, then what the location does next. A read that +ran out of time or memory retries a smaller page; any other failure retries the same page after a +wait that doubles each time, and the fifth in a row stops the sync: + +```text +Warehouse history read failed for statistics (100-relation page): no reply within 130 s. Sync incomplete; retaining mirror and retrying a smaller page in 30 s. +Warehouse history read failed for catalog (2000-row page): permission denied for table _sqlbuild_catalog. Sync incomplete; retaining mirror and retrying in 60 s. +Warehouse history read failed for catalog (2000-row page): permission denied for table _sqlbuild_catalog. Synchronization stopped after 5 failed reads in a row; it resumes after the next run, on request or after a publish. +``` + +| Cause | Printed as | +| --- | --- | +| The location couldn't start its reader | `the metadata reader did not start:` and the error | +| The warehouse refused the connection or the query | The warehouse's message | +| No reply in time | `no reply within N s`: 120 s for a page, or 30 s plus 1 s per relation for a statistics page | + +The location's page shows a running sync, for example +`syncing history 9/14 · catalog · 1,640 rows`, and the last full sync's kinds, rows and time. + +A location is ready as soon as it has compiled and published; nothing waits for the sync. Lineage, +jobs, the catalog and code work at once. Plan previews, node history, check results and costs read +the server's copy of the warehouse, so until the location's first sync completes they show +**Syncing warehouse history** with its progress, and fill in when it does. + +The location's page shows a failing sync as **failing** with when it last synced; its tooltip lists +each failing kind with the cause, the page size and the warehouse's message, shortened and with +secrets removed. The Connections page shows a stopped sync as **stopped after failed reads**. + ## Skip overlapping runs and retry failures Declare who owns a job, whether scheduled requests wait for a running copy, and how often a failed @@ -413,16 +492,16 @@ owner = "Orders team" overlap = "skip" schedule = { cron = "0 2 * * *", timezone = "UTC" } -[on_failure] -retries = 2 -retry_delay_seconds = 600 +[retry] +attempts = 3 +delay_seconds = 600 ``` | Setting | Values | Default | Effect | | --- | --- | --- | --- | | `owner` | Any non-blank text | None | Shown on the job's page and in the job list | | `overlap` | `queue` or `skip` | `queue` | With `skip`, a schedule tick or sensor request that arrives while a run of the job is queued or running starts no run | -| `on_failure` | `retries` from 1 to 10, `retry_delay_seconds` from 0 to 86400 | None | A failed or partial run reruns its failed and skipped nodes after the delay, until it succeeds or the retries run out | +| `retry` | `attempts` from 2 to 11, `delay_seconds` from 0 to 86400 | None | A failed or partial run reruns its failed and skipped nodes after the delay, until it succeeds or has run `attempts` times in all | A skipped tick shows as **skipped** in the trigger's tick log, with the run it yielded to, and creates no run. A sensor check lists the requests it skipped the same way. @@ -440,9 +519,16 @@ Details: paused, still queues. - A retry waits in the queue until its delay ends, and the run list shows it as a rerun of the run that failed. Cancelling a run stops its retries. -- A group retries at group level, with the group's own `on_failure`: each retry is one rerun of the - group's failed and skipped nodes. A member job's `on_failure` applies only when that job runs on - its own. +- A member job's `[retry]` also applies inside a group run. When the member fails, the group runs it + again after `delay_seconds`, inside the same group run, and the jobs that depend on it wait + instead of skipping. The group keeps its pool slots meanwhile. +- A group's own `[retry]` is a final sweep: after the group run ends `failed` or `partial`, it + queues one new linked rerun of the group's failed and skipped nodes. +- So a retry of a whole run, a job's or a group's, is a new linked run, and a retry of a member + stays inside its group run. +- Sensors and alerts see a member's final attempt only. The group run's timeline shows each retry, + such as `orders_job failed, retrying in 15 min (attempt 2 of 3)`, the member's row shows its + attempt, and the run's header counts the members it retried. - An invalid value is a `D017` error at its line, such as ``orchestration/jobs/nightly_orders_job.toml:6:11: unknown variant `coalesce`, expected `queue` or `skip` ``. @@ -498,6 +584,32 @@ cpus = 2 - An invalid value fails publication with the job's name, such as `job nightly_orders_job resources: cpus must be a positive number`. +## Cap pools + +A project declares each [pool](../concepts/concurrency.md#jobs-sharing-something-else-pools)'s +limit. To keep one launcher's runs below it, lower the limit with `pool_caps`: + +```toml +[[launchers.local]] +location = "orders" +target = "production" +server_url = "http://127.0.0.1:8000" +max_workers = 4 +project_dir = "/srv/orders" +executable = "/usr/local/bin/sqb" + +[launchers.local.pool_caps] +warehouse_heavy = 1 +``` + +- A cap lowers a limit and never raises it. A pool without a cap keeps the project's limit. +- A job taking more slots than the cap takes the whole capped pool. +- A cap applies to its launcher's location and target only. +- A cap for a pool the project doesn't declare has no effect. + +`max_workers`, write claims, `overlap` and pools are compared on +[Concurrency](../concepts/concurrency.md). + ## Run ad-hoc commands People with `run.adhoc` can run a `sqb` command on the current publication without declaring a diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md index 8ee38ea28f..c2c51a1835 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md @@ -92,9 +92,11 @@ Details: ## Python tests -Python tests are pytest modules named `test_*.py` under `tests/python/` and `tests/providers/`. Each -sits in the folder that mirrors the code it tests, so the tests of -`python/orders/storefront_import/node.py` live in `tests/python/orders/storefront_import/`. +Python tests are pytest modules named `test_*.py` under `tests/python/`, `tests/providers/` and +`tests/orchestration/`. Each sits in the folder that mirrors the code it tests, so the tests of +`python/orders/storefront_import/node.py` live in `tests/python/orders/storefront_import/`, and +the tests of the sensor `orchestration/sensors/orders/restock.py` in +`tests/orchestration/sensors/orders/test_restock.py`. | Command | SQL unit tests | Python tests | |---------|----------------|--------------| @@ -105,7 +107,8 @@ sits in the folder that mirrors the code it tests, so the tests of | `sqb build` | Yes | Never | pytest runs from the project directory with the same Python as SQLBuild and with -`--import-mode=importlib`, so tests import project code as `python.orders...` or `providers...`. +`--import-mode=importlib`, so tests import project code as `python.orders...`, `providers...` or +`orchestration.sensors...`. Its output appears after the SQL test summary, followed by `Python tests passed` or `Python tests failed`. `sqb test` exits non-zero when either kind fails. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md index 7d39789499..0535db467e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md @@ -20,6 +20,7 @@ Online: https://sqlbuild.com/docs/concepts/audits/ - Source audits - Seed audits - Severity +- Ignoring audit failures - Run scope - Skipping SQL analysis - Running audits standalone @@ -154,7 +155,9 @@ Native warehouse adapters best-effort append confirmed audit outcomes to `_sqlbuild_audit_results`. Rows are immutable and use deterministic IDs, so retrying the same result is idempotent. Projection failure is reported separately and does not change the audit outcome or command exit code. A results table created before `audit_meta_json` existed gains that nullable -column the first time a result with `meta` is recorded. Lifecycle sinks can also consume the corresponding `audit_completed` fact, which +column the first time a result with `meta` is recorded. A failure a build [ignored](#ignoring-audit-failures) is +recorded with outcome `error` and `{"ignored": true}` in `metadata_json`; its `audit_completed` +event carries `"ignored": true`. Lifecycle sinks can also consume the corresponding `audit_completed` fact, which is published as each audit finishes. ## Built-in audits @@ -428,7 +431,7 @@ the build continues. A seed audit cannot read a model built from that seed (`P00 | Severity | Behavior | |----------|----------| -| `error` | Blocks the build. Staging table is not promoted, DML is not applied. | +| `error` | Blocks the build. Staging table is not promoted, DML is not applied, unless the build [ignores audit failures](#ignoring-audit-failures). | | `warn` | Reports a warning but allows the build to continue. | An audit whose query raises a warehouse error, for example because a relation it reads is missing, @@ -450,6 +453,55 @@ columns ( ), ``` +## Ignoring audit failures + +Sometimes a build must go on even though an audit fails, for example to load a day's data before +anyone has looked at the failure. `--ignore-audit-failures` on `sqb build` lets every failure +through: + +```bash +sqb build --ignore-audit-failures +``` + +Every audit still runs at the same point and records its outcome, but no failure holds anything +back: staged tables are promoted, incremental batches are applied, a source whose audit fails +doesn't block the models that read it, and everything downstream runs. It +applies to every audit, including ones whose query raises an error. Unit tests are not affected; use +`--no-tests` to skip them. + +The build says so before and after it runs. The plan starts with `Audit failures won't stop this +build`, each ignored failure is listed with its model and audit and marked `(ignored)`, and the final +line reads `Completed successfully · 3 audit failures ignored` instead of plain success. The exit code +is 0 unless something else failed. + +Set it for every build of a project, of one target, or of one job instead of passing the flag: + +```toml +# sqlbuild_project.toml +[settings] +ignore_audit_failures = true + +[targets.prod] +ignore_audit_failures = false +``` + +```toml +# orchestration/jobs/nightly_job.toml +[settings] +ignore_audit_failures = true +``` + +The flag wins over a job, a job over its target and a target over `[settings]`. The default is +`false`. `--no-ignore-audit-failures` makes failures block even where a setting ignores them, and a +job's `ignore_audit_failures = false` does the same for its runs. Each job in a +[group](orchestration.md#group-jobs) uses its own setting, so members can differ. +`--ignore-audit-failures` with `--no-audits` is an error (`C419`): no audit runs, so there is no +failure to ignore. + +On the server, a run that ignored failures still ends `success` or `warn`, and the run records how +many it ignored: a group adds up its members'. Each ignored failure is recorded with outcome `error`, +marked as ignored: see [Result history](#result-history). + ## Run scope Audits on incremental models can run at different lifecycle phases: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/concurrency.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/concurrency.md new file mode 100644 index 0000000000..957db31cb5 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/concurrency.md @@ -0,0 +1,176 @@ + + +# Concurrency + +> Choose how many models and runs SQLBuild works on at once: concurrency, max_workers, write claims, overlap and pools. + +Online: https://sqlbuild.com/docs/concepts/concurrency/ + +## Contents + +- Models at once in a run: `concurrency` +- Runs at once on a server: `max_workers` +- Runs writing the same tables: write claims +- A job triggered while it runs: `overlap` +- Jobs sharing something else: pools + +SQLBuild has one setting for each kind of "not too much at once". Pick the row that matches what +you want: + +| I want… | Use | +| --- | --- | +| More or fewer models building at once in a run | [`concurrency`](#models-at-once-in-a-run-concurrency) | +| At most N runs on this server | [`max_workers`](#runs-at-once-on-a-server-max_workers) | +| Two runs never writing the same tables at once | Nothing: automatic ([write claims](#runs-writing-the-same-tables-write-claims)) | +| A job that's triggered while already running to wait, or be dropped | [`overlap = "queue"` or `"skip"`](#a-job-triggered-while-it-runs-overlap) | +| Jobs that don't share tables but share something else (an API with a rate limit, an SSH host, a heavy warehouse) never to exceed N at once | [A pool](#jobs-sharing-something-else-pools) | + +## Models at once in a run: `concurrency` + +`concurrency` is how many models, tests and other nodes one command or run builds at once. Set a +project default, change it per target, or give one job its own value: + +```toml +# sqlbuild_project.toml +[settings] +concurrency = 4 + +[targets.prod] +concurrency = 8 +``` + +```toml +# orchestration/jobs/nightly_orders_job.toml +[settings] +concurrency = 2 +``` + +`--concurrency` on the command line wins over all of them; see +[Target concurrency](project-configuration.md#target-concurrency) for the full order. The job's page shows the value its runs use on the target and where it came from. + +## Runs at once on a server: `max_workers` + +Each launcher in `server.toml` runs at most `max_workers` runs of its location and target at once. +Further runs wait in the queue: + +```toml +# server.toml +[[launchers.local]] +location = "orders" +target = "production" +server_url = "http://127.0.0.1:8000" +max_workers = 4 +project_dir = "/srv/orders" +executable = "/usr/local/bin/sqb" +``` + +The server's total is the sum of its launchers' `max_workers`. See +[Configure the server](../cli/server.md#configure-the-server). + +## Runs writing the same tables: write claims + +Nothing to configure. Before a run starts, it claims every node it writes in its target. A run that +writes a node another run has claimed waits until that run ends, and its queue entry names the run +and node it waits for. Runs that write different nodes run side by side. + +## A job triggered while it runs: `overlap` + +```toml +# orchestration/jobs/nightly_orders_job.toml +overlap = "skip" +``` + +With `queue` (the default), a schedule tick or sensor request that arrives while the job is queued +or running queues another run, which starts after the first. With `skip`, it starts no run. Runs +people start, reruns and backfills always queue. See +[Skip overlapping runs](../cli/server.md#skip-overlapping-runs-and-retry-failures). + +## Jobs sharing something else: pools + +Write claims keep runs off the same tables. A pool limits jobs that share something else: a +supplier API with a rate limit, an SSH host, a warehouse that slows down under load. + +**A pool is just a name and a number that SQLBuild counts.** It never looks at the API, host or +warehouse. It only makes sure the runs taking the pool never hold more slots than its limit. + +Declare each pool with its limit in `sqlbuild_project.toml`, and change a limit for one target: + +```toml +# sqlbuild_project.toml +[pools] +supplier_api = 2 +warehouse_heavy = 4 + +[targets.dev.pools] +warehouse_heavy = 1 +``` + +Two loader jobs share the supplier API, which allows two connections at once. Each takes one slot, +so both may run together, but a third job taking `supplier_api` waits: + +```toml +# orchestration/jobs/load_suppliers_job.toml +name = "load_suppliers_job" +description = "Load supplier records from the supplier API" +select = "raw_suppliers" +pools = { supplier_api = 1 } +``` + +```toml +# orchestration/jobs/load_prices_job.toml +name = "load_prices_job" +description = "Load supplier prices from the supplier API" +select = "raw_prices" +pools = { supplier_api = 1 } +``` + +A job that needs more of the pool takes more slots. This backfill uses both API connections, so it +runs only when no other job holds a `supplier_api` slot, and nothing else takes one while it runs: + +```toml +# orchestration/jobs/backfill_prices_job.toml +name = "backfill_prices_job" +description = "Reload every supplier price" +select = "raw_prices" +pools = { supplier_api = 2 } +``` + +A job may take several pools. It starts only when every one has room, and takes all its slots at +once: + +```toml +# orchestration/jobs/price_report_job.toml +name = "price_report_job" +description = "Fetch the latest prices and rebuild the heavy price report" +select = "+price_report" +pools = { supplier_api = 1, warehouse_heavy = 2 } +``` + +| Field | Value | +| --- | --- | +| `[pools]` in `sqlbuild_project.toml` | Each pool's limit, a whole number of slots from `1` to `10000` | +| `[targets..pools]` | The target's limit for pools `[pools]` declares | +| `pools` on a job or job group | Each pool it takes, with its amount: a whole number of slots from `1` to its limit | + +Details: + +- Every pool is written with an amount: `pools = { supplier_api = 1 }`. +- A pool the project doesn't declare fails `sqb compile` with `D017`, naming the job and the + declared pools. +- An amount above the pool's limit fails `sqb compile`. A job with `targets` is checked against + those targets' limits, and a job without `targets` against the project's limit and each target's. +- A target can change only the limit of a declared pool. Naming another pool fails with `D001`, + such as `targets.prod.pools.supplier_apy changes a pool [pools] does not declare`. +- Limits are project policy, so `sqlbuild_local.toml` can't set them. +- Slots belong to their location and target: runs of another location or target never take them. +- A run that waits for a pool takes none of its slots, in that pool or any other, until all have + room. An earlier queued run taking the same pool goes first. +- A job group takes every pool its members take, each at the largest amount, when it starts. Its + members then use the group's slots, one member per pool at a time. +- The server can lower a limit for one launcher with `pool_caps`, never raise it; see + [Cap pools](../cli/server.md#cap-pools). A job whose amount is above the cap takes the whole + capped pool, and the location's health shows a warning naming it, such as `job + backfill_prices_job takes 2 of pool supplier_api; this server caps it at 1, so it runs alone in + that pool`. +- The job's page shows each pool it takes, its limit and the other jobs taking it. A queued run's + wait names the full pool and the runs holding it. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md index 2dd7a15b20..4b6cc3d451 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md @@ -64,12 +64,12 @@ exclude = "tag:deprecated" targets = ["production"] owner = "Orders team" overlap = "skip" -concurrency_groups = ["warehouse_heavy"] +pools = { warehouse_heavy = 1 } schedule = { cron = "0 2 * * *", timezone = "Europe/London" } -[on_failure] -retries = 2 -retry_delay_seconds = 600 +[retry] +attempts = 3 +delay_seconds = 600 [settings] concurrency = 4 @@ -89,10 +89,10 @@ cpus = 2 | `targets` | No | The targets the job exists in; all targets when omitted | | `owner` | No | Shown on the job's page and in the job list | | `overlap` | No | `queue` (default) or `skip`; see [Skip overlapping runs](../cli/server.md#skip-overlapping-runs-and-retry-failures) | -| `concurrency_groups` | No | Server concurrency limits the job's runs share | +| `pools` | No | Slots the job takes of each [pool](concurrency.md#jobs-sharing-something-else-pools), such as `{ warehouse_heavy = 1 }` | | `schedule` | No | When the server starts the job; see below | -| `[on_failure]` | No | `retries` from 1 to 10, `retry_delay_seconds` from 0 to 86400 | -| `[settings]` | No | Build options for each run, such as `concurrency`, `vars`, `full_refresh` or `changed` | +| `[retry]` | No | `attempts`, the most runs in all, from 2 to 11, and `delay_seconds` from 0 to 86400; see [Retry failures](../cli/server.md#skip-overlapping-runs-and-retry-failures) | +| `[settings]` | No | Build options for each run, such as `concurrency`, `vars`, `full_refresh`, `changed` or `ignore_audit_failures` ([Ignoring audit failures](audits.md#ignoring-audit-failures)) | | `[resources]` | No | Docker worker limits; see [Limit worker containers](../cli/server.md#limit-worker-containers) | A job's `[settings]` cannot set `select` or `exclude`: the job's own selection is the run's @@ -105,7 +105,10 @@ reports a job or sensor `select` that names more than three nodes one by one; se ## Group jobs A job group runs its jobs as one run. A job waits for another job in the group when it reads a node -that job builds. Jobs that wait on each other, directly or through other members, are a cycle, and +that job builds, directly or through nodes no job in the group builds. It waits only for the +nearest such jobs: what they read is their own wait, so a node further upstream, such as a function +many jobs build, adds no more waits. A job never waits for the other jobs that build a node it +builds itself. Jobs that wait on each other, directly or through other members, are a cycle, and `sqb compile` reports every cycle in the group as one `D017` error. ```toml @@ -117,12 +120,54 @@ schedule = { cron = "0 3 * * *", timezone = "UTC" } ``` A group takes `name`, `description` and `jobs`, and the same `targets`, `owner`, `overlap`, -`concurrency_groups`, `schedule` and `[on_failure]` as a job. Its members are jobs, never other +`pools`, `schedule` and `[retry]` as a job. Its members are jobs, never other groups. +When a location publishes, it records why each member waits, node by node: a node of the earlier +job feeds a node of the waiting job, directly or through nodes no member selects, such as a source +between a loader job and the models that read it. The job's page shows these reasons. + A job that reads a seed or a node that several members build waits for all of those members. Give seeds and shared nodes their own job, so the jobs that read them wait only for that job. +### Choose what happens after a failure + +`[dependencies]` sets how a member reacts when a job it depends on fails: + +```toml +# orchestration/job_groups/nightly_job_group.toml +name = "nightly_job_group" +description = "Orders, inventory and the partner export" +jobs = ["orders_job", "inventory_job", "partner_export_job"] +schedule = { cron = "0 21 * * *", timezone = "UTC" } + +[dependencies] +wait_for = "job" +on_failure = "partial" + +[dependencies.partner_export_job] +on_failure = "skip" +``` + +| Setting | Value | Meaning | +| --- | --- | --- | +| `wait_for` | `job` (default) | A member starts once the jobs it reads from have finished | +| `on_failure` | `partial` (default) | Build what can still be built; nodes downstream of a failed node are skipped | +| | `skip` | Don't run the member; all its nodes end skipped and the member ends `skipped` | +| | `run` | Run every node of the member regardless | + +- `[dependencies.]` overrides the group's settings for one member, and must name a job in + `jobs`. Overrides live in the group file because a job can belong to several groups. +- A skipped member raises no failure alert and fires `run_event` sensors with `kinds=["skipped"]`, + not `"failed"`. The jobs that depend on it react to it as they would to a failure, since what + they read was not built. +- A member job's own `[retry]` runs it again inside the group run, and the jobs that depend on it + wait for its final attempt before `on_failure` applies. A group's `[retry]` reruns the whole + group's failures once the group run ends; see + [Retry failures](../cli/server.md#skip-overlapping-runs-and-retry-failures). +- `wait_for = "node"`, which will start each node as soon as the nodes it reads are built, is + planned but not available yet; `sqb compile` refuses it, so use `"job"` for now. + ## Schedule a job `schedule` takes a cron expression or the name of a calendar, and always a timezone. @@ -182,11 +227,18 @@ on_new_orders = sensor( | Condition | Starts a run for | | --- | --- | | `platform_event(kinds=[...])` | Server events, such as an overdue tick | -| `run_event(job="...", kinds=[...])` | Outcomes of another job or group, optionally only when `executed_successfully` nodes ran and succeeded | +| `run_event(job="...", kinds=[...])` | Outcomes of another job or group, including the job's runs inside groups, optionally only when `executed_successfully` nodes ran and succeeded | | `node_event(select="...", kinds=[...])` | Each success, failure or recovery of the selected nodes in any run | | `source_new_data(select="...")` | Each new version of the selected sources | | `source_stale(select="...", after_seconds=...)` | Each source version older than `after_seconds` | +A `run_event` on a job also fires for that job's runs inside a job group. Each member of a group +run ends with its own outcome as soon as it finishes, while the rest of the group may still be +running, and `executed_successfully` checks that member's nodes. Moving a job into a group therefore +keeps every sensor that watches it. A sensor on the group fires once, when the whole group run ends. +`kinds` takes `success`, `warn`, `partial`, `failed`, `cancelled`, `skipped` (a member its group +did not run) or `completed` for any of them. + A sensor without `when` decorates a function that polls and returns `RunRequest(run_key=...)` values or a `SkipReason`. A sensor also takes `minimum_interval_seconds` (default 30), `default_status` and `targets`. @@ -229,6 +281,8 @@ Every problem with an orchestration file is a `D017` error at its file, line and | `declares a second sensor; each module declares exactly one` | Split the module | | `schedule cron: invalid cron expression: Pattern must have 5 fields when seconds are disallowed.` | Write a five-field cron, such as `0 2 * * *` | | `job nightly_orders_job resources: cpus must be a positive number` | Set `cpus` above zero | +| `job 'nightly_orders_job' uses pool 'warehouse_hevy', which [pools] in sqlbuild_project.toml does not declare` | Fix the name, or declare the pool in [`[pools]`](project-configuration.md#pools) | +| `job 'nightly_orders_job' takes 2 slots of pool 'supplier_api', more than its limit of 1 on target 'production'` | Take fewer slots, or raise the limit in `[targets.production.pools]` | | `sensor 'orders_sensor' is declared here, but importing the module does not define it` | Assign the sensor to a module-level name or list, not a dictionary or function body | | `group dependencies contain a cycle in job group daily_orders_job_group: orders_marts_job waits for staging_orders_job (model:orders reads model:stg_orders); staging_orders_job waits for orders_marts_job (model:stg_orders reads seed:regions)` | Move each node to one job, and give seeds and nodes that several members read their own job that runs first | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 1136c6544e..ed51b47f92 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -23,6 +23,7 @@ Online: https://sqlbuild.com/docs/concepts/project-configuration/ - Scenario - Git - Review +- Pools - dbt - Skills - Project Python @@ -186,6 +187,7 @@ A target is a named build context - the database and schema you build into, plus | `missing_migration_origin` | What a build does when a declared `migrate_from` origin does not exist in this target: `allow` (default), `require_confirmation`, or `deny`. See [Missing migration origins](models/migrations.md#missing-migration-origins) | | `warehouses` | Snowflake warehouse per command group, `build` and `query`. See [Command-group warehouses](adapters/snowflake.md#command-group-warehouses) | | `concurrency` | Worker limit for commands run against this target, replacing `[settings] concurrency` (see below) | +| `ignore_audit_failures` | Whether builds on this target let audit failures through, replacing `[settings] ignore_audit_failures`; see [Ignoring audit failures](audits.md#ignoring-audit-failures) | ```toml [connections.warehouse] @@ -256,7 +258,7 @@ still takes precedence for that invocation. Targets often run on very different warehouses. A target can set its own worker limit, so local and CI runs use many workers while production stays within what its warehouse runs without -queuing: +queuing. To limit runs rather than models, see [Concurrency](concurrency.md): ```toml [settings] @@ -518,6 +520,7 @@ delta_compile = true | `delta_compile` | `true` | After an edit to model SQL or SQL test files only, compile just the changed files against the previous compile. See [Compiling after an edit](../cli/compile.md#compiling-after-an-edit). | | `allow_missing_secrets` | `false` | Let `compile`, `rules run`, `format`, `dag`, `lineage` and `scope` run without provider secrets. `sqlbuild_local.toml` only. See [Secrets](python-nodes/providers.md#secrets). | | `allow_warehouse_round_trips` | `false` | Allow a loader to depend on a task or asset that reads SQL, so data can leave the warehouse and come back through a source. Project file only. See [The SQL boundary](python-nodes/overview.md#warehouse-round-trips). | +| `ignore_audit_failures` | `false` | Let builds go on past failing audits: they still run and record `fail`, but promote, apply and build downstream as if they passed. A target, a job and `--ignore-audit-failures` / `--no-ignore-audit-failures` override it. See [Ignoring audit failures](audits.md#ignoring-audit-failures). | ### Unit test and scenario limits @@ -640,7 +643,7 @@ keep_in_use = false | `archive_retention_days` | `14` | How many days an archive is kept before it is deleted. `0` deletes it in the same run. | | `delete_tracked_only` | `true` | Only clean relations that appear in fingerprint tracking | | `exclude_patterns` | `[]` | Glob patterns for relations to skip | -| `direct_state_history_versions` | `20` | How many state-history rows to keep per identity in `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness`, and how many builds of each model and seed to keep in `_sqlbuild_consumption` | +| `direct_state_history_versions` | `20` | How many state-history rows to keep per identity in `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness`, how many builds of each model and seed to keep in `_sqlbuild_consumption`, and how many results of each check to keep in `_sqlbuild_audit_results` | | `in_use_window` | `"7d"` | How far back the janitor looks for reads of each relation it would archive. A fixed duration such as `"7d"` or `"36h"`. See [Relations in use](../cli/janitor.md#relations-in-use) | | `keep_in_use` | `false` | Keep relations read within `in_use_window` instead of archiving them. `--keep-in-use` turns it on for one run. Stops with `C503` when usage can't be read in full | @@ -724,6 +727,27 @@ critical_tags = ["critical"] | `schema_prefix` | `"review"` | Prefix of the schema each review creates and drops; the schema name is lowercase | | `critical_tags` | `["critical"]` | Model tags that make a model critical, in addition to `critical true` in its `MODEL` header | +## Pools + +`[pools]` declares each pool with its limit, and `[targets..pools]` changes a limit for one +target. Jobs and job groups take slots of a pool with `pools = { supplier_api = 1 }`. + +```toml +[pools] +supplier_api = 2 + +[targets.dev.pools] +supplier_api = 1 +``` + +| Field | Value | +|-------|-------| +| `[pools]` | Each pool's limit, a whole number of slots from `1` to `10000` | +| `[targets..pools]` | The target's limit for pools `[pools]` declares | + +When to use a pool, and how amounts, targets and server caps work, is on +[Concurrency](concurrency.md#jobs-sharing-something-else-pools). + ## dbt `[dbt]` points SQLBuild at an existing dbt project. See @@ -772,7 +796,7 @@ Compilation rejects Python anywhere else with `D016`, so indirectly importable m an unofficial project structure. The project's `tests/` directory holds SQL tests and scenarios, and pytest tests of project Python -under `tests/python/` and `tests/providers/`, which `sqb test` runs; see +under `tests/python/`, `tests/providers/` and `tests/orchestration/`, which `sqb test` runs; see [Python tests](../cli/test.md#python-tests). Orchestrator code such as Dagster or Rivers definitions lives outside the SQLBuild project directory and points at it; see [Dagster](../integrations/dagster.md) and [Rivers](../integrations/rivers.md). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md index 95b637cf5e..1cb2bc4c52 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md @@ -211,6 +211,11 @@ Rules run in every `sqb compile`, `sqb plan` and `sqb build`, after SQLBuild's o Rule fails, nothing runs in the warehouse. See [Execution and caching](rules/execution-and-caching.md). +A code location, including the one `sqb dev` starts, runs the Rules when it loads the project, as +part of its compile. A finding fails the load, so the snapshot is not published and runs keep using +the last good one. Scheduled runs and plan previews on a published snapshot don't run the Rules +again. See [Check a location's health](../cli/location.md#check-a-locations-health). + ## Rules compared with tests, audits, and contracts Rules do not replace every form of validation: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md index 81dae29d92..ebe9518280 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md @@ -60,9 +60,9 @@ passes, the case fails with `custom rule is not deterministic`. The usual module-level state, such as a set of already-seen models, that carries over from one subject to the next. Compute findings from the subject and `ctx` alone. -Keep Rule tests under the project's `tests/` directory, outside `tests/python/` and -`tests/providers/`, such as in `tests/rules/`. `sqb test` runs only those two folders, so run Rule -tests with pytest. +Keep Rule tests under the project's `tests/` directory, outside `tests/python/`, `tests/providers/` +and `tests/orchestration/`, such as in `tests/rules/`. `sqb test` runs only those three folders, so +run Rule tests with pytest. ## Parallel evaluation diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md index eb800afc84..061fb110f3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md @@ -227,7 +227,8 @@ Providers importing `orchestration/` are reported by `SQBRPYTHON015` instead. ## Testing Python -`sqb test` runs the pytest modules under `tests/python/` and `tests/providers/`; see +`sqb test` runs the pytest modules under `tests/python/`, `tests/providers/` and +`tests/orchestration/`; see [Python tests](../../cli/test.md#python-tests). `SQBRPYTHON021` keeps each test beside a mirror of the code it tests, and `SQBRPYTHON701` to `SQBRPYTHON724` keep tests table-driven: @@ -279,7 +280,7 @@ def test_given_a_status_when_mapping_then_returns_its_label(test_case: RowMappin | Code | Finding | Fix | |---|---|---| -| `SQBRPYTHON021` | A test module whose folder does not mirror one under `python/` or `providers/`, or a `test_.py` with no `.py` or `/` package beside the code it tests | Move the test to the mirrored folder, or rename it for the module it tests | +| `SQBRPYTHON021` | A test module whose folder does not mirror one under `python/`, `providers/` or `orchestration/`, or a `test_.py` with no `.py` or `/` package beside the code it tests | Move the test to the mirrored folder, or rename it for the module it tests | | `SQBRPYTHON701` to `SQBRPYTHON711` | A test folder or module whose layout, imports, helpers, conditionals or test-case dataclasses break the pattern above | Follow the pattern above | | `SQBRPYTHON712` to `SQBRPYTHON724` | A test function or `pytest.mark.parametrize` that is not named `test_given__when__then_`, or does not take dataclass cases through `test_case` with ids from each `description` | Follow the pattern above | @@ -435,7 +436,7 @@ remediation. | Code | Name | Files | Default | Checks | |---|---|---|---|---| -| `SQBRPYTHON021` | python-test-placement | `tests/` | yes | Python tests must mirror an existing module under `python/` or `providers/` | +| `SQBRPYTHON021` | python-test-placement | `tests/` | yes | Python tests must mirror an existing module under `python/`, `providers/` or `orchestration/` | | `SQBRPYTHON701` | test-init-module-empty | `tests/` | yes | Test package `__init__.py` files must be empty or docstring-only | | `SQBRPYTHON702` | test-absolute-imports | `tests/` | yes | Tests must use absolute imports | | `SQBRPYTHON703` | test-no-top-level-helpers | `tests/` | yes | Test modules may contain only tests, imports, and declarations | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md index bef44c5e3f..9a25d769ef 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md @@ -91,7 +91,8 @@ sqb build --select +kind:asset ``` The kinds are `model`, `source`, `seed`, `udf`, `table_fn`, `task`, `asset`, `loader` and -`check`, as the server's catalog names them. Kind selectors compose with `+`, `,` and `--exclude` +`check`, as the server's catalog names them, and `function`, which selects both `udf` and +`table_fn`. Kind selectors compose with `+`, `,` and `--exclude` like tag selectors. An unknown kind is an error that names the closest kinds, and a kind your project has no nodes of is an error, as an unused tag is. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md index 77f54892c8..5a92a346ca 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md @@ -536,8 +536,8 @@ sqb test sqb test --select stg_orders ``` -`sqb test` also runs the project's pytest modules under `tests/python/` and `tests/providers/`; -`sqb build` never runs them. See [Python tests](../cli/test.md#python-tests). +`sqb test` also runs the project's pytest modules under `tests/python/`, `tests/providers/` and +`tests/orchestration/`; `sqb build` never runs them. See [Python tests](../cli/test.md#python-tests). ## More test features diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md b/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md index 4ae00c287e..1f509c9bd3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md @@ -238,6 +238,7 @@ audit_results (view): one row per evaluated audit; its evidence rows are in arti evidence_count ANY evidence_truncated ANY sample_digest ANY + ignored ANY conflicts (view): record keys reused with different content; the first row is kept dataset TEXT diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 4ef0bc4743..de71fe3845 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -793,6 +793,7 @@ class WarehouseSnapshot(NativeRecord): existing_columns: dict[str, tuple[ColumnInfo, ...]] fingerprints: WarehouseFingerprints cursor_snapshots: dict[str, ModelCursorSnapshot] + cursor_inputs_at_run: bool source_freshness_state_schemas: frozenset[str] column_dialect: str | None renamed_models: frozenset[str] @@ -806,6 +807,7 @@ class WarehouseSnapshot(NativeRecord): existing_columns: dict[str, tuple[ColumnInfo, ...]] = ..., fingerprints: WarehouseFingerprints = ..., cursor_snapshots: dict[str, ModelCursorSnapshot] = ..., + cursor_inputs_at_run: bool = ..., source_freshness_state_schemas: frozenset[str] = ..., column_dialect: str | None = ..., renamed_models: frozenset[str] = ..., @@ -1234,12 +1236,33 @@ class PlanProviderUsage(NativeRecord): ) -> None: ... def __replace__(self, **changes: Any) -> PlanProviderUsage: ... +class Dialect: + """How a plan writes SQL: the adapter's syntax, with no way to reach the warehouse. + + Attributes that only write SQL or describe the dialect (`render_*`, `supports_*` and + dialect facts) are the adapter's own; anything reaching the warehouse raises `AttributeError`. + """ + + def __init__(self, adapter: BaseAdapter) -> None: ... + def __getattr__(self, name: str) -> Any: ... + +class WarehouseReads: + """The warehouse reads one plan runs through: live, or a preview's synced state.""" + +def live_reads( + *, + adapter: BaseAdapter, + connection: Any, + connection_config: Any = ..., + execute: Callable[..., Any] | None = ..., +) -> WarehouseReads: ... + class PlannerRuntime: """Resolved execution environment shared by planner phases.""" project: CompiledProjectView - adapter: BaseAdapter - connection: Any + dialect: Dialect + reads: WarehouseReads project_config: ProjectConfig | None local_config: LocalConfig | None on_progress: Callable[[str], None] | None @@ -1249,8 +1272,8 @@ class PlannerRuntime: self, *, project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, + dialect: Dialect, + reads: WarehouseReads, project_config: ProjectConfig | None = ..., local_config: LocalConfig | None = ..., on_progress: Callable[[str], None] | None = ..., @@ -1999,7 +2022,7 @@ def normalize_cursor_snapshot_grain( ) -> ModelCursorSnapshot: ... def resolve_model_sql( *, - adapter: BaseAdapter, + dialect: Dialect, model: CompiledModelView, snapshot: WarehouseSnapshot, context: ModelPlanContext, @@ -2012,7 +2035,7 @@ def resolve_model_sql( ) -> str: ... def resolve_function_sql( *, - adapter: BaseAdapter, + dialect: Dialect, function: CompiledFunctionView, model_locations: dict[str, CompiledRelationLocation], seed_locations: dict[str, CompiledRelationLocation], @@ -2024,7 +2047,7 @@ def resolve_function_sql( def resolve_model_hook_entries( *, model: CompiledModelView, - adapter: BaseAdapter, + dialect: Dialect, context: ModelPlanContext, external_sql_reference_resolver: ExternalSqlReferenceResolver | None, ) -> tuple[object, object]: ... @@ -3481,7 +3504,7 @@ def build_relation_fixture_context( def build_relation_fixture_completion( *, project: CompiledProjectView, - adapter: BaseAdapter, + dialect: Dialect, request: tuple[tuple[str, ...], FixtureGroups], planning_context: RelationFixturePlanningContext | None = None, ) -> RelationFixtureCompletion: ... @@ -4488,7 +4511,7 @@ def runtime_project_relations( *, relations: Mapping[SqlResourceRef, str], dialect: str | None ) -> ProjectRelationIndex: ... def render_source_relation(*, entry: SourceEntry, adapter: StrictAdapter | None = None) -> str: ... -def render_source_read_relation(*, adapter: BaseAdapter, source_entry: SourceEntry) -> str: ... +def render_source_read_relation(*, dialect: Dialect, source_entry: SourceEntry) -> str: ... def resolve_ref_references( *, query_sql: str, @@ -4496,7 +4519,7 @@ def resolve_ref_references( seed_locations: dict[str, CompiledRelationLocation], cursor_bounds: CursorBounds | None, cursor_filter_inputs: dict[str, str], - adapter: BaseAdapter, + dialect: Dialect, cursor_type: str | None, lower_bound_inclusive: bool, ) -> str: ... @@ -4508,7 +4531,7 @@ def resolve_source_references( star_exclude_keyword: str, cursor_bounds: CursorBounds | None, cursor_filter_inputs: dict[str, str], - adapter: BaseAdapter, + dialect: Dialect, cursor_type: str | None, lower_bound_inclusive: bool, ) -> str: ... @@ -4521,13 +4544,13 @@ def resolve_udf_references( *, query_sql: str, function_locations: dict[str, CompiledRelationLocation], - adapter: BaseAdapter, + dialect: Dialect, ) -> str: ... def resolve_table_function_references( *, query_sql: str, function_locations: dict[str, CompiledRelationLocation], - adapter: BaseAdapter, + dialect: Dialect, ) -> str: ... def apply_deferred_locations( *, @@ -4945,7 +4968,7 @@ def build_cli_scenario_plan( *, scenario: CompiledSqlScenario, pipeline_result: CompilePipelineResult, - adapter: BaseAdapter, + dialect: Dialect, project_name: str, source_lexical_syntax: SqlLexicalSyntax, ) -> ScenarioExecutionPlan: ... @@ -4981,7 +5004,7 @@ def resolve_scenario_check_sql( *, sql: str, relation_plan: ScenarioRelationPlan, - adapter: BaseAdapter, + dialect: Dialect, source_label: str, source_lexical_syntax: SqlLexicalSyntax, ) -> str: ... @@ -4990,7 +5013,7 @@ def build_scenario_fixture_plans( scenario: CompiledSqlScenario, graph_plan: ScenarioGraphPlan, relation_plan: ScenarioRelationPlan, - adapter: BaseAdapter, + dialect: Dialect, source_lexical_syntax: SqlLexicalSyntax, fixture_sql_overrides: dict[tuple[CompiledResourceType, str], str] | None = None, ) -> tuple[ScenarioFixturePlan, ...]: ... @@ -4998,7 +5021,7 @@ def build_scenario_execution_plan( *, scenario: CompiledSqlScenario, project: CompiledProjectView, - adapter: BaseAdapter, + dialect: Dialect, graph_plan: ScenarioGraphPlan, relation_plan: ScenarioRelationPlan, source_lexical_syntax: SqlLexicalSyntax, @@ -5432,8 +5455,7 @@ class MigrationStateInspection: def __init__( self, *, - adapter: BaseAdapter, - connection: Any, + reads: WarehouseReads, database: str | None, known_fingerprints: WarehouseFingerprints | None = None, ) -> None: ... @@ -5583,19 +5605,19 @@ def append_eligible_direct_source_freshness_records( run_id: str, ) -> None: ... def observe_configured_source_freshness( - *, adapter: StrictAdapter, connection: object, source: SourceEntry, observed_at: datetime + *, dialect: Dialect, reads: WarehouseReads, source: SourceEntry, observed_at: datetime ) -> SourceFreshnessObservation: ... def observe_adapter_sources_freshness( *, - adapter: StrictAdapter, - connection: object, + dialect: Dialect, + reads: WarehouseReads, sources: tuple[SourceEntry, ...], observed_at: datetime, ) -> AdapterSourceFreshnessBatch: ... def build_direct_source_freshness_planning_result( *, - adapter: StrictAdapter, - connection: object, + dialect: Dialect, + reads: WarehouseReads, sources: tuple[SourceEntry, ...], state_database: str | None, state_schemas: tuple[str, ...], @@ -5666,7 +5688,7 @@ def apply_maximum_start_policy( def build_selected_test_entries( *, project: CompiledProjectView, - adapter: BaseAdapter, + dialect: Dialect, selected_keys: frozenset[CompiledObjectKey], sql_test_selection: SqlTestSelection, case_name: str | None = None, @@ -5677,7 +5699,7 @@ def resolve_static_command_scope( def resolve_static_relation_context( *, project: CompiledProjectView, - adapter: BaseAdapter, + dialect: Dialect, scope: PlannerScope, deferral: DeferralInputs | None = None, project_config: ProjectConfig | None = None, @@ -5687,14 +5709,14 @@ def resolve_static_relation_context( def build_audit_command_plan( *, project: CompiledProjectView, - adapter: BaseAdapter, + dialect: Dialect, scope: PlannerScope, relations: PlannerRelationsContext, ) -> PlanOutput: ... def build_test_command_plan( *, project: CompiledProjectView, - adapter: BaseAdapter, + dialect: Dialect, scope: PlannerScope, relations: PlannerRelationsContext, case_name: str | None = None, @@ -5707,7 +5729,7 @@ def build_seed_command_plan( fingerprints: dict[str, Fingerprint] | None = None, ) -> PlanOutput: ... def read_selected_seed_fingerprints( - *, adapter: BaseAdapter, connection: Any, seeds: tuple[CompiledSeedView, ...] + *, reads: WarehouseReads, seeds: tuple[CompiledSeedView, ...] ) -> dict[str, Fingerprint]: ... def resolve_clone_boundary[K]( *, @@ -5729,8 +5751,8 @@ def sql_test_model_chain_names_by_key( def build_execution_plan( *, project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, + dialect: Dialect, + reads: WarehouseReads, selection: PlannerSelection, overrides: PlannerOverrides, deferral: DeferralInputs, @@ -5749,7 +5771,7 @@ class SqlTestPlanning: def start_sql_test_planning( *, project: CompiledProjectView, - adapter: BaseAdapter, + dialect: Dialect, selected_keys: frozenset[CompiledObjectKey], sql_test_selection: SqlTestSelection, enabled: bool, @@ -5802,8 +5824,8 @@ def resolve_cursor_overrides( def build_planner_relations_context( *, project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, + dialect: Dialect, + reads: WarehouseReads, scope: PlannerScope, deferral: DeferralInputs | None = None, project_config: ProjectConfig | None = None, @@ -6103,7 +6125,7 @@ def validate_source_cursor_input_columns( def build_cursor_input_relations( *, model: CompiledModelView, - adapter: BaseAdapter, + dialect: Dialect, model_locations: dict[str, CompiledRelationLocation], models_by_name: dict[str, CompiledModelView], functions_by_name: dict[str, CompiledFunctionView] | None = None, @@ -6124,7 +6146,7 @@ def plan_audit( model_locations: dict[str, CompiledRelationLocation], seed_locations: dict[str, CompiledRelationLocation], source_map: dict[str, SourceEntry], - adapter: BaseAdapter, + dialect: Dialect, lineage_upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], lineage_downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], model_materializations: dict[str, str], @@ -6311,9 +6333,8 @@ class MigrationFingerprintCache: def gather_warehouse_snapshot( *, project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, - execute: AdapterExecute[Any, Any], + dialect: Dialect, + reads: WarehouseReads, selected_keys: frozenset[CompiledObjectKey] | None = None, full_refresh: bool = False, full_refresh_model_names: frozenset[str] | None = None, @@ -6325,15 +6346,13 @@ def watermark_type_is_compatible(*, declared_type: str, cursor_type: str | None) def gather_source_columns( *, project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, + reads: WarehouseReads, source_entries: tuple[SourceEntry, ...] | None = None, ) -> dict[str, tuple[ColumnInfo, ...]]: ... def get_semantic_source_columns( *, project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, + reads: WarehouseReads, source_read_map: dict[str, SourceEntry], columns: dict[str, tuple[ColumnInfo, ...]], selected_keys: frozenset[CompiledObjectKey], @@ -6341,16 +6360,16 @@ def get_semantic_source_columns( def check_selected_source_tables_exist( *, project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, + dialect: Dialect, + reads: WarehouseReads, scope: PlannerScope, relations: PlannerRelationsContext, ) -> None: ... def build_planner_source_freshness_result( *, project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, + dialect: Dialect, + reads: WarehouseReads, scope: PlannerScope, relations: PlannerRelationsContext, freshness_state_schemas: frozenset[str] | None = None, @@ -6387,8 +6406,7 @@ def planner_metadata_name_filter( ) -> tuple[str, ...] | None: ... def list_source_candidates( *, - adapter: BaseAdapter, - connection: Any, + reads: WarehouseReads, database: str | None, entries: tuple[SourceEntry, ...], ) -> tuple[RelationInfo, ...]: ... @@ -6566,6 +6584,7 @@ class BuildCommandRequest(NativeRecord): cli_vars: dict[str, object] | None run_tests: bool run_audits: bool + ignore_audit_failures: bool | None manifest: bool json_output: bool json_output_path: Path | None @@ -6607,6 +6626,7 @@ class BuildCommandRequest(NativeRecord): cli_vars: dict[str, object] | None = None, run_tests: bool = True, run_audits: bool = True, + ignore_audit_failures: bool | None = None, manifest: bool = False, json_output: bool = False, json_output_path: Path | None = None, @@ -6967,6 +6987,8 @@ class PlanCommandRequest(NativeRecord): no_cache: bool max_microbatches: int | None selection_diagnostics: bool + timings: bool + ignore_audit_failures: bool | None def __init__( self, project_dir: Path | None = None, @@ -6991,6 +7013,8 @@ class PlanCommandRequest(NativeRecord): no_cache: bool = False, max_microbatches: int | None = None, selection_diagnostics: bool = False, + timings: bool = False, + ignore_audit_failures: bool | None = None, ) -> None: ... class PlaygroundCommandRequest(NativeRecord): @@ -10208,6 +10232,7 @@ class CompilePipelineOptions(NativeRecord): accepts_unit_test_selectors: bool sql_test_selection: SqlTestSelection source_lexical_syntax: SqlLexicalSyntax | None + ignore_audit_failures: bool def __init__( self, selected_target: str | None = ..., @@ -10235,6 +10260,7 @@ class CompilePipelineOptions(NativeRecord): accepts_unit_test_selectors: bool = ..., sql_test_selection: SqlTestSelection = ..., source_lexical_syntax: SqlLexicalSyntax | None = ..., + ignore_audit_failures: bool = ..., ) -> None: ... class PythonPlanEntry(NativeRecord): @@ -10490,12 +10516,14 @@ class PlannerPolicies(NativeRecord): selection_diagnostics: bool plan_sql_tests: bool record_migration_fingerprints: bool + ignore_audit_failures: bool def __init__( self, auto_load_sources: bool = ..., selection_diagnostics: bool = ..., plan_sql_tests: bool = ..., record_migration_fingerprints: bool = ..., + ignore_audit_failures: bool = ..., ) -> None: ... class DiscoveredPythonLoaderMetadata(NativeRecord): @@ -11957,6 +11985,7 @@ class TargetConfig(NativeRecord): warehouses: TargetWarehousesConfig concurrency: int | None catalog_connection: str | None + ignore_audit_failures: bool | None def __init__( self, connection: dict[str, object] = ..., @@ -11981,6 +12010,7 @@ class TargetConfig(NativeRecord): warehouses: TargetWarehousesConfig = ..., concurrency: int | None = ..., catalog_connection: str | None = ..., + ignore_audit_failures: bool | None = ..., ) -> None: ... class LocalTargetConfig(NativeRecord): @@ -12008,6 +12038,7 @@ class LocalTargetConfig(NativeRecord): warehouses: TargetWarehousesConfig concurrency: int | None catalog_connection: str | None + ignore_audit_failures: bool | None def __init__( self, connection: dict[str, object] = ..., @@ -12033,6 +12064,7 @@ class LocalTargetConfig(NativeRecord): warehouses: TargetWarehousesConfig = ..., concurrency: int | None = ..., catalog_connection: str | None = ..., + ignore_audit_failures: bool | None = ..., ) -> None: ... class SettingsConfig(NativeRecord): @@ -12057,6 +12089,7 @@ class SettingsConfig(NativeRecord): delta_compile: bool allow_missing_secrets: bool allow_warehouse_round_trips: bool + ignore_audit_failures: bool def __init__( self, sql_analysis: bool = ..., @@ -12078,6 +12111,7 @@ class SettingsConfig(NativeRecord): delta_compile: bool = ..., allow_missing_secrets: bool = ..., allow_warehouse_round_trips: bool = ..., + ignore_audit_failures: bool = ..., ) -> None: ... @property def sql_validation(self) -> bool: ... @@ -14271,6 +14305,7 @@ class AuditPlanEntry(NativeRecord): description: str | None reads_outside_target_lineage: bool meta: dict[str, object] | None + ignore_failure: bool def __init__( self, key: CompiledObjectKey, @@ -14299,6 +14334,7 @@ class AuditPlanEntry(NativeRecord): description: str | None = ..., reads_outside_target_lineage: bool = ..., meta: dict[str, object] | None = ..., + ignore_failure: bool = ..., ) -> None: ... class ChainStep(NativeRecord): @@ -14524,6 +14560,7 @@ class PlanOutput(NativeRecord): cursor_overrides: CursorOverrides | None existing_model_names: frozenset[str] end_audit_source_keys: frozenset[CompiledObjectKey] + ignore_audit_failures: bool def __init__( self, execution_order: tuple[CompiledObjectKey, ...] = ..., @@ -14560,6 +14597,7 @@ class PlanOutput(NativeRecord): cursor_overrides: CursorOverrides | None = ..., existing_model_names: frozenset[str] = ..., end_audit_source_keys: frozenset[CompiledObjectKey] = ..., + ignore_audit_failures: bool = ..., ) -> None: ... @property def python_source_entries(self) -> dict[str, SourceEntry]: ... diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index f0c625af14..12e9f34c50 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -39,6 +39,7 @@ UnsupportedTypedSqlRenderingError, ) from sqlbuild.adapter.contract.main.column_name_key import column_name_key +from sqlbuild.adapter.contract.main.estimates_one_at_a_time import estimates_one_at_a_time from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition from sqlbuild.adapter.contract.main.unsupported_relation_usage import unsupported_relation_usage @@ -119,6 +120,7 @@ class BaseAdapter(RetentionAdapterMixin, StrictAdapter): metadata_inspection_concurrency: ClassVar[int] = 1 concurrent_ddl_limit: ClassVar[int] = 1 supports_session_warehouse: ClassVar[bool] = False + history_reads_wake_compute: ClassVar[bool] = False state_timestamp_type: ClassVar[str | None] = None untyped_column_type: ClassVar[str | None] = None _snapshot_sql_dialect: ClassVar[SnapshotSqlDialect] = SnapshotSqlDialect( @@ -1685,6 +1687,18 @@ def estimate_relation_row_count( detail=f"adapter '{self.adapter_name}' does not report row counts from metadata", ) + def estimate_relation_row_counts( + self, *, connection: Any, database: str | None, schema: str, names: tuple[str, ...] + ) -> tuple[RelationRowCountEstimate, ...]: + """Metadata-only rows and bytes of `names` in one schema, one relation at a time.""" + + return estimates_one_at_a_time( + estimate=self.estimate_relation_row_count, + connection=connection, + location=(database, schema), + names=names, + ) + def _build_row_diff_relation_ctes( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/diff.py b/src/sqlbuild/adapter/contract/classes/diff.py index 376cc6debe..688ee52341 100644 --- a/src/sqlbuild/adapter/contract/classes/diff.py +++ b/src/sqlbuild/adapter/contract/classes/diff.py @@ -89,6 +89,13 @@ def estimate_relation_row_count( """Estimate counts, forbidding data scans when metadata_only is requested.""" ... + @abstractmethod + def estimate_relation_row_counts( + self, *, connection: Any, database: str | None, schema: str, names: tuple[str, ...] + ) -> tuple[RelationRowCountEstimate, ...]: + """Metadata-only estimates of `names` in one schema, in order, never scanning data.""" + ... + @abstractmethod def sample_unequal_rows( self, diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 9060b04c15..6929cd572c 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -51,6 +51,7 @@ ) from sqlbuild.adapter.contract.exceptions import AdapterUserError, ViewRenameRefusedError from sqlbuild.adapter.contract.main.column_name_key import column_name_key +from sqlbuild.adapter.contract.main.estimates_one_at_a_time import estimates_one_at_a_time from sqlbuild.adapter.contract.main.is_quoted_identifier import is_quoted_identifier from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition @@ -2320,6 +2321,18 @@ def estimate_relation_row_count( ) return RelationRowCountEstimate(row_count=int(row[0])) + def estimate_relation_row_counts( + self, *, connection: Any, database: str | None, schema: str, names: tuple[str, ...] + ) -> tuple[RelationRowCountEstimate, ...]: + """Metadata-only estimates of `names` in one schema, one relation at a time.""" + + return estimates_one_at_a_time( + estimate=self.estimate_relation_row_count, + connection=connection, + location=(database, schema), + names=names, + ) + def diff_schema( self, *, diff --git a/src/sqlbuild/adapter/contract/main/estimates_by_name.py b/src/sqlbuild/adapter/contract/main/estimates_by_name.py new file mode 100644 index 0000000000..ab05468aa6 --- /dev/null +++ b/src/sqlbuild/adapter/contract/main/estimates_by_name.py @@ -0,0 +1,20 @@ +"""Row count estimates of named relations from one catalog query's rows.""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from typing import Any + +from sqlbuild.adapter.contract.models import RelationRowCountEstimate + + +def estimates_by_name( + *, + rows: Sequence[Sequence[Any]], + names: tuple[str, ...], + estimate: Callable[[tuple[Any, ...] | None], RelationRowCountEstimate], +) -> tuple[RelationRowCountEstimate, ...]: + """Each name's estimate, in order, from the row leading with that name, or from `None`.""" + + by_name: dict[str, tuple[Any, ...]] = {str(row[0]): tuple(row[1:]) for row in rows} + return tuple(estimate(by_name.get(name)) for name in names) diff --git a/src/sqlbuild/adapter/contract/main/estimates_one_at_a_time.py b/src/sqlbuild/adapter/contract/main/estimates_one_at_a_time.py new file mode 100644 index 0000000000..fb311e23e4 --- /dev/null +++ b/src/sqlbuild/adapter/contract/main/estimates_one_at_a_time.py @@ -0,0 +1,28 @@ +"""Metadata-only row count estimates of named relations, read one relation at a time.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from sqlbuild.adapter.contract.models import RelationRowCountEstimate + + +def estimates_one_at_a_time( + *, + estimate: Callable[..., RelationRowCountEstimate], + connection: Any, + location: tuple[str | None, str], + names: tuple[str, ...], +) -> tuple[RelationRowCountEstimate, ...]: + """Each name's metadata-only `estimate` in `location`'s `(database, schema)`, in order.""" + + database: str | None + schema: str + database, schema = location + return tuple( + estimate( + connection=connection, database=database, schema=schema, name=name, metadata_only=True + ) + for name in names + ) diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index c8ae2ec073..589a6b5478 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -55,6 +55,7 @@ from sqlbuild.adapter.contract.main.complete_table_freshness_results import ( complete_table_freshness_results, ) +from sqlbuild.adapter.contract.main.estimates_one_at_a_time import estimates_one_at_a_time from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition from sqlbuild.adapter.contract.models import ( @@ -2646,6 +2647,18 @@ def estimate_relation_row_count( byte_count_basis="tables_api_num_bytes" if isinstance(byte_count, int) else None, ) + def estimate_relation_row_counts( + self, *, connection: Any, database: str | None, schema: str, names: tuple[str, ...] + ) -> tuple[RelationRowCountEstimate, ...]: + """Metadata-only estimates of `names` in one dataset, one table at a time.""" + + return estimates_one_at_a_time( + estimate=self.estimate_relation_row_count, + connection=connection, + location=(database, schema), + names=names, + ) + def inspect_row_diff_coverage( self, *, diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 854c772a33..48dc9308b3 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -39,6 +39,7 @@ from sqlbuild.adapter.contract.classes.unkeyed_diff import UnkeyedDiffMixin from sqlbuild.adapter.contract.constants import DIFF_LEFT_SIDE, DIFF_RIGHT_SIDE from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapter.contract.main.estimates_by_name import estimates_by_name from sqlbuild.adapter.contract.main.normalize_seed_csv_value import normalize_seed_csv_value from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition @@ -175,6 +176,7 @@ class ClickHouseAdapter( max_identifier_length: ClassVar[int] = MAX_IDENTIFIER_LENGTH relation_grants_supported: ClassVar[bool] = False state_timestamp_type: ClassVar[str | None] = STATE_TIMESTAMP_TYPE + history_reads_wake_compute: ClassVar[bool] = True _snapshot_sql_dialect: ClassVar[SnapshotSqlDialect] = SnapshotSqlDialect( timestamp_type="Nullable(DateTime64(6, 'UTC'))", distinct_condition=render_is_distinct_from, @@ -494,14 +496,25 @@ def estimate_relation_row_count( + _database_filter(schema) + f" AND name = {quote_string(name)}", ).fetchone() - if row is None: - return RelationRowCountEstimate(row_count=None, detail="relation not found") - byte_count: int | None = int(row[2]) if metadata_only and row[2] is not None else None - return RelationRowCountEstimate( - row_count=int(row[1]) if row[1] is not None else None, - detail=f"engine {row[0]} does not report row counts" if row[1] is None else None, - byte_count=byte_count, - byte_count_basis="system_tables_total_bytes" if byte_count is not None else None, + return _row_count_estimate(row=row, metadata_only=metadata_only) + + def estimate_relation_row_counts( + self, *, connection: Any, database: str | None, schema: str, names: tuple[str, ...] + ) -> tuple[RelationRowCountEstimate, ...]: + """Read every named table's rows and bytes from ``system.tables`` in one query.""" + + del database + if not names: + return () + return estimates_by_name( + rows=self.execute( + connection=connection, + sql="SELECT name, engine, total_rows, total_bytes FROM system.tables WHERE " + + _database_filter(schema) + + f" AND name IN ({', '.join(quote_string(name) for name in names)})", + ).fetchall(), + names=names, + estimate=lambda row: _row_count_estimate(row=row, metadata_only=True), ) def read_view_definition( @@ -2824,3 +2837,17 @@ def _prune_history_sql( f"PARTITION BY {partition} ORDER BY {order}) AS __sqlbuild_history_rank FROM {table}) " f"WHERE __sqlbuild_history_rank > {retain_versions}) SETTINGS transform_null_in = 1" ) + + +def _row_count_estimate(*, row: Any, metadata_only: bool) -> RelationRowCountEstimate: + """The estimate of one ``(engine, total_rows[, total_bytes])`` row, or of no row.""" + + if row is None: + return RelationRowCountEstimate(row_count=None, detail="relation not found") + byte_count: int | None = int(row[2]) if metadata_only and row[2] is not None else None + return RelationRowCountEstimate( + row_count=int(row[1]) if row[1] is not None else None, + detail=f"engine {row[0]} does not report row counts" if row[1] is None else None, + byte_count=byte_count, + byte_count_basis="system_tables_total_bytes" if byte_count is not None else None, + ) diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index af1bd65b96..2696200e7d 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -53,6 +53,7 @@ from sqlbuild.adapter.contract.main.complete_table_freshness_results import ( complete_table_freshness_results, ) +from sqlbuild.adapter.contract.main.estimates_one_at_a_time import estimates_one_at_a_time from sqlbuild.adapter.contract.main.normalize_seed_csv_value import normalize_seed_csv_value from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition @@ -171,6 +172,7 @@ def get_columns_for_relations( execution_duration_limit_seconds: ClassVar[int | None] = CANCELLABLE_DURATION_LIMIT_SECONDS statement_cancellation_is_asynchronous: ClassVar[bool] = True adapter_name: ClassVar[str] = BuiltinAdapter.DATABRICKS.value + history_reads_wake_compute: ClassVar[bool] = True sql_analysis_dialect_name: ClassVar[str | None] = "databricks" sql_lexical_syntax: ClassVar[SqlLexicalSyntax] = SqlLexicalSyntax( backslash_escape_quotes=frozenset({"'", '"'}), @@ -2646,6 +2648,18 @@ def estimate_relation_row_count( byte_count_basis="describe_table_statistics_bytes" if bytes_match is not None else None, ) + def estimate_relation_row_counts( + self, *, connection: Any, database: str | None, schema: str, names: tuple[str, ...] + ) -> tuple[RelationRowCountEstimate, ...]: + """Metadata-only estimates of `names` in one schema, one relation at a time.""" + + return estimates_one_at_a_time( + estimate=self.estimate_relation_row_count, + connection=connection, + location=(database, schema), + names=names, + ) + def inspect_row_diff_coverage( self, *, diff --git a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py index ed78457cd2..5567677291 100644 --- a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py +++ b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py @@ -27,6 +27,7 @@ class MotherDuckAdapter( """MotherDuck adapter backed by DuckDB's MotherDuck connection support.""" adapter_name: ClassVar[str] = BuiltinAdapter.MOTHERDUCK.value + history_reads_wake_compute: ClassVar[bool] = True sql_analysis_dialect_name: ClassVar[str | None] = "duckdb" def render_loader_rows_select( diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 442ec02f9d..13e44efa93 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -51,6 +51,7 @@ QUALIFIED_NAME_SEPARATOR, ) from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapter.contract.main.estimates_by_name import estimates_by_name from sqlbuild.adapter.contract.main.normalize_seed_csv_value import normalize_seed_csv_value from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition @@ -2434,6 +2435,38 @@ def estimate_relation_row_count( f"WHERE {schema_filter} AND relation.relname = {_quote_sql_string(name)}" ), ).fetchone() + return self._row_count_estimate(row=row, metadata_only=metadata_only) + + def estimate_relation_row_counts( + self, *, connection: Any, database: str | None, schema: str, names: tuple[str, ...] + ) -> tuple[RelationRowCountEstimate, ...]: + """Read every named relation's ``reltuples`` and size with one catalog query.""" + + del database + if not names: + return () + listed: str = ", ".join(_quote_sql_string(name) for name in names) + return estimates_by_name( + rows=self.execute( + connection=connection, + sql=( + "SELECT relation.relname, relation.relkind, relation.reltuples, " + "pg_relation_size(relation.oid) FROM pg_class AS relation " + "JOIN pg_namespace AS namespace ON namespace.oid = relation.relnamespace " + f"WHERE namespace.nspname = {_quote_sql_string(schema)} " + f"AND relation.relname IN ({listed})" + ), + ).fetchall(), + names=names, + estimate=lambda row: self._row_count_estimate(row=row, metadata_only=True), + ) + + @staticmethod + def _row_count_estimate( + *, row: tuple[Any, ...] | None, metadata_only: bool + ) -> RelationRowCountEstimate: + """The estimate of one ``(relkind, reltuples, size)`` catalog row, or of no row.""" + if row is None: return RelationRowCountEstimate(row_count=None, detail="relation not found") if str(row[0]) not in ROW_COUNT_RELATION_KINDS: diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 59199946b8..a0afac9244 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -6,6 +6,7 @@ import json import logging import os +import time from collections.abc import Callable from dataclasses import replace from datetime import date, datetime @@ -188,6 +189,8 @@ _STATE_TABLE_RETENTION_ATTRIBUTE: str = "_state_table_retention_fallbacks_state" _METADATA_INSPECTION_CONCURRENCY: int = 8 _SHOW_RESULT_LIMIT: int = 10_000 +_SCHEMA_OBJECTS_ATTRIBUTE: str = "_schema_objects_listings" +_SCHEMA_OBJECTS_REUSE_SECONDS: float = 60.0 _SHOW_OBJECT_RELATION_TYPES: dict[str, str] = {"TABLE": "base table", "VIEW": "view"} _COST_LOGIN_TIMEOUT_SECONDS: int = 5 _COST_NETWORK_TIMEOUT_SECONDS: int = 15 @@ -3663,6 +3666,67 @@ def estimate_relation_row_count( byte_count_basis=basis if byte_count is not None else None, ) + def estimate_relation_row_counts( + self, + *, + connection: _SnowflakeConnection, + database: str | None, + schema: str, + names: tuple[str, ...], + ) -> tuple[RelationRowCountEstimate, ...]: + """Rows and bytes from one SHOW OBJECTS IN SCHEMA a minute, which needs no warehouse.""" + + objects: dict[str, dict[str, object]] = self._schema_objects( + connection=connection, database=database, schema=schema + ) + estimates: list[RelationRowCountEstimate] = [] + name: str + for name in names: + row: dict[str, object] | None = objects.get(self._information_schema_identifier(name)) + if row is None: + estimates.append( + RelationRowCountEstimate(row_count=None, detail="relation not found") + ) + continue + is_table: bool = str(row.get("kind")) == SHOW_OBJECTS_TABLE_KIND + rows: object = row.get("rows") if is_table else None + byte_count: object = row.get("bytes") if is_table else None + estimates.append( + RelationRowCountEstimate( + row_count=None if rows is None else int(cast(int, rows)), + detail="no row count (view)" if rows is None else None, + byte_count=None if byte_count is None else int(cast(int, byte_count)), + byte_count_basis=None if byte_count is None else "show_objects_bytes", + ) + ) + return tuple(estimates) + + def _schema_objects( + self, *, connection: _SnowflakeConnection, database: str | None, schema: str + ) -> dict[str, dict[str, object]]: + """One schema's SHOW OBJECTS rows by stored name, listed again after a minute.""" + + listings: dict[tuple[int, str | None, str], tuple[float, dict[str, dict[str, object]]]] + listings = getattr(self, _SCHEMA_OBJECTS_ATTRIBUTE, None) or {} + key: tuple[int, str | None, str] = (id(connection), database, schema) + now: float = time.monotonic() + cached: tuple[float, dict[str, dict[str, object]]] | None = listings.get(key) + if cached is not None and now - cached[0] < _SCHEMA_OBJECTS_REUSE_SECONDS: + return cached[1] + rows: list[dict[str, object]] = self._fetch_paged_show_rows( + connection=connection, + query="SHOW OBJECTS IN SCHEMA " + + self._show_schema_scope(connection=connection, database=database, schema=schema), + show_database=self._show_database( + connection=connection, database=database, schema=schema + ), + schema=schema, + ) + objects: dict[str, dict[str, object]] = {str(row["name"]): row for row in rows} + listings[key] = (now, objects) + setattr(self, _SCHEMA_OBJECTS_ATTRIBUTE, listings) + return objects + def _show_object_row_count( self, *, connection: Any, database: str | None, schema: str, name: str ) -> tuple[Any, Any] | None: diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py index e8759a3ed2..664fac4c02 100644 --- a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py @@ -55,6 +55,7 @@ DIFF_RIGHT_SIDE, ) from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapter.contract.main.estimates_one_at_a_time import estimates_one_at_a_time from sqlbuild.adapter.contract.main.normalize_seed_csv_value import normalize_seed_csv_value from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition @@ -1551,6 +1552,18 @@ def estimate_relation_row_count( ).fetchone() return RelationRowCountEstimate(row_count=int(count[0])) + def estimate_relation_row_counts( + self, *, connection: Any, database: str | None, schema: str, names: tuple[str, ...] + ) -> tuple[RelationRowCountEstimate, ...]: + """Metadata-only estimates of `names` in one schema, one relation at a time.""" + + return estimates_one_at_a_time( + estimate=self.estimate_relation_row_count, + connection=connection, + location=(database, schema), + names=names, + ) + def diff_schema( self, *, diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 26c30be512..274308f4df 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -54,6 +54,7 @@ AdapterUserError, UnsupportedTypedSqlRenderingError, ) +from sqlbuild.adapter.contract.main.estimates_by_name import estimates_by_name from sqlbuild.adapter.contract.main.normalize_seed_csv_value import normalize_seed_csv_value from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition @@ -1404,11 +1405,30 @@ def estimate_relation_row_count( "GROUP BY objects.type" ), ).fetchone() - if row is None: - return RelationRowCountEstimate(row_count=None, detail="relation not found") - if str(row[0]).strip().upper() != USER_TABLE_OBJECT_TYPE or row[1] is None: - return RelationRowCountEstimate(row_count=None, detail="no row count (view)") - return RelationRowCountEstimate(row_count=int(row[1])) + return _row_count_estimate(row) + + def estimate_relation_row_counts( + self, *, connection: Any, database: str | None, schema: str, names: tuple[str, ...] + ) -> tuple[RelationRowCountEstimate, ...]: + """Sum every named table's partition rows with one catalog query.""" + + if not names: + return () + catalog: str = "" if database is None else f"{self.render_identifier(database)}." + listed: str = ", ".join(f"N{_quote_sql_string(name)}" for name in names) + rows: list[tuple[Any, ...]] = self.execute( + connection=connection, + sql=( + "SELECT objects.name, objects.type, SUM(partitions.rows) " + f"FROM {catalog}sys.objects AS objects " + f"JOIN {catalog}sys.schemas AS schemas ON schemas.schema_id = objects.schema_id " + f"LEFT JOIN {catalog}sys.partitions AS partitions " + "ON partitions.object_id = objects.object_id AND partitions.index_id IN (0, 1) " + f"WHERE schemas.name = N{_quote_sql_string(schema)} AND objects.name IN ({listed}) " + "GROUP BY objects.name, objects.type" + ), + ).fetchall() + return estimates_by_name(rows=rows, names=names, estimate=_row_count_estimate) def inspect_row_diff_coverage( self, @@ -3108,3 +3128,13 @@ def load_seed( cursor.executemany(insert_sql, rows) finally: cursor.close() + + +def _row_count_estimate(row: tuple[Any, ...] | None) -> RelationRowCountEstimate: + """The estimate of one ``(object type, partition rows)`` catalog row, or of no row.""" + + if row is None: + return RelationRowCountEstimate(row_count=None, detail="relation not found") + if str(row[0]).strip().upper() != USER_TABLE_OBJECT_TYPE or row[1] is None: + return RelationRowCountEstimate(row_count=None, detail="no row count (view)") + return RelationRowCountEstimate(row_count=int(row[1])) diff --git a/src/sqlbuild/executor/audit_results/constants.py b/src/sqlbuild/executor/audit_results/constants.py index 0e9f68eacc..94ddfe7b1e 100644 --- a/src/sqlbuild/executor/audit_results/constants.py +++ b/src/sqlbuild/executor/audit_results/constants.py @@ -97,3 +97,5 @@ COLUMN_MINIMUM_SAMPLES: StateSqlValueType.INTEGER, COLUMN_EVIDENCE_COUNT: StateSqlValueType.INTEGER, } + +IGNORED_FAILURE_METADATA_JSON: str = '{"ignored": true}' diff --git a/src/sqlbuild/executor/auditing/_helpers/execution.py b/src/sqlbuild/executor/auditing/_helpers/execution.py index d22010c7c1..ded4ca0625 100644 --- a/src/sqlbuild/executor/auditing/_helpers/execution.py +++ b/src/sqlbuild/executor/auditing/_helpers/execution.py @@ -281,6 +281,7 @@ def _base_result( sample_unit=audit.sample_unit, minimum_samples=audit.minimum_samples, thresholds=audit.thresholds, + ignored=audit.ignore_failure and outcome == AuditOutcome.ERROR, ) diff --git a/src/sqlbuild/executor/auditing/_helpers/result_projection.py b/src/sqlbuild/executor/auditing/_helpers/result_projection.py index a6a8f91cb6..c48889f2a5 100644 --- a/src/sqlbuild/executor/auditing/_helpers/result_projection.py +++ b/src/sqlbuild/executor/auditing/_helpers/result_projection.py @@ -23,7 +23,10 @@ ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditEvaluationMode, ThresholdOperator -from sqlbuild.executor.audit_results.constants import AUDIT_RESULT_SCHEMA_VERSION +from sqlbuild.executor.audit_results.constants import ( + AUDIT_RESULT_SCHEMA_VERSION, + IGNORED_FAILURE_METADATA_JSON, +) from sqlbuild.executor.audit_results.exceptions import AuditResultStorageError from sqlbuild.executor.audit_results.main._write import write_audit_result_records from sqlbuild.executor.audit_results.models import ( @@ -363,7 +366,7 @@ def _build_record( # noqa: PLR0913 evidence_sql=result.evidence_sql, executed_sql=result.executed_sql, sql_digest=hashlib.sha256(result.executed_sql.encode("utf-8")).hexdigest(), - metadata_json=None, + metadata_json=IGNORED_FAILURE_METADATA_JSON if result.ignored else None, reused=False, ) @@ -398,6 +401,8 @@ def _publish_audit_completed(record: AuditResultRecord) -> None: "reused": False, "result_id": record.result_id, } + if record.metadata_json == IGNORED_FAILURE_METADATA_JSON: + payload["ignored"] = True optional: dict[str, JSONValue | None] = { "attached_target_kind": record.attached_target_kind, "attached_target_name": record.attached_target_name, diff --git a/src/sqlbuild/executor/auditing/models.py b/src/sqlbuild/executor/auditing/models.py index eaf0f4dd21..8dceb6e2e7 100644 --- a/src/sqlbuild/executor/auditing/models.py +++ b/src/sqlbuild/executor/auditing/models.py @@ -75,3 +75,10 @@ class AuditExecutionResult: execution_error: str | None = None audit_meta: Mapping[str, object] | None = None skip_reason: str | None = None + ignored: bool = False + + @property + def blocks(self) -> bool: + """Whether this failure holds back what depends on it; an ignored failure holds nothing.""" + + return self.outcome == AuditOutcome.ERROR and not self.ignored diff --git a/src/sqlbuild/executor/build/_helpers/source_audits.py b/src/sqlbuild/executor/build/_helpers/source_audits.py index 1170fbe316..08035a99a7 100644 --- a/src/sqlbuild/executor/build/_helpers/source_audits.py +++ b/src/sqlbuild/executor/build/_helpers/source_audits.py @@ -93,7 +93,7 @@ def run_pending_source_audits( lifecycle.failed() audit_results.append(result) publish_completed_audit_results((result,)) - if result.outcome == AuditOutcome.ERROR: + if result.blocks: failed_names.append(source_name) newly_blocked.update( downstream_blocked_keys( diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 84a73638cf..538d3971b9 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -1457,7 +1457,7 @@ def _with_seed_audits( quality_scope="seed", ) self._seed_audit_results[key] = results - if not any(audit.outcome == AuditOutcome.ERROR for audit in results): + if not any(audit.blocks for audit in results): return result return dataclasses.replace( result, diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index 5905b2702a..2883e40c8f 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -235,6 +235,7 @@ class BuildExecutionResult: skipped_count: int = 0 warning_count: int = 0 insufficient_count: int = 0 + ignored_audit_failure_count: int = 0 audit_result_projection_attempted_count: int = 0 audit_result_projection_written_count: int = 0 audit_result_projection_failed_count: int = 0 diff --git a/src/sqlbuild/executor/pipeline/_helpers/scenario.py b/src/sqlbuild/executor/pipeline/_helpers/scenario.py index 372658a443..e2b083a049 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/scenario.py +++ b/src/sqlbuild/executor/pipeline/_helpers/scenario.py @@ -20,6 +20,7 @@ from sqlbuild._native import ( CompiledSqlScenario, CompilePipelineResult, + Dialect, NativeProgressProjector, ScenarioExecutionPlan, SqlLexicalSyntax, @@ -701,7 +702,7 @@ def _run_scenario[ResultT: _ScenarioPipelineResult]( scenario_plan = build_scenario_plan( scenario=scenario, pipeline_result=pipeline_result, - adapter=adapter, + dialect=Dialect(adapter), project_name=project_name, source_lexical_syntax=source_lexical_syntax, ) @@ -753,7 +754,7 @@ def select_scenario_snapshot_capture_candidates( scenario_plan: ScenarioExecutionPlan = build_scenario_plan( scenario=scenario, pipeline_result=pipeline_result, - adapter=adapter, + dialect=Dialect(adapter), project_name=project_name, source_lexical_syntax=source_lexical_syntax, ) diff --git a/src/sqlbuild/executor/run/_helpers/execution/final_audits.py b/src/sqlbuild/executor/run/_helpers/execution/final_audits.py index 7f0ab92c27..c175101d88 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/final_audits.py +++ b/src/sqlbuild/executor/run/_helpers/execution/final_audits.py @@ -6,7 +6,7 @@ from sqlbuild._native import AuditPlanEntry, CompiledRelationLocation, Fingerprint, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope +from sqlbuild.compiler.auditing.types import AuditRunScope from sqlbuild.executor.auditing.main._execute import execute_audit from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.reuse.audit import ( @@ -59,7 +59,7 @@ def run_final_model_audits( quality_scope="final", ) results.append(result) - if result.outcome == AuditOutcome.ERROR: + if result.blocks: has_error = True return FinalAuditRun(results=tuple(results), has_error=has_error) @@ -106,6 +106,6 @@ def run_delta_scope_audits( quality_scope="delta", ) results.append(result) - if result.outcome == AuditOutcome.ERROR: + if result.blocks: has_error = True return FinalAuditRun(results=tuple(results), has_error=has_error) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/view.py b/src/sqlbuild/executor/run/_helpers/materializations/view.py index 69ea65c384..1ab396dd2c 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/view.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/view.py @@ -19,7 +19,7 @@ from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) -from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope +from sqlbuild.compiler.auditing.types import AuditRunScope from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.auditing.main._execute import execute_audit from sqlbuild.executor.auditing.models import AuditExecutionResult @@ -146,7 +146,7 @@ def execute_view_entry( quality_scope="final", ) audit_results.append(result) - if result.outcome == AuditOutcome.ERROR: + if result.blocks: audit_error = True if audit_error: diff --git a/src/sqlbuild/orchestration_declarations/types.py b/src/sqlbuild/orchestration_declarations/types.py index 8149b173a6..b8b1151cf0 100644 --- a/src/sqlbuild/orchestration_declarations/types.py +++ b/src/sqlbuild/orchestration_declarations/types.py @@ -23,6 +23,7 @@ class RunEventKind(StrEnum): PARTIAL = "partial" FAILED = "failed" CANCELLED = "cancelled" + SKIPPED = "skipped" class NodeEventKind(StrEnum): diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 653d10a8bf..8defaec979 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1634,6 +1634,59 @@ class PythonUpstreamResultE2ETestCase: expected_error: str +@dataclass(frozen=True) +class WarehouseRoundTripE2ETestCase: + """A build of orders -> score task -> partner loader -> source -> scored model.""" + + description: str + project_settings: str + task_body: str + expected_exit_code: int + expected_output_order: tuple[str, ...] + expected_output_fragments: tuple[str, ...] + expected_scored_rows: tuple[tuple[object, ...], ...] | None + expected_scored_table_exists: bool = True + + +@dataclass(frozen=True) +class ChangedRoundTripE2ETestCase: + """A second round-trip build with `build_flags`, after changing `orders` and `extra_files`.""" + + description: str + orders_sql: str + expected_output_fragments: tuple[str, ...] + expected_scored_rows: tuple[tuple[object, ...], ...] + build_flags: tuple[str, ...] = ("--changed",) + extra_files: dict[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class IgnoredAuditFailureE2ETestCase: + """One blocking path built with audit failures held back or ignored.""" + + description: str + project_files: dict[str, str] + seed_sql: str + settings_toml: str + build_args: tuple[str, ...] + expected_exit_code: int + expected_tables: dict[str, int] + expected_audit_results: tuple[tuple[str, str, bool], ...] + expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class IgnoredAuditFailureCommandE2ETestCase: + """A command whose output says whether audit failures are ignored, or refuses the flags.""" + + description: str + settings_toml: str + command: tuple[str, ...] + expected_exit_code: int + expected_fragments: tuple[str, ...] + expected_missing_fragments: tuple[str, ...] + + @dataclass(frozen=True) class PythonTargetConfigBuildE2ETestCase: """Test case for the active target's settings seen by Python nodes and loaders.""" @@ -1718,29 +1771,3 @@ class AdapterDefaultResultsFallbackE2ETestCase: kept_loader: str reading_task: str expected_payload_keys: tuple[str, str] - - -@dataclass(frozen=True) -class WarehouseRoundTripE2ETestCase: - """A build of orders -> score task -> partner loader -> source -> scored model.""" - - description: str - project_settings: str - task_body: str - expected_exit_code: int - expected_output_order: tuple[str, ...] - expected_output_fragments: tuple[str, ...] - expected_scored_rows: tuple[tuple[object, ...], ...] | None - expected_scored_table_exists: bool = True - - -@dataclass(frozen=True) -class ChangedRoundTripE2ETestCase: - """A second round-trip build with `build_flags`, after changing `orders` and `extra_files`.""" - - description: str - orders_sql: str - expected_output_fragments: tuple[str, ...] - expected_scored_rows: tuple[tuple[object, ...], ...] - build_flags: tuple[str, ...] = ("--changed",) - extra_files: dict[str, str] = field(default_factory=dict) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 64a8fdd8ce..447fecc6c7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -25,6 +25,7 @@ execute_duckdb, prepare_inline_project, query_duckdb, + row_count, run_sqb, ) from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import duckshop_orchestration @@ -1492,49 +1493,6 @@ def changed_build_steps(warehouse: ChangedWarehouse) -> tuple[ChangedBuildStep, ) -def node_result_source_yml(location: str) -> str: - """A managed `raw_orders` source at `location`, for node result location tests.""" - - return ( - "sources:\n - name: raw_orders\n description: Raw orders.\n" - f" managed: true\n{location} write_strategy: table\n" - " columns:\n - name: order_id\n type: INTEGER\n" - ) - - -def node_result_project(*, tmp_path: Path, target_lines: str, files: dict[str, str]) -> Path: - """A DuckDB orders project whose `dev` target has `target_lines`, plus `files`.""" - - return prepare_inline_project( - tmp_path=tmp_path, - project_name="orders", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' - '[connection]\ndatabase = "dev.duckdb"\n\n' - f'[defaults]\ndatabase = "dev"\nschema = "mart"\n\n[targets.dev]\n{target_lines}' - ), - "models/orders.sql": ( - "MODEL (description 'Orders.', materialized table);\n\n" - 'SELECT order_id FROM __source("raw_orders")\n' - ), - } - | files, - ) - - -def node_results_in(*, db_path: Path, schema: str, node_type: str) -> list[tuple[str, str, str]]: - """`(node_name, run_id, payload_json_b64)` of one node type's results in `schema`.""" - - return query_duckdb( - db_path=db_path, - sql=( - f"SELECT node_name, run_id, payload_json_b64 FROM {schema}._sqlbuild_node_results " - f"WHERE node_type = '{node_type}' ORDER BY node_name" - ), - ) - - ALLOW_ROUND_TRIPS: str = "\n[settings]\nallow_warehouse_round_trips = true\n" SCORING_TASK: str = ( ' relation = ctx.relation(model("orders"))\n' @@ -1647,3 +1605,145 @@ def build_round_trip_project( assert positions == sorted(positions), output assert all(fragment in output for fragment in test_case.expected_output_fragments), output return project_dir / "scoring.duckdb", output + + +# ---- builds that ignore audit failures ---- + +IGNORED_AUDITS_DATABASE: str = "shop.duckdb" +_IGNORED_AUDITS_PROJECT_TOML: str = ( + 'name = "ignored_audits"\nadapter = "duckdb"\n\n' + f'[connection]\ndatabase = "{IGNORED_AUDITS_DATABASE}"\n' +) +_EXPRESSION_AUDIT: str = "AUDIT ();\n\nSELECT * FROM @relation WHERE NOT (@expression)\n" +_RAW_ORDERS_SQL: str = ( + "CREATE TABLE main.raw_orders AS SELECT * FROM (VALUES (1, 100), (2, 200)) t(id, amount_cents)" +) +# A table whose final audit fails, and a model that reads it. +STAGED_AUDIT_FILES: dict[str, str] = { + "audits/generic/expression_is_true.sql": _EXPRESSION_AUDIT, + "models/daily.sql": ( + "MODEL (description 'Daily totals.', materialized table, audits [expression_is_true " + '(name "total_is_negative", expression "total_cents < 0", severity error)]);\n\n' + "SELECT 1 AS day, 300 AS total_cents\n" + ), + "models/daily_summary.sql": ( + "MODEL (description 'Reads daily.', materialized table);\n\n" + 'SELECT day FROM __ref("daily")\n' + ), +} +# An incremental model whose delta audit fails, over one row it already holds. +INCREMENTAL_AUDIT_FILES: dict[str, str] = { + "audits/generic/expression_is_true.sql": _EXPRESSION_AUDIT, + "sources/raw.yml": ( + "sources:\n - name: raw_orders\n description: Raw orders.\n" + " schema: main\n table: raw_orders\n" + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', materialized incremental, incremental_strategy merge, " + 'unique_key [id], audits [expression_is_true (name "amount_is_negative", ' + 'expression "amount_cents < 0", severity error, run_scope delta_and_final)]);\n\n' + 'SELECT id, amount_cents FROM __source("raw_orders")\n' + ), +} +INCREMENTAL_AUDIT_SEED_SQL: str = ( + f"{_RAW_ORDERS_SQL}; CREATE TABLE main.orders AS SELECT 0 AS id, 50 AS amount_cents" +) +# A source whose audit fails, and a model that reads it. +SOURCE_AUDIT_FILES: dict[str, str] = { + "audits/generic/expression_is_true.sql": _EXPRESSION_AUDIT, + "sources/raw.yml": ( + "sources:\n - name: raw_orders\n description: Raw orders.\n" + " schema: main\n table: raw_orders\n audits:\n" + " - expression_is_true:\n name: raw_amount_is_negative\n" + ' expression: "amount_cents < 0"\n severity: error\n' + ), + "models/order_count.sql": ( + "MODEL (description 'Reads raw orders.', materialized table);\n\n" + 'SELECT count(*) AS order_count FROM __source("raw_orders")\n' + ), +} +SOURCE_AUDIT_SEED_SQL: str = _RAW_ORDERS_SQL + + +def ignored_audits_project( + tmp_path: Path, files: dict[str, str], seed_sql: str, settings_toml: str +) -> Path: + """A DuckDB project of `files`, `settings_toml` after its connection, its warehouse seeded.""" + + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="ignored_audits", + repo_files={"sqlbuild_project.toml": _IGNORED_AUDITS_PROJECT_TOML + settings_toml, **files}, + ) + execute_duckdb(db_path=project_dir / IGNORED_AUDITS_DATABASE, sql=seed_sql) + return project_dir + + +def warehouse_table_rows(db_path: Path) -> dict[str, int]: + """Each project table in `main` with its row count, leaving out SQLBuild's own state tables.""" + + names: list[tuple[Any, ...]] = query_duckdb( + db_path=db_path, + sql=( + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'main' " + "AND table_name NOT LIKE '\\_sqlbuild%' ESCAPE '\\' ORDER BY table_name" + ), + ) + return {str(name): row_count(db_path=db_path, table_name=str(name)) for (name,) in names} + + +def recorded_audit_outcomes(db_path: Path) -> tuple[tuple[str, str, bool], ...]: + """Each stored audit result's name, outcome and whether its build ignored it, by name.""" + + rows: list[tuple[Any, ...]] = query_duckdb( + db_path=db_path, + sql=( + "SELECT audit_name, outcome, " + "coalesce(json_extract(metadata_json, '$.ignored')::BOOLEAN, false) " + "FROM main._sqlbuild_audit_results ORDER BY audit_name, run_scope_phase" + ), + ) + return tuple((str(name), str(outcome), bool(ignored)) for name, outcome, ignored in rows) + + +def node_result_source_yml(location: str) -> str: + """A managed `raw_orders` source at `location`, for node result location tests.""" + + return ( + "sources:\n - name: raw_orders\n description: Raw orders.\n" + f" managed: true\n{location} write_strategy: table\n" + " columns:\n - name: order_id\n type: INTEGER\n" + ) + + +def node_result_project(*, tmp_path: Path, target_lines: str, files: dict[str, str]) -> Path: + """A DuckDB orders project whose `dev` target has `target_lines`, plus `files`.""" + + return prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "dev.duckdb"\n\n' + f'[defaults]\ndatabase = "dev"\nschema = "mart"\n\n[targets.dev]\n{target_lines}' + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', materialized table);\n\n" + 'SELECT order_id FROM __source("raw_orders")\n' + ), + } + | files, + ) + + +def node_results_in(*, db_path: Path, schema: str, node_type: str) -> list[tuple[str, str, str]]: + """`(node_name, run_id, payload_json_b64)` of one node type's results in `schema`.""" + + return query_duckdb( + db_path=db_path, + sql=( + f"SELECT node_name, run_id, payload_json_b64 FROM {schema}._sqlbuild_node_results " + f"WHERE node_type = '{node_type}' ORDER BY node_name" + ), + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_ignored_audit_failures.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_ignored_audit_failures.py new file mode 100644 index 0000000000..57eba8bddd --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_ignored_audit_failures.py @@ -0,0 +1,239 @@ +"""Real-CLI coverage of builds that let failing audits through instead of holding work back.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + IgnoredAuditFailureCommandE2ETestCase, + IgnoredAuditFailureE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( + IGNORED_AUDITS_DATABASE, + INCREMENTAL_AUDIT_FILES, + INCREMENTAL_AUDIT_SEED_SQL, + SOURCE_AUDIT_FILES, + SOURCE_AUDIT_SEED_SQL, + STAGED_AUDIT_FILES, + ignored_audits_project, + recorded_audit_outcomes, + warehouse_table_rows, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb + +_NO_SEED: str = "SELECT 1" +_IGNORING_SETTINGS: str = "\n[settings]\nignore_audit_failures = true\n" +_DEV_TARGET: str = '\n[targets.dev]\nschema = "main"\nignore_audit_failures = {value}\n' +_PLAN_NOTICE: str = "Audit failures won't stop this build" +_FAILED_LINE: str = "Completed with errors" +_STAGED_HELD: dict[str, int] = {"daily__staging": 1} +_STAGED_BUILT: dict[str, int] = {"daily": 1, "daily_summary": 1} +_STAGED_FAILED: tuple[tuple[str, str, bool], ...] = (("total_is_negative", "error", False),) +_STAGED_IGNORED: tuple[tuple[str, str, bool], ...] = (("total_is_negative", "error", True),) +_STAGED_IGNORED_LINES: tuple[str, ...] = ( + "audit total_is_negative returned 1 row (ignored)", + "Completed successfully · 1 audit failure ignored", +) + + +@pytest.mark.parametrize( + "test_case", + [ + IgnoredAuditFailureE2ETestCase( + description="a failing final audit keeps a staged table from replacing its target", + project_files=STAGED_AUDIT_FILES, + seed_sql=_NO_SEED, + settings_toml="", + build_args=(), + expected_exit_code=1, + expected_tables=_STAGED_HELD, + expected_audit_results=_STAGED_FAILED, + expected_fragments=(_FAILED_LINE,), + ), + IgnoredAuditFailureE2ETestCase( + description="an ignored final audit failure promotes the staged table and builds downstream", + project_files=STAGED_AUDIT_FILES, + seed_sql=_NO_SEED, + settings_toml="", + build_args=("--ignore-audit-failures",), + expected_exit_code=0, + expected_tables=_STAGED_BUILT, + expected_audit_results=_STAGED_IGNORED, + expected_fragments=(_PLAN_NOTICE, *_STAGED_IGNORED_LINES), + ), + IgnoredAuditFailureE2ETestCase( + description="a failing delta audit keeps an incremental batch out of its target", + project_files=INCREMENTAL_AUDIT_FILES, + seed_sql=INCREMENTAL_AUDIT_SEED_SQL, + settings_toml="", + build_args=(), + expected_exit_code=1, + expected_tables={"orders": 1, "orders__delta": 2, "raw_orders": 2}, + expected_audit_results=(("amount_is_negative", "error", False),), + expected_fragments=(_FAILED_LINE,), + ), + IgnoredAuditFailureE2ETestCase( + description="an ignored delta audit failure applies the incremental batch", + project_files=INCREMENTAL_AUDIT_FILES, + seed_sql=INCREMENTAL_AUDIT_SEED_SQL, + settings_toml="", + build_args=("--ignore-audit-failures",), + expected_exit_code=0, + expected_tables={"orders": 3, "raw_orders": 2}, + expected_audit_results=( + ("amount_is_negative", "error", True), + ("amount_is_negative", "error", True), + ), + expected_fragments=( + _PLAN_NOTICE, + "audit amount_is_negative returned 2 rows (ignored)", + "Completed successfully · 2 audit failures ignored", + ), + ), + IgnoredAuditFailureE2ETestCase( + description="a failing source audit blocks the models that read the source", + project_files=SOURCE_AUDIT_FILES, + seed_sql=SOURCE_AUDIT_SEED_SQL, + settings_toml="", + build_args=(), + expected_exit_code=1, + expected_tables={"raw_orders": 2}, + expected_audit_results=(("raw_amount_is_negative", "error", False),), + expected_fragments=(_FAILED_LINE,), + ), + IgnoredAuditFailureE2ETestCase( + description="an ignored source audit failure lets the models that read the source build", + project_files=SOURCE_AUDIT_FILES, + seed_sql=SOURCE_AUDIT_SEED_SQL, + settings_toml="", + build_args=("--ignore-audit-failures",), + expected_exit_code=0, + expected_tables={"order_count": 1, "raw_orders": 2}, + expected_audit_results=(("raw_amount_is_negative", "error", True),), + expected_fragments=( + "audit raw_amount_is_negative returned 2 rows (ignored)", + "Completed successfully · 1 audit failure ignored", + ), + ), + IgnoredAuditFailureE2ETestCase( + description="the project setting ignores audit failures without the flag", + project_files=STAGED_AUDIT_FILES, + seed_sql=_NO_SEED, + settings_toml=_IGNORING_SETTINGS, + build_args=(), + expected_exit_code=0, + expected_tables=_STAGED_BUILT, + expected_audit_results=_STAGED_IGNORED, + expected_fragments=(_PLAN_NOTICE, *_STAGED_IGNORED_LINES), + ), + IgnoredAuditFailureE2ETestCase( + description="the negated flag lets audit failures block over the project setting", + project_files=STAGED_AUDIT_FILES, + seed_sql=_NO_SEED, + settings_toml=_IGNORING_SETTINGS, + build_args=("--no-ignore-audit-failures",), + expected_exit_code=1, + expected_tables=_STAGED_HELD, + expected_audit_results=_STAGED_FAILED, + expected_fragments=(_FAILED_LINE,), + ), + IgnoredAuditFailureE2ETestCase( + description="a target that keeps audit failures blocking overrides the project setting", + project_files=STAGED_AUDIT_FILES, + seed_sql=_NO_SEED, + settings_toml=_IGNORING_SETTINGS + _DEV_TARGET.format(value="false"), + build_args=("--target", "dev"), + expected_exit_code=1, + expected_tables=_STAGED_HELD, + expected_audit_results=_STAGED_FAILED, + expected_fragments=(_FAILED_LINE,), + ), + IgnoredAuditFailureE2ETestCase( + description="the flag ignores audit failures over a target that keeps them blocking", + project_files=STAGED_AUDIT_FILES, + seed_sql=_NO_SEED, + settings_toml=_DEV_TARGET.format(value="false"), + build_args=("--target", "dev", "--ignore-audit-failures"), + expected_exit_code=0, + expected_tables=_STAGED_BUILT, + expected_audit_results=_STAGED_IGNORED, + expected_fragments=_STAGED_IGNORED_LINES, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_failing_audit_when_building_then_it_blocks_unless_its_failure_is_ignored( + test_case: IgnoredAuditFailureE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = ignored_audits_project( + tmp_path, test_case.project_files, test_case.seed_sql, test_case.settings_toml + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", *test_case.build_args), project_dir=project_dir + ) + + output: str = result.stdout + result.stderr + database: Path = project_dir / IGNORED_AUDITS_DATABASE + assert result.returncode == test_case.expected_exit_code, output + assert warehouse_table_rows(database) == test_case.expected_tables, output + assert recorded_audit_outcomes(database) == test_case.expected_audit_results, output + assert all(fragment in output for fragment in test_case.expected_fragments), output + + +@pytest.mark.parametrize( + "test_case", + [ + IgnoredAuditFailureCommandE2ETestCase( + description="ignoring audit failures while running no audits is refused", + settings_toml="", + command=("build", "--ignore-audit-failures", "--no-audits"), + expected_exit_code=1, + expected_fragments=("C419", "cannot be combined with --no-audits"), + expected_missing_fragments=("Plan ready",), + ), + IgnoredAuditFailureCommandE2ETestCase( + description="a plan made with the flag says so at the top", + settings_toml="", + command=("plan", "--ignore-audit-failures"), + expected_exit_code=0, + expected_fragments=(f"Plan ready 2 selected\n{_PLAN_NOTICE}",), + expected_missing_fragments=(), + ), + IgnoredAuditFailureCommandE2ETestCase( + description="a plan made under the project setting says so at the top", + settings_toml=_IGNORING_SETTINGS, + command=("plan",), + expected_exit_code=0, + expected_fragments=(_PLAN_NOTICE,), + expected_missing_fragments=(), + ), + IgnoredAuditFailureCommandE2ETestCase( + description="a plan without it says nothing about audit failures", + settings_toml="", + command=("plan",), + expected_exit_code=0, + expected_fragments=("Plan ready",), + expected_missing_fragments=(_PLAN_NOTICE,), + ), + ], + ids=lambda case: case.description, +) +def test_given_ignored_audit_failures_when_planning_or_combining_flags_then_the_command_says_so( + test_case: IgnoredAuditFailureCommandE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = ignored_audits_project( + tmp_path, STAGED_AUDIT_FILES, _NO_SEED, test_case.settings_toml + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", *test_case.command), project_dir=project_dir + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + assert all(fragment in output for fragment in test_case.expected_fragments), output + assert not any(fragment in output for fragment in test_case.expected_missing_fragments), output diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 9e61257383..ba09c7b216 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1445,13 +1445,6 @@ class ManagedSourceLoaderLocationTestCase: expected_rows: tuple[tuple[object, ...], ...] -@dataclass(frozen=True) -class JobGroupCyclesCase: - description: str - fixed_group_toml: str - expected_messages: tuple[str, ...] - - @dataclass(frozen=True) class GraphBoundaryCompileTestCase: """A round-trip project edited into a graph every plan refuses, and what compile reports.""" @@ -1461,3 +1454,10 @@ class GraphBoundaryCompileTestCase: edited_files: dict[str, str] expected_exit_code: int expected_output_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class JobGroupCyclesCase: + description: str + fixed_group_toml: str + expected_messages: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py index 067e857ff1..ac1eafad7f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py @@ -72,3 +72,32 @@ class PlanExitCodeE2ETestCase: plan_args: tuple[str, ...] expected_exit_code: int expected_output_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class PreviewPerformanceTestCase: + """A synced 10K-model warehouse and the budgets a warm `--changed` preview of it meets.""" + + description: str + built_share: float + expected_unbuilt: tuple[int, int] + expected_definition_changes: tuple[int, int] + expected_max_warm_seconds: float + #: The warm preview's share of the location's publish, a linear 10K compile on one machine. + expected_max_warm_to_publish: float + expected_max_check_seconds: float + + +@dataclass(frozen=True) +class PlanTimingsE2ETestCase: + description: str + plan_args: tuple[str, ...] + expected_first_step: str + expected_last_step: str + + +@dataclass(frozen=True) +class PlanWithoutTimingsE2ETestCase: + description: str + plan_args: tuple[str, ...] + expected_table_printed: bool diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/conftest.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/conftest.py new file mode 100644 index 0000000000..9ce428a86d --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/conftest.py @@ -0,0 +1,36 @@ +"""The generated 10K-model DuckShop project the plan preview performance guard plans.""" + +import subprocess +import sys +from collections.abc import Iterator +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( + DUCKSHOP_10K_GENERATOR, + DUCKSHOP_10K_SETUP_SECONDS, + served_duckshop_location, + timed_plan, +) + + +@pytest.fixture(scope="module") +def duckshop_10k(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Path, Path, Path]]: + """The generated project, a planner capture of one plan of it, and the snapshot it published.""" + root: Path = tmp_path_factory.mktemp("duckshop_10k_preview") + project: Path = root / "project" + generated: subprocess.CompletedProcess[str] = subprocess.run( + (sys.executable, str(DUCKSHOP_10K_GENERATOR / "generate.py"), "--out", str(project)), + capture_output=True, + text=True, + check=False, + timeout=DUCKSHOP_10K_SETUP_SECONDS, + ) + assert generated.returncode == 0, generated.stdout + generated.stderr + capture: Path = root / "capture" + with served_duckshop_location( + root / "calibration", project, env={"SQLBUILD_PLANNER_CAPTURE_DIR": str(capture)} + ) as served: + _ = timed_plan(served, [], {}) + yield project, next(capture.glob("*plan_output.json")), served.control diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py index bce8148381..5e42aa6063 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py @@ -2,15 +2,40 @@ from __future__ import annotations +import json +import os +import re +import sqlite3 import subprocess +import sys +import time +from bisect import bisect_right +from collections.abc import Iterator +from contextlib import closing, contextmanager +from dataclasses import dataclass +from itertools import accumulate from pathlib import Path +from typing import Any +from urllib.request import Request, urlopen +from uuid import uuid4 from tests.e2e.src.sqlbuild.cli.commands.main.lineage.helpers import DIAMOND_LAYERS from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + REPO_ROOT, execute_duckdb, prepare_inline_project, run_sqb, ) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + SnapshotApi, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.catalog.helpers import ( + request_plan, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + location_process, + snapshot_api, +) DIAMOND_DATABASE_FILE: str = "orders.duckdb" _DIAMOND_PROJECT_TOML: str = ( @@ -216,3 +241,241 @@ def migrated_project(tmp_path: Path) -> Path: ) assert migrated.returncode == 0, migrated.stdout + migrated.stderr return project_dir + + +PLAN_TIMINGS_HEADER: str = "Plan timings" +_TIMING_STEP_ROW: re.Pattern[str] = re.compile(r"^ (\S.*?)\s+(\d+\.\d)\s+\d+%$", re.MULTILINE) +_TIMING_TOTAL_ROW: re.Pattern[str] = re.compile(r"^ Total\s+(\d+\.\d)$", re.MULTILINE) + + +@dataclass(frozen=True) +class PlanTimingTable: + """A `--timings` table: its header words, every row line, the parsed steps and the total.""" + + header: list[str] + row_count: int + steps: list[tuple[str, float]] + total_seconds: float + + +def migrated_plan_output(tmp_path: Path, plan_args: tuple[str, ...]) -> str: + """`sqb plan` output over the migrated project; the plan must succeed.""" + + project_dir: Path = migrated_project(tmp_path) + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "plan", *plan_args), + project_dir=project_dir, + ) + output: str = result.stdout + result.stderr + assert result.returncode == 0, output + return output + + +def plan_timing_table(output: str) -> PlanTimingTable: + """The `--timings` table at the end of `output`.""" + + table: str = output[output.index(PLAN_TIMINGS_HEADER) :].rstrip("\n") + lines: list[str] = table.splitlines() + total: re.Match[str] | None = _TIMING_TOTAL_ROW.search(table) + assert total is not None, output + return PlanTimingTable( + header=lines[0].split(), + row_count=len(lines) - 2, + steps=[(name, float(seconds)) for name, seconds in _TIMING_STEP_ROW.findall(table)], + total_seconds=float(total.group(1)), + ) + + +# ---- the DuckShop 10K plan preview guard ---- + +DUCKSHOP_10K_TARGET: str = "dev" +DUCKSHOP_10K_GENERATOR: Path = REPO_ROOT / "benchmarks" / "duckshop-10k" +DUCKSHOP_10K_SETUP_SECONDS: int = 600 +PREVIEW_PUBLISHED: str = "Location snapshot published." +PREVIEW_DONE: str = "Location work completed." +_PAGE_RECORDS: int = 2000 +_PAGE_BYTES: int = 6 * 1024 * 1024 +_REQUIRED_SYNCS: tuple[str, ...] = ("catalog", "fingerprints", "consumption") + + +@dataclass(frozen=True) +class ServedLocation: + api: SnapshotApi + control: Path + log: Path + process: subprocess.Popen[bytes] + #: Seconds from starting the location until it published its compiled project. + published_seconds: float + + +@contextmanager +def served_duckshop_location( + directory: Path, project: Path, *, env: dict[str, str] | None = None +) -> Iterator[ServedLocation]: + """A control server and `sqb location serve` of the project, once the location has published.""" + directory.mkdir(parents=True) + with snapshot_api( + directory / "control", + server_arguments=("--password-origin", "https://control.example.com"), + ) as api: + log: Path = directory / "location.log" + started: float = time.monotonic() + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + DUCKSHOP_10K_TARGET, + ], + log, + directory / "location.stdout", + env=os.environ | (env or {}) | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, + ) as process: + control: Path = directory / "control/.sqlbuild/control.sqlite3" + publishing: ServedLocation = ServedLocation(api, control, log, process, 0.0) + published: float = wait_for_log_count(publishing, PREVIEW_PUBLISHED, 1, 900.0) + yield ServedLocation(api, control, log, process, published - started) + + +def wait_for_log_count(served: ServedLocation, text: str, count: int, timeout: float) -> float: + """When the location log first holds `count` lines of `text`.""" + deadline: float = time.monotonic() + timeout + while (content := served.log.read_text(errors="replace")).count(text) < count: + assert served.process.poll() is None, content[-4000:] + assert time.monotonic() < deadline, content[-4000:] + time.sleep(0.02) + return time.monotonic() + + +def published_keys(control: Path) -> list[str]: + """Every node a `sqb build` of the published project runs, as the server freezes it.""" + with closing(sqlite3.connect(control)) as database: + return [ + row[0] + for row in database.execute( + "SELECT n.node_key FROM sqb_snapshot_nodes n JOIN sqb_current_snapshots c " + "ON c.snapshot_id=n.snapshot_id WHERE n.node_kind IN " + "('model','source','seed','udf','table_fn','task','asset','loader')" + ) + ] + + +def location_session(control: Path) -> tuple[str, int]: + """The serving location's session and its published generation, which metadata pages name.""" + with closing(sqlite3.connect(control)) as database: + session: str = database.execute( + "SELECT instance_id FROM sqb_location_instances WHERE target=?", (DUCKSHOP_10K_TARGET,) + ).fetchone()[0] + generation: int = database.execute( + "SELECT s.generation FROM sqb_current_snapshots c JOIN sqb_snapshots s " + "ON s.snapshot_id=c.snapshot_id WHERE c.target=?", + (DUCKSHOP_10K_TARGET,), + ).fetchone()[0] + return str(session), int(generation) + + +def _location_post(api: SnapshotApi, path: str, body: dict[str, Any]) -> None: + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}{path}", + data=json.dumps(body).encode(), + headers={"Authorization": f"Bearer {api.bearer}", "Content-Type": "application/json"}, + ) + with urlopen(request, timeout=120) as response: + _ = response.read() + + +def _pages(records: list[dict[str, Any]]) -> Iterator[list[dict[str, Any]]]: + """Pages of at most `_PAGE_RECORDS` records and `_PAGE_BYTES`, each holding at least one.""" + sizes: list[int] = [len(json.dumps(record)) for record in records] + start: int = 0 + while start < len(records): + totals: list[int] = list(accumulate(sizes[start : start + _PAGE_RECORDS])) + end: int = start + max(1, bisect_right(totals, _PAGE_BYTES)) + yield records[start:end] + start = end + + +def _namespace_order(item: Any) -> str: + """A stable order over state namespaces, whose database may be unset.""" + return json.dumps(item, default=str) + + +def mirror_synced_state(served: ServedLocation, batches: dict[str, list[dict[str, Any]]]) -> None: + """Deliver the synced state as a location's sync does: pages, complete scans, clean syncs.""" + session, generation = location_session(served.control) + path: str = f"/targets/{DUCKSHOP_10K_TARGET}/metadata" + + def page_of(kind: str, records: list[dict[str, Any]]) -> dict[str, Any]: + return {"kind": kind, "session_id": session, "generation": generation, "records": records} + + namespaces: set[tuple[str | None, str]] = set() + for kind in ("fingerprints", "consumption", "observations"): + for page in _pages(batches[kind]): + _location_post(served.api, path, page_of(kind, page)) + namespaces.update((record["state_database"], record["state_schema"]) for record in page) + by_namespace: dict[tuple[str | None, str], list[dict[str, Any]]] = {} + for record in batches["catalog"]: + by_namespace.setdefault((record["state_database"], record["state_schema"]), []).append( + record + ) + for (database, schema), records in sorted(by_namespace.items(), key=_namespace_order): + scan: str = str(uuid4()) + pages: list[list[dict[str, Any]]] = list(_pages(records)) + for index, page in enumerate(pages): + body: dict[str, Any] = page_of("catalog", page) + body["scan"] = { + "scan_id": scan, + "location": {"database": database, "schema": schema}, + "complete": index == len(pages) - 1, + } + _location_post(served.api, path, body) + namespaces.add((database, schema)) + for database, schema in sorted(namespaces, key=_namespace_order): + for table in _REQUIRED_SYNCS: + _location_post( + served.api, + "/health", + { + "event_id": str(uuid4()), + "session_id": session, + "target": DUCKSHOP_10K_TARGET, + "health": { + "kind": "warehouse_sync", + "generation": generation, + "location": {"database": database, "schema": schema}, + "table": table, + "failure": None, + "records_sent": 1, + "elapsed_millis": 1, + }, + }, + ) + + +def timed_plan( + served: ServedLocation, selection: list[str], settings: dict[str, Any] +) -> tuple[float, dict[str, Any]]: + """A person's plan from the moment the server queues it until the location stored it.""" + started: float = time.monotonic() + request_id: str = request_plan(served.control, selection, settings) + deadline: float = started + 600.0 + result: str | None = None + while result is None: + assert served.process.poll() is None, served.log.read_text(errors="replace")[-4000:] + assert time.monotonic() < deadline, served.log.read_text(errors="replace")[-4000:] + time.sleep(0.02) + with closing(sqlite3.connect(served.control)) as database: + row: tuple[str | None] | None = database.execute( + "SELECT result_json FROM sqb_location_work WHERE request_id=?", (request_id,) + ).fetchone() + result = (row or (None,))[0] + elapsed: float = time.monotonic() - started + document: dict[str, Any] = json.loads(result)["outcome"]["document"] + return elapsed, document diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py new file mode 100644 index 0000000000..bbb2512559 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py @@ -0,0 +1,142 @@ +"""Guard that a location previews a `--changed` build of the 10K-model DuckShop project quickly. + +The location plans from the server's synced copy of a warehouse that built the project: about 13K +relations, a fingerprint and `--changed` history per built model, written by +`benchmarks/duckshop-10k/synced_state.py` with the hashes the planner computes for the project, so +only the edited models and what reads them run. A quadratic lookup over those 10K models or 13K +relations, in planning or in the copy it plans from, shows here first. + +It runs in the 10,000-model cell of the dense compile job, after the compile guards. +""" + +import json +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.plan._test_types import PreviewPerformanceTestCase +from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( + DUCKSHOP_10K_GENERATOR, + DUCKSHOP_10K_SETUP_SECONDS, + DUCKSHOP_10K_TARGET, + PREVIEW_DONE, + mirror_synced_state, + published_keys, + served_duckshop_location, + timed_plan, + wait_for_log_count, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.person.helpers import ( + person_login, + person_request, + provision_person, +) + + +@pytest.mark.performance +@pytest.mark.preview_performance +@pytest.mark.parametrize( + "test_case", + ( + PreviewPerformanceTestCase( + description="every model built, one in a hundred edited since", + built_share=1.0, + expected_unbuilt=(0, 0), + expected_definition_changes=(40, 200), + expected_max_warm_seconds=30.0, + expected_max_warm_to_publish=0.45, + expected_max_check_seconds=0.5, + ), + PreviewPerformanceTestCase( + description="one model in twenty never built", + built_share=0.95, + expected_unbuilt=(300, 700), + expected_definition_changes=(40, 200), + expected_max_warm_seconds=30.0, + expected_max_warm_to_publish=0.45, + expected_max_check_seconds=0.5, + ), + ), + ids=lambda case: case.description, +) +def test_given_a_synced_10k_model_warehouse_when_previewing_a_changed_build_then_the_warm_preview_is_fast( + test_case: PreviewPerformanceTestCase, + duckshop_10k: tuple[Path, Path, Path], + tmp_path: Path, +) -> None: + project, capture, published = duckshop_10k + batches_path: Path = tmp_path / "synced_state.json" + written: subprocess.CompletedProcess[str] = subprocess.run( + ( + sys.executable, + str(DUCKSHOP_10K_GENERATOR / "synced_state.py"), + str(published), + str(capture), + str(batches_path), + "--built-share", + str(test_case.built_share), + ), + capture_output=True, + text=True, + check=False, + timeout=DUCKSHOP_10K_SETUP_SECONDS, + ) + assert written.returncode == 0, written.stdout + written.stderr + batches: dict[str, list[dict[str, Any]]] = json.loads(batches_path.read_text()) + with served_duckshop_location(tmp_path / "served", project) as served: + mirror_synced_state(served, batches) + selection: list[str] = published_keys(served.control) + cold, _ = timed_plan(served, selection, {"changed": True}) + warm, document = timed_plan(served, selection, {"changed": True}) + _ = wait_for_log_count(served, PREVIEW_DONE, 2, 60.0) + admin: dict[str, Any] = provision_person( + served.control.parents[1], "create-user", "plan-admin" + ) + session: tuple[str, str] = person_login(served.api, admin["name"]) + scope: dict[str, Any] = { + "location_id": served.api.location_id, + "target": DUCKSHOP_10K_TARGET, + } + checks: list[float] = [] + for _ in range(5): + started: float = time.monotonic() + check: dict[str, Any] = person_request( + served.api, + "/api/control/runs/adhoc/check", + body=scope | {"command": ["build", "--changed"], "priority": 0}, + session=session, + ) + checks.append(time.monotonic() - started) + reasons: list[str] = [run["reason"] for run in document["changed"]["runs"]] + unbuilt: int = reasons.count("relation missing") + definition_changes: int = sum( + reason.startswith(("definition changed", "planner change")) for reason in reasons + ) + check_seconds: float = sorted(checks)[len(checks) // 2] + warm_to_publish: float = warm / served.published_seconds + print( + f"duckshop 10k preview ({test_case.description}): cold={cold:.2f}s warm={warm:.2f}s " + f"publish={served.published_seconds:.2f}s warm_to_publish={warm_to_publish:.2f} " + f"check_median={check_seconds * 1000:.0f}ms runs={len(reasons)} unbuilt={unbuilt} " + f"definition_changes={definition_changes} models={len(document['models'])}", + flush=True, + ) + assert document.get("error") is None and len(document["models"]) >= 10_000 + assert check["error"] is None + assert test_case.expected_unbuilt[0] <= unbuilt <= test_case.expected_unbuilt[1], ( + "the synced state builds what the case says" + ) + assert ( + test_case.expected_definition_changes[0] + <= definition_changes + <= test_case.expected_definition_changes[1] + ), "only the edited models changed definition: the synced hashes match the planner's" + assert warm < test_case.expected_max_warm_seconds + assert warm_to_publish < test_case.expected_max_warm_to_publish, ( + "planning grew faster than compiling: look for a lookup over every model or relation" + ) + assert check_seconds < test_case.expected_max_check_seconds diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_plan_timings.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_plan_timings.py new file mode 100644 index 0000000000..1409cc61c8 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_plan_timings.py @@ -0,0 +1,67 @@ +"""E2E coverage for `sqb plan --timings`.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.plan._test_types import ( + PlanTimingsE2ETestCase, + PlanWithoutTimingsE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( + PLAN_TIMINGS_HEADER, + PlanTimingTable, + migrated_plan_output, + plan_timing_table, +) + + +@pytest.mark.parametrize( + "test_case", + [ + PlanTimingsE2ETestCase( + description="--timings prints each step with Other last, then the total", + plan_args=("--timings",), + expected_first_step="Compiling the project", + expected_last_step="Other", + ), + ], + ids=lambda case: case.description, +) +def test_given_timings_when_planning_then_the_steps_add_up_to_the_total( + test_case: PlanTimingsE2ETestCase, + tmp_path: Path, +) -> None: + output: str = migrated_plan_output(tmp_path, test_case.plan_args) + + table: PlanTimingTable = plan_timing_table(output) + assert table.header == ["Plan", "timings", "seconds", "share"], output + assert len(table.steps) == table.row_count, output + assert table.steps[0][0] == test_case.expected_first_step, output + assert table.steps[-1][0] == test_case.expected_last_step, output + assert "Reading build history" not in output, output + assert sum(seconds for _, seconds in table.steps) == pytest.approx( + table.total_seconds, abs=0.05 * len(table.steps) + ), output + + +@pytest.mark.parametrize( + "test_case", + [ + PlanWithoutTimingsE2ETestCase( + description="a plan without --timings prints no table", + plan_args=(), + expected_table_printed=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_no_timings_flag_when_planning_then_no_table_is_printed( + test_case: PlanWithoutTimingsE2ETestCase, + tmp_path: Path, +) -> None: + output: str = migrated_plan_output(tmp_path, test_case.plan_args) + + assert (PLAN_TIMINGS_HEADER in output) == test_case.expected_table_printed, output diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_python_tests.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_python_tests.py index 52396e05f5..a051644451 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_python_tests.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_python_tests.py @@ -60,6 +60,17 @@ ), } +ORCHESTRATION_PYTHON_TEST: dict[str, str] = { + "orchestration/sensors/orders/_helpers/restock_window.py": ( + "def restock_due(hours_since_publish):\n return hours_since_publish >= 6\n" + ), + "tests/orchestration/sensors/orders/_helpers/test_restock_window.py": ( + "from orchestration.sensors.orders._helpers.restock_window import restock_due\n\n\n" + "def test_given_six_hours_when_checking_then_restock_is_due():\n" + " assert restock_due(6)\n" + ), +} + FAILING_PYTHON_TEST: dict[str, str] = { "tests/python/inventory/stock_sync/test_node.py": ( "def test_given_stock_when_syncing_then_fails():\n assert False\n" @@ -118,6 +129,14 @@ def __getattr__(self, name): expected_output_fragments=(SQL_PASSED, PYTHON_PASSED, "1 passed"), expected_absent_fragments=(), ), + PythonTestsE2ETestCase( + description="an orchestration test mirroring its sensor code runs", + command=("--no-color", "test", "--python"), + extra_repo_files=ORCHESTRATION_PYTHON_TEST, + expected_exit_code=0, + expected_output_fragments=(PYTHON_PASSED, "1 passed"), + expected_absent_fragments=(SQL_PASSED,), + ), PythonTestsE2ETestCase( description="a failing Python test fails the command after the SQL tests", command=("--no-color", "test"), diff --git a/tests/integration/scripts/fensu_policy/_test_types.py b/tests/integration/scripts/fensu_policy/_test_types.py index 33d3fb804d..01e7dff793 100644 --- a/tests/integration/scripts/fensu_policy/_test_types.py +++ b/tests/integration/scripts/fensu_policy/_test_types.py @@ -18,3 +18,12 @@ class ObservationBoundaryTestCase: planner: str expected_returncode: int expected_faults: tuple[str, ...] + + +@dataclass(frozen=True) +class PlannerReadsTestCase: + description: str + files: dict[str, str] + planner_paths: tuple[str, ...] + expected_returncode: int + expected_faults: tuple[str, ...] diff --git a/tests/integration/scripts/fensu_policy/helpers.py b/tests/integration/scripts/fensu_policy/helpers.py index e0592a79f3..10e874e666 100644 --- a/tests/integration/scripts/fensu_policy/helpers.py +++ b/tests/integration/scripts/fensu_policy/helpers.py @@ -28,8 +28,7 @@ def write_observation_project( ) (crate / "src/lib.rs").write_text("mod observations;\nmod planner;\n") (crate / "src/observations/mod.rs").write_text( - "pub fn table() -> &'static str { OBSERVATIONS_TABLE_NAME }\n" - "pub struct SyncedCopyAdapter;\n" + "pub fn table() -> &'static str { OBSERVATIONS_TABLE_NAME }\npub struct SyncedReads;\n" ) (crate / "src/planner.rs").write_text(planner) entries: str = ", ".join(f'"{path}=Writes observations."' for path in allowed) @@ -41,6 +40,39 @@ def write_observation_project( ) +def write_planner_project( + *, root: Path, planner_paths: tuple[str, ...], files: dict[str, str] +) -> None: + """A crate of the given Rust `files`, with XSB080 covering `planner_paths`.""" + + repository: Path = Path(__file__).resolve().parents[4] + shutil.copytree( + repository / "scripts/fensu_policy", + root / "scripts/fensu_policy", + ignore=shutil.ignore_patterns("__pycache__"), + ) + (root / "scripts/__init__.py").write_text("") + (root / "Cargo.toml").write_text( + '[workspace]\nmembers = ["crates/sqlbuild-python"]\nresolver = "2"\n' + ) + crate: Path = root / "crates/sqlbuild-python" + (crate / "src").mkdir(parents=True) + (crate / "Cargo.toml").write_text( + '[package]\nname = "sqlbuild-python"\nversion = "0.1.0"\nedition = "2021"\n' + ) + modules: str = "".join(f"mod {Path(name).stem};\n" for name in files) + (crate / "src/lib.rs").write_text(modules) + for name, source in files.items(): + (crate / "src" / name).write_text(source) + entries: str = ", ".join(f'"{path}=Plans through WarehouseReads."' for path in planner_paths) + (root / "fensu.toml").write_text( + '[targets.rust]\nanalyzer = "rust"\nroots = ["crates"]\n' + 'rule_modules = ["scripts.fensu_policy.rules.planner_reads"]\n' + 'select = ["XSB080"]\n\n' + f"[targets.rust.rule_options.XSB080]\nplanner_paths = [{entries}]\n" + ) + + def check_rust(*, root: Path) -> subprocess.CompletedProcess[str]: """Run `fensu check` on the Rust target without a cache.""" diff --git a/tests/integration/scripts/fensu_policy/test_observation_boundary.py b/tests/integration/scripts/fensu_policy/test_observation_boundary.py index 0b153492fd..017257a60f 100644 --- a/tests/integration/scripts/fensu_policy/test_observation_boundary.py +++ b/tests/integration/scripts/fensu_policy/test_observation_boundary.py @@ -66,20 +66,20 @@ expected_faults=("allowed path crates/sqlbuild-python/src/planner.rs no longer names",), ), ObservationBoundaryTestCase( - description="live planner naming the synced copy only in a comment", + description="live planner naming the synced reads only in a comment", rule="XSB079", allowed=("crates/sqlbuild-python/src/observations",), - planner="// SyncedCopyAdapter serves plan previews only\npub fn plan() {}\n", + planner="// SyncedReads serves plan previews only\npub fn plan() {}\n", expected_returncode=0, expected_faults=(), ), ObservationBoundaryTestCase( - description="live planner branching on the synced copy", + description="live planner branching on the synced reads", rule="XSB079", allowed=("crates/sqlbuild-python/src/observations",), planner=( "pub fn plan() {}\n" - "pub fn preview(adapter: &Any) -> bool { adapter.is::() }\n" + "pub fn preview(reads: &Any) -> bool { reads.is::() }\n" ), expected_returncode=1, expected_faults=(" --> crates/sqlbuild-python/src/planner.rs:2:0",), diff --git a/tests/integration/scripts/fensu_policy/test_planner_reads.py b/tests/integration/scripts/fensu_policy/test_planner_reads.py new file mode 100644 index 0000000000..12643d0e01 --- /dev/null +++ b/tests/integration/scripts/fensu_policy/test_planner_reads.py @@ -0,0 +1,92 @@ +"""Exercise the planner's warehouse-reads boundary through Fensu's real Rust analyzer.""" + +import subprocess +from pathlib import Path + +import pytest + +from tests.integration.scripts.fensu_policy._test_types import PlannerReadsTestCase +from tests.integration.scripts.fensu_policy.helpers import check_rust, write_planner_project + +_PLANNER: tuple[str, ...] = ("crates/sqlbuild-python/src/planner.rs",) + + +@pytest.mark.parametrize( + "test_case", + [ + PlannerReadsTestCase( + description="planner reading through reads and writing SQL through its dialect", + files={ + "planner.rs": ( + "// The adapter and connection stay behind WarehouseReads.\n" + "pub fn plan(reads: &Reads, dialect: &Dialect) {\n" + ' call(&[("adapter", dialect.clone())]);\n' + " reads.relations();\n" + "}\n" + ), + }, + planner_paths=_PLANNER, + expected_returncode=0, + expected_faults=(), + ), + PlannerReadsTestCase( + description="planner taking the adapter", + files={"planner.rs": "pub fn plan() {}\npub fn list(adapter: &Any) {}\n"}, + planner_paths=_PLANNER, + expected_returncode=1, + expected_faults=(" --> crates/sqlbuild-python/src/planner.rs:2:0",), + ), + PlannerReadsTestCase( + description="planner reading the runtime's adapter by name", + files={ + "planner.rs": 'pub fn plan(runtime: &Any) {\n runtime.getattr("adapter");\n}\n' + }, + planner_paths=_PLANNER, + expected_returncode=1, + expected_faults=(" --> crates/sqlbuild-python/src/planner.rs:2:0",), + ), + PlannerReadsTestCase( + description="planner calling an adapter warehouse method by name", + files={ + "planner.rs": 'pub fn plan(value: &Any) {\n field(value, "list_relations");\n}\n' + }, + planner_paths=_PLANNER, + expected_returncode=1, + expected_faults=(" --> crates/sqlbuild-python/src/planner.rs:2:0",), + ), + PlannerReadsTestCase( + description="code outside the planner taking the adapter", + files={ + "planner.rs": "pub fn plan() {}\n", + "command.rs": "pub fn run(adapter: &Any, connection: &Any) {}\n", + }, + planner_paths=_PLANNER, + expected_returncode=0, + expected_faults=(), + ), + PlannerReadsTestCase( + description="planner path that matches no file", + files={"planner.rs": "pub fn plan() {}\n"}, + planner_paths=(*_PLANNER, "crates/sqlbuild-python/src/removed.rs"), + expected_returncode=1, + expected_faults=("planner path crates/sqlbuild-python/src/removed.rs matches no",), + ), + ], + ids=lambda case: case.description, +) +def test_given_planner_code_when_checking_then_only_warehouse_access_outside_reads_faults( + tmp_path: Path, test_case: PlannerReadsTestCase +) -> None: + write_planner_project( + root=tmp_path, planner_paths=test_case.planner_paths, files=test_case.files + ) + + result: subprocess.CompletedProcess[str] = check_rust(root=tmp_path) + + assert result.returncode == test_case.expected_returncode, result.stdout + result.stderr + assert result.stdout.count("XSB080") == len(test_case.expected_faults), result.stdout + assert all(fault in result.stdout for fault in test_case.expected_faults), result.stdout + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 75ee9b8409..afaa800b44 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -288,6 +288,17 @@ class PythonCheckJobRejectionTestCase: expected_message: str +@dataclass(frozen=True) +class SnapshotConcurrencyTestCase: + description: str + project_tables: str + local_files: dict[str, str] + job_settings: str + selected_target: str | None + expected_job: dict[str, object] + expected_group: dict[str, object] + + @dataclass(frozen=True) class SnapshotScheduleTestCase: description: str @@ -311,6 +322,14 @@ class SnapshotDeclarationRejectionTestCase: copies: int = 1 +@dataclass(frozen=True) +class SnapshotRetryTestCase: + description: str + tables: str + expected_on_failure: dict[str, int] + expected_warnings: tuple[str, ...] + + @dataclass(frozen=True) class SnapshotDiscoveryTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index c41b849efa..77e2fe33ad 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -106,6 +106,12 @@ 'SELECT order_id, amount\nFROM __ref("stg_orders")\n' ), } +# The type-proof project with its contract finding fixed, so a location's load passes its rules. +PUBLISHABLE_FILES: dict[str, str] = TYPE_PROOF_FILES | { + "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( + "SELECT order_id, amount", "SELECT order_id, CAST(amount AS BIGINT) AS amount" + ), +} _FACTS_CONSUMERS: tuple[ModuleType, ...] = (_native,) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 78536260c1..dfeb9594bc 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -132,7 +132,7 @@ class WorkerExecutionTestCase: files_after_publication: dict[str, str] = field(default_factory=dict) store_setup: str = "" store_edit: Callable[[sqlite3.Connection], None] = lambda _: None - concurrency_limits: dict[str, int] = field(default_factory=dict) + pools: dict[str, int] = field(default_factory=dict) launcher: Literal["native", "uncooperative", "docker"] = "native" # A Docker launcher's [launchers.docker.run] settings and the worker HostConfig they produce. launcher_settings: dict[str, Any] = field(default_factory=dict) @@ -219,6 +219,25 @@ class WarehouseHistoryTestCase: append_fingerprints: Callable[..., None] +@dataclass(frozen=True) +class NewestVersionsTestCase: + description: str + adapter_name: str + connection_key: str + newest_versions: int | None + expected_runs: dict[str, list[str]] + + +@dataclass(frozen=True) +class PageBytesTestCase: + description: str + adapter_name: str + connection_key: str + wide_rows: int + definition_bytes: int + expected_min_pages: int + + @dataclass(frozen=True) class CatalogConnectionTestCase: description: str @@ -505,3 +524,63 @@ class PlanPreviewTestCase: expected_window_models: tuple[str, ...] expected_unplanned_cursor_models: tuple[str, ...] expected_added_unknowns: tuple[str, ...] + + +@dataclass(frozen=True) +class HeldPlanTestCase: + description: str + settings: tuple[dict[str, object], ...] + expected_runs: tuple[tuple[str, ...], ...] + expected_compiles: int + expected_held_plans: int + + +@dataclass(frozen=True) +class RulesAtLoadTestCase: + description: str + model_sql: str + expected_health: dict[str, Any] = field(default_factory=dict) + expected_terminal: str = "" + expected_rule_evaluations: int = 0 + + +@dataclass(frozen=True) +class UnknownSourceAccessTestCase: + description: str + added_files: dict[str, str] + selection: list[str] + expected_unknown: str + + +@dataclass(frozen=True) +class SyncedFreshnessTestCase: + description: str + files: dict[str, str] + expected_absent: str + expected_line: str + + +@dataclass(frozen=True) +class FailedPlanTestCase: + description: str + files: dict[str, str] + edited_file: str + edit: tuple[str, str] + selection: list[str] + expected_code: str + expected_reason: str + + +@dataclass(frozen=True) +class ChangedBuildPublicationTestCase: + description: str + edit: tuple[str, str] + expected_published: dict[str, list[str]] + + +@dataclass(frozen=True) +class ChangedBuildReplanTestCase: + description: str + edit: tuple[str, str] + expected_published: dict[str, list[str]] + expected_rebuilt: dict[str, list[str]] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index ac4e2c0948..a2ed27e6c1 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -443,10 +443,11 @@ def warehouse_history_page( table: str, after: list[str | None] | None = None, observed_since: str | None = None, + **options: int | None, ) -> Any: """Exercise the real isolated reader without a server, state database or declaration import.""" result: subprocess.CompletedProcess[str] = warehouse_history_read( - project, table, after, observed_since + project, table, after, observed_since, **options ) assert result.returncode == 0, result.stderr return json.loads(result.stdout) @@ -457,8 +458,14 @@ def warehouse_history_read( table: str, after: list[str | None] | None = None, observed_since: str | None = None, + **options: int | None, ) -> subprocess.CompletedProcess[str]: """One isolated reader exchange, with the reader's own report of a failed read.""" + read: dict[str, Any] = { + "location": {"database": None, "schema": "main"}, + "after": after, + "observed_since": observed_since, + } | options return subprocess.run( [ str(Path(sys.executable).with_name("sqb")), @@ -473,16 +480,7 @@ def warehouse_history_read( "--target", "development", ], - input=json.dumps( - { - table: { - "location": {"database": None, "schema": "main"}, - "after": after, - "observed_since": observed_since, - } - } - ) - + "\n", + input=json.dumps({table: read}) + "\n", text=True, capture_output=True, timeout=20, @@ -492,14 +490,16 @@ def warehouse_history_read( def warehouse_history_pages( - project: Path, table: str, observed_since: str | None = None + project: Path, table: str, observed_since: str | None = None, **options: int | None ) -> list[dict[str, Any]]: """Follow bounded source pages, including empty filtered pages with a continuation.""" pages: list[dict[str, Any]] = [] after: list[str | None] | None = None more: bool = True while more: - page: dict[str, Any] = warehouse_history_page(project, table, after, observed_since) + page: dict[str, Any] = warehouse_history_page( + project, table, after, observed_since, **options + ) assert "failed" not in page, page pages.append(page) after = page["next_after"] @@ -549,6 +549,18 @@ def wait_for_log( time.sleep(0.02) +def wait_for_log_count( + process: subprocess.Popen[bytes], log: Path, text: str, count: int, *, timeout: float = 45 +) -> str: + """Wait until `text` has been logged `count` times; returns the log.""" + deadline: float = time.monotonic() + timeout + while (logged := log.read_text()).count(text) < count: + assert process.poll() is None, logged + assert time.monotonic() < deadline, logged + time.sleep(0.05) + return logged + + def wait_for_development_ready( process: subprocess.Popen[bytes], log: Path, *, timeout: float = 60 ) -> tuple[str, str]: @@ -602,7 +614,7 @@ def wait_for_idle_sync(process: subprocess.Popen[bytes], log: Path) -> None: deadline: float = time.monotonic() + 45 text: str = log.read_text() - while text.rfind("synchronization completed.") < text.rfind("Reading warehouse history"): + while text.rfind("synchronization completed") < text.rfind("Reading warehouse history"): assert process.poll() is None, text assert time.monotonic() < deadline, text time.sleep(0.05) @@ -652,7 +664,7 @@ def wait_for_warehouse_sync_health( health = report["report"]["health"] assert health["generation"] == 1 assert health["location"] == {"database": None, "schema": "main"} - assert 0 <= health["records_sent"] <= 100 + assert 0 <= health["records_sent"] <= 2000 assert health["elapsed_millis"] > 0 assert bool(health.get("error")) == (expected["failure"] == "read"), health return reports @@ -667,6 +679,7 @@ def wait_for_location_health( *, minimum_reports: int = 1, ) -> dict[str, Any]: + """Wait for reports matching every field of `health`; fields it leaves out may differ.""" deadline: float = time.monotonic() + 15 + 30 * (minimum_reports - 1) key: tuple[str, str] = (target, json.dumps(health, sort_keys=True)) observations: dict[tuple[str, str], dict[str, dict[str, Any]]] = {} @@ -675,15 +688,44 @@ def wait_for_location_health( assert time.monotonic() < deadline, log.read_text() for report in reversed(location_health_reports(api)): event_id: str = report["report"]["event_id"] + reported: dict[str, Any] = report["report"]["health"] observed_key: tuple[str, str] = ( report["report"]["target"], - json.dumps(report["report"]["health"], sort_keys=True), + json.dumps({name: reported.get(name) for name in health}, sort_keys=True), ) observations.setdefault(observed_key, {})[event_id] = report time.sleep(0.02) return list(observations[key].values())[-1] +def published_connection_health( + api: SnapshotApi, + expected: tuple[str, bool], + published: tuple[dict[str, Any], str], + process: subprocess.Popen[bytes], + log: Path, +) -> dict[str, Any]: + """The timed connection check after a snapshot published, on a local DuckDB connection.""" + target, healthy = expected + snapshot, trigger = published + health: dict[str, Any] = wait_for_location_health( + api, + target, + { + "kind": "connection", + "generation": snapshot["record"]["generation"], + "healthy": healthy, + "trigger": trigger, + }, + process, + log, + ) + assert isinstance(health["report"]["health"]["elapsed_millis"], int) + connection: dict[str, Any] = snapshot["documents"]["connection"] + assert (connection["adapter"], connection["auth_method"]) == ("duckdb", "none") + return health + + def wait_for_interpreter(process: subprocess.Popen[bytes], pidfile: Path, log: Path) -> int: deadline: float = time.monotonic() + 15 while not pidfile.exists(): @@ -868,6 +910,8 @@ def scheduled_worker( 'name = "orders"\nadapter = "duckdb"\n' '[connections.local]\ndatabase = "warehouse.duckdb"\n' '[targets.development]\nschema = "main"\n' + "[pools]\n" + + "".join(f"{name} = {limit}\n" for name, limit in test_case.pools.items()) ), "models/orders.sql": ( "MODEL (description 'Orders.', materialized table);\n" + test_case.model_sql + "\n" @@ -931,12 +975,7 @@ def scheduled_worker( ) write_server_config( configuration, - { - "launchers": { - deployment.kind: [launcher], - "concurrency_limits": test_case.concurrency_limits, - } - }, + {"launchers": {deployment.kind: [launcher]}}, ) control_store: Path = directory / "control/.sqlbuild/control.sqlite3" with sqlite3.connect(control_store) as control: @@ -1119,7 +1158,6 @@ def claimed_worker(directory: Path, api: SnapshotApi, publication: dict[str, Any "selection": selection, "write_set": selection, "settings": {"full_refresh": True}, - "concurrency_groups": [], } with sqlite3.connect(directory / ".sqlbuild/control.sqlite3") as connection: _ = connection.execute( @@ -1771,3 +1809,192 @@ def plan_from_copy( outcome: dict[str, Any] = result["result_json"]["outcome"] assert outcome["kind"] == "plan", outcome return refused, outcome["document"] + + +def changed_build_and_plan_from_copy( + tmp_path: Path, project: Path, selection: list[str] +) -> tuple[dict[str, Any], dict[str, Any]]: + """Serve `project` until its first sync completes. + + Returns the changed build the location published after that sync and the location's + `--changed` plan of `selection` from the same synced copy. + """ + control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" + log: Path = tmp_path / "location.log" + started: datetime = datetime.now(UTC) + with ( + snapshot_api(tmp_path / "control") as api, + location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, + ) as location, + ): + _ = wait_for_generation(api, "development", 1, location, log) + synced: datetime = max( + datetime.fromisoformat(report["received_at"]) + for report in wait_for_warehouse_sync_health( + api, + {"failure": None, "tables": ["catalog", "consumption", "fingerprints"]}, + location, + log, + after=started, + ) + ) + deadline: float = time.monotonic() + 45 + published: tuple[str, str] | None = None + while published is None or datetime.fromisoformat(published[0]) <= synced: + assert location.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + time.sleep(0.05) + with closing(sqlite3.connect(control)) as database: + published = database.execute( + "SELECT reported_at, report_json FROM sqb_changed_builds" + ).fetchone() + request_id: str = request_plan(control, selection, {"changed": True}) + result: dict[str, Any] = wait_for_location_work(api, request_id, location, log) + outcome: dict[str, Any] = result["result_json"]["outcome"] + assert outcome["kind"] == "plan", outcome + return json.loads(published[1]), outcome["document"] + + +def plans_from_copy( + tmp_path: Path, project: Path, requests: list[tuple[list[str], dict[str, object]]] +) -> list[dict[str, Any]]: + """Serve `project` until its first sync completes, then plan each request from the copy.""" + control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" + log: Path = tmp_path / "location.log" + started: datetime = datetime.now(UTC) + documents: list[dict[str, Any]] = [] + with ( + snapshot_api(tmp_path / "control") as api, + location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, + ) as location, + ): + _ = wait_for_generation(api, "development", 1, location, log) + _ = wait_for_warehouse_sync_health( + api, + {"failure": None, "tables": ["catalog", "consumption", "fingerprints"]}, + location, + log, + after=started, + ) + for selection, settings in requests: + request_id: str = request_plan(control, selection, settings) + result: dict[str, Any] = wait_for_location_work(api, request_id, location, log) + outcome: dict[str, Any] = result["result_json"]["outcome"] + assert outcome["kind"] == "plan", outcome + documents.append(outcome["document"]) + return documents + + +def changed_runs(plan: dict[str, Any]) -> tuple[str, ...]: + """The nodes a changed plan runs.""" + return tuple(run["node"] for run in plan["changed"]["runs"]) + + +def changed_build_report(plan: dict[str, Any]) -> dict[str, list[str]]: + """What a changed build of `plan` reports: the nodes it runs, split by why they run.""" + runs: list[dict[str, Any]] = plan["changed"]["runs"] + changed: list[dict[str, Any]] = list(filter(lambda run: run["downstream_of"] is None, runs)) + downstream: list[dict[str, Any]] = list( + filter(lambda run: run["downstream_of"] is not None, runs) + ) + return { + "changed": [run["node"] for run in changed], + "downstream": [run["node"] for run in downstream], + } + + +def failed_load(tmp_path: Path, project: Path) -> tuple[dict[str, Any], int, str]: + """Serve `project` until its first load fails: its health, snapshot API status and log.""" + log: Path = tmp_path / "location.log" + with ( + snapshot_api(tmp_path / "control") as api, + location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, + ) as location, + ): + report: dict[str, Any] = wait_for_location_health( + api, "development", {"kind": "compile_failed"}, location, log + ) + with pytest.raises(HTTPError) as missing: + _ = current_snapshot(api, "development") + return report["report"]["health"], missing.value.code, log.read_text() + + +def plan_before_sync( + tmp_path: Path, project: Path, settings: dict[str, object] +) -> tuple[dict[str, Any], str]: + """Serve `project` until it publishes, then plan from its copy without waiting for a sync.""" + control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" + log: Path = tmp_path / "location.log" + with ( + snapshot_api(tmp_path / "control") as api, + location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "dev", + ], + log, + tmp_path / "location.stdout", + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, + ) as location, + ): + _ = wait_for_generation(api, "dev", 1, location, log) + request_id: str = request_plan(control, [], settings) + result: dict[str, Any] = wait_for_location_work(api, request_id, location, log) + return result["result_json"]["outcome"], log.read_text() diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py index 829ce52e75..082f789c27 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py @@ -14,7 +14,7 @@ import pytest from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( - TYPE_PROOF_FILES, + PUBLISHABLE_FILES, duckshop_orchestration, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( @@ -71,9 +71,9 @@ def test_given_invalid_calendar_results_when_publishing_through_the_cli_then_val project: Path = tmp_path / "project" write_files( project_dir=project, - files=TYPE_PROOF_FILES + files=PUBLISHABLE_FILES | { - "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + "sqlbuild_project.toml": PUBLISHABLE_FILES["sqlbuild_project.toml"] + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + '\n[targets.development]\nschema = "main"\n', "orchestration/jobs/order_reporting_job.toml": ( @@ -139,9 +139,9 @@ def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refr monkeypatch.setenv("SQLBUILD_CALENDAR_EVALUATIONS", str(evaluations)) write_files( project_dir=project, - files=TYPE_PROOF_FILES + files=PUBLISHABLE_FILES | { - "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + "sqlbuild_project.toml": PUBLISHABLE_FILES["sqlbuild_project.toml"] + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + '\n[targets.development]\nschema = "main"\n', **{ @@ -254,7 +254,7 @@ def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refr assert [row["phase"] for row in observations] == ["setup", "evaluate", "teardown"] * 3 data.write_text(json.dumps({"offsets": [1]})) (project / "models/stg_orders.sql").write_text( - TYPE_PROOF_FILES["models/stg_orders.sql"].replace( + PUBLISHABLE_FILES["models/stg_orders.sql"].replace( "CAST(5 AS INTEGER)", "CAST(19 AS INTEGER)" ) ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev_terminal.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev_terminal.py index 5ecf93410f..576fcf1772 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev_terminal.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev_terminal.py @@ -9,7 +9,7 @@ import pytest -from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import PUBLISHABLE_FILES from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( DevelopmentTerminalTestCase, ) @@ -67,10 +67,10 @@ def test_given_an_interactive_adapter_when_developing_then_sign_in_happens_once_ connects: Path = tmp_path / "connects.txt" write_files( project_dir=project, - files=TYPE_PROOF_FILES + files=PUBLISHABLE_FILES | { "sqlbuild_project.toml": 'default_target = "development"\n' - + TYPE_PROOF_FILES["sqlbuild_project.toml"].replace( + + PUBLISHABLE_FILES["sqlbuild_project.toml"].replace( 'adapter = "duckdb"', 'adapter = "duckdb_sign_in"' ) + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py index 0feaabe881..bdf18ce294 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py @@ -112,7 +112,7 @@ def test_given_a_running_local_worker_when_scheduler_dies_then_restart_adopts_an "failed", [], launcher="uncooperative", - concurrency_limits={"orders_limit": 1}, + pools={"orders_limit": 1}, files={ **duckshop_orchestration("local_recovery_orders"), }, @@ -142,7 +142,9 @@ def test_given_an_unstarted_orphan_when_recovery_publication_fails_then_claims_w with sqlite3.connect(worker.control_store) as control: assert control.execute( "SELECT group_key, run_id FROM sqb_concurrency_slots" - ).fetchall() == [("orders_limit", active["run_id"])] + ).fetchall() == [ + (f"{worker.api.location_id}/development/orders_limit", active["run_id"]) + ] retained: dict[str, list[tuple[Any, ...]]] = { claim: control.execute(claim).fetchall() for claim in claims } diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py index 349e372f57..7409df00d4 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py @@ -1,7 +1,11 @@ """A location plans from the server's synced copy and never opens the warehouse to do it.""" import json +import os +import sqlite3 import subprocess +import sys +from contextlib import closing from pathlib import Path from textwrap import dedent from typing import Any @@ -10,13 +14,33 @@ import pytest from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ChangedBuildPublicationTestCase, + ChangedBuildReplanTestCase, ChangedPlanPreviewTestCase, + FailedPlanTestCase, + HeldPlanTestCase, PlanPreviewTestCase, + SyncedFreshnessTestCase, + UnknownSourceAccessTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.catalog.helpers import ( + request_plan, + request_warehouse_sync, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + changed_build_and_plan_from_copy, + changed_build_report, + changed_runs, live_plan, + location_process, + plan_before_sync, plan_from_copy, + plans_from_copy, run_sqb, + snapshot_api, + wait_for_generation, + wait_for_location_work, + wait_for_log_count, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -116,11 +140,44 @@ "retention", "column rename support", "sources outside mirrored schemas", + "recorded migrations", ] +_HELD: str = "Planning from the published compiled project." +_SYNCED_READING: str = "Reading the synced warehouse state..." _REPLAYED: str = ( "cursor window of daily_orders (the run starts it away from the last build's cursor)" ) +_SYNCED: str = "Warehouse history synchronization completed" +_PLANNED: str = "Planned what changed" + + +_FUNCTION_CALLER_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n[connections.local]\ndatabase = "warehouse.db"\n' + '[targets.development]\nschema = "main"\n' + ), + "seeds/orders.csv": "id,ordered_at,amount\n1,2026-09-01,10\n2,2026-09-02,20\n", + "seeds/orders.yml": ( + "seeds:\n- name: orders\n description: Orders.\n columns:\n" + " - name: id\n type: INTEGER\n - name: ordered_at\n type: DATE\n" + " - name: amount\n type: INTEGER\n" + ), + "functions/sql/udf__adjust_amount.sql": ( + "FUNCTION (description 'Adjusted amount.',\n arguments (amount INTEGER),\n" + " returns INTEGER,\n);\n\namount + 1\n" + ), + "models/order_amounts.sql": ( + "MODEL (description 'Adjusted order amounts.',\n materialized incremental,\n" + " incremental_strategy merge,\n unique_key [id],\n cursor ordered_at,\n" + " cursor_type timestamp,\n cursor_grain day,\n" + " cursor_inputs (\n orders ordered_at,\n ),\n" + " replay_on_change full,\n full_refresh false,\n);\n\n" + "SELECT id, CAST(ordered_at AS DATE) AS ordered_at,\n" + ' __udf("udf__adjust_amount")(amount) AS adjusted_amount\nFROM __seed("orders")\n' + ), +} + @pytest.mark.parametrize( "test_case", @@ -185,6 +242,15 @@ def test_given_a_synced_location_when_a_plan_is_requested_then_it_plans_from_the assert "synced_copy" in preview, log.read_text() synced: dict[str, Any] = preview.pop("synced_copy") assert synced["synced_at"] is not None + timings: dict[str, Any] = preview.pop("timings") + assert "timings" not in live + assert timings["planned_from"] == "synced_state" + step_names: list[str] = [step["name"] for step in timings["steps"]] + assert step_names[-1] == "Other" + assert "Other" not in step_names[:-1] + assert sum(step["seconds"] for step in timings["steps"]) == pytest.approx( + timings["total_seconds"], abs=0.01 + ) assert synced["unknown_until_run"] == [*_UNKNOWN, *test_case.expected_added_unknowns] live_models: dict[str, dict[str, Any]] = {model["name"]: model for model in live.pop("models")} preview_models: dict[str, dict[str, Any]] = { @@ -263,3 +329,294 @@ def test_given_an_edited_model_when_a_changed_plan_is_requested_then_the_copy_na tmp_path / "location.log" ).read_text() assert preview["changed"] == json.loads(live_run.stdout)["changed"] + + +@pytest.mark.parametrize( + "test_case", + [ + ChangedBuildPublicationTestCase( + description="an edited model publishes as changed and its reader as downstream", + edit=("AS order_count", "AS order_count, 1 AS batch"), + expected_published={ + "changed": ["model:order_count"], + "downstream": ["model:order_summary"], + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_an_edited_model_when_the_location_plans_what_changed_then_it_publishes_what_the_changed_plan_runs( + test_case: ChangedBuildPublicationTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + summary: str = ( + 'MODEL (description "Order count summary.", materialized table);\n\n' + 'SELECT order_count FROM __ref("order_count")\n' + ) + write_files(project_dir=project, files=_PROJECT_FILES | {"models/order_summary.sql": summary}) + built: subprocess.CompletedProcess[str] = run_sqb(project, "build", "--target", "development") + assert built.returncode == 0, built.stderr + edited: str = _PROJECT_FILES["models/order_count.sql"].replace(*test_case.edit) + write_files(project_dir=project, files={"models/order_count.sql": edited}) + # The selection the server freezes `sqb build --changed` to: every node a build runs. + selection: list[str] = sorted([*_SELECTION, "model:order_summary", "seed:raw_orders"]) + + published, plan = changed_build_and_plan_from_copy(tmp_path, project, selection) + + assert published == changed_build_report(plan), (tmp_path / "location.log").read_text() + assert published == test_case.expected_published + + +@pytest.mark.parametrize( + "test_case", + [ + ChangedBuildReplanTestCase( + description="a publication and a sync that mirrors a new build each plan again", + edit=("AS order_count", "AS order_count, 1 AS batch"), + expected_published={"changed": ["model:order_count"], "downstream": []}, + expected_rebuilt={"changed": [], "downstream": []}, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_served_location_when_syncs_complete_then_it_replans_only_after_publication_or_an_applied_change( + test_case: ChangedBuildReplanTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + write_files(project_dir=project, files=_PROJECT_FILES) + built: subprocess.CompletedProcess[str] = run_sqb(project, "build", "--target", "development") + assert built.returncode == 0, built.stderr + edited: str = _PROJECT_FILES["models/order_count.sql"].replace(*test_case.edit) + write_files(project_dir=project, files={"models/order_count.sql": edited}) + control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" + log: Path = tmp_path / "location.log" + + def changed_build() -> dict[str, list[str]]: + with closing(sqlite3.connect(control)) as database: + row: tuple[str] | None = database.execute( + "SELECT report_json FROM sqb_changed_builds" + ).fetchone() + assert row is not None, "the location reported its changed build" + return json.loads(row[0]) + + with ( + snapshot_api(tmp_path / "control") as api, + location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, + ) as location, + ): + _ = wait_for_generation(api, "development", 1, location, log) + first: str = wait_for_log_count(location, log, _PLANNED, 1) + assert first.index(_SYNCED) < first.index(_PLANNED), ( + "a publication plans what changed once its sync completes, not from the old copy" + ) + assert changed_build() == test_case.expected_published + + _ = wait_for_location_work(api, request_warehouse_sync(control), location, log) + _ = wait_for_log_count(location, log, _SYNCED, 2) + # A later plan request runs on a later turn, after any plan the completed sync started. + _ = wait_for_location_work(api, request_plan(control, ["model:orders"]), location, log) + assert log.read_text().count(_PLANNED) == 1, "a sync that applied nothing keeps the count" + + rebuilt: subprocess.CompletedProcess[str] = run_sqb( + project, "build", "--target", "development" + ) + assert rebuilt.returncode == 0, rebuilt.stderr + _ = wait_for_location_work(api, request_warehouse_sync(control), location, log) + _ = wait_for_log_count(location, log, _PLANNED, 2) + assert changed_build() == test_case.expected_rebuilt, ( + "a sync that mirrored the new build plans again" + ) + + +@pytest.mark.parametrize( + "test_case", + [ + HeldPlanTestCase( + description=( + "only the plan that sets variables compiles the project again; the other two " + "and the changed-build count plan from the published project" + ), + settings=( + {"changed": True}, + {"changed": True, "vars": {"batch": "2"}}, + {"changed": True}, + ), + expected_runs=((), ("model:order_count",), ()), + expected_compiles=1, + expected_held_plans=3, + ), + ], + ids=lambda case: case.description, +) +def test_given_plan_requests_when_the_location_plans_them_then_only_variables_compile_the_project_again( + test_case: HeldPlanTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + files: dict[str, str] = _PROJECT_FILES | { + "sqlbuild_project.toml": _PROJECT_FILES["sqlbuild_project.toml"] + '[vars]\nbatch = "1"\n', + "models/order_count.sql": _PROJECT_FILES["models/order_count.sql"].replace( + "AS order_count", "AS order_count, @@batch AS batch" + ), + } + write_files(project_dir=project, files=files) + built: subprocess.CompletedProcess[str] = run_sqb(project, "build", "--target", "development") + assert built.returncode == 0, built.stderr + requests: list[tuple[list[str], dict[str, object]]] = [ + (["model:order_count"], settings) for settings in test_case.settings + ] + + plans: list[dict[str, Any]] = plans_from_copy(tmp_path, project, requests) + + log: str = (tmp_path / "location.log").read_text() + assert tuple(map(changed_runs, plans)) == test_case.expected_runs, log + assert log.count("Compiling project...") == test_case.expected_compiles, log + assert log.count(_HELD) == test_case.expected_held_plans, log + assert log.count(_SYNCED_READING) == test_case.expected_held_plans + 1, log + assert "Warehouse connected" not in log, "a synced copy opens no warehouse connection" + + +@pytest.mark.parametrize( + "test_case", + [ + UnknownSourceAccessTestCase( + description="a source the synced state does not list is unknown, not probed", + added_files={ + "sources/customers.yml": dedent( + """ + sources: + - name: raw_customers + description: Customers loaded outside this project. + schema: main + table: raw_customers + columns: + - name: customer_id + type: INTEGER + """ + ).strip() + + "\n", + "models/customers.sql": ( + 'MODEL (description "Customers.", materialized table);\n' + 'SELECT customer_id FROM __source("raw_customers")\n' + ), + }, + selection=["model:customers"], + expected_unknown="read access to these sources", + ), + ], + ids=lambda case: case.description, +) +def test_given_an_unlisted_source_when_the_location_previews_a_plan_then_its_read_access_is_unknown( + test_case: UnknownSourceAccessTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + write_files(project_dir=project, files=_PROJECT_FILES) + built: subprocess.CompletedProcess[str] = run_sqb(project, "build", "--target", "development") + assert built.returncode == 0, built.stderr + write_files(project_dir=project, files=test_case.added_files) + + _, preview = plan_from_copy(tmp_path, project, test_case.selection, {}) + + assert "error" not in preview, preview + assert test_case.expected_unknown in preview["synced_copy"]["unknown_until_run"], preview + + +@pytest.mark.parametrize( + "test_case", + [ + SyncedFreshnessTestCase( + description="a warehouse source's metadata freshness is unknown, not a failed lookup", + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "snowflake"\ndefault_target = "dev"\n' + '[connections.warehouse]\naccount = "example-account"\nuser = "example_user"\n' + '[targets.dev]\nconnection = "warehouse"\ndatabase = "ANALYTICS"\n' + 'schema = "DEV"\n' + ), + "sources/raw.yml": dedent( + """ + sources: + - name: raw_orders + description: Raw orders. + database: RAW + schema: SHOP + table: ORDERS + columns: + - name: order_id + type: VARCHAR + """ + ).strip() + + "\n", + "models/orders.sql": ( + "MODEL (description 'Orders.', materialized table);\n" + 'SELECT order_id FROM __source("raw_orders")\n' + ), + }, + expected_absent="Freshness metadata FAIL", + expected_line="Freshness metadata OK (1 source, 1 unknown)", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_synced_copy_when_a_preview_reads_source_freshness_then_it_is_unknown_without_a_failure( + test_case: SyncedFreshnessTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + write_files(project_dir=project, files=test_case.files) + + outcome, log = plan_before_sync(tmp_path, project, {"changed": True}) + + assert outcome["kind"] == "plan", log + assert "error" not in outcome["document"], outcome + assert test_case.expected_absent not in log, log + assert test_case.expected_line in log, log + + +@pytest.mark.parametrize( + "test_case", + [ + FailedPlanTestCase( + description="a full_refresh false caller of a changed function cannot rebuild", + files=_FUNCTION_CALLER_FILES, + edited_file="functions/sql/udf__adjust_amount.sql", + edit=("amount + 1", "amount + 100"), + selection=["model:order_amounts"], + expected_code="S203", + expected_reason="model 'order_amounts' sets full_refresh false", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_plan_that_fails_when_the_location_plans_it_then_its_own_error_is_stored( + test_case: FailedPlanTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + write_files(project_dir=project, files=test_case.files) + built: subprocess.CompletedProcess[str] = run_sqb(project, "build", "--target", "development") + assert built.returncode == 0, built.stderr + edited: Path = project / test_case.edited_file + _ = edited.write_text( + edited.read_text(encoding="utf-8").replace(*test_case.edit), encoding="utf-8" + ) + + published, plan = changed_build_and_plan_from_copy(tmp_path, project, test_case.selection) + + assert plan["error"]["code"] == test_case.expected_code, plan + assert test_case.expected_reason in plan["error"]["message"], plan + assert published["changed"] == [] and published["downstream"] == [], published + assert published["failing"] == plan["error"]["message"], published diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py index a4e413d3e4..a1cd6af2d2 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -26,7 +26,7 @@ from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( METADATA, - TYPE_PROOF_FILES, + PUBLISHABLE_FILES, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( PlainHttpWarningTestCase, @@ -64,7 +64,7 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical project_dir: Path = tmp_path / "project" duckshop: Path = Path("benchmarks/duckshop/project") files: dict[str, str] = ( - TYPE_PROOF_FILES + PUBLISHABLE_FILES | { path.relative_to(duckshop).as_posix(): path.read_text() for path in [ @@ -73,7 +73,10 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical ] } | { - "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( + # DuckShop's edge-case jobs take these pools, which its project file declares. + "sqlbuild_project.toml": PUBLISHABLE_FILES["sqlbuild_project.toml"] + + "\n[pools]\nedge_inventory_limit = 1\nedge_products_limit = 1\n", + "models/orders.sql": PUBLISHABLE_FILES["models/orders.sql"].replace( "MODEL (", "MODEL (tags ['edge_case'],", 1 ), } @@ -101,7 +104,7 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical assert frozenset(initial["nodes"]) == test_case.expected_nodes assert initial["nodes"]["model:orders"]["definition"]["deps"] == ["model:stg_orders"] edited: dict[str, str] = files | { - "models/stg_orders.sql": TYPE_PROOF_FILES["models/stg_orders.sql"].replace( + "models/stg_orders.sql": PUBLISHABLE_FILES["models/stg_orders.sql"].replace( "CAST(5 AS INTEGER)", "CAST(9 AS INTEGER)" ) } @@ -195,8 +198,8 @@ def test_given_a_code_location_when_publishing_from_cli_then_compilation_isolate capsys: pytest.CaptureFixture[str], ) -> None: project_dir: Path = tmp_path / "project" - files: dict[str, str] = TYPE_PROOF_FILES | { - "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + files: dict[str, str] = PUBLISHABLE_FILES | { + "sqlbuild_project.toml": PUBLISHABLE_FILES["sqlbuild_project.toml"] + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + f'\n[targets.{json.dumps(test_case.target)}]\nschema = "main"\n', "orchestration/jobs/daily_orders_job.toml": ( @@ -339,9 +342,9 @@ def test_given_a_plain_http_server_when_a_location_publishes_then_only_a_non_loo project_dir: Path = tmp_path / "project" write_files( project_dir=project_dir, - files=TYPE_PROOF_FILES + files=PUBLISHABLE_FILES | { - "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + "sqlbuild_project.toml": PUBLISHABLE_FILES["sqlbuild_project.toml"] + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + '\n[targets.development]\nschema = "main"\n', }, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py index e2562c2f3e..bc9730285e 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py @@ -13,7 +13,7 @@ import duckdb import pytest -from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import PUBLISHABLE_FILES from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( LocationDatabaseTestCase, LocationReloadTestCase, @@ -22,6 +22,7 @@ from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( current_snapshot, location_process, + published_connection_health, snapshot_api, wait_for_generation, wait_for_interpreter, @@ -53,7 +54,7 @@ ], ids=lambda case: case.description, ) -def test_given_a_served_location_when_inputs_change_or_restart_then_generations_reload_without_losing_last_good( +def test_given_a_served_location_when_inputs_change_or_restart_then_generations_reload_without_losing_last_good( # noqa: PLR0915 test_case: LocationReloadTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: project: Path = tmp_path / "project" @@ -72,8 +73,8 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ 'name = "daily_orders_job"\ndescription = "Build orders."\nselect = "orders"\n' ), } - files: dict[str, str] = TYPE_PROOF_FILES | { - "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + files: dict[str, str] = PUBLISHABLE_FILES | { + "sqlbuild_project.toml": PUBLISHABLE_FILES["sqlbuild_project.toml"] + f"\n[connections.local]\ndatabase = {json.dumps(test_case.database)}\n" + f'\n[targets.{json.dumps(test_case.target)}]\nschema = "main"\n', **orchestration, @@ -108,10 +109,10 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ heartbeat: dict[str, Any] = wait_for_location_health( api, test_case.target, {"kind": "heartbeat"}, process, log ) - initial_health: dict[str, Any] = wait_for_location_health( + initial_health: dict[str, Any] = published_connection_health( api, - test_case.target, - {"kind": "connection", "generation": 1, "healthy": test_case.expected_connection}, + (test_case.target, test_case.expected_connection), + (first, "startup"), process, log, ) @@ -128,6 +129,13 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ assert int(compiler_pid.read_text()) == first_interpreter assert "CAST(13 AS INTEGER)" in second["nodes"]["model:stg_orders"]["raw_sql"] assert {change["subject_key"] for change in second["changes"]} == {"model:stg_orders"} + _ = published_connection_health( + api, + (test_case.target, test_case.expected_connection), + (second, "publish"), + process, + log, + ) _ = staging.write_text("SELECT FROM invalid SQL !\n") _ = wait_for_location_health( @@ -204,10 +212,10 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ ) assert int(compiler_pid.read_text()) != replacement_interpreter assert restarted["nodes"] == configured["nodes"] - restarted_health: dict[str, Any] = wait_for_location_health( + restarted_health: dict[str, Any] = published_connection_health( api, - test_case.target, - {"kind": "connection", "generation": 7, "healthy": test_case.expected_connection}, + (test_case.target, test_case.expected_connection), + (restarted, "startup"), process, log, ) @@ -256,9 +264,9 @@ def test_given_a_busy_location_when_stopped_then_its_interpreter_is_reaped( child_pid: Path = tmp_path / "interpreter.pid" write_files( project_dir=project, - files=TYPE_PROOF_FILES + files=PUBLISHABLE_FILES | { - "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + "sqlbuild_project.toml": PUBLISHABLE_FILES["sqlbuild_project.toml"] + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + '\n[targets.development]\nschema = "main"\n', "adapters/slow.py": ( @@ -325,9 +333,9 @@ def test_given_a_location_using_a_local_database_when_it_is_created_and_written_ project: Path = tmp_path / "project" write_files( project_dir=project, - files=TYPE_PROOF_FILES + files=PUBLISHABLE_FILES | { - "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + "sqlbuild_project.toml": PUBLISHABLE_FILES["sqlbuild_project.toml"] + f"\n[connections.local]\n{test_case.connection_toml}\n" + '\n[targets.development]\nschema = "main"\n', }, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rules_at_load.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rules_at_load.py new file mode 100644 index 0000000000..54dffdc1d4 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rules_at_load.py @@ -0,0 +1,135 @@ +"""A location load is a full compile: its rules run once, at load, and never again on its snapshot.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + RulesAtLoadTestCase, + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + failed_load, + plans_from_copy, + run_sqb, + scheduled_worker, + worker_result, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + +_PROJECT: str = ( + 'name = "orders"\nadapter = "duckdb"\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + '[targets.development]\nschema = "main"\n' +) +_SELECT_STAR_RULE: str = '[rules]\nselect = ["SQBRMODEL102"]\n' +_NO_RULES: str = "[rules]\nselect = []\n" +_MODEL: str = "MODEL (description 'Orders.', materialized table);\n" +_EVALUATING: str = "Evaluating configured rules..." + + +@pytest.mark.parametrize( + "test_case", + [ + RulesAtLoadTestCase( + description="a SELECT * model fails the load as a compile failure with its finding", + model_sql="SELECT * FROM (SELECT 1 AS amount) AS amounts", + expected_health={ + "kind": "compile_failed", + "generation": 1, + "rules": { + "finding_count": 1, + "findings": [ + { + "code": "SQBRMODEL102", + "message": "SELECT * is allowed only inside dependency import CTEs", + "path": "models/orders.sql", + "line": 2, + } + ], + }, + }, + expected_terminal="Compile failed: 1 rule finding", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_rule_finding_when_a_location_loads_then_nothing_is_published_and_health_lists_it( + test_case: RulesAtLoadTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + write_files( + project_dir=project, + files={ + "sqlbuild_project.toml": _PROJECT + _SELECT_STAR_RULE, + "models/orders.sql": _MODEL + test_case.model_sql + "\n", + }, + ) + + health, snapshot_status, log = failed_load(tmp_path, project) + + assert health == test_case.expected_health, log + assert snapshot_status == 404, log + assert test_case.expected_terminal in log, log + + +@pytest.mark.parametrize( + "test_case", + [ + RulesAtLoadTestCase( + description="plan previews on the published snapshot leave the rules to the load", + model_sql="SELECT 1 AS amount", + expected_rule_evaluations=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_published_snapshot_when_plans_are_previewed_then_rules_ran_only_at_load( + test_case: RulesAtLoadTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + write_files( + project_dir=project, + files={ + "sqlbuild_project.toml": _PROJECT + _SELECT_STAR_RULE, + "models/orders.sql": _MODEL + test_case.model_sql + "\n", + }, + ) + built: subprocess.CompletedProcess[str] = run_sqb(project, "build", "--target", "development") + assert built.returncode == 0, built.stderr + + plans: list[dict[str, Any]] = plans_from_copy( + tmp_path, project, [(["model:orders"], {}), (["model:orders"], {"changed": True})] + ) + + log: str = (tmp_path / "location.log").read_text() + assert len(plans) == 2, log + assert log.count(_EVALUATING) == test_case.expected_rule_evaluations, log + assert "Configured rules passed." in log, log + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + description="a scheduled run builds what was published though its rules now fail", + model_sql="SELECT * FROM (SELECT 1 AS amount) AS amounts", + warehouse_query="SELECT amount FROM main.orders", + expected_status="success", + expected_rows=[(1,)], + files={"sqlbuild_project.toml": _PROJECT + _NO_RULES}, + files_after_publication={"sqlbuild_project.toml": _PROJECT + _SELECT_STAR_RULE}, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_published_snapshot_when_a_scheduled_run_executes_then_it_does_not_evaluate_rules( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = worker_result(worker, test_case) + + assert run["status"] == test_case.expected_status, worker.log.read_text() diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py index 4b61ff1e7c..670d19029b 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py @@ -62,7 +62,7 @@ def test_given_real_catalog_when_reading_pages_then_columns_and_physical_types_a ) adapter.close(connection) before: str = hashlib.sha256((project / "warehouse.db").read_bytes()).hexdigest() - pages: list[dict[str, Any]] = warehouse_history_pages(project, "catalog") + pages: list[dict[str, Any]] = warehouse_history_pages(project, "catalog", limit=100) assert tuple(len(page["records"]) for page in pages) == (100, 7) records: list[dict[str, Any]] = [] for page in pages: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index fdfe4ea104..20dd587f77 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -12,6 +12,7 @@ import time from contextlib import closing from datetime import UTC, datetime, timedelta, timezone +from itertools import chain from pathlib import Path from typing import Any from urllib.request import Request, urlopen @@ -37,12 +38,14 @@ from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( METADATA, - TYPE_PROOF_FILES, + PUBLISHABLE_FILES, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( AutomaticHistoryTestCase, CatalogConnectionTestCase, FingerprintHistoryTestCase, + NewestVersionsTestCase, + PageBytesTestCase, ReadFailureTestCase, StateEventsTestCase, SyncPacingTestCase, @@ -269,7 +272,7 @@ def test_given_external_state_events_when_reading_pages_then_identities_text_and for column in legacy_columns: _ = adapter.execute(connection=connection, sql=f"ALTER TABLE {table} DROP COLUMN {column}") adapter.close(connection) - pages: list[dict[str, Any]] = warehouse_history_pages(project, fixture["kind"]) + pages: list[dict[str, Any]] = warehouse_history_pages(project, fixture["kind"], limit=100) rows: list[dict[str, Any]] = [] for page in pages: rows.extend(page["records"]) @@ -285,7 +288,7 @@ def test_given_external_state_events_when_reading_pages_then_identities_text_and ) assert rows[-1] == expected | fixture["last_record"] | dict.fromkeys(legacy_columns) recent_pages: list[dict[str, Any]] = warehouse_history_pages( - project, fixture["kind"], fixture["later_time"] + project, fixture["kind"], fixture["later_time"], limit=100 ) recent_rows: list[dict[str, Any]] = [] for page in recent_pages: @@ -591,7 +594,7 @@ def test_given_populated_predecessor_tables_when_writers_add_ids_then_history_pr assert predecessor.execute("PRAGMA journal_mode=DELETE").fetchone() == (journal_mode,) def page(after: list[str | None] | None = None, observed_since: str | None = None) -> Any: - return warehouse_history_page(project, "source_freshness", after, observed_since) + return warehouse_history_page(project, "source_freshness", after, observed_since, limit=100) old: dict[str, Any] = page() assert len(old["records"]) == 1 and old["records"][0]["event_id"] is None @@ -832,7 +835,7 @@ def test_given_external_warehouse_retention_when_a_complete_scan_arrives_then_mi ) adapter.close(connection) declarations: Path = tmp_path / "declarations" - write_files(project_dir=declarations, files=TYPE_PROOF_FILES) + write_files(project_dir=declarations, files=PUBLISHABLE_FILES) discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=declarations) compiled: CompiledProjectView = build_compiled_project( discovered_inputs=discovered, @@ -1099,7 +1102,11 @@ def test_given_durable_mirror_when_late_rows_arrive_then_incremental_overlap_pre project_dir=project, files={"models/orders.sql": "MODEL (description 'Orders.'); SELECT 17 AS amount"}, ) - with snapshot_api(tmp_path / "control") as api: + # Every build of the one node stays, so the overlap, not retention, decides what is mirrored. + with snapshot_api( + tmp_path / "control", + server_arguments=("--state-history-retention-versions", "65535"), + ) as api: control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" log: Path = tmp_path / "location.log" with location_process( @@ -1159,7 +1166,7 @@ def test_given_durable_mirror_when_late_rows_arrive_then_incremental_overlap_pre assert observed <= expected, (table, stage, observed) remaining |= observed != expected remaining |= ( - log.read_text().count("Warehouse history synchronization completed.") + log.read_text().count("Warehouse history synchronization completed") < stage["completed_cycles"] ) time.sleep(0.05) @@ -1179,19 +1186,17 @@ def test_given_durable_mirror_when_late_rows_arrive_then_incremental_overlap_pre datetime.fromisoformat(report["received_at"]) for report in reports ) time.sleep(6) - completed: int = log.read_text().count("Warehouse history synchronization completed.") + completed: int = log.read_text().count("Warehouse history synchronization completed") time.sleep(12) assert ( - log.read_text().count("Warehouse history synchronization completed.") == completed + log.read_text().count("Warehouse history synchronization completed") == completed ), "an idle location read the warehouse without a run ending" work: dict[str, Any] = wait_for_location_work( api, request_warehouse_sync(control), location, log ) assert work["result_json"]["outcome"] == {"kind": "sync_started"} deadline = time.monotonic() + 45 - while ( - log.read_text().count("Warehouse history synchronization completed.") == completed - ): + while log.read_text().count("Warehouse history synchronization completed") == completed: assert location.poll() is None, log.read_text() assert time.monotonic() < deadline, log.read_text() time.sleep(0.05) @@ -1271,7 +1276,7 @@ def test_given_fingerprint_history_when_read_in_isolated_pages_then_legacy_retri adapter.close(connection) def page(after: list[str | None] | None = None, observed_since: str | None = None) -> Any: - return warehouse_history_page(project, "fingerprints", after, observed_since) + return warehouse_history_page(project, "fingerprints", after, observed_since, limit=100) legacy: dict[str, Any] = page() assert legacy["next_after"] is None @@ -1645,8 +1650,7 @@ def synced(completed: int) -> None: assert paced == test_case.added_rows, test_case.description assert elapsed < test_case.expected_max_seconds, ( f"{test_case.description}: {elapsed:.1f} s for " - f"{test_case.added_rows // 100} pages; a tick per page needs at least " - f"{test_case.added_rows / 400:.0f} s" + f"{test_case.added_rows} rows; pages must run back to back" ) calls.write_text("") @@ -1656,3 +1660,116 @@ def synced(completed: int) -> None: f"{test_case.description}: an idle location ran git {len(idle_calls)} times " f"in {test_case.idle_seconds} s" ) + + +@pytest.mark.parametrize( + "test_case", + [ + NewestVersionsTestCase( + "DuckDB first read keeps each node's newest two builds", + "duckdb", + "database", + 2, + { + "customers": ["customers-run-3", "customers-run-4"], + "orders": ["orders-run-3", "orders-run-4"], + }, + ), + NewestVersionsTestCase( + "SQLite first read keeps each node's newest build", + "sqlite", + "path", + 1, + {"customers": ["customers-run-4"], "orders": ["orders-run-4"]}, + ), + NewestVersionsTestCase( + "a read without a limit delivers every build", + "duckdb", + "database", + None, + { + "customers": [f"customers-run-{build}" for build in range(5)], + "orders": ["legacy-run"] + [f"orders-run-{build}" for build in range(5)], + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_many_builds_per_node_when_a_full_read_has_a_version_limit_then_only_the_newest_arrive( + test_case: NewestVersionsTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + for node in ("orders", "customers"): + for build in range(5): + _ = adapter.execute( + connection=connection, + sql=( + "INSERT INTO main._sqlbuild_fingerprints SELECT node_type," + f"'{node}',target_database,target_schema,'{node}','{node}-run-{build}'," + "definition_hash,version_hash,schema_fingerprint,definition_b64," + f"metadata_json_b64,'2026-03-0{build + 1} 00:00:00' " + "FROM main._sqlbuild_fingerprints WHERE run_id='legacy-run'" + ), + ) + adapter.close(connection) + + pages: list[dict[str, Any]] = warehouse_history_pages( + project, "fingerprints", newest_versions=test_case.newest_versions + ) + records: list[dict[str, Any]] = list(chain.from_iterable(page["records"] for page in pages)) + runs: dict[str, list[str]] = {} + for record in records: + runs.setdefault(record["node_name"], []).append(record["run_id"]) + + assert {node: sorted(found) for node, found in runs.items()} == test_case.expected_runs + + +@pytest.mark.parametrize( + "test_case", + [ + PageBytesTestCase("DuckDB wide builds", "duckdb", "database", 24, 1_000_000, 3), + PageBytesTestCase("SQLite wide builds", "sqlite", "path", 24, 1_000_000, 3), + ], + ids=lambda case: case.description, +) +def test_given_wide_builds_when_reading_large_pages_then_the_byte_budget_ends_each_page_and_no_row_is_lost( + test_case: PageBytesTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + definition: str = base64.b64encode( + ("SELECT 1 /*" + "x" * test_case.definition_bytes + "*/").encode() + ).decode() + for index in range(test_case.wide_rows): + _ = adapter.execute( + connection=connection, + sql=( + "INSERT INTO main._sqlbuild_fingerprints SELECT node_type," + f"'inventory_{index:03}',target_database,target_schema,target_name," + f"'wide-run-{index:03}',definition_hash,version_hash,schema_fingerprint," + f"'{definition}',metadata_json_b64,ts " + "FROM main._sqlbuild_fingerprints WHERE run_id='legacy-run'" + ), + ) + adapter.close(connection) + + pages: list[dict[str, Any]] = warehouse_history_pages(project, "fingerprints") + + records: list[dict[str, Any]] = list(chain.from_iterable(page["records"] for page in pages)) + runs: list[str] = [record["run_id"] for record in records] + assert sorted(runs) == sorted( + ["legacy-run"] + [f"wide-run-{index:03}" for index in range(test_case.wide_rows)] + ), "every build arrives exactly once across byte-bounded pages" + assert len(pages) >= test_case.expected_min_pages, [len(page["records"]) for page in pages] + for page in pages: + encoded: int = len(json.dumps(page["records"]).encode()) + assert encoded <= 8 * 1024 * 1024 + 2 * test_case.definition_bytes, encoded diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_work.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_work.py index 12f864c08d..3f43358ad8 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_work.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_work.py @@ -13,7 +13,7 @@ import pytest -from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import PUBLISHABLE_FILES from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( LocationWorkTestCase, ) @@ -51,9 +51,9 @@ def test_given_a_published_location_when_an_owner_requests_work_then_results_ret child_pid: Path = tmp_path / "debug.pid" write_files( project_dir=project, - files=TYPE_PROOF_FILES + files=PUBLISHABLE_FILES | { - "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + "sqlbuild_project.toml": PUBLISHABLE_FILES["sqlbuild_project.toml"] + f"\n[connections.local]\ndatabase = {json.dumps(test_case.database)}\n" + '\n[targets."preview #1"]\nschema = "main"\n', "adapters/slow.py": ( @@ -152,7 +152,7 @@ def test_given_a_published_location_when_an_owner_requests_work_then_results_ret ) _ = wait_for_interpreter(process, child_pid, log) _ = (project / "models/stg_orders.sql").write_text( - TYPE_PROOF_FILES["models/stg_orders.sql"].replace( + PUBLISHABLE_FILES["models/stg_orders.sql"].replace( "CAST(5 AS INTEGER)", "CAST(17 AS INTEGER)" ) ) @@ -182,9 +182,9 @@ def test_given_person_roles_when_requesting_connection_tests_then_live_scope_and project: Path = tmp_path / "project" write_files( project_dir=project, - files=TYPE_PROOF_FILES + files=PUBLISHABLE_FILES | { - "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + "sqlbuild_project.toml": PUBLISHABLE_FILES["sqlbuild_project.toml"] + f"\n[connections.local]\ndatabase = {json.dumps(test_case.database)}\n" + '\n[targets.development]\nschema = "main"\n', }, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 344395a439..59b3f4edd6 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -587,12 +587,13 @@ def test_given_a_failing_seed_audit_when_a_worker_builds_then_its_seed_fails_wit "test_case", [ WorkerExecutionTestCase( - description="opposing parent and independent child limits", + description="opposing parent and independent child limits, capped by the launcher", model_sql="SELECT 11 AS amount", warehouse_query="SELECT amount FROM orders UNION ALL SELECT amount FROM inventory ORDER BY amount", expected_status="success", expected_rows=[(11,), (23,)], - concurrency_limits={"orders_limit": 1, "inventory_limit": 1}, + pools={"orders_limit": 2, "inventory_limit": 2}, + launcher_settings={"pool_caps": {"orders_limit": 1, "inventory_limit": 1}}, files={ "models/inventory.sql": ( "MODEL (description 'Inventory.', materialized table);\nSELECT 23 AS amount\n" @@ -601,26 +602,26 @@ def test_given_a_failing_seed_audit_when_a_worker_builds_then_its_seed_fails_wit 'name = "orders_job"\n' 'description = "Build orders."\n' 'select = "orders"\n' - 'concurrency_groups = ["inventory_limit"]\n' + "pools = { inventory_limit = 1 }\n" ), "orchestration/jobs/inventory_job.toml": ( 'name = "inventory_job"\n' 'description = "Build inventory."\n' 'select = "inventory"\n' - 'concurrency_groups = ["orders_limit"]\n' + "pools = { orders_limit = 1 }\n" ), "orchestration/job_groups/annual_orders_job_group.toml": ( 'name = "annual_orders_job_group"\n' 'description = "Orders group."\n' 'jobs = ["orders_job"]\n' - 'concurrency_groups = ["orders_limit"]\n' + "pools = { orders_limit = 1 }\n" 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), "orchestration/job_groups/annual_inventory_job_group.toml": ( 'name = "annual_inventory_job_group"\n' 'description = "Inventory group."\n' 'jobs = ["inventory_job"]\n' - 'concurrency_groups = ["inventory_limit"]\n' + "pools = { inventory_limit = 1 }\n" 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, @@ -647,10 +648,14 @@ def test_given_opposing_group_limits_when_scheduled_then_each_group_holds_every_ ) assert orders["status"] == inventory["status"] == "success", worker.log.read_text() with sqlite3.connect(worker.control_store) as control: - assert control.execute( - "SELECT r.job_key, c.group_key, max(c.occupied) FROM test_slot_claims c " - "JOIN sqb_runs r ON r.run_id=c.run_id GROUP BY r.job_key, c.group_key ORDER BY r.job_key" - ).fetchall() == [ + assert [ + (job_key, group_key.rsplit("/", 1)[-1], occupied) + for job_key, group_key, occupied in control.execute( + "SELECT r.job_key, c.group_key, max(c.occupied) FROM test_slot_claims c " + "JOIN sqb_runs r ON r.run_id=c.run_id GROUP BY r.job_key, c.group_key " + "ORDER BY r.job_key, c.group_key" + ) + ] == [ ("annual_inventory_job_group", "inventory_limit", 1), ("annual_inventory_job_group", "orders_limit", 1), ("annual_orders_job_group", "inventory_limit", 1), diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 4a60108d7c..d38ad1f21e 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -29,6 +29,7 @@ PythonJobGroupTestCase, PythonJobSelectionTestCase, SnapshotCatalogTestCase, + SnapshotConcurrencyTestCase, SnapshotDeclarationRejectionTestCase, SnapshotDiscoveryRejectionTestCase, SnapshotDiscoveryTestCase, @@ -39,6 +40,7 @@ SnapshotProcessReuseTestCase, SnapshotProvidersTestCase, SnapshotRejectionTestCase, + SnapshotRetryTestCase, SnapshotScheduleTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( @@ -886,6 +888,9 @@ def snapshot(schedule: str) -> dict[str, Any]: write_files( project_dir=tmp_path, files={ + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + "\n[pools]\norder_reporting = 3\n" + + "\n[targets.development.pools]\norder_reporting = 2\n", "orchestration/jobs/nightly_orders_job.toml": ( 'name = "nightly_orders_job"\n' 'description = "Refresh order reporting."\n' @@ -894,7 +899,7 @@ def snapshot(schedule: str) -> dict[str, Any]: 'targets = ["development"]\n' 'owner = "Orders team"\n' 'overlap = "skip"\n' - 'concurrency_groups = ["order_reporting"]\n' + "pools = { order_reporting = 2 }\n" f"schedule = {schedule}\n" "\n[on_failure]\nretries = 2\nretry_delay_seconds = 600\n" '\n[settings]\nrun_tests = false\nvars = { batch = 3 }\nwarehouse = "reporting"\n' @@ -929,7 +934,8 @@ def snapshot(schedule: str) -> dict[str, Any]: "vars": {"batch": 3}, "warehouse": "reporting", }, - "concurrency_groups": ["order_reporting"], + "pools": {"order_reporting": {"amount": 2, "limit": 2}}, + "resolved_concurrency": {"limit": 4, "source": "job"}, "select": "+orders", "exclude": "stg_orders", "targets": ["development"], @@ -956,6 +962,160 @@ def snapshot(schedule: str) -> dict[str, Any]: assert second["nodes"] == first["nodes"] +@pytest.mark.parametrize( + "test_case", + [ + SnapshotConcurrencyTestCase( + description="nothing set uses SQLBuild's default", + project_tables="", + local_files={}, + job_settings="", + selected_target=None, + expected_job={"limit": 1, "source": "default"}, + expected_group={"limit": 1, "source": "default"}, + ), + SnapshotConcurrencyTestCase( + description="the project's setting", + project_tables="\n[settings]\nconcurrency = 3\n", + local_files={}, + job_settings="", + selected_target=None, + expected_job={"limit": 3, "source": "project"}, + expected_group={"limit": 3, "source": "project"}, + ), + SnapshotConcurrencyTestCase( + description="the location's local setting over the project's", + project_tables="\n[settings]\nconcurrency = 3\n", + local_files={"sqlbuild_local.toml": "[settings]\nconcurrency = 5\n"}, + job_settings="", + selected_target=None, + expected_job={"limit": 5, "source": "local"}, + expected_group={"limit": 5, "source": "local"}, + ), + SnapshotConcurrencyTestCase( + description="the target's concurrency over the settings", + project_tables='\n[settings]\nconcurrency = 3\n\n[targets.development]\nschema = "main"\nconcurrency = 6\n', + local_files={}, + job_settings="", + selected_target="development", + expected_job={"limit": 6, "source": "target"}, + expected_group={"limit": 6, "source": "target"}, + ), + SnapshotConcurrencyTestCase( + description="the job's own setting over the target's; its group keeps the target's", + project_tables='\n[targets.development]\nschema = "main"\nconcurrency = 6\n', + local_files={}, + job_settings="\n[settings]\nconcurrency = 2\n", + selected_target="development", + expected_job={"limit": 2, "source": "job"}, + expected_group={"limit": 6, "source": "target"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_concurrency_layers_when_publishing_then_each_job_and_group_carries_its_resolved_concurrency( + tmp_path: Path, test_case: SnapshotConcurrencyTestCase +) -> None: + write_files( + project_dir=tmp_path, + files=TYPE_PROOF_FILES + | test_case.local_files + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + test_case.project_tables, + "orchestration/jobs/nightly_orders_job.toml": ( + 'name = "nightly_orders_job"\n' + 'description = "Refresh order reporting."\n' + 'select = "+orders"\n' + test_case.job_settings + ), + "orchestration/job_groups/nightly_job_group.toml": ( + 'name = "nightly_job_group"\n' + 'description = "Everything nightly."\n' + 'jobs = ["nightly_orders_job"]\n' + ), + }, + ) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discovered, + adapter=DuckDbAdapter(), + selected_target=test_case.selected_target, + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + + documents: dict[str, Any] = json.loads( + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + discovered_inputs=discovered, + ).to_json() + )["documents"] + + assert ( + documents["jobs"]["nightly_orders_job"]["resolved_concurrency"], + documents["groups"]["nightly_job_group"]["resolved_concurrency"], + ) == (test_case.expected_job, test_case.expected_group) + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotRetryTestCase( + description="current retry names", + tables="[retry]\nattempts = 3\ndelay_seconds = 900\n", + expected_on_failure={"retries": 2, "retry_delay_seconds": 900}, + expected_warnings=(), + ), + SnapshotRetryTestCase( + description="deprecated on_failure names", + tables="[on_failure]\nretries = 2\nretry_delay_seconds = 600\n", + expected_on_failure={"retries": 2, "retry_delay_seconds": 600}, + expected_warnings=( + "Warning: orchestration/jobs/daily_build_job.toml:6:1: " + "`[on_failure]` is deprecated; rename it to `[retry]`", + "Warning: orchestration/jobs/daily_build_job.toml:7:11: " + "`retries` is deprecated; rename it to `attempts`", + "Warning: orchestration/jobs/daily_build_job.toml:8:23: " + "`retry_delay_seconds` is deprecated; rename it to `delay_seconds`", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_job_retry_table_when_compiling_then_both_names_publish_one_policy_and_old_ones_warn( + tmp_path: Path, capfd: pytest.CaptureFixture[str], test_case: SnapshotRetryTestCase +) -> None: + text: str = "".join(f"{key} = {value}\n" for key, value in _DAILY_BUILD.items()) + write_files( + project_dir=tmp_path, + files=TYPE_PROOF_FILES + | {"orchestration/jobs/daily_build_job.toml": text + "\n" + test_case.tables}, + ) + capfd.readouterr() + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + warnings: list[str] = list( + filter(lambda line: line.startswith("Warning: "), capfd.readouterr().err.splitlines()) + ) + snapshot: dict[str, Any] = json.loads( + compile_snapshot( + project=build_compiled_project( + discovered_inputs=discovered, + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ), + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + discovered_inputs=discovered, + ).to_json() + ) + + assert snapshot["documents"]["jobs"]["daily_build_job"]["on_failure"] == ( + test_case.expected_on_failure + ) + assert tuple(warnings) == test_case.expected_warnings + + @pytest.mark.parametrize( "test_case", [ @@ -1172,6 +1332,8 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( "MODEL (", "MODEL (tags ['edge_case'],", 1 ), + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + "\n[pools]\nedge_inventory_limit = 1\nedge_products_limit = 1\n", }, expected_jobs=( "edge_snapshot_inventory_job", @@ -1293,6 +1455,46 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject } }, ), + SnapshotDiscoveryTestCase( + description="a run_event on a group member can watch for the group skipping it", + files={ + "orchestration/jobs/source_orders_job.toml": ( + 'name = "source_orders_job"\ndescription = "Orders."\nselect = "orders"\n' + ), + "orchestration/jobs/inventory_job.toml": ( + 'name = "inventory_job"\ndescription = "Inventory."\nselect = "stg_orders"\n' + ), + "orchestration/job_groups/source_job_group.toml": ( + 'name = "source_job_group"\ndescription = "Source orders."\n' + 'jobs = ["source_orders_job"]\n' + ), + "orchestration/sensors/on_orders_sensor.py": ( + "from sqlbuild.orchestration import RunEventKind, run_event, sensor\n" + "on_orders = sensor(\n" + " 'on_orders_sensor',\n" + " description='Build inventory when the group skips orders.',\n" + " when=run_event(job='source_orders_job', kinds=[RunEventKind.SKIPPED]),\n" + " run='inventory_job',\n" + ")\n" + ), + }, + expected_jobs=("inventory_job", "source_orders_job"), + expected_groups={"source_job_group": {"source_orders_job": []}}, + expected_sensors={ + "on_orders_sensor": { + "description": "Build inventory when the group skips orders.", + "job_key": "inventory_job", + "when": { + "kind": "run_event", + "job_key": "source_orders_job", + "kinds": ["skipped"], + }, + "minimum_interval_seconds": 30, + "default_status": "running", + "declared_at": {"path": "orchestration/sensors/on_orders_sensor.py", "line": 2}, + } + }, + ), SnapshotDiscoveryTestCase( description="any-run node selectors freeze canonical keys without a source job reference", files={**duckshop_orchestration("node_orders")}, @@ -1897,6 +2099,41 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects expected_error=ValueError, expected_message="unknown variant", ), + SnapshotDiscoveryRejectionTestCase( + description="a job using a pool the project does not declare", + files={ + "orchestration/jobs/nightly_orders_job.toml": ( + 'name = "nightly_orders_job"\n' + 'description = "Refresh order reporting."\n' + 'select = "orders"\n' + "pools = { nightly_window = 1 }\n" + ) + }, + expected_error=ValueError, + expected_message=( + r"job 'nightly_orders_job' uses pool 'nightly_window', which " + r"\[pools\] in sqlbuild_project.toml does not declare" + ), + ), + SnapshotDiscoveryRejectionTestCase( + description="a job taking more of a pool than its limit on a target", + files={ + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + "\n[pools]\nsupplier_api = 3\n" + + "\n[targets.production.pools]\nsupplier_api = 1\n", + "orchestration/jobs/nightly_orders_job.toml": ( + 'name = "nightly_orders_job"\n' + 'description = "Refresh order reporting."\n' + 'select = "orders"\n' + "pools = { supplier_api = 2 }\n" + ), + }, + expected_error=ValueError, + expected_message=( + r"job 'nightly_orders_job' takes 2 slots of pool 'supplier_api', more than its " + r"limit of 1 on target 'production'" + ), + ), ], ids=lambda case: case.description, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py index d185c972c0..31ba55728a 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py @@ -5,6 +5,7 @@ from sqlbuild._native import ( CompiledResourceType, + Dialect, RelationFixtureCompletion, RelationFixturePlanningContext, build_compiled_project, @@ -65,7 +66,7 @@ def test_given_unrelated_project_types_when_completing_fixture_then_eager_work_i fixture_key: tuple[CompiledResourceType, str] = (CompiledResourceType.SOURCE, "raw_orders") completed: RelationFixtureCompletion = build_relation_fixture_completion( project=project, - adapter=adapter, + dialect=Dialect(adapter), request=( ("orders",), ((CompiledResourceType.SOURCE, {"raw_orders": "SELECT 1 AS order_id"}),), diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py index ebfacda47b..c52642fb62 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -19,6 +19,7 @@ CompiledSource, CompileModelConfig, CompileSqlReference, + Dialect, DiscoveredSchemaFile, DiscoveredSeedFile, DiscoveredSourceFile, @@ -30,6 +31,7 @@ WarehouseSnapshot, compiled_model_record, gather_warehouse_snapshot, + live_reads, project_with, ) from sqlbuild.adapter.contract.models import ColumnInfo @@ -371,9 +373,8 @@ def gather_with_catalog( with open_inspection_catalog(adapter=adapter, connection=connection): return gather_warehouse_snapshot( project=project, - adapter=adapter, - connection=connection, - execute=execute, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection, execute=execute), on_progress=progress.append, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index a9f03644ae..50736f687b 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -18,6 +18,7 @@ CompileModelConfig, CompileSqlReference, CursorOverridePair, + Dialect, ModelCursorSnapshot, ModelPlanContext, SourceEntry, @@ -182,7 +183,7 @@ def resolve_and_execute( """Resolve model SQL and execute it against a real connection.""" resolved_sql: str = resolve_model_sql( - adapter=DuckDbAdapter(), + dialect=Dialect(DuckDbAdapter()), model=model, snapshot=snapshot, context=ModelPlanContext( diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py index 910416c611..3a34891fd2 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild._native import SourceColumnEntry, SourceEntry, resolve_source_references +from sqlbuild._native import Dialect, SourceColumnEntry, SourceEntry, resolve_source_references from sqlbuild.adapter.contract.models import ColumnInfo from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( AdapterOverrideTestCase, @@ -45,7 +45,7 @@ def test_given_custom_adapter_when_resolving_source_then_executes_overridden_ren star_exclude_keyword="EXCLUDE", cursor_bounds=None, cursor_filter_inputs={}, - adapter=ResolutionOverrideAdapter(), + dialect=Dialect(ResolutionOverrideAdapter()), cursor_type=None, lower_bound_inclusive=True, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py index 6d9d293942..b890bb77df 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py @@ -10,6 +10,7 @@ BackfillAction, BackfillResult, CursorOverridePair, + Dialect, ModelPlanContext, SqlHookEntry, WarehouseSnapshot, @@ -67,7 +68,7 @@ def test_given_compiled_dependencies_when_resolving_then_adapter_overrides_execu ) connection.execute("CREATE TABLE staging.actual_orders AS SELECT 9 AS order_id") sql: str = resolve_model_sql( - adapter=adapter, + dialect=Dialect(adapter), model=model, snapshot=WarehouseSnapshot(existing_relations={}), context=context, @@ -79,7 +80,7 @@ def test_given_compiled_dependencies_when_resolving_then_adapter_overrides_execu assert adapter.calls == test_case.expected_model_calls adapter.calls.clear() function_sql: str = resolve_function_sql( - adapter=adapter, + dialect=Dialect(adapter), function=project.functions[0], model_locations=context.model_locations, seed_locations=context.seed_locations, @@ -93,7 +94,7 @@ def test_given_compiled_dependencies_when_resolving_then_adapter_overrides_execu assert adapter.calls == test_case.expected_relation_calls adapter.calls.clear() pre_hooks, post_hooks = resolve_model_hook_entries( - model=model, adapter=adapter, context=context, external_sql_reference_resolver=None + model=model, dialect=Dialect(adapter), context=context, external_sql_reference_resolver=None ) assert isinstance(pre_hooks, tuple | list) assert len(pre_hooks) == 1 diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py index dd4fe0765c..a037846175 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py @@ -11,6 +11,7 @@ CompiledObjectKey, CompiledRelationLocation, CompiledResourceType, + Dialect, DiscoveredAuditBlock, DiscoveredAuditFile, SourceEntry, @@ -118,7 +119,7 @@ def test_given_audit_when_executing_resolved_sql_then_returns_expected_rows( model_locations=model_locations, seed_locations={}, source_map=test_case.source_map, - adapter=adapter, + dialect=Dialect(adapter), lineage_upstream_deps={}, lineage_downstream_deps={}, model_materializations={}, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py index 233cc82020..05edf41a93 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( CompiledResourceType, + Dialect, DiscoveredProjectInputs, RelationFixtureCompletion, RelationFixturePlanningContext, @@ -105,7 +106,7 @@ def test_given_partial_fixture_when_compiling_and_planning_then_completes_requir entries, warnings = build_selected_test_entries( sql_test_selection=SqlTestSelection(), project=project, - adapter=adapter, + dialect=Dialect(adapter), selected_keys=frozenset((project.models[0].key,)), ) @@ -179,7 +180,7 @@ def test_given_upper_case_reads_through_star_import_when_planning_then_fills_eac entries, warnings = build_selected_test_entries( sql_test_selection=SqlTestSelection(), project=project, - adapter=adapter, + dialect=Dialect(adapter), selected_keys=frozenset((project.models[0].key,)), ) @@ -249,7 +250,7 @@ def test_given_untyped_null_fixture_when_planning_then_applies_authoritative_typ entries, warnings = build_selected_test_entries( sql_test_selection=SqlTestSelection(), project=project, - adapter=adapter, + dialect=Dialect(adapter), selected_keys=frozenset((project.models[0].key,)), ) @@ -303,7 +304,7 @@ def test_given_source_contract_when_completing_fixture_then_respects_authoritati key: tuple[CompiledResourceType, str] = (CompiledResourceType.SOURCE, "raw_orders") completed: RelationFixtureCompletion = build_relation_fixture_completion( project=project, - adapter=adapter, + dialect=Dialect(adapter), request=( ("orders",), ((CompiledResourceType.SOURCE, {"raw_orders": "SELECT 1 AS ORDER_ID, 2 AS extra"}),), diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py index 873ec2eb22..ff7c1d48d2 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py @@ -9,10 +9,12 @@ CompiledObjectKey, CompiledResourceType, CursorSnapshotScope, + Dialect, ModelCursorSnapshot, PlannerInputError, WarehouseSnapshot, gather_warehouse_snapshot, + live_reads, ) from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -192,9 +194,14 @@ def test_given_failing_eligible_max_read_when_reading_in_parallel_then_plan_fail "cursor_start_max_ahead": "1d", }, ), - adapter=adapter, - connection=connection, - execute=TrackingCursorExecute(failing_eligible_relations=frozenset({"marts.daily_2"})), + dialect=Dialect(adapter), + reads=live_reads( + adapter=adapter, + connection=connection, + execute=TrackingCursorExecute( + failing_eligible_relations=frozenset({"marts.daily_2"}) + ), + ), cursor_scope=CursorSnapshotScope( model_keys=keys, runtime_producer_keys=keys, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py index e77bcf29ae..0ed6547cc9 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py @@ -12,6 +12,7 @@ CompiledResourceType, CursorOverrides, CursorSnapshotScope, + Dialect, Fingerprint, ModelCursorSnapshot, PlannerInputError, @@ -22,6 +23,7 @@ compiled_model_record, gather_source_columns, gather_warehouse_snapshot, + live_reads, project_with, write_fingerprint, ) @@ -254,9 +256,8 @@ def test_given_warehouse_state_when_gathering_snapshot_then_returns_expected( ) snapshot: WarehouseSnapshot = gather_warehouse_snapshot( project=project, - adapter=adapter, - connection=connection, - execute=execute, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection, execute=execute), selected_keys=test_case.selected_keys, ) @@ -294,9 +295,8 @@ def test_given_no_target_schemas_when_gathering_snapshot_then_returns_empty( project: CompiledProjectView = build_project_with_targets(model_locations={}, seed_locations={}) snapshot: WarehouseSnapshot = gather_warehouse_snapshot( project=project, - adapter=adapter, - connection=connection, - execute=execute, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection, execute=execute), ) assert len(snapshot.existing_relations) == test_case.expected_relation_count @@ -333,8 +333,7 @@ def test_given_sources_when_gathering_source_columns_then_filters_metadata_names columns: dict[str, tuple[ColumnInfo, ...]] = gather_source_columns( project=project, - adapter=adapter, - connection=connection, + reads=live_reads(adapter=adapter, connection=connection), ) assert frozenset(columns.keys()) == test_case.expected_source_names @@ -487,9 +486,8 @@ def _track_progress(message: str) -> None: ) snapshot: WarehouseSnapshot = gather_warehouse_snapshot( project=project, - adapter=adapter, - connection=connection, - execute=execute, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection, execute=execute), selected_keys=test_case.selected_keys, full_refresh=test_case.full_refresh, on_progress=_track_progress, @@ -577,9 +575,8 @@ def _track_progress(message: str) -> None: ) snapshot: WarehouseSnapshot = gather_warehouse_snapshot( project=project, - adapter=adapter, - connection=connection, - execute=execute, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection, execute=execute), selected_keys=test_case.selected_keys, full_refresh=test_case.full_refresh, on_progress=_track_progress, @@ -645,9 +642,8 @@ def _record_execute(*, connection: Any, sql: str) -> Any: snapshot: WarehouseSnapshot = gather_warehouse_snapshot( project=project, - adapter=adapter, - connection=connection, - execute=_record_execute, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection, execute=_record_execute), ) assert snapshot.cursor_snapshots == { @@ -700,9 +696,8 @@ def test_given_timezone_cursor_text_when_gathering_snapshot_then_normalizes_ware snapshot: WarehouseSnapshot = gather_warehouse_snapshot( project=project, - adapter=adapter, - connection=connection, - execute=execute, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection, execute=execute), ) assert snapshot.cursor_snapshots["daily_events"] == test_case.expected_cursor_snapshot @@ -758,9 +753,12 @@ def _fail_eligible_execute(*, connection: Any, sql: str) -> Any: snapshot: WarehouseSnapshot = gather_warehouse_snapshot( project=project, - adapter=_FailEligibleCursorAdapter(), - connection=connection, - execute=_fail_eligible_execute, + dialect=Dialect(_FailEligibleCursorAdapter()), + reads=live_reads( + adapter=_FailEligibleCursorAdapter(), + connection=connection, + execute=_fail_eligible_execute, + ), cursor_scope=CursorSnapshotScope( model_keys=frozenset({daily_events_key}), runtime_producer_keys=frozenset({daily_events_key}), @@ -835,9 +833,8 @@ def _record_execute(*, connection: Any, sql: str) -> Any: snapshot: WarehouseSnapshot = gather_warehouse_snapshot( project=project, - adapter=adapter, - connection=connection, - execute=_record_execute, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection, execute=_record_execute), cursor_scope=CursorSnapshotScope( model_keys=frozenset({hourly_key}), runtime_producer_keys=frozenset({hourly_key}), @@ -925,9 +922,8 @@ def _record_execute(*, connection: Any, sql: str) -> Any: snapshot: WarehouseSnapshot = gather_warehouse_snapshot( project=project, - adapter=adapter, - connection=connection, - execute=_record_execute, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection, execute=_record_execute), selected_keys=metadata_keys, cursor_scope=CursorSnapshotScope( model_keys=frozenset({selected_cursor_key}), @@ -990,9 +986,8 @@ def test_given_selected_invalid_cursor_model_when_gathering_snapshot_then_raises with pytest.raises(PlannerInputError, match=test_case.expected_error_fragment) as exc_info: gather_warehouse_snapshot( project=project, - adapter=adapter, - connection=connection, - execute=execute, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection, execute=execute), selected_keys=frozenset(model.key for model in project.models), cursor_scope=CursorSnapshotScope( model_keys=frozenset({invalid_cursor_key}), @@ -1061,9 +1056,8 @@ def test_given_full_refresh_with_time_only_watermark_contract_when_gathering_the with pytest.raises(PlannerInputError, match=test_case.expected_error_fragment): gather_warehouse_snapshot( project=project, - adapter=adapter, - connection=connection, - execute=execute, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection, execute=execute), selected_keys=frozenset(model.key for model in project.models), full_refresh_model_names=frozenset({"daily_events"}), cursor_scope=CursorSnapshotScope( diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 5db6a59333..ecde09533b 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -26,6 +26,7 @@ CompilePipelineResult, CompileSqlReference, DeferralInputs, + Dialect, DiscoveredProjectInputs, DiscoveredSchemaFile, DiscoveredSeedFile, @@ -45,6 +46,7 @@ compiled_model_record, compute_query_hash, discover_project_inputs, + live_reads, project_with, run_compile_pipeline, write_fingerprint, @@ -457,7 +459,11 @@ def grouped(model: type) -> dict[str, Any]: overrides: PlannerOverrides = PlannerOverrides(**grouped(PlannerOverrides)) deferral: DeferralInputs = DeferralInputs(**grouped(DeferralInputs)) policies: PlannerPolicies = PlannerPolicies(**grouped(PlannerPolicies)) + adapter: Any = kwargs.pop("adapter") + connection: Any = kwargs.pop("connection") return build_execution_plan( + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection), selection=selection, overrides=overrides, deferral=deferral, diff --git a/tests/integration/src/sqlbuild/executor/scenario/test_snapshot_fingerprint.py b/tests/integration/src/sqlbuild/executor/scenario/test_snapshot_fingerprint.py index fb90c504ae..c9e4466680 100644 --- a/tests/integration/src/sqlbuild/executor/scenario/test_snapshot_fingerprint.py +++ b/tests/integration/src/sqlbuild/executor/scenario/test_snapshot_fingerprint.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( CompiledSqlScenario, CompilePipelineResult, + Dialect, ScenarioExecutionPlan, build_cli_scenario_plan, ) @@ -61,7 +62,7 @@ def test_given_capture_and_local_adapters_when_fingerprinting_snapshot_then_matc build_cli_scenario_plan( scenario=scenario, pipeline_result=pipeline, - adapter=adapter, + dialect=Dialect(adapter), project_name="shop", source_lexical_syntax=capture_adapter.sql_lexical_syntax, ) diff --git a/tests/native_field_fallbacks.txt b/tests/native_field_fallbacks.txt index c51c8afde5..e0b3a92884 100644 --- a/tests/native_field_fallbacks.txt +++ b/tests/native_field_fallbacks.txt @@ -56,12 +56,14 @@ crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs CliNamespace.format crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs CliNamespace.format_fix crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs CliNamespace.format_fixtures_only crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs CliNamespace.from_target +crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs CliNamespace.ignore_audit_failures crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs CliNamespace.json crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs CliNamespace.left_query crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs CliNamespace.left_query_file crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs CliNamespace.project_dir crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs CliNamespace.right_query crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs CliNamespace.right_query_file +crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs CliNamespace.run_audits crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs CliNamespace.scenario_command crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs CliNamespace.scenario_local crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs CliNamespace.scenario_refresh diff --git a/tests/unit/src/sqlbuild/adapters/_test_types.py b/tests/unit/src/sqlbuild/adapters/_test_types.py index 1ef3802c5a..e140d0c35f 100644 --- a/tests/unit/src/sqlbuild/adapters/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/_test_types.py @@ -197,3 +197,22 @@ class AdapterCursorLiteralEscapingTestCase: adapter: StrictAdapter expected_literal: str expected_delete_bound: str + + +@dataclass(frozen=True) +class HistoryComputeTestCase: + description: str + adapter: type[BaseAdapter] + expected_wakes_compute: bool + + +@dataclass(frozen=True) +class SchemaRowCountEstimatesTestCase: + description: str + adapter: BaseAdapter + metadata_rows: tuple[tuple[object, ...], ...] + pages: tuple[tuple[str, ...], ...] + expected_estimates: tuple[tuple[int | None, str | None, int | None], ...] + expected_reads: int + expected_sql_fragment: str + metadata_columns: tuple[str, ...] = () diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/test_final_reads.py b/tests/unit/src/sqlbuild/adapters/clickhouse/test_final_reads.py index a1a392a98f..f1c89bfdd8 100644 --- a/tests/unit/src/sqlbuild/adapters/clickhouse/test_final_reads.py +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/test_final_reads.py @@ -2,7 +2,8 @@ import pytest -from sqlbuild._native import SourceEntry, render_source_read_relation +from sqlbuild._native import Dialect, SourceEntry, render_source_read_relation +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.discovery.main.builtins import builtin_adapter_classes from tests.unit.src.sqlbuild.adapters.clickhouse._test_types import FinalSourceReadTestCase @@ -23,10 +24,10 @@ def test_given_final_source_when_rendering_on_clickhouse_then_read_is_deduplicated( test_case: FinalSourceReadTestCase, ) -> None: - adapter: object = builtin_adapter_classes()[test_case.adapter_name]() + adapter: BaseAdapter = builtin_adapter_classes()[test_case.adapter_name]() entry: SourceEntry = SourceEntry(name="customers", schema="raw", table="customers", final=True) - assert render_source_read_relation(adapter=adapter, source_entry=entry) == ( + assert render_source_read_relation(dialect=Dialect(adapter), source_entry=entry) == ( test_case.expected_relation ) @@ -52,10 +53,10 @@ def test_given_final_source_when_rendering_on_clickhouse_then_read_is_deduplicat def test_given_final_source_when_rendering_on_other_adapter_then_coded_error( test_case: FinalSourceReadTestCase, ) -> None: - adapter: object = builtin_adapter_classes()[test_case.adapter_name]() + adapter: BaseAdapter = builtin_adapter_classes()[test_case.adapter_name]() entry: SourceEntry = SourceEntry(name="customers", schema="raw", table="customers", final=True) with pytest.raises(AdapterUserError) as error: - render_source_read_relation(adapter=adapter, source_entry=entry) + render_source_read_relation(dialect=Dialect(adapter), source_entry=entry) assert error.value.code == test_case.expected_code diff --git a/tests/unit/src/sqlbuild/adapters/helpers.py b/tests/unit/src/sqlbuild/adapters/helpers.py index de65f3f303..4bba1c29a1 100644 --- a/tests/unit/src/sqlbuild/adapters/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/helpers.py @@ -6,10 +6,14 @@ import time from collections.abc import Callable, Iterator from dataclasses import dataclass +from itertools import chain from typing import Any import pytest +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import RelationRowCountEstimate + class FetchedRows: """A cursor result that returns fixed rows.""" @@ -347,3 +351,18 @@ def watchdog() -> None: thread: threading.Thread = threading.Thread(target=watchdog) thread.start() return thread + + +def estimate_pages( + *, adapter: BaseAdapter, connection: MetadataConnection, pages: tuple[tuple[str, ...], ...] +) -> tuple[RelationRowCountEstimate, ...]: + """Estimate each page of names in schema ``analytics.dev``, as a statistics sync does.""" + + return tuple( + chain.from_iterable( + adapter.estimate_relation_row_counts( + connection=connection, database="analytics", schema="dev", names=page + ) + for page in pages + ) + ) diff --git a/tests/unit/src/sqlbuild/adapters/test_history_compute.py b/tests/unit/src/sqlbuild/adapters/test_history_compute.py new file mode 100644 index 0000000000..385e75ab64 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/test_history_compute.py @@ -0,0 +1,39 @@ +"""Which built-in adapters start or wake billed compute when a code location reads history.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.clickhouse.classes.clickhouse_adapter import ClickHouseAdapter +from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.motherduck.classes.motherduck_adapter import MotherDuckAdapter +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from sqlbuild.adapters.sqlite.classes.sqlite_adapter import SqliteAdapter +from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter +from tests.unit.src.sqlbuild.adapters._test_types import HistoryComputeTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + HistoryComputeTestCase("Databricks starts a SQL warehouse", DatabricksAdapter, True), + HistoryComputeTestCase("ClickHouse wakes an idle service", ClickHouseAdapter, True), + HistoryComputeTestCase("MotherDuck runs every query on compute", MotherDuckAdapter, True), + HistoryComputeTestCase( + "Snowflake skips unchanged tables with metadata", SnowflakeAdapter, False + ), + HistoryComputeTestCase("BigQuery has no warehouse to resume", BigQueryAdapter, False), + HistoryComputeTestCase("PostgreSQL is a running server", PostgresAdapter, False), + HistoryComputeTestCase("SQL Server is a running server", SqlServerAdapter, False), + HistoryComputeTestCase("DuckDB reads a local file", DuckDbAdapter, False), + HistoryComputeTestCase("SQLite reads a local file", SqliteAdapter, False), + ], + ids=lambda case: case.description, +) +def test_given_an_adapter_when_a_location_reads_history_then_it_declares_whether_compute_wakes( + test_case: HistoryComputeTestCase, +) -> None: + assert test_case.adapter.history_reads_wake_compute is test_case.expected_wakes_compute diff --git a/tests/unit/src/sqlbuild/adapters/test_relation_row_count_estimate.py b/tests/unit/src/sqlbuild/adapters/test_relation_row_count_estimate.py index f2c2bacc01..b1c4ba90af 100644 --- a/tests/unit/src/sqlbuild/adapters/test_relation_row_count_estimate.py +++ b/tests/unit/src/sqlbuild/adapters/test_relation_row_count_estimate.py @@ -19,12 +19,14 @@ BigQueryRowCountEstimateTestCase, DuckDbRowCountEstimateTestCase, RelationRowCountEstimateTestCase, + SchemaRowCountEstimatesTestCase, ) from tests.unit.src.sqlbuild.adapters.helpers import ( FakeBigQueryClient, FakeBigQueryConnection, FakeBigQueryTable, MetadataConnection, + estimate_pages, execute_through_cursor, ) @@ -440,3 +442,81 @@ def test_given_duckdb_relation_when_estimating_row_count_then_reads_catalog_size if __name__ == "__main__": pytest.main([__file__, "-vv"]) + + +@pytest.mark.parametrize( + "test_case", + [ + SchemaRowCountEstimatesTestCase( + description="snowflake lists the schema once for every page within a minute", + adapter=SnowflakeAdapter(), + metadata_rows=(("ORDERS", "TABLE", 5000, 65536), ("CUSTOMERS", "VIEW", 0, 0)), + metadata_columns=_SNOWFLAKE_SHOW_OBJECTS_COLUMNS, + pages=(("orders", "customers"), ("missing",)), + expected_estimates=( + (5000, None, 65536), + (None, "no row count (view)", None), + (None, "relation not found", None), + ), + expected_reads=1, + expected_sql_fragment="SHOW OBJECTS IN SCHEMA", + ), + SchemaRowCountEstimatesTestCase( + description="postgres reads each page with one catalog query", + adapter=PostgresAdapter(), + metadata_rows=(("orders", "r", 1234.0, 8192), ("customers", "v", -1.0, 0)), + pages=(("orders", "customers"), ("missing",)), + expected_estimates=( + (1234, None, 8192), + (None, "no table statistics (view or partitioned table)", None), + (None, "relation not found", None), + ), + expected_reads=2, + expected_sql_fragment="relation.relname IN ('orders', 'customers')", + ), + SchemaRowCountEstimatesTestCase( + description="clickhouse reads each page with one system.tables query", + adapter=ClickHouseAdapter(), + metadata_rows=(("orders", "MergeTree", 10, 100), ("customers", "View", None, None)), + pages=(("orders", "customers"), ("missing",)), + expected_estimates=( + (10, None, 100), + (None, "engine View does not report row counts", None), + (None, "relation not found", None), + ), + expected_reads=2, + expected_sql_fragment="name IN ('orders', 'customers')", + ), + SchemaRowCountEstimatesTestCase( + description="sql server reads each page with one catalog query", + adapter=SqlServerAdapter(), + metadata_rows=(("orders", "U ", 777), ("customers", "V ", None)), + pages=(("orders", "customers"), ("missing",)), + expected_estimates=( + (777, None, None), + (None, "no row count (view)", None), + (None, "relation not found", None), + ), + expected_reads=2, + expected_sql_fragment="objects.name IN (N'orders', N'customers')", + ), + ], + ids=lambda case: case.description, +) +def test_given_pages_of_names_when_estimating_schema_row_counts_then_no_read_is_per_relation( + test_case: SchemaRowCountEstimatesTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + connection: MetadataConnection = MetadataConnection( + test_case.metadata_rows, test_case.metadata_columns + ) + monkeypatch.setattr(test_case.adapter, "execute", execute_through_cursor) + + estimates: tuple[RelationRowCountEstimate, ...] = estimate_pages( + adapter=test_case.adapter, connection=connection, pages=test_case.pages + ) + + assert [ + (estimate.row_count, estimate.detail, estimate.byte_count) for estimate in estimates + ] == list(test_case.expected_estimates), test_case.description + assert len(connection.executed) == test_case.expected_reads, connection.executed + assert test_case.expected_sql_fragment in connection.executed[0], connection.executed diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 85bd322bbb..680120704a 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -29,6 +29,7 @@ DeclarationResolutionContext, DeclarationRuntimeProjection, DeclarationScopeResolver, + Dialect, DiscoveredMacroFile, DiscoveredProjectInputs, InferredNullability, @@ -349,7 +350,7 @@ def resolve_model_references_for_adapter(*, sql: str, adapter_name: str) -> str: seed_locations={}, cursor_bounds=None, cursor_filter_inputs={}, - adapter=adapters[adapter_name](), + dialect=Dialect(adapters[adapter_name]()), cursor_type=None, lower_bound_inclusive=True, ) diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py index 48a52f2a35..c1735ad88d 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py @@ -117,3 +117,11 @@ class StoredColumnRowTestCase: description: str decisions: tuple[str, ...] expected_decisions: tuple[str, ...] + + +@dataclass(frozen=True) +class NamedFingerprintTestCase: + description: str + model_name: str + """The `schema.name` the newest fingerprint built, when there is one.""" + expected_targets: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py b/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py index 0667c33bbb..a98491455e 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( ColumnMigrationDecision, ColumnMigrationEvent, + Fingerprint, MigrationDecision, MigrationDiscovery, MigrationEvent, @@ -103,3 +104,22 @@ def old_name_fact( archive_name=archive_name, grants_copied=grants_copied, ) + + +def built_fingerprint(node_type: str, node_name: str, schema: str, day: int) -> Fingerprint: + """Return a fingerprint of `node_name` built into `schema` at noon on January `day`, 2026.""" + + return Fingerprint( + node_type=node_type, + node_name=node_name, + target_database=None, + target_schema=schema, + target_name=node_name, + run_id=f"run_{day:03d}", + definition_hash="definition_hash", + version_hash="version_hash", + schema_fingerprint="schema_hash", + definition="SELECT 1", + metadata_json="{}", + ts=datetime(2026, 1, day, 12, 0, 0), + ) diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_named_fingerprints.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_named_fingerprints.py new file mode 100644 index 0000000000..7ca90a6f0a --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_named_fingerprints.py @@ -0,0 +1,89 @@ +"""Unit coverage for finding a model's newest fingerprint by name across inspected schemas.""" + +from __future__ import annotations + +import duckdb +import pytest + +from sqlbuild._native import Fingerprint, MigrationStateInspection, live_reads, write_fingerprints +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import NamedFingerprintTestCase +from tests.unit.src.sqlbuild.compiler.migrations.main.helpers import built_fingerprint + +_FINGERPRINTS: dict[str, tuple[Fingerprint, ...]] = { + "main": ( + built_fingerprint("model", "orders", "main", 2), + built_fingerprint("model", "customers", "main", 5), + built_fingerprint("seed", "currencies", "main", 3), + ), + "archive": ( + built_fingerprint("model", "orders", "archive", 4), + built_fingerprint("model", "customers", "archive", 1), + ), +} + + +@pytest.mark.parametrize( + "test_case", + [ + NamedFingerprintTestCase( + description="a model built in two schemas has its newest build in the later schema", + model_name="orders", + expected_targets=("archive.orders",), + ), + NamedFingerprintTestCase( + description="a model built in two schemas has its newest build in the earlier schema", + model_name="customers", + expected_targets=("main.customers",), + ), + NamedFingerprintTestCase( + description="a name in another case is another model", + model_name="Orders", + expected_targets=(), + ), + NamedFingerprintTestCase( + description="a seed's fingerprint is not a model's", + model_name="currencies", + expected_targets=(), + ), + NamedFingerprintTestCase( + description="a model never built has none", + model_name="refunds", + expected_targets=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_inspected_schemas_when_a_model_is_named_then_its_newest_fingerprint_is_found( + test_case: NamedFingerprintTestCase, +) -> None: + adapter: DuckDbAdapter = DuckDbAdapter() + connection: duckdb.DuckDBPyConnection = duckdb.connect(":memory:") + _ = connection.execute("CREATE SCHEMA archive") + schema: str + fingerprints: tuple[Fingerprint, ...] + for schema, fingerprints in _FINGERPRINTS.items(): + write_fingerprints( + connection=connection, + execute=adapter.execute, + database=None, + schema=schema, + fingerprints=fingerprints, + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + ) + inspection: MigrationStateInspection = MigrationStateInspection( + reads=live_reads(adapter=adapter, connection=connection), database=None + ) + + inspection.inspect_schemas(schemas={"main"}) + inspection.inspect_schemas(schemas={"archive"}) + + found: Fingerprint | None = inspection.named_fingerprint(model_name=test_case.model_name) + assert ( + tuple( + f"{fingerprint.target_schema}.{fingerprint.target_name}" + for fingerprint in filter(None, (found,)) + ) + == test_case.expected_targets + ) diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py index 7c2bb93826..92c8b741fb 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py @@ -13,6 +13,7 @@ OldNameViewEvent, OldNameViewEventType, OldNameViewHistory, + live_reads, project_old_name_views, read_migration_state_events, stored_old_name_view_columns, @@ -234,7 +235,7 @@ def test_given_state_table_without_optional_columns_when_planning_then_absent_co for column in test_case.absent_columns: _ = connection.execute(f"ALTER TABLE main._sqlbuild_old_name_views DROP COLUMN {column}") inspection: MigrationStateInspection = MigrationStateInspection( - adapter=adapter, connection=connection, database=None + reads=live_reads(adapter=adapter, connection=connection), database=None ) inspection.inspect_old_name_views(schemas={"main"}) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py index 3adc1d2436..b879b91b7d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py @@ -16,6 +16,7 @@ CompiledResourceType, CompileModelConfig, DeferralInputs, + Dialect, Fingerprint, PlannerInputError, PlannerOverrides, @@ -23,6 +24,7 @@ PlannerScope, WarehouseFingerprints, WarehouseSnapshot, + live_reads, plan_column_migrations, ) from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo @@ -69,8 +71,8 @@ def test_given_untokenizable_tracked_model_when_planning_column_migrations_then_ effective_vars={}, models=(model,), ), - adapter=adapter, - connection=object(), + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=object()), ) scope: PlannerScope = PlannerScope( upstream_deps={model.key: ()}, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py index 84ff684bb7..29ef26a7c7 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py @@ -11,6 +11,7 @@ CompiledRelationLocation, CompiledResourceType, CompileModelConfig, + Dialect, LocalConfig, PlannerRelationsContext, PlannerRuntime, @@ -23,6 +24,7 @@ TargetConfig, TimeTravelRetentionSource, WarehouseSnapshot, + live_reads, ) from sqlbuild.adapter.contract.classes.retention_adapter import RetentionAdapterMixin from sqlbuild.adapter.contract.models import RelationInfo @@ -84,8 +86,8 @@ def build_retention_planner_inputs( return ( PlannerRuntime( project=project, - adapter=adapter, - connection=object(), + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=object()), project_config=ProjectConfig( name="test", adapter=str(adapter.adapter_name), diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py index 69a15cd0b7..fc28f0cdff 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py @@ -755,8 +755,8 @@ def test_given_missing_bigquery_dataset_when_planning_then_defers_retention_unti ) runtime: PlannerRuntime = PlannerRuntime( project=runtime.project, - adapter=runtime.adapter, - connection=runtime.connection, + dialect=runtime.dialect, + reads=runtime.reads, invocation_time=runtime.invocation_time, project_config=ProjectConfig( name="test", diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py index 7e9c286e8b..ff549ce031 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py @@ -6,6 +6,7 @@ BackfillAction, BackfillResult, CursorOverridePair, + Dialect, ModelCursorSnapshot, PlannerInputError, WarehouseSnapshot, @@ -69,7 +70,7 @@ def test_given_cursor_interval_when_resolving_then_renders_expected_sql( model: CompiledModelView = build_cursor_intrinsic_model(config_values=test_case.model_config) resolved_sql: str = resolve_model_sql( - adapter=DuckDbAdapter(), + dialect=Dialect(DuckDbAdapter()), model=model, snapshot=WarehouseSnapshot(cursor_snapshots=test_case.cursor_snapshots), context=build_empty_model_plan_context(), @@ -108,7 +109,7 @@ def test_given_missing_cursor_interval_when_resolving_then_rejects_intrinsics( with pytest.raises(PlannerInputError, match=test_case.expected_error_fragment) as raised: resolve_model_sql( - adapter=DuckDbAdapter(), + dialect=Dialect(DuckDbAdapter()), model=model, snapshot=WarehouseSnapshot(), context=build_empty_model_plan_context(), diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py index 461182ea37..e327804d7d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py @@ -5,6 +5,7 @@ import pytest from sqlbuild._native import ( + Dialect, ModelPlanContext, PythonHookEntry, SourceEntry, @@ -70,7 +71,7 @@ def test_given_sql_hook_references_when_planning_then_they_resolve_like_model_sq pre_hooks, post_hooks = resolve_model_hook_entries( model=model, - adapter=DuckDbAdapter(), + dialect=Dialect(DuckDbAdapter()), context=_CONTEXT, external_sql_reference_resolver=None, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py index 1c5a9349c5..58a18b977e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py @@ -11,6 +11,7 @@ CursorBounds, DbtCompileReferenceResolver, DbtManifestIndex, + Dialect, apply_deferred_locations, build_dbt_manifest_index, resolve_dbt_ref_references, @@ -135,7 +136,7 @@ def test_given_refs_without_cursor_when_resolving_then_returns_expected_sql( seed_locations=_SEED_TARGETS, cursor_bounds=None, cursor_filter_inputs={}, - adapter=DuckDbAdapter(), + dialect=Dialect(DuckDbAdapter()), cursor_type=None, lower_bound_inclusive=True, ) @@ -205,7 +206,7 @@ def test_given_refs_with_cursor_when_resolving_then_returns_expected_sql( seed_locations=_SEED_TARGETS, cursor_bounds=test_case.cursor_bounds, cursor_filter_inputs=_CURSOR_INPUTS, - adapter=DuckDbAdapter(), + dialect=Dialect(DuckDbAdapter()), cursor_type=test_case.cursor_type, lower_bound_inclusive=True, ) @@ -239,7 +240,7 @@ def test_given_refs_with_exclusive_cursor_when_resolving_then_returns_expected_s seed_locations=_SEED_TARGETS, cursor_bounds=test_case.cursor_bounds, cursor_filter_inputs=_CURSOR_INPUTS, - adapter=DuckDbAdapter(), + dialect=Dialect(DuckDbAdapter()), cursor_type=test_case.cursor_type, lower_bound_inclusive=False, ) @@ -326,7 +327,7 @@ def test_given_table_function_marker_when_resolving_then_returns_expected_sql( result: str = resolve_table_function_references( query_sql=test_case.query_sql, function_locations=_FUNCTION_TARGETS, - adapter=DuckDbAdapter(), + dialect=Dialect(DuckDbAdapter()), ) assert result == test_case.expected_sql @@ -366,7 +367,7 @@ def test_given_udf_marker_when_resolving_then_returns_expected_sql( result: str = resolve_udf_references( query_sql=test_case.query_sql, function_locations=_FUNCTION_TARGETS, - adapter=DuckDbAdapter(), + dialect=Dialect(DuckDbAdapter()), ) assert result == test_case.expected_sql @@ -389,7 +390,7 @@ def test_given_custom_adapter_when_resolving_udf_marker_then_uses_adapter_render result: str = resolve_udf_references( query_sql=test_case.query_sql, function_locations=_FUNCTION_TARGETS, - adapter=BracketUdfCallAdapter(), + dialect=Dialect(BracketUdfCallAdapter()), ) assert result == test_case.expected_sql @@ -412,7 +413,7 @@ def test_given_custom_adapter_when_resolving_table_function_marker_then_uses_ada result: str = resolve_table_function_references( query_sql=test_case.query_sql, function_locations=_FUNCTION_TARGETS, - adapter=BracketTableFunctionCallAdapter(), + dialect=Dialect(BracketTableFunctionCallAdapter()), ) assert result == test_case.expected_sql diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py index 5a8031c023..3281e0d061 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py @@ -4,7 +4,13 @@ import pytest -from sqlbuild._native import CursorBounds, SourceColumnEntry, SourceEntry, resolve_source_references +from sqlbuild._native import ( + CursorBounds, + Dialect, + SourceColumnEntry, + SourceEntry, + resolve_source_references, +) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -335,7 +341,7 @@ def test_given_source_references_when_resolving_then_returns_expected_sql( star_exclude_keyword=test_case.star_exclude_keyword, cursor_bounds=test_case.cursor_bounds, cursor_filter_inputs=test_case.cursor_inputs, - adapter=DuckDbAdapter(), + dialect=Dialect(DuckDbAdapter()), cursor_type=test_case.cursor_type, lower_bound_inclusive=test_case.lower_bound_inclusive, ) @@ -404,7 +410,7 @@ def test_given_adapter_specific_source_references_when_resolving_then_returns_ad star_exclude_keyword="EXCEPT", cursor_bounds=None, cursor_filter_inputs={}, - adapter=BigQueryAdapter(), + dialect=Dialect(BigQueryAdapter()), cursor_type=None, lower_bound_inclusive=True, ) @@ -453,7 +459,7 @@ def test_given_reserved_source_column_when_resolving_then_quotes_contract_projec star_exclude_keyword="EXCLUDE", cursor_bounds=None, cursor_filter_inputs={}, - adapter=SnowflakeAdapter(), + dialect=Dialect(SnowflakeAdapter()), cursor_type=None, lower_bound_inclusive=True, ) @@ -549,7 +555,7 @@ def test_given_sqlserver_alias_required_sources_when_resolving_then_aliases_deri star_exclude_keyword=test_case.star_exclude_keyword, cursor_bounds=test_case.cursor_bounds, cursor_filter_inputs=test_case.cursor_inputs, - adapter=SqlServerAdapter(), + dialect=Dialect(SqlServerAdapter()), cursor_type=test_case.cursor_type, lower_bound_inclusive=test_case.lower_bound_inclusive, ) @@ -593,7 +599,7 @@ def test_given_sqlserver_timestamp_bounds_when_resolving_then_uses_datetime2_lit star_exclude_keyword=test_case.star_exclude_keyword, cursor_bounds=test_case.cursor_bounds, cursor_filter_inputs=test_case.cursor_inputs, - adapter=SqlServerAdapter(), + dialect=Dialect(SqlServerAdapter()), cursor_type=test_case.cursor_type, lower_bound_inclusive=test_case.lower_bound_inclusive, ) @@ -630,7 +636,7 @@ def test_given_postgres_sources_when_resolving_then_avoids_unsupported_star_excl star_exclude_keyword=test_case.star_exclude_keyword, cursor_bounds=test_case.cursor_bounds, cursor_filter_inputs=test_case.cursor_inputs, - adapter=PostgresAdapter(), + dialect=Dialect(PostgresAdapter()), cursor_type=test_case.cursor_type, lower_bound_inclusive=test_case.lower_bound_inclusive, ) @@ -797,7 +803,7 @@ def test_given_invalid_source_references_when_resolving_then_raises_clear_error( star_exclude_keyword=test_case.star_exclude_keyword, cursor_bounds=None, cursor_filter_inputs={}, - adapter=DuckDbAdapter(), + dialect=Dialect(DuckDbAdapter()), cursor_type=None, lower_bound_inclusive=True, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index 89f8aa2ce3..9fa5d39433 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -16,6 +16,7 @@ CompiledResourceType, CompiledSqlTest, CompileSqlTestCte, + Dialect, DiscoveredSqlTestBlock, DiscoveredSqlTestFile, NativeSqlTestPlan, @@ -1098,11 +1099,11 @@ def test_given_udf_sql_test_when_planning_then_compares_resolved_actual_to_expec ) command_plan: PlanOutput = build_test_command_plan( project=project, - adapter=DuckDbAdapter(), + dialect=Dialect(DuckDbAdapter()), scope=scope, relations=resolve_static_relation_context( project=project, - adapter=DuckDbAdapter(), + dialect=Dialect(DuckDbAdapter()), scope=scope, ), ) @@ -1232,11 +1233,11 @@ def test_given_table_function_sql_test_when_planning_then_compares_resolved_actu ) command_plan: PlanOutput = build_test_command_plan( project=project, - adapter=DuckDbAdapter(), + dialect=Dialect(DuckDbAdapter()), scope=scope, relations=resolve_static_relation_context( project=project, - adapter=DuckDbAdapter(), + dialect=Dialect(DuckDbAdapter()), scope=scope, ), ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py index 5af837bb96..1ae841f3b1 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py @@ -7,6 +7,7 @@ CompiledObjectKey, CompiledRelationLocation, CompileSqlReference, + Dialect, SourceEntry, SqlReferenceKind, plan_audit, @@ -72,7 +73,7 @@ def test_given_logically_attached_audit_when_planning_then_scheduling_and_identi ), seed_locations={}, source_map={}, - adapter=PlannerTestAdapter(), + dialect=Dialect(PlannerTestAdapter()), lineage_upstream_deps=upstream, lineage_downstream_deps=downstream, model_materializations={test_case.attached_target_name: "incremental"}, @@ -152,7 +153,7 @@ def test_given_audit_when_planning_then_resolves_sql( model_locations=model_locations, seed_locations={}, source_map=source_map, - adapter=PlannerTestAdapter(), + dialect=Dialect(PlannerTestAdapter()), lineage_upstream_deps={}, lineage_downstream_deps={}, model_materializations={}, @@ -191,7 +192,7 @@ def test_given_audit_with_unresolved_marker_when_planning_then_it_raises_clear_e model_locations=model_locations, seed_locations={}, source_map=source_map, - adapter=PlannerTestAdapter(), + dialect=Dialect(PlannerTestAdapter()), lineage_upstream_deps={}, lineage_downstream_deps={}, model_materializations={}, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index 752f165308..8ed38de96e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -10,6 +10,7 @@ CompiledResourceType, CompiledSqlTest, CompileSqlReference, + Dialect, PlanWarning, RelationFixtureCompletion, RelationFixturePlanningContext, @@ -98,7 +99,7 @@ def test_given_many_tests_when_building_entries_then_project_fixture_metadata_is entries, warnings = build_selected_test_entries( sql_test_selection=SqlTestSelection(), project=test_project, - adapter=PlannerTestAdapter(), + dialect=Dialect(PlannerTestAdapter()), selected_keys=frozenset((model.key,)), ) @@ -220,7 +221,7 @@ def test_given_complete_authoritative_fixture_when_completing_then_skips_column_ completed: RelationFixtureCompletion = fixture_completion.build_relation_fixture_completion( project=project_with(project=project, models=(model,)), - adapter=PlannerTestAdapter(), + dialect=Dialect(PlannerTestAdapter()), request=( (model.name,), ((CompiledResourceType.SOURCE, {"raw_orders": test_case.fixture_sql}),), diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py index d96a4723c5..67aaf2113e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( CompiledSqlScenario, CompilePipelineResult, + Dialect, ScenarioArtifactName, ScenarioExecutionPlan, SqlLexicalSyntax, @@ -52,7 +53,7 @@ def test_given_adapter_identifier_limit_when_building_cli_scenario_plan_then_use result: ScenarioExecutionPlan = build_cli_scenario_plan( scenario=scenario, pipeline_result=pipeline_result, - adapter=WideIdentifierPlannerTestAdapter(), + dialect=Dialect(WideIdentifierPlannerTestAdapter()), project_name="scenario_demo", source_lexical_syntax=SqlLexicalSyntax(), ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py index 39157307aa..ebbe6f1b10 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( CompiledSqlScenario, CompileSqlScenarioCte, + Dialect, DiscoveredHookFunction, PlannerInputError, ScenarioExecutionPlan, @@ -209,7 +210,7 @@ def test_given_scenario_helpers_when_building_fixture_plans_then_fixtures_are_se scenario=build_scenario_relation_test_scenario(), graph_plan=test_case.graph_plan, relation_plan=relation_plan, - adapter=PlannerTestAdapter(), + dialect=Dialect(PlannerTestAdapter()), source_lexical_syntax=SqlLexicalSyntax(), ) @@ -296,7 +297,7 @@ def test_given_project_source_ref_in_scenario_fixture_when_building_fixture_plan scenario=scenario, graph_plan=test_case.graph_plan, relation_plan=relation_plan, - adapter=PlannerTestAdapter(), + dialect=Dialect(PlannerTestAdapter()), source_lexical_syntax=SqlLexicalSyntax(), ) @@ -414,7 +415,7 @@ def notify() -> None: result, warnings = build_scenario_execution_plan( scenario=build_scenario_relation_test_scenario(), project=project, - adapter=PlannerTestAdapter(), + dialect=Dialect(PlannerTestAdapter()), source_lexical_syntax=SqlLexicalSyntax(), graph_plan=test_case.graph_plan, relation_plan=relation_plan, @@ -512,7 +513,7 @@ def test_given_required_unmocked_seed_when_building_execution_plan_then_loads_pr result, warnings = build_scenario_execution_plan( scenario=build_scenario_relation_test_scenario(include_seed_fixture=False), project=project, - adapter=PlannerTestAdapter(), + dialect=Dialect(PlannerTestAdapter()), source_lexical_syntax=SqlLexicalSyntax(), graph_plan=test_case.graph_plan, relation_plan=relation_plan, @@ -612,7 +613,7 @@ def test_given_scenario_check_sql_when_resolving_then_uses_scenario_relations( result: str = resolve_scenario_check_sql( sql=test_case.sql, relation_plan=relation_plan, - adapter=PlannerTestAdapter(), + dialect=Dialect(PlannerTestAdapter()), source_lexical_syntax=SqlLexicalSyntax(), source_label=SCENARIO_PATH, ) @@ -753,7 +754,7 @@ def test_given_dialect_lexical_syntax_when_resolving_check_sql_then_replaces_eve resolve_scenario_check_sql( sql=test_case.sql, relation_plan=relation_plan, - adapter=PlannerTestAdapter(), + dialect=Dialect(PlannerTestAdapter()), source_lexical_syntax=lexical_syntax, source_label=SCENARIO_PATH, ) @@ -819,7 +820,7 @@ def test_given_unresolvable_check_sql_when_resolving_then_raises_scenario_error( resolve_scenario_check_sql( sql=test_case.sql, relation_plan=relation_plan, - adapter=PlannerTestAdapter(), + dialect=Dialect(PlannerTestAdapter()), source_lexical_syntax=lexical_syntax, source_label=SCENARIO_PATH, ) @@ -877,7 +878,7 @@ def test_given_unresolvable_fixture_sql_when_planning_fixtures_then_raises_namin scenario=scenario, graph_plan=graph_plan, relation_plan=relation_plan, - adapter=PlannerTestAdapter(), + dialect=Dialect(PlannerTestAdapter()), source_lexical_syntax=test_case.lexical_syntax, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py index 6069d383cb..d75c2f1f48 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py @@ -11,6 +11,7 @@ CompiledProject, CompiledResourceType, CursorInputRelation, + Dialect, PlannerInputError, PlannerRelationsContext, PlannerScope, @@ -19,6 +20,7 @@ build_cursor_input_relations, build_planner_relations_context, gather_source_columns, + live_reads, runtime_cursor_producer_names, validate_source_cursor_input_columns, ) @@ -159,7 +161,8 @@ def test_given_sources_when_gathering_columns_then_returns_expected_source_colum adapter: _RecordingAdapter = _RecordingAdapter(test_case.adapter_column_names) result: dict[str, tuple[ColumnInfo, ...]] = gather_source_columns( - project=project, adapter=adapter, connection=None + project=project, + reads=live_reads(adapter=adapter, connection=None), ) assert tuple(adapter.queried_sql) == test_case.expected_queried_sql @@ -200,8 +203,7 @@ def test_given_multi_database_sources_when_gathering_columns_then_matches_curren result: dict[str, tuple[ColumnInfo, ...]] = gather_source_columns( project=project, - adapter=adapter, - connection=None, + reads=live_reads(adapter=adapter, connection=None), source_entries=source_entries, ) @@ -267,8 +269,8 @@ def test_given_known_source_columns_when_building_relations_context_then_reuses_ context: PlannerRelationsContext = build_planner_relations_context( project=project, - adapter=adapter, - connection=None, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=None), scope=scope, known_source_columns=known_source_columns, ) @@ -445,7 +447,7 @@ def test_given_view_filter_and_transitive_watermark_when_planning_then_relation_ relations: tuple[CursorInputRelation, ...] = build_cursor_input_relations( model=model, - adapter=DuckDbAdapter(), + dialect=Dialect(DuckDbAdapter()), model_locations={view_model.name: view_model.destination}, models_by_name=models_by_name, seed_locations={}, @@ -606,7 +608,7 @@ def test_given_table_function_and_transitive_watermark_when_planning_then_source ) relations: tuple[CursorInputRelation, ...] = build_cursor_input_relations( model=model, - adapter=DuckDbAdapter(), + dialect=Dialect(DuckDbAdapter()), model_locations={name: upstream.destination for name, upstream in models_by_name.items()}, models_by_name=models_by_name, functions_by_name=functions_by_name, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py index 47e077aed9..50dbbc19b7 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py @@ -6,6 +6,7 @@ from sqlbuild._native import ( CompiledProject, + Dialect, DirectSourceFreshnessPlanningResult, PlannerRelationsContext, PlannerScope, @@ -14,6 +15,7 @@ SourceFreshnessStrategy, SourceFreshnessValueKind, build_planner_source_freshness_result, + live_reads, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( @@ -44,8 +46,8 @@ def test_given_source_deferral_context_when_building_source_freshness_then_uses_ effective_connection={}, effective_vars={}, ), - adapter=adapter, - connection=connection, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection), scope=PlannerScope( upstream_deps={}, downstream_deps={}, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py index a564cb4f92..ab6f66dfcd 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py @@ -11,11 +11,13 @@ CompiledProject, CompiledResourceType, CompiledSource, + Dialect, DiscoveredSourceFile, PlannerRelationsContext, PlannerScope, SourceEntry, check_selected_source_tables_exist, + live_reads, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo @@ -57,8 +59,8 @@ def check_one_source_table( ) check_selected_source_tables_exist( project=cast(CompiledProject, SimpleNamespace(sources=(compiled_source,))), - adapter=adapter, - connection=connection, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection), scope=PlannerScope( upstream_deps={model_key: (source_key,)}, downstream_deps={source_key: (model_key,)}, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py index f82650e67c..e902fa4d8d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py @@ -16,6 +16,7 @@ discover_project_inputs, get_semantic_source_columns, list_source_candidates, + live_reads, ) from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog @@ -70,8 +71,7 @@ def test_given_multiple_missing_source_schemas_when_inspecting_then_metadata_is_ connection.execute("CREATE TABLE orders(id INTEGER); CREATE TABLE customers(id INTEGER)") result: dict[str, tuple[ColumnInfo, ...]] = get_semantic_source_columns( project=project, - adapter=adapter, - connection=connection, + reads=live_reads(adapter=adapter, connection=connection), source_read_map={source.name: source.source_entry for source in project.sources}, columns={}, selected_keys=frozenset(model.key for model in project.models), @@ -128,8 +128,7 @@ def test_given_source_schema_case_differs_when_inspecting_then_columns_are_found with scope: result: dict[str, tuple[ColumnInfo, ...]] = get_semantic_source_columns( project=project, - adapter=adapter, - connection=connection, + reads=live_reads(adapter=adapter, connection=connection), source_read_map={source.name: source.source_entry for source in project.sources}, columns={}, selected_keys=frozenset(model.key for model in project.models), @@ -187,8 +186,7 @@ def test_given_snowflake_source_schema_when_listing_candidates_then_finds_relati with open_inspection_catalog(adapter=adapter, connection=connection): candidates: tuple[RelationInfo, ...] = list_source_candidates( - adapter=adapter, - connection=connection, + reads=live_reads(adapter=adapter, connection=connection), database="analytics", entries=tuple( SourceEntry( diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/classes/_test_types.py index d494672a34..f751002080 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/_test_types.py @@ -40,3 +40,17 @@ class PlanningExitTestCase: script: str expected_returncode: int expected_output: str + + +@dataclass(frozen=True) +class DialectAttributeTestCase: + description: str + attribute: str + expected_value: object + + +@dataclass(frozen=True) +class DialectRefusalTestCase: + description: str + attribute: str + expected_message: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_dialect.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_dialect.py new file mode 100644 index 0000000000..62d957a97e --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_dialect.py @@ -0,0 +1,68 @@ +"""Unit coverage for the planner's SQL dialect: the adapter's syntax without warehouse access.""" + +from __future__ import annotations + +import pytest + +from sqlbuild._native import Dialect +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from tests.unit.src.sqlbuild.compiler.planner.classes._test_types import ( + DialectAttributeTestCase, + DialectRefusalTestCase, +) + +_ADAPTER: DuckDbAdapter = DuckDbAdapter() + + +@pytest.mark.parametrize( + "test_case", + [ + DialectAttributeTestCase( + description="a render method is the adapter's own", + attribute="render_qualified_name", + expected_value=_ADAPTER.render_qualified_name, + ), + DialectAttributeTestCase( + description="a dialect fact is the adapter's own", + attribute="adapter_name", + expected_value=_ADAPTER.adapter_name, + ), + ], + ids=lambda case: case.description, +) +def test_given_syntax_attribute_when_read_through_dialect_then_matches_adapter( + test_case: DialectAttributeTestCase, +) -> None: + assert getattr(Dialect(_ADAPTER), test_case.attribute) == test_case.expected_value + + +@pytest.mark.parametrize( + "test_case", + [ + DialectRefusalTestCase( + description="executing SQL", + attribute="execute", + expected_message="`execute` would reach the warehouse", + ), + DialectRefusalTestCase( + description="listing relations", + attribute="list_relations", + expected_message="`list_relations` would reach the warehouse", + ), + DialectRefusalTestCase( + description="opening a connection", + attribute="connect", + expected_message="`connect` would reach the warehouse", + ), + ], + ids=lambda case: case.description, +) +def test_given_warehouse_attribute_when_read_through_dialect_then_refuses( + test_case: DialectRefusalTestCase, +) -> None: + with pytest.raises(AttributeError, match=test_case.expected_message): + getattr(Dialect(_ADAPTER), test_case.attribute) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py index 6a71ddcb5e..20001ac955 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledResourceType, + Dialect, PlannedSqlTests, PlannerInputError, SqlTestPlanning, @@ -51,7 +52,7 @@ def read_tests() -> tuple[object, ...]: planning: SqlTestPlanning = start_sql_test_planning( sql_test_selection=SqlTestSelection(), project=SqlTestsProject(read_tests), # ty: ignore[invalid-argument-type] - adapter=object(), # ty: ignore[invalid-argument-type] + dialect=Dialect(object()), # ty: ignore[invalid-argument-type] selected_keys=_KEYS, enabled=test_case.enabled, ) @@ -82,7 +83,7 @@ def read_tests() -> tuple[object, ...]: planning: SqlTestPlanning = start_sql_test_planning( sql_test_selection=SqlTestSelection(), project=SqlTestsProject(read_tests), # ty: ignore[invalid-argument-type] - adapter=object(), # ty: ignore[invalid-argument-type] + dialect=Dialect(object()), # ty: ignore[invalid-argument-type] selected_keys=_KEYS, enabled=test_case.enabled, ) @@ -117,7 +118,7 @@ def read_tests() -> tuple[object, ...]: planning: SqlTestPlanning = start_sql_test_planning( sql_test_selection=SqlTestSelection(), project=SqlTestsProject(read_tests), # ty: ignore[invalid-argument-type] - adapter=object(), # ty: ignore[invalid-argument-type] + dialect=Dialect(object()), # ty: ignore[invalid-argument-type] selected_keys=_KEYS, enabled=test_case.enabled, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py index f1b6418b8c..a5be2b5415 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py @@ -60,7 +60,7 @@ def sql_tests(self): native.start_sql_test_planning( project=SlowSqlTestsProject(), - adapter=None, + dialect=native.Dialect(DuckDbAdapter()), selected_keys=frozenset(), sql_test_selection=SqlTestSelection(), enabled=True, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py index ba62c9e9da..8f720e8ac6 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py @@ -13,6 +13,7 @@ CompiledResourceType, CompileModelConfig, DeferralInputs, + Dialect, Fingerprint, PlannerOverrides, PlannerPolicies, @@ -25,6 +26,7 @@ build_execution_plan, build_version_identity_metadata_json, compute_query_hash, + live_reads, write_fingerprint, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -51,7 +53,11 @@ def grouped(model: type) -> dict[str, Any]: overrides: PlannerOverrides = PlannerOverrides(**grouped(PlannerOverrides)) deferral: DeferralInputs = DeferralInputs(**grouped(DeferralInputs)) policies: PlannerPolicies = PlannerPolicies(**grouped(PlannerPolicies)) + adapter: Any = kwargs.pop("adapter") + connection: Any = kwargs.pop("connection") return build_execution_plan( + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection), selection=selection, overrides=overrides, deferral=deferral, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py index cd9d9916e4..25c7b68d35 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py @@ -11,6 +11,7 @@ from sqlbuild._native import ( CompileModelConfig, + Dialect, DiscoveredSeedFile, Fingerprint, PlannerRelationsContext, @@ -23,6 +24,7 @@ build_seed_command_plan, build_test_command_plan, compiled_model_record, + live_reads, project_with, read_selected_seed_fingerprints, resolve_static_command_scope, @@ -76,12 +78,12 @@ def test_given_incremental_attached_audit_when_planning_then_no_warehouse_method ) relations: PlannerRelationsContext = resolve_static_relation_context( project=project, - adapter=adapter, + dialect=Dialect(adapter), scope=scope, ) plan: PlanOutput = build_audit_command_plan( project=project, - adapter=adapter, + dialect=Dialect(adapter), scope=scope, relations=relations, ) @@ -110,13 +112,13 @@ def test_given_sql_test_with_function_dependency_when_planning_then_required_fun ) relations: PlannerRelationsContext = resolve_static_relation_context( project=project, - adapter=adapter, + dialect=Dialect(adapter), scope=scope, ) plan: PlanOutput = build_test_command_plan( project=project, - adapter=adapter, + dialect=Dialect(adapter), scope=scope, relations=relations, ) @@ -156,7 +158,7 @@ def test_given_selected_seed_when_planning_then_identity_and_reason_are_preserve ) relations: PlannerRelationsContext = resolve_static_relation_context( project=project, - adapter=adapter, + dialect=Dialect(adapter), scope=scope, ) @@ -223,8 +225,7 @@ def test_given_selected_seed_without_state_table_when_reading_reason_then_only_s connection: object = object() fingerprints: dict[str, Fingerprint] = read_selected_seed_fingerprints( - adapter=adapter, - connection=connection, + reads=live_reads(adapter=adapter, connection=connection), seeds=(seed,), ) diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py index 558e7417bb..7550aad2da 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py @@ -6,12 +6,14 @@ import pytest from sqlbuild._native import ( + Dialect, SourceEntry, SourceFreshnessConfig, SourceFreshnessObservation, SourceFreshnessObservationError, SourceFreshnessStrategy, SourceFreshnessValueKind, + live_reads, normalize_source_freshness_data_version, observe_configured_source_freshness, ) @@ -90,8 +92,8 @@ def test_given_source_freshness_config_when_observing_then_returns_data_version( for statement in test_case.setup_sql: adapter.execute(connection=connection, sql=statement) observation: SourceFreshnessObservation = observe_configured_source_freshness( - adapter=adapter, - connection=connection, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection), source=SourceEntry( name=test_case.source_name, table=test_case.table, @@ -181,8 +183,8 @@ def test_given_invalid_source_freshness_result_when_observing_then_raises_clear_ SourceFreshnessObservationError, match=test_case.expected_error_fragment ): observe_configured_source_freshness( - adapter=adapter, - connection=connection, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection), source=source, observed_at=datetime(2026, 1, 1, 12, 0, 0, tzinfo=UTC), ) @@ -222,8 +224,8 @@ def test_given_adapter_freshness_is_unsupported_when_observing_then_metadata_is_ SourceFreshnessObservationError, match=test_case.expected_error_fragment ): observe_configured_source_freshness( - adapter=adapter, - connection=connection, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection), source=source, observed_at=datetime(2026, 1, 1, 12, 0, 0, tzinfo=UTC), ) diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py index 5097f6fddc..fae622b5da 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py @@ -7,11 +7,13 @@ from sqlbuild._native import ( AdapterSourceFreshnessBatch, + Dialect, DirectSourceFreshnessPlanningResult, SourceFreshnessConfig, SourceFreshnessStrategy, SourceFreshnessUnknownReason, build_direct_source_freshness_planning_result, + live_reads, observe_adapter_sources_freshness, ) from sqlbuild.adapter.contract.models import TableFreshnessMetadata @@ -85,8 +87,8 @@ def test_given_batch_with_unobservable_table_when_observing_then_isolates_each_s with invocation_scope("freshness-invocation"), dispatcher_scope(dispatcher): batch: AdapterSourceFreshnessBatch = observe_adapter_sources_freshness( - adapter=adapter, - connection=object(), + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=object()), sources=physical_table_sources( freshness=SourceFreshnessConfig(strategy=SourceFreshnessStrategy.ADAPTER) ), @@ -132,8 +134,8 @@ def test_given_batch_with_absent_table_metadata_when_planning_then_records_other connection: Any = adapter.connect({"database": ":memory:"}) try: result: DirectSourceFreshnessPlanningResult = build_direct_source_freshness_planning_result( - adapter=adapter, - connection=connection, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection), sources=physical_table_sources(freshness=None), state_database=None, state_schemas=("state_schema",), diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py index 49003234d5..434b5259be 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py @@ -6,11 +6,13 @@ import pytest from sqlbuild._native import ( + Dialect, SourceEntry, SourceFreshnessConfig, SourceFreshnessObservation, SourceFreshnessStrategy, SourceFreshnessValueKind, + live_reads, observe_configured_source_freshness, ) from sqlbuild.adapter.contract.models import QueryResult, TableFreshnessMetadata @@ -92,8 +94,8 @@ def test_given_adapter_freshness_metadata_when_observing_then_uses_expected_obse connection: Any = adapter.connect({"database": ":memory:"}) try: observation: SourceFreshnessObservation = observe_configured_source_freshness( - adapter=adapter, - connection=connection, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection), source=SourceEntry( name="raw.orders", table="orders", @@ -145,8 +147,8 @@ def record_event(event: LifecycleEvent) -> None: dispatcher.subscribe_lifecycle(subscriber=record_event, accepts_opaque=False) with invocation_scope("freshness-invocation"), dispatcher_scope(dispatcher): observe_configured_source_freshness( - adapter=adapter, - connection=object(), + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=object()), source=SourceEntry( name="raw.orders", database=test_case.source_database, @@ -190,8 +192,8 @@ def test_given_column_freshness_expression_source_when_observing_then_uses_subqu adapter: CapturingQueryDuckDbAdapter = CapturingQueryDuckDbAdapter() observe_configured_source_freshness( - adapter=adapter, - connection=object(), + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=object()), source=SourceEntry( name="raw_orders", expression=test_case.expression, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py index 38faa71844..b2f9b6264b 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py @@ -7,6 +7,7 @@ import pytest from sqlbuild._native import ( + Dialect, DirectSourceFreshnessPlanningResult, SourceEntry, SourceFreshnessAgePolicy, @@ -16,6 +17,7 @@ SourceFreshnessStrategy, SourceFreshnessValueKind, build_direct_source_freshness_planning_result, + live_reads, source_freshness_data_version_hash, write_source_freshness_records, ) @@ -136,8 +138,8 @@ def test_given_direct_source_freshness_state_when_planning_then_classifies_hash_ ) result: DirectSourceFreshnessPlanningResult = build_direct_source_freshness_planning_result( - adapter=adapter, - connection=connection, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection), sources=(source,), state_database=None, state_schemas=("state_schema",), @@ -227,8 +229,8 @@ def test_given_source_freshness_age_policy_when_planning_then_records_age_status ) result: DirectSourceFreshnessPlanningResult = build_direct_source_freshness_planning_result( - adapter=adapter, - connection=connection, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection), sources=(source,), state_database=None, state_schemas=("state_schema",), @@ -287,8 +289,8 @@ def test_given_adapter_metadata_age_policy_when_planning_then_records_age_status ) result: DirectSourceFreshnessPlanningResult = build_direct_source_freshness_planning_result( - adapter=adapter, - connection=connection, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection), sources=(source,), state_database=None, state_schemas=("state_schema",), @@ -379,8 +381,8 @@ def test_given_timestamp_lag_tolerance_when_planning_then_classifies_tolerated_m ) result: DirectSourceFreshnessPlanningResult = build_direct_source_freshness_planning_result( - adapter=adapter, - connection=connection, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection), sources=( SourceEntry( name="raw.orders", @@ -428,8 +430,8 @@ def test_given_unconfigured_source_without_adapter_metadata_when_planning_then_m connection: Any = adapter.connect({"database": ":memory:"}) try: result: DirectSourceFreshnessPlanningResult = build_direct_source_freshness_planning_result( - adapter=adapter, - connection=connection, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection), sources=(SourceEntry(name="raw.orders", table="orders"),), state_database=None, state_schemas=("state_schema",), @@ -469,8 +471,8 @@ def test_given_column_freshness_expression_when_planning_then_observes_subquery( connection: Any = adapter.connect({"database": ":memory:"}) try: result: DirectSourceFreshnessPlanningResult = build_direct_source_freshness_planning_result( - adapter=adapter, - connection=connection, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection), sources=( SourceEntry( name="raw_orders", @@ -521,8 +523,8 @@ def test_given_adapter_metadata_support_when_planning_unconfigured_source_then_o connection: Any = adapter.connect({"database": ":memory:"}) try: result: DirectSourceFreshnessPlanningResult = build_direct_source_freshness_planning_result( - adapter=adapter, - connection=connection, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection), sources=( SourceEntry(name="raw.orders", schema="raw", table="orders"), SourceEntry(name="raw.customers", schema="raw", table="customers"), @@ -566,8 +568,8 @@ def test_given_managed_source_when_planning_source_freshness_then_skips_observat connection: Any = adapter.connect({"database": ":memory:"}) try: result: DirectSourceFreshnessPlanningResult = build_direct_source_freshness_planning_result( - adapter=adapter, - connection=connection, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection), sources=(SourceEntry(name="raw.orders", schema="raw", table="orders", managed=True),), state_database=None, state_schemas=("state_schema",), @@ -626,8 +628,8 @@ def test_given_multiple_state_schemas_when_planning_then_merges_previous_records data_version="2", ) result: DirectSourceFreshnessPlanningResult = build_direct_source_freshness_planning_result( - adapter=adapter, - connection=connection, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection), sources=( SourceEntry( name="raw.orders", @@ -698,8 +700,8 @@ def test_given_duplicate_state_schema_records_when_planning_then_uses_newest_obs ) result: DirectSourceFreshnessPlanningResult = build_direct_source_freshness_planning_result( - adapter=adapter, - connection=connection, + dialect=Dialect(adapter), + reads=live_reads(adapter=adapter, connection=connection), sources=( SourceEntry( name="raw.orders", diff --git a/tests/unit/src/sqlbuild/executor/auditing/main/_test_types.py b/tests/unit/src/sqlbuild/executor/auditing/main/_test_types.py index 33913b8547..caef707168 100644 --- a/tests/unit/src/sqlbuild/executor/auditing/main/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/auditing/main/_test_types.py @@ -54,3 +54,11 @@ class AuditInterruptCase: description: str error: BaseException expected_error_type: type[BaseException] + + +@dataclass(frozen=True) +class IgnoredFailureProjectionCase: + description: str + ignored: bool + expected_metadata_json: str | None + expected_payload_ignored: bool | None diff --git a/tests/unit/src/sqlbuild/executor/auditing/main/test_project_results.py b/tests/unit/src/sqlbuild/executor/auditing/main/test_project_results.py index 36fd549e5c..3a6f55d821 100644 --- a/tests/unit/src/sqlbuild/executor/auditing/main/test_project_results.py +++ b/tests/unit/src/sqlbuild/executor/auditing/main/test_project_results.py @@ -26,7 +26,10 @@ from sqlbuild.runtime.observability.main.dispatcher_scope import dispatcher_scope from sqlbuild.runtime.observability.main.identity_scope import identity_scope from sqlbuild.runtime.observability.models import ExecutionIdentity, LifecycleEvent -from tests.unit.src.sqlbuild.executor.auditing.main._test_types import AuditExecutionCase +from tests.unit.src.sqlbuild.executor.auditing.main._test_types import ( + AuditExecutionCase, + IgnoredFailureProjectionCase, +) from tests.unit.src.sqlbuild.executor.auditing.main.helpers import ( build_projection_entry, build_projection_result, @@ -93,6 +96,73 @@ def write_records(**kwargs: object) -> None: assert events[0].payload["audit_description"] == "Valid row percentage" +@pytest.mark.parametrize( + "test_case", + [ + IgnoredFailureProjectionCase( + description="an ignored failure is recorded as a failure marked ignored", + ignored=True, + expected_metadata_json='{"ignored": true}', + expected_payload_ignored=True, + ), + IgnoredFailureProjectionCase( + description="a failure that blocked carries no ignored mark", + ignored=False, + expected_metadata_json=None, + expected_payload_ignored=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_failed_audit_when_projected_then_history_and_event_say_whether_it_was_ignored( + test_case: IgnoredFailureProjectionCase, + monkeypatch: pytest.MonkeyPatch, +) -> None: + records: list[AuditResultRecord] = [] + + def write_records(**kwargs: object) -> None: + records.extend(cast(tuple[AuditResultRecord, ...], kwargs["records"])) + + monkeypatch.setattr(projection_module, "write_audit_result_records", write_records) + dispatcher: EventDispatcher = EventDispatcher() + events: list[LifecycleEvent] = [] + dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) + plan: PlanOutput = PlanOutput( + audit_entries=(build_projection_entry(),), + model_locations={ + "orders": CompiledRelationLocation( + database="warehouse", schema="analytics", name="orders", qualified_name=None + ) + }, + ) + # Writer is replaced, so only method attributes passed into it are required. + adapter: Any = writer_adapter() + + with ( + identity_scope(ExecutionIdentity(invocation_id="invocation", run_id="run")), + dispatcher_scope(dispatcher), + ): + _ = project_audit_result_batch( + plan=plan, + results=( + replace( + build_projection_result(), + outcome=AuditOutcome.ERROR, + ignored=test_case.ignored, + ), + ), + adapter=adapter, + connection=object(), + ) + + assert [(record.outcome, record.metadata_json) for record in records] == [ + (AuditOutcome.ERROR.value, test_case.expected_metadata_json) + ] + assert [event.payload.get("ignored") for event in events] == [ + test_case.expected_payload_ignored + ] + + @pytest.mark.parametrize( "test_case", [AuditExecutionCase("reused projection", AuditOutcome.WARN)], diff --git a/tests/unit/src/sqlbuild/presentation/classes/_test_types.py b/tests/unit/src/sqlbuild/presentation/classes/_test_types.py index 068f788a87..adc4af0cd7 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/presentation/classes/_test_types.py @@ -60,3 +60,10 @@ class StatusReporterExitTestCase: description: str script: str expected_returncode: int + + +@dataclass(frozen=True) +class ConsoleLineTestCase: + description: str + project_toml: str + expected_events: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/presentation/classes/test_transient_line_coordinator.py b/tests/unit/src/sqlbuild/presentation/classes/test_transient_line_coordinator.py index 42f18c13f2..f1c446d868 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/test_transient_line_coordinator.py +++ b/tests/unit/src/sqlbuild/presentation/classes/test_transient_line_coordinator.py @@ -1,9 +1,17 @@ from __future__ import annotations +import sys +from pathlib import Path + import pytest -from sqlbuild._native import TransientLineCoordinator +from sqlbuild._native import ( + TransientLineCoordinator, + discover_project_inputs, + shared_transient_line_coordinator, +) from tests.unit.src.sqlbuild.presentation.classes._test_types import ( + ConsoleLineTestCase, TransientLineCoordinatorTestCase, ) from tests.unit.src.sqlbuild.presentation.classes.helpers import ( @@ -103,3 +111,37 @@ def test_given_live_spinner_when_clearing_active_line_then_later_writes_do_not_r if __name__ == "__main__": pytest.main([__file__, "-vv"]) + + +@pytest.mark.parametrize( + "test_case", + ( + ConsoleLineTestCase( + description="a native console line clears and then redraws the live spinner", + project_toml=( + 'name = "orders"\nadapter = "duckdb"\n[settings]\ncolumn_contract_mode = "implicit"\n' + ), + expected_events=( + "clear", + "write:Warning: settings.column_contract_mode in sqlbuild_project.toml is " + "deprecated; replace it with settings.check_declared_columns = true\n", + "redraw", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_live_spinner_when_native_code_writes_a_console_line_then_line_never_joins_spinner( + test_case: ConsoleLineTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + events: list[str] = [] + owner: RecordingSpinnerOwner = RecordingSpinnerOwner(events=events) + coordinator: TransientLineCoordinator = shared_transient_line_coordinator() + _ = (tmp_path / "sqlbuild_project.toml").write_text(test_case.project_toml, encoding="utf-8") + monkeypatch.setattr(sys, "stderr", RecordingStream(events=events, tty=True)) + coordinator.claim(stream=RecordingStream(events=[], tty=True), owner=owner) + + _ = discover_project_inputs(project_dir=tmp_path) + coordinator.release(owner=owner) + + assert tuple(events) == test_case.expected_events diff --git a/ui/.gitignore b/ui/.gitignore index dccd4e2705..388877c712 100644 --- a/ui/.gitignore +++ b/ui/.gitignore @@ -3,3 +3,5 @@ node_modules/ build/ .vite/ vite.config.ts.timestamp-* +test-results/ +playwright-report/ diff --git a/ui/COPY.md b/ui/COPY.md new file mode 100644 index 0000000000..551d90ec29 --- /dev/null +++ b/ui/COPY.md @@ -0,0 +1,88 @@ +# UI copy style + +Copy in the UI states facts. Explanations go in help tips and the docs. A test +(`src/lib/copy-style/`) checks the rules it can. + +## Rules + +1. Labels, column headers, keys, tabs and buttons are nouns or short verb phrases + of 1–4 words. Use sentence case and no final period. +2. Values and cells are facts: a state, a number, a time or a name. Never write a + sentence about what the system does. +3. Don't use "will", "is recorded", "so that", "this means", "fails if", "after + writing" or "you" in labels, values or cells. +4. Explanations go in a help tip (`Panel help`, `HelpTip`, a `KeyValues` row's + `help`, a `title`) or in the docs, never in `note` or a value. +5. A `Panel note` is a short fact (a count, a time, a scope) of at most 6 words. + It never restates the title. +6. Put one fact per value. Join facts with " · ", at most 3 segments, and keep + each segment verbless. +7. Status words come from the shared label maps (`lib/build-area/labels.ts`, + `lib/operate/labels.ts`, `lib/platform/labels.ts`). The scheduler's or CLI's + own name for a setting goes in the tooltip as "(name)", not in the value. +8. Empty states say what is missing in a few words ("No runs yet."). How-to + guidance goes to the docs. A second sentence must add a consequence, not + repeat the first. +9. Permission gaps get a lock icon and "No access (permission.id)", not a + sentence. +10. Audited reads get a lock icon with the tooltip "Viewing is audited" + (`Panel audited`), not inline text. +11. Warnings about consequences stay inline and lead with the consequence + ("Replaces all data."). They are the only inline sentences allowed. +12. Confirmation dialogs for irreversible actions may use full sentences. +13. Write "CLI" for "from the command line" and "Manual" for "started by hand". +14. Never put user data, API field names or code identifiers in prose. Show them + as `code` values. + +Count with `plural`, `noun` and `nodeCount` from `src/lib/format/plural.ts`, not +local helpers. SQL audits are "audits" and Python `@check` nodes are "Python +checks". + +## The copy check + +`src/lib/copy-style/copy-style.test.ts` runs with the unit tests. It reads +`src/routes` and `src/lib` (not tests, mocks, `lib/api`, `components/ui`) and +checks these copy sites, with each `{…}` or `${…}` counted as one token: + +- values of `Record<…, string>` maps in `labels.ts`, `lib/status.ts` and + `lib/platform/maintenance.ts`, except maps named `…Help`, `…Hint`, + `…Technical` or `…Tone` +- `Panel` `title` and `note`, `StatusLabel label`, `label:` and `value:` + literals (key-value rows, columns, filter options, toolbar counts) +- literal ``, `
` and `
` text +- `empty=` attributes and `class="empty"` / `class="msg"` text + +Rules it applies: + +- length: labels 4 words, values 5, notes 6, an empty state's first sentence 8 +- the banned phrases of rule 3, plus "when it", in labels, values and notes +- no `;`, no `: ` followed by a lowercase clause, at most 3 " · " segments +- a note sharing 2 or more content words with its panel title +- an empty state's second sentence without a consequence word (replace, + delete, lost, can't, irreversible) + +### Exceptions + +Mark a deliberate exception on its line or the line before: + +```svelte + +

Rebuilds every model from scratch, replacing all data. Audited.

+``` + +In TypeScript use `// copy-allow: `. The reason is required: a bare +`copy-allow` is itself a finding. The test logs how many sites are allowed. + +### Baseline + +The check is report-only: it logs every finding and fails only on a finding +missing from `src/lib/copy-style/baseline.txt`, or on a baseline entry that no +longer occurs. The baseline can therefore only shrink. Entries are +`file | kind | rule | text`, so they survive lines moving. The baseline is +empty today; keep it that way by fixing new findings or marking exceptions. + +To rewrite the baseline after fixing findings: + +```sh +COPY_BASELINE=write npx vitest run src/lib/copy-style +``` diff --git a/ui/PERF.md b/ui/PERF.md new file mode 100644 index 0000000000..6b793bb4c1 --- /dev/null +++ b/ui/PERF.md @@ -0,0 +1,36 @@ +# UI performance rules + +The UI must stay responsive on a modest machine: a few cores and an old, low-end graphics card +driving large screens. The layout and performance end-to-end tests run with the CPU slowed 6× and +fail when a page breaks these budgets; keep them passing rather than raising the budgets. + +## Budgets (6× slower CPU, realistic data) + +- Opening a list page, first or repeat visit: settled within 2 s. +- Graph pan, wheel zoom, trackpad momentum and pinch: 90% of frames ≤ 50 ms, none over 200 ms. +- Typing in a search or selection field: suggestions update without a visible stall. +- The plan page on a real-sized plan: switching section, or Show more, has its rows on screen within 1 s. + A large section loads one page of rows when it is opened, never all of them. + +## Rules + +1. **No work proportional to options × rows.** Count filter options in one pass (`countBy` in + `lib/list/count-by.ts`); never `options.map(o => rows.filter(...).length)`. A unit check flags it. +2. **Draw only what is visible.** Lists above ~200 rows use the shared `DataTable` windowing; big + option lists use the windowed `ChoiceFilter` list; graphs draw at the visible level (closed boxes, + aggregated edges). +3. **Pan and zoom move a picture; they don't redraw it.** During a gesture only the viewport + transform changes (a promoted layer). Anything that depends on zoom (label scale, compact cards) is + one CSS custom property or class on the container, applied after the view settles, in coarse steps. + No per-node state, style or data updates per zoom step; no SVG background repainted per frame. + A card's zoomed-out pill (its name's tail among its siblings, and its size per compact level) is + worked out once per card, and again only when its font loads, and shown by the class. + Edge routes are worked out when the nodes change (a layout, a measure, a drop), never per frame. +4. **No DOM measuring in reactive paths** that run on scroll, pan or zoom. +5. **Fetch once, derive locally.** Per-snapshot data (node lists, selection vocabulary) is cached per + location and snapshot; debounce server evaluation of typed input (~150 ms). +6. **Measure on realistic data.** Mock and benchmark data must be sized like a real project (thousands + of nodes, long names, populated state). An optimisation proved only on small or empty data does not + count. +7. **Prove a guard can fail.** When adding a performance check, break the thing it protects once and + confirm the check fails, then revert. diff --git a/ui/README.md b/ui/README.md index e2bcbc77ac..85d7309d61 100644 --- a/ui/README.md +++ b/ui/README.md @@ -21,6 +21,16 @@ npm run check npm run build # writes static assets to build/ ``` +## Copy + +UI text follows the style guide in [COPY.md](COPY.md); a unit test checks it +against a baseline that may only shrink. + +## Performance + +Pages, lists and graphs follow the budgets and rules in [PERF.md](PERF.md); the end-to-end +performance tests enforce them with the CPU slowed 6×. + ## Generated files The run panel's verbs and flags come from the CLI's command table diff --git a/ui/e2e/choice-filter.spec.ts b/ui/e2e/choice-filter.spec.ts new file mode 100644 index 0000000000..4e1e88d754 --- /dev/null +++ b/ui/e2e/choice-filter.spec.ts @@ -0,0 +1,127 @@ +import { expect, test, type Page } from '@playwright/test'; + +// The choice filter's list with thousands of options: the Checks page with awkward mock data has +// 5,000 checks on about 2,000 long-named nodes, picked from the selection bar's Node chip. Its picker +// shows a search box, draws a bounded number of rows, keeps every label inside its row and every +// count inside the panel, and ticking options adds them to the selection, which narrows the list. +// Set CHOICE_FILTER_SHOT to a path to save a screenshot. + +test.use({ viewport: { width: 1440, height: 1000 } }); + +const FRAGMENT = 'enough_to_show_17'; +/** A node of the mock project with checks; the awkward copies keep the originals' names. */ +const ORIGINAL = 'fact_orders'; + +/** + * The list's drawn data rows: not the spacers, and not the loading row shown until the checks arrive. + * The mock answers after a delay, which `networkidle` doesn't wait for (it makes no requests), so tests + * wait for these rows rather than for the network. + */ +const DATA_ROWS = 'table.sb-table tbody tr:not(.spacer):not(.load-row)'; + +/** Opens Checks, `query` added to the URL, once its checks are drawn. */ +async function openChecks(page: Page, query = '') { + await page.goto(`/checks?mock&awkward${query}`); + // Thousands of awkward checks can take a while to arrive on a busy machine. + await expect(page.locator(DATA_ROWS).first()).toBeVisible({ timeout: 20_000 }); +} + +/** The "Attached to" cell's full text (`node` or `node.column`) of each drawn row. */ +const nodeCells = (page: Page) => page.locator(`${DATA_ROWS} td:nth-child(3)`).evaluateAll((cells) => cells.map((c) => (c as HTMLElement).title)); +const onNode = (cell: string, node: string) => cell === node || cell.startsWith(`${node}.`); + +/** Problems with the open panel's rows: labels spilling out of their row, counts outside the panel. */ +async function panelProblems(page: Page) { + return page.evaluate(() => { + const problems: string[] = []; + const panel = document.querySelector('.choice-panel'); + if (!panel) return ['no open panel']; + const box = panel.getBoundingClientRect(); + if (box.right > window.innerWidth - 4) problems.push(`panel ends at ${Math.round(box.right)}px`); + for (const row of panel.querySelectorAll('.cf-row')) { + const r = row.getBoundingClientRect(); + const label = row.querySelector('.cf-label')!.getBoundingClientRect(); + const count = row.querySelector('.cf-count'); + const name = row.title.slice(-24); + if (row.scrollWidth > row.clientWidth + 1) problems.push(`row …${name} overflows by ${row.scrollWidth - row.clientWidth}px`); + if (label.right > r.right + 1) problems.push(`label …${name} ends past its row`); + if (!count) { + problems.push(`row …${name} has no count`); + continue; + } + const c = count.getBoundingClientRect(); + if (!c.width || c.left < label.right - 1 || c.right > box.right + 1) problems.push(`count of …${name} is not inside the panel beside its label`); + } + return problems; + }); +} + +test('a picker with thousands of options searches, windows and takes several values', async ({ page }) => { + // The Node chip's picker on Checks is the same list as the toolbar's filters, with every node. + await openChecks(page, `&select=${ORIGINAL}`); + await page.locator('.selection-bar .sel-chip .chip-face').first().click(); + + const search = page.getByRole('textbox', { name: 'Search Node' }); + await expect(search).toBeFocused(); + const rows = page.locator('.choice-panel .cf-row'); + await expect(rows.first()).toBeVisible(); + expect(await rows.count(), 'the list is windowed').toBeLessThan(100); + expect(await panelProblems(page)).toEqual([]); + + await search.fill(FRAGMENT); + await expect(rows.first()).toBeVisible(); + const titles = await rows.evaluateAll((els) => els.map((e) => (e as HTMLElement).title)); + expect(titles.length).toBeGreaterThan(2); + for (const t of titles) expect(t.toLowerCase()).toContain(FRAGMENT); + expect(await panelProblems(page)).toEqual([]); + + await rows.nth(0).click(); + await rows.nth(1).click(); + await expect(search, 'focus stays in the search box').toBeFocused(); + await expect(rows.nth(0).locator('input')).toBeChecked(); + // Each ticked name joins the selection as another node, which sqb unions. + await expect(page.locator('.selection-bar .sel-chip')).toHaveCount(3); + + if (process.env.CHOICE_FILTER_SHOT) { + // The bar and the open picker, with some of the page around them. + const [a, b] = [await page.locator('.selection-bar').boundingBox(), await page.locator('.choice-panel').boundingBox()]; + const top = Math.max(0, Math.min(a!.y, b!.y) - 24); + const bottom = Math.min(1000, Math.max(a!.y + a!.height, b!.y + b!.height) + 24); + await page.screenshot({ path: process.env.CHOICE_FILTER_SHOT, clip: { x: 0, y: top, width: 1440, height: bottom - top } }); + } + + await page.keyboard.press('Escape'); + await expect(page.locator('.choice-panel')).toHaveCount(0); + expect(new URL(page.url()).searchParams.get('select')?.split(' ')).toEqual([ORIGINAL, ...titles.slice(0, 2)]); +}); + +test('a filter with few options has no search box and works from the keyboard', async ({ page }) => { + await openChecks(page); + await page.getByRole('button', { name: /^Attached to/ }).click(); + await expect(page.locator('.choice-panel .cf-row').first()).toBeVisible(); + await expect(page.locator('.choice-panel input[type="search"], .choice-panel .cf-search')).toHaveCount(0); + await page.keyboard.press('ArrowDown'); + await page.keyboard.press(' '); + await expect(page.locator('.choice-panel .cf-row').first()).toHaveAttribute('aria-selected', 'true'); + await expect(page.getByRole('button', { name: /^Attached to/ })).toContainText('Attached to:'); +}); + +test('an old single-node link still filters to that node, and a second node joins it', async ({ page }) => { + await page.goto(`/checks?mock&node=${ORIGINAL}`); + await expect(page.locator(DATA_ROWS).first()).toBeVisible({ timeout: 20_000 }); + await expect(page.locator('.selection-bar .sel-chip')).toHaveText([new RegExp(ORIGINAL)]); + const url = new URL(page.url()); + expect([url.searchParams.get('select'), url.searchParams.get('node')]).toEqual([ORIGINAL, null]); + await expect.poll(async () => (await nodeCells(page)).every((c) => onNode(c, ORIGINAL))).toBe(true); + + // Tick a second node in the chip's picker: the list shows both. + await page.locator('.selection-bar .sel-chip .chip-face').click(); + const other = page.locator('.choice-panel .cf-row[aria-selected="false"]').first(); + const second = (await other.getAttribute('title'))!; + await other.click(); + await page.keyboard.press('Escape'); + await expect.poll(async () => { + const cells = await nodeCells(page); + return cells.length > 0 && cells.every((c) => onNode(c, ORIGINAL) || onNode(c, second)) && cells.some((c) => onNode(c, second)); + }).toBe(true); +}); diff --git a/ui/e2e/compact-shots.spec.ts b/ui/e2e/compact-shots.spec.ts new file mode 100644 index 0000000000..7b0ae9e67e --- /dev/null +++ b/ui/e2e/compact-shots.spec.ts @@ -0,0 +1,78 @@ +import { test, type Page } from '@playwright/test'; +import { zoomTo } from './zoom'; + +// Photographs of the zoomed-out pills, and how many names are cut off, in the pills' font (IBM Plex +// Mono) and in Inter, for choosing between them. Runs only with SHOTS set: +// SHOTS=/some/dir npx playwright test e2e/compact-shots.spec.ts --project=layout +const shots = process.env.SHOTS; +test.use({ viewport: { width: 1600, height: 900 } }); + +const PAGES = [ + { name: 'group', path: '/jobs/warehouse_nightly?mock' }, + { name: 'group-open', path: '/jobs/warehouse_nightly?mock', click: 'Expand all' }, + { name: 'lineage', path: '/lineage?mock&large' }, + { name: 'lineage-open', path: '/lineage?mock&large', click: 'Expand all' } +]; + +/** Pills wholly in view, and those whose name is cut off. */ +const cutNames = () => { + const flow = document.querySelector('.svelte-flow')!.getBoundingClientRect(); + let shown = 0; + let cut = 0; + for (const pill of document.querySelectorAll('.cface .pill')) { + const b = pill.getBoundingClientRect(); + if (b.left < flow.left || b.right > flow.right || b.top < flow.top || b.bottom > flow.bottom) continue; + const name = [...pill.querySelectorAll('.nm')].find((n) => getComputedStyle(n).display !== 'none'); + if (!name) continue; + shown++; + if (name.scrollWidth > name.clientWidth + 0.5) cut++; + } + return { shown, cut }; +}; + +async function open(page: Page, path: string, click: string | undefined, font: 'mono' | 'inter') { + // Set once the page is parsed and before the app's scripts measure anything. + if (font === 'inter') + await page.addInitScript(() => + document.addEventListener('readystatechange', () => { + if (document.readyState === 'interactive') document.head.appendChild(Object.assign(document.createElement('style'), { textContent: 'html:root { --font-node-compact: var(--font-sans); }' })); + }) + ); + await page.goto(path); + await page.locator('.graph-canvas:not(.unfitted) .svelte-flow__node').first().waitFor({ timeout: 20_000 }); + if (click) await page.getByRole('button', { name: click }).click(); + await page.waitForTimeout(800); +} + +for (const { name, path, click } of PAGES) + for (const font of ['mono', 'inter'] as const) + test(`${name} in ${font}`, async ({ page }) => { + test.skip(!shots, 'SHOTS is not set'); + test.setTimeout(120_000); + await open(page, path, click, font); + const counts: string[] = []; + for (const zoom of [0.7, 0.5]) { + await zoomTo(page, zoom); + await page.evaluate(() => document.fonts.ready); + await page.waitForTimeout(300); + const { shown, cut } = await page.evaluate(cutNames); + counts.push(`${zoom}: ${cut} of ${shown} cut`); + const z = `z0${Math.round(zoom * 100)}`; + if (font === 'mono') await page.screenshot({ path: `${shots}/b2-${name}-${z}.png` }); + if (zoom === 0.5) await page.screenshot({ path: `${shots}/b2-${name}-${z}-${font}.png` }); + } + console.info(`CUT ${name} ${font}: ${counts.join('; ')}`); + }); + +test.describe('close-up', () => { + // Four device pixels to each pixel, so the ring and the triangle's centring can be seen. + test.use({ deviceScaleFactor: 4 }); + test('of a job pill', async ({ page }) => { + test.skip(!shots, 'SHOTS is not set'); + await open(page, '/jobs/warehouse_nightly?mock', undefined, 'mono'); + await zoomTo(page, 0.7); + const pill = page.locator('.cface .pill[data-kind="job"]').first(); + const b = (await pill.boundingBox())!; + await page.screenshot({ path: `${shots}/b2-job-pill-closeup.png`, clip: { x: b.x - 10, y: b.y - 10, width: b.width + 20, height: b.height + 20 } }); + }); +}); diff --git a/ui/e2e/compact.spec.ts b/ui/e2e/compact.spec.ts new file mode 100644 index 0000000000..d2af2e7c79 --- /dev/null +++ b/ui/e2e/compact.spec.ts @@ -0,0 +1,131 @@ +import { expect, test } from '@playwright/test'; +import { zoomTo } from './zoom'; + +// Zoomed out, cards take their compact form: a one-line pill, a ring in the colour of the card's state +// holding its kind's icon, then the part of its name that tells it apart from its siblings. The name +// never wraps, shows at 12 px or more on screen, shrinks to fit before it is cut, and is cut only at its +// end. Cards are drawn in full down to 0.82. +// With SHOTS set, each graph is also photographed at each zoom: SHOTS=/some/dir npx playwright test e2e/compact.spec.ts +const shots = process.env.SHOTS; + +const GRAPHS: { name: string; path: string; click?: string }[] = [ + { name: 'group', path: '/jobs/warehouse_nightly?mock&awkward' }, + { name: 'group-members', path: '/jobs/warehouse_nightly?mock&awkward', click: 'Folders' }, + { name: 'lineage-overview', path: '/lineage?mock&large' }, + // Opened, its models' names share endings across layers and folders. + { name: 'lineage-overview-open', path: '/lineage?mock&large', click: 'Expand all' }, + { name: 'lineage', path: '/lineage?mock' } +]; + +/** Each compact pill wholly in view that breaks a rule, as problems. */ +function pillProblems(): string[] { + const problems: string[] = []; + const flow = document.querySelector('.svelte-flow')!.getBoundingClientRect(); + const zoom = Number(/scale\(([\d.]+)\)/.exec(document.querySelector('.svelte-flow__viewport')!.style.transform)?.[1] ?? 1); + const root = getComputedStyle(document.documentElement); + // Colours compared as the browser resolves them. + const probe = document.body.appendChild(document.createElement('i')); + const colour = (value: string) => ((probe.style.color = ''), (probe.style.color = value.trim()), getComputedStyle(probe).color); + const token = (name: string) => colour(root.getPropertyValue(name)); + const tones = Object.fromEntries(['--sb-success', '--sb-error', '--sb-running', '--sb-warning', '--sb-stale', '--sb-behind', '--sb-text-faint'].map((t) => [t, token(t)])); + /** The card's state colour, as its full card shows it: its last outcome, unless it needs building. */ + function expected(card: HTMLElement): string | null { + const status = card.querySelector(':scope > .status'); + if (card.classList.contains('group')) { + const dots = [...card.querySelectorAll('.status i')].map((i) => i.style.background); + const has = (t: string) => dots.some((d) => d.includes(t)); + return has('--sb-error') ? tones['--sb-error'] : has('--sb-stale') ? tones['--sb-stale'] : has('--sb-success') ? tones['--sb-success'] : tones['--sb-text-faint']; + } + if (!status) return null; + const tone = colour(getComputedStyle(status).getPropertyValue('--tone')); + const outcomeFirst = tone === tones['--sb-error'] || tone === tones['--sb-running'] || (card.querySelector('.needs') && tone !== tones['--sb-success'] && tone !== tones['--sb-warning']); + if (outcomeFirst) return tone; + if (card.querySelector('.drift.behind, .needs .behind:only-child')) return tones['--sb-behind']; + if (card.querySelector('.drift, .needs .changed')) return tones['--sb-stale']; + return tone; + } + const visible: string[] = []; + for (const node of document.querySelectorAll('.svelte-flow__node')) { + const card = node.firstElementChild as HTMLElement; + const pill = card?.querySelector(':scope > .cface .pill'); + if (!pill) continue; + const box = pill.getBoundingClientRect(); + if (box.left < flow.left || box.right > flow.right || box.top < flow.top || box.bottom > flow.bottom) continue; + const full = pill.title; + const name = [...pill.querySelectorAll('.nm')].find((n) => getComputedStyle(n).display !== 'none'); + if (!name) { + problems.push(`${full}: no name shown`); + continue; + } + visible.push(`${pill.dataset.parent ?? ''} / ${name.textContent ?? ''}`); + const px = parseFloat(getComputedStyle(name).fontSize) * zoom; + if (px < 11.95) problems.push(`${full}: its name shows at ${px.toFixed(2)} px`); + if (Math.abs(box.height - 30) > 1) problems.push(`${full}: its pill is ${box.height.toFixed(1)} px tall`); + // One line: never wrapped, one line box. + if (getComputedStyle(name).whiteSpace !== 'nowrap' || name.getClientRects().length !== 1 || name.getBoundingClientRect().height > px * 1.6) problems.push(`${full}: its name wraps`); + // Cut only once it can shrink no further, and then at its end. + if (name.scrollWidth > name.clientWidth + 0.5) { + if (px > 12.05) problems.push(`${full}: cut at ${px.toFixed(2)} px, before shrinking to 12`); + if (getComputedStyle(name).textOverflow !== 'ellipsis') problems.push(`${full}: cut without an ellipsis at its end`); + } + if (!full.endsWith(name.textContent ?? '\0')) problems.push(`${full}: shows "${name.textContent}", not an ending of its name`); + const ring = pill.querySelector('.ring')!; + const want = expected(card); + if (want && getComputedStyle(ring).borderTopColor !== want) problems.push(`${full}: its ring is ${getComputedStyle(ring).borderTopColor}, its state ${want}`); + } + probe.remove(); + const repeated = visible.filter((t, i) => visible.indexOf(t) !== i); + if (repeated.length) problems.push(`names shown more than once among siblings: ${[...new Set(repeated)].join(', ')}`); + if (!visible.length) problems.push('no compact card in view'); + return problems; +} + +test.use({ viewport: { width: 1600, height: 900 } }); + +for (const { name, path, click } of GRAPHS) { + test(`${name}: zoomed out, each card is a one-line pill with its state's ring`, async ({ page }) => { + test.setTimeout(90_000); + await page.goto(path); + await page.locator('.graph-canvas:not(.unfitted) .svelte-flow__node').first().waitFor({ timeout: 20_000 }); + if (click) await page.getByRole('button', { name: click }).click(); + await page.waitForTimeout(800); + const canvas = page.locator('.graph-canvas').first(); + + // Just above the compact zoom, cards are still drawn in full. + await zoomTo(page, 0.85); + await expect(canvas).not.toHaveClass(/\bcompact\b/); + if (shots) await page.screenshot({ path: `${shots}/compact-${name}-z085-full.png` }); + + for (const zoom of [0.7, 0.5, 0.4]) { + await zoomTo(page, zoom); + await expect(canvas).toHaveClass(/\bcompact\b/); + if (shots) await page.screenshot({ path: `${shots}/compact-${name}-z0${Math.round(zoom * 100)}.png` }); + expect(await page.evaluate(pillProblems), `${name} at ${zoom}`).toEqual([]); + } + }); +} + +test('a job pill’s play triangle sits in the middle of its ring', async ({ page }) => { + await page.goto('/jobs/warehouse_nightly?mock'); + await page.locator('.graph-canvas:not(.unfitted) .svelte-flow__node').first().waitFor({ timeout: 20_000 }); + await page.waitForTimeout(800); + for (const zoom of [0.7, 0.5]) { + await zoomTo(page, zoom); + const offsets = await page.evaluate(() => + [...document.querySelectorAll('.pill[data-kind="job"] .ring')].map((ring) => { + // The triangle's corners: two on its left edge, one at its right tip; its centroid a third of the way across. + const path = ring.querySelector('svg path')!; + const b = path.getBBox(); + const m = path.getScreenCTM()!; + const centroid = new DOMPoint(b.x + b.width / 3, b.y + b.height / 2).matrixTransform(m); + const r = ring.getBoundingClientRect(); + return { dx: centroid.x - (r.left + r.width / 2), dy: centroid.y - (r.top + r.height / 2), size: r.width }; + }) + ); + expect(offsets.length, 'job pills in view').toBeGreaterThan(0); + for (const o of offsets) { + expect(Math.abs(o.dx), `horizontally, in a ${o.size.toFixed(1)} px ring at ${zoom}`).toBeLessThanOrEqual(0.5); + expect(Math.abs(o.dy), `vertically, at ${zoom}`).toBeLessThanOrEqual(0.5); + } + } +}); diff --git a/ui/e2e/concurrency.spec.ts b/ui/e2e/concurrency.spec.ts new file mode 100644 index 0000000000..937dee9f10 --- /dev/null +++ b/ui/e2e/concurrency.spec.ts @@ -0,0 +1,67 @@ +import { expect, test, type Page } from '@playwright/test'; + +// The Concurrency panel on job and group pages: plain facts about what runs alongside what. With +// SHOTS set, each page is photographed: SHOTS=/some/dir npx playwright test e2e/concurrency.spec.ts +const shots = process.env.SHOTS; +test.use({ viewport: { width: 1600, height: 1000 } }); + +/** The panel's lines, as text, once the page has loaded; a group page opens its details first. */ +async function panel(page: Page, job: string, group = false): Promise { + await page.goto(`/jobs/${job}?mock`); + if (group) await page.getByRole('button', { name: 'Toggle details' }).click({ timeout: 20_000 }); + const lines = page.locator('ul.concurrency > li'); + await expect(lines.first()).toBeVisible({ timeout: 20_000 }); + await lines.last().scrollIntoViewIfNeeded(); + if (shots) await page.screenshot({ path: `${shots}/concurrency-panel-${job}.png` }); + // Each fact, without the runs holding a pool now, which change as the mock's runs do. + return (await lines.allInnerTexts()).map((t) => t.replace(/\s+/g, ' ').replace(/ Running now: .*$/, '').trim()); +} + +test('a job states its run concurrency and where it comes from, overlap, and its pool', async ({ page }) => { + const lines = await panel(page, 'core_build'); + expect(lines).toEqual([ + 'Inside a run: 8 at once (job setting)', + 'Never alongside runs writing the same tables (automatic)', + 'If triggered while running: is skipped', + 'warehouse_heavy: takes 1 of 1 (capped on this server; the project allows 3), shared with reporting_exports, close_of_business' + ]); + await expect(page.getByRole('link', { name: 'How concurrency works' })).toHaveAttribute('href', 'https://sqlbuild.com/docs/concepts/concurrency/'); + await expect(page.locator('ul.concurrency a', { hasText: 'reporting_exports' })).toHaveAttribute('href', /\/jobs\/reporting_exports/); +}); + +test('each layer a run concurrency can come from is named', async ({ page }) => { + for (const [job, line] of [ + ['marts_only', 'Inside a run: 6 at once (target prod)'], + ['activity_microbatch', 'Inside a run: 2 at once (local setting)'], + ['revenue_refresh', 'Inside a run: 4 at once (project setting)'], + ['load_sources', 'Inside a run: 4 at once (SQLBuild default)'] + ] as const) { + expect((await panel(page, job))[0], job).toBe(line); + } +}); + +test('a job in two pools lists each, and a job that waits says so', async ({ page }) => { + const lines = await panel(page, 'reporting_exports'); + expect(lines).toContain('If triggered while running: waits'); + expect(lines).toContain('supplier_api: takes 1 of 2, shared with load_sources'); + expect(lines.filter((l) => l.includes(': takes '))).toHaveLength(2); +}); + +test('a group capped below what it asks takes the whole pool, shown as a warning', async ({ page }) => { + const lines = await panel(page, 'close_of_business', true); + expect(lines).toContain('Takes the whole warehouse_heavy pool on this server (capped at 1; the project asks for 3), shared with core_build, reporting_exports'); + await expect(page.locator('ul.concurrency li.pool.clamped')).toHaveCount(1); +}); + +test('a group in no pool says so', async ({ page }) => { + expect(await panel(page, 'nightly_all', true)).toContain('Pools: none'); +}); + +test("the location shows the server's pool-cap warning", async ({ page }) => { + await page.goto('/locations/0d9c6a52-3f1e-4b7a-9c43-5b8e2f61a7d4?mock'); + const caps = page.locator('.pool-caps'); + await expect(caps).toContainText('Pool caps on this server', { timeout: 20_000 }); + await expect(caps).toContainText('group close_of_business takes 3 of pool warehouse_heavy; capped at 1, so it runs alone in that pool'); + await expect(caps.getByRole('link', { name: 'close_of_business' })).toHaveAttribute('href', /\/jobs\/close_of_business/); + if (shots) await page.screenshot({ path: `${shots}/concurrency-panel-location.png` }); +}); diff --git a/ui/e2e/edges.spec.ts b/ui/e2e/edges.spec.ts new file mode 100644 index 0000000000..5eea6454a4 --- /dev/null +++ b/ui/e2e/edges.spec.ts @@ -0,0 +1,67 @@ +import { expect, test } from '@playwright/test'; + +// Edges, plain or highlighted, are drawn beneath every card and are hit only on their stroke, so a +// card under an edge can always be clicked. The library raises edges from inside an open box above +// the cards outside it, where they took a card's clicks. +test.use({ viewport: { width: 1600, height: 900 } }); + +const EDGE = 'group:seeds/marts-group:customers'; +const CARD = 'group:orders'; + +/** The middle of an edge's path, in page coordinates. */ +function middle(id: string): { x: number; y: number } { + const path = document.querySelector(`.svelte-flow__edge[data-id="${id}"] path.svelte-flow__edge-path`)!; + const m = path.getScreenCTM()!; + const p = path.getPointAtLength(path.getTotalLength() / 2); + return { x: m.a * p.x + m.e, y: m.d * p.y + m.f }; +} + +test('a card under a highlighted edge from an open box is clicked, not the edge', async ({ page }) => { + await page.goto('/lineage?mock&large'); + await page.locator('.graph-canvas:not(.unfitted) .svelte-flow__node').first().waitFor({ timeout: 20_000 }); + await page.waitForTimeout(800); + // Open a folder: edges from inside it are raised above the closed cards. + await page.locator('.svelte-flow__node[data-id="group:seeds"]').click(); + await expect(page.locator(`.svelte-flow__edge[data-id="${EDGE}"]`)).toHaveCount(1); + await page.waitForTimeout(800); + + // Drag a closed folder's card so its centre lies on that edge. + const card = page.locator(`.svelte-flow__node[data-id="${CARD}"]`); + const on = await page.evaluate(middle, EDGE); + let centre = { x: 0, y: 0 }; + // A drag falls a little short of the pointer; a few drags of what's left bring the card there. + for (let attempt = 0; attempt < 4; attempt++) { + const box = (await card.boundingBox())!; + centre = { x: box.x + box.width / 2, y: box.y + box.height / 2 }; + if (Math.hypot(centre.x - on.x, centre.y - on.y) < 6) break; + await page.mouse.move(centre.x, centre.y); + await page.mouse.down(); + for (let i = 1; i <= 10; i++) await page.mouse.move(centre.x + ((on.x - centre.x) * i) / 10, centre.y + ((on.y - centre.y) * i) / 10); + await page.mouse.up(); + await page.waitForTimeout(300); + } + expect(Math.hypot(centre.x - on.x, centre.y - on.y), 'the card lies on the edge').toBeLessThan(6); + + // Hover the edge's source, which highlights it, then click the card's centre: it opens the folder. + const source = (await page.locator('.svelte-flow__node[data-id="group:seeds/marts"]').boundingBox())!; + await page.mouse.move(source.x + source.width / 2, source.y + source.height / 2); + await page.waitForTimeout(200); + const under = await page.evaluate(({ x, y }) => document.elementFromPoint(x, y)?.closest('.svelte-flow__node, .svelte-flow__edge')?.getAttribute('data-id') ?? '', centre); + expect(under, 'what is under the card centre').toBe(CARD); + await page.mouse.click(centre.x, centre.y); + await expect(page.locator('.svelte-flow__node[data-id="folder:orders"]'), 'the folder opened').toHaveCount(1); +}); + +test('an edge is hit on its stroke only, never on its bounding box', async ({ page }) => { + await page.goto('/jobs/warehouse_nightly?mock'); + await page.locator('.graph-canvas:not(.unfitted) .svelte-flow__edge').first().waitFor({ timeout: 20_000 }); + const styles = await page.evaluate(() => + [...document.querySelectorAll('.svelte-flow__edge')].flatMap((edge) => [ + ['svg', getComputedStyle(edge.closest('svg')!).pointerEvents], + ['edge', getComputedStyle(edge).pointerEvents], + ...[...edge.querySelectorAll('path')].map((p) => [p.getAttribute('class') ?? 'path', getComputedStyle(p).pointerEvents]) + ]) + ); + const wrong = styles.filter(([what, value]) => (what === 'svg' ? value !== 'none' : !['stroke', 'visiblestroke'].includes(value.toLowerCase()))); + expect(wrong).toEqual([]); +}); diff --git a/ui/e2e/ignored-audits.spec.ts b/ui/e2e/ignored-audits.spec.ts new file mode 100644 index 0000000000..ed35b31407 --- /dev/null +++ b/ui/e2e/ignored-audits.spec.ts @@ -0,0 +1,78 @@ +import { expect, test } from '@playwright/test'; + +// Audit failures a build lets through (`ignore_audit_failures`): a success with a warning, never a +// failure. In the mock, core_build ignores audit failures and its run run-2b70 let two through. +// With SHOTS set, each surface is photographed: SHOTS=/some/dir npx playwright test e2e/ignored-audits.spec.ts +const shots = process.env.SHOTS; +test.use({ viewport: { width: 1600, height: 1000 } }); + +test('the run list reads a run that let audits fail as a success with a warning', async ({ page }) => { + await page.goto('/runs?mock&q=2b70'); + const row = page.locator('tr', { has: page.locator('a', { hasText: '2b70' }) }); + const status = row.getByTestId('run-status'); + await expect(status).toHaveText('Succeeded · 2 audit failures ignored', { timeout: 20_000 }); + // Warning-toned, not red, and unlike a plain success. + const colours = (e: Element) => ({ tone: getComputedStyle(e).getPropertyValue('--tone').trim(), warning: getComputedStyle(document.documentElement).getPropertyValue('--sb-warning').trim(), error: getComputedStyle(document.documentElement).getPropertyValue('--sb-error').trim() }); + const ignored = await status.evaluate(colours); + expect(ignored.tone).toBe(ignored.warning); + expect(ignored.tone).not.toBe(ignored.error); + if (shots) await page.screenshot({ path: `${shots}/ignored-audits-runs.png` }); + await page.goto('/runs?mock&q=91cd'); + const plain = page.locator('tr', { has: page.locator('a', { hasText: '91cd' }) }).getByTestId('run-status'); + await expect(plain).toHaveText('Succeeded', { timeout: 20_000 }); + const success = await plain.evaluate(colours); + expect(success.tone).not.toBe(success.warning); +}); + +test('the run page lists each ignored failure by model and audit', async ({ page }) => { + await page.goto('/runs/run-2b70?mock'); + await expect(page.getByTestId('run-status')).toHaveText('Succeeded · 2 audit failures ignored', { timeout: 20_000 }); + const panel = page.getByTestId('ignored-audits'); + await expect(panel.locator('.head')).toContainText('2 audit failures ignored'); + await expect(panel.locator('li')).toHaveCount(2); + await expect(panel.locator('li', { hasText: 'fact_orders' })).toContainText('unique(order_id)'); + await expect(panel.locator('li', { hasText: 'dim_customers' })).toContainText('unique(customer_id)'); + await expect(panel.locator('li .tag')).toHaveText(['ignored', 'ignored']); + if (shots) await page.screenshot({ path: `${shots}/ignored-audits-run.png` }); + // A run that let nothing through has no such list. + await page.goto('/runs/run-91cd?mock'); + await expect(page.getByTestId('run-status')).toHaveText('Succeeded', { timeout: 20_000 }); + await expect(page.getByTestId('ignored-audits')).toHaveCount(0); +}); + +test('the Checks page filters to the failures builds let through', async ({ page }) => { + await page.goto('/checks?mock'); + await page.getByRole('tab', { name: /^Ignored failures/ }).click({ timeout: 20_000 }); + const rows = page.locator('table.checks tbody tr, .checks tbody tr'); + await expect(rows).toHaveCount(2); + await expect(rows.locator('.ignored')).toHaveText(['ignored', 'ignored']); + if (shots) await page.screenshot({ path: `${shots}/ignored-audits-checks.png` }); +}); + +test('a job that ignores audit failures says so in its settings', async ({ page }) => { + await page.goto('/jobs/core_build?mock'); + await expect(page.getByTestId('job-ignores-audit-failures')).toHaveText("Audit failures don't stop runs", { timeout: 20_000 }); + await page.goto('/jobs/marts_only?mock'); + await expect(page.locator('ul.concurrency')).toBeVisible({ timeout: 20_000 }); + await expect(page.getByTestId('job-ignores-audit-failures')).toHaveCount(0); +}); + +test('the plan says when audit failures won’t stop the build', async ({ page }) => { + await page.goto(`/plan?mock&cmd=${encodeURIComponent('sqb build --ignore-audit-failures')}`); + await expect(page.getByTestId('plan-ignores-audit-failures')).toHaveText('Audit failures won’t stop this build'.replace('’', "'"), { timeout: 30_000 }); + if (shots) await page.screenshot({ path: `${shots}/ignored-audits-plan.png` }); + await page.goto(`/plan?mock&cmd=${encodeURIComponent('sqb build')}`); + await expect(page.getByRole('button', { name: 'Plan again' })).toBeVisible({ timeout: 30_000 }); + await expect(page.getByTestId('plan-ignores-audit-failures')).toHaveCount(0); +}); + +test('the run list shows the whole label, unclipped', async ({ page }) => { + await page.goto('/runs?mock&q=2b70'); + const status = page.locator('tr', { has: page.locator('a', { hasText: '2b70' }) }).getByTestId('run-status'); + await expect(status).toBeVisible({ timeout: 20_000 }); + const cell = status.locator('xpath=ancestor::td[1]'); + const [s, c] = [await status.boundingBox(), await cell.boundingBox()]; + expect(s!.x + s!.width).toBeLessThanOrEqual(c!.x + c!.width); + expect(await status.evaluate((e) => e.scrollWidth <= e.clientWidth)).toBe(true); + expect(await cell.evaluate((e) => e.scrollWidth <= e.clientWidth), 'the cell is not cut off').toBe(true); +}); diff --git a/ui/e2e/layout.spec.ts b/ui/e2e/layout.spec.ts new file mode 100644 index 0000000000..c5f5b1398e --- /dev/null +++ b/ui/e2e/layout.spec.ts @@ -0,0 +1,362 @@ +import { expect, test, type Page } from '@playwright/test'; + +// Every page, with every list stretched to 5,000 rows and very long names (`?mock&awkward`), at two +// common desktop widths: nothing overflows the window sideways, header cells line up with their +// columns, long lists draw a bounded number of rows, and sticky headers cover the rows under them. + +const WIDTHS = [1440, 1920]; +/** Most rows a windowed list may have in the page at once. */ +const MAX_ROWS = 400; + +const PAGES = [ + '/', + '/admin/audit', + '/admin/check-access', + '/admin/people', + '/admin/roles', + '/catalog', + '/changes', + '/checks', + '/connections', + '/jobs', + '/lineage', + '/locations', + '/maintenance', + '/plan', + '/providers', + '/runs', + '/settings', + '/triggers' +]; +/** Item pages, reached through the first row of their list. */ +const ITEMS = [ + { list: '/jobs', link: 'a[href*="/jobs/"]' }, + { list: '/catalog', link: 'a[href*="/catalog/"]' }, + { list: '/checks', link: 'a[href*="/checks/"]' }, + { list: '/triggers', link: 'a[href*="/triggers/"]' }, + { list: '/runs', link: 'a[href*="/runs/"]' }, + { list: '/providers', link: 'a[href*="/providers/"]' }, + { list: '/admin/roles', link: 'a[href*="/admin/roles/"]' } +]; + +/** The mock's large job group (src/lib/api/mock/big-group.ts). */ +const BIG_GROUP = 'warehouse_nightly'; + +const awkward = (path: string) => `${path}${path.includes('?') ? '&' : '?'}mock&awkward`; + +async function settle(page: Page) { + await page.waitForLoadState('networkidle'); + await page.waitForTimeout(500); + // A graph is drawn once its data loads and shows once its first fit is done; the layout check fails + // one still hidden. Wait for every graph to have cards and be fitted, or to say it has too many to draw: + // a canvas without cards yet isn't marked unfitted, so waiting for that alone could stop too early. + await page + .waitForFunction( + () => + [...document.querySelectorAll('.graph-canvas')].every((c) => (c.querySelector('.svelte-flow__node') ? !c.classList.contains('unfitted') : !!c.parentElement?.querySelector('.graph-too-many'))), + null, + { timeout: 30_000 } + ) + .catch(() => {}); +} + +/** In the page: each node card's status line that wraps or cuts a part off, as problems. */ +function statusLines(): string[] { + const problems: string[] = []; + // A node card's status line is one line, each part whole: none wraps, none is cut off. + for (const line of document.querySelectorAll('.svelte-flow__node .status')) { + const box = line.getBoundingClientRect(); + if (!box.height || getComputedStyle(line).visibility === 'hidden') continue; + const name = line.closest('.svelte-flow__node')?.dataset.id; + const parts = [...line.children].filter((c) => c.getBoundingClientRect().width > 0); + // Wrapped: a run of a part's text over two lines (its line boxes, from a range over each text + // node; icons beside the text sit at their own height), or parts on different rows. + const lines = (p: Element) => { + let most = 0; + const walk = document.createTreeWalker(p, NodeFilter.SHOW_TEXT); + for (let text = walk.nextNode(); text; text = walk.nextNode()) { + if (!text.textContent?.trim()) continue; + const range = document.createRange(); + range.selectNodeContents(text); + most = Math.max(most, new Set([...range.getClientRects()].filter((r) => r.width > 0).map((r) => Math.round(r.top))).size); + } + return most; + }; + const middles = parts.map((p) => p.getBoundingClientRect().top + p.getBoundingClientRect().height / 2); + const wrapped = parts.some((p) => lines(p) > 1) || Math.max(...middles) - Math.min(...middles) > 3; + const cut = parts.some((p) => p.getBoundingClientRect().right > box.right - 1); + if (wrapped || cut || line.scrollWidth > line.clientWidth + 1) { + problems.push(`${name}: its status line ${wrapped ? 'wraps' : 'is cut off'} ("${(line.textContent ?? '').replace(/\s+/g, ' ').trim()}")`); + } + } + return problems; +} + +/** In the page: each fitted graph's start card (`data-fit-anchor`) not wholly inside its canvas, as problems. */ +function anchorOutside(): string[] { + const found: string[] = []; + for (const flow of document.querySelectorAll('.svelte-flow[data-fit-anchor]')) { + const id = flow.dataset.fitAnchor; + const card = id ? [...flow.querySelectorAll('.svelte-flow__node')].find((n) => n.dataset.id === id) : null; + if (!card) continue; + const [pane, box] = [flow.getBoundingClientRect(), card.getBoundingClientRect()]; + if (box.left < pane.left || box.right > pane.right || box.top < pane.top || box.bottom > pane.bottom) { + found.push(`the card the graph opens on (${id}) is cut off: ${Math.round(box.left - pane.left)}px from the left, ${Math.round(box.top - pane.top)}px from the top`); + } + // Unless it centres on an open box, it starts on the leftmost column, so that column is shown whole. + if (card.classList.contains('svelte-flow__node-folderBox') || card.classList.contains('svelte-flow__node-memberBox')) continue; + // Nodes are siblings in the page; a box's contents start right of the box, so any node starting + // further left than half a card is a column left of the start card. + const further = [...flow.querySelectorAll('.svelte-flow__node')].find((n) => n.getBoundingClientRect().left < box.left - box.width / 2); + if (further) found.push(`the graph opens beside its first column: ${further.dataset.id} starts ${Math.round(box.left - further.getBoundingClientRect().left)}px left of ${id}`); + } + return found; +} + +/** The graph's pan and zoom, as Svelte Flow sets it on its viewport. */ +const viewOf = (page: Page) => page.locator('.svelte-flow__viewport').first().evaluate((el) => (el as HTMLElement).style.transform); + +/** Zooming in and pressing Reset comes back to the view the graph opened on. */ +async function expectResetToInitialView(page: Page, opened?: string) { + await page.waitForTimeout(400); + const initial = opened ?? (await viewOf(page)); + await page.locator('.svelte-flow__controls-zoomin').first().click(); + await expect.poll(() => viewOf(page)).not.toBe(initial); + await page.getByRole('button', { name: 'Reset' }).click(); + await expect.poll(() => viewOf(page), { message: 'the view after Reset' }).toBe(initial); +} + +async function checkLayout(page: Page, label: string) { + // `settle` can return before a graph has mounted (no canvas yet counts as settled), so a graph may + // still be mid-fit here on a busy machine. Give it time; one that never fits still fails below. + await page.waitForFunction(() => !document.querySelector('.graph-canvas.unfitted'), null, { timeout: 30_000 }).catch(() => {}); + const found = await page.evaluate((maxRows) => { + const problems: string[] = []; + const width = window.innerWidth; + // The page itself never scrolls sideways; lists scroll inside their own box. + if (document.documentElement.scrollWidth > width + 1) problems.push(`page is ${document.documentElement.scrollWidth}px wide in a ${width}px window`); + // Page headers and toolbars stay inside the window. + for (const el of document.querySelectorAll('.app-topbar, .list-toolbar, .head, .toolbar, article > header, .page-body-inner > header')) { + // Cards on a graph's canvas pan out of view by design; the canvas clips them. + if (el.closest('.svelte-flow')) continue; + const box = el.getBoundingClientRect(); + if (box.width && box.right > width + 1) problems.push(`${el.className.split(' ')[0]} ends at ${Math.round(box.right)}px`); + for (const child of el.querySelectorAll('*')) { + const c = child.getBoundingClientRect(); + if (c.width && c.right > width + 1 && getComputedStyle(child).visibility !== 'hidden') { + problems.push(`${el.className.split(' ')[0]}: "${(child.textContent ?? '').trim().slice(0, 40)}" ends at ${Math.round(c.right)}px`); + break; + } + } + } + // Every graph has finished its first fit: an unfitted graph stays hidden. + if (document.querySelector('.graph-canvas.unfitted')) problems.push('a graph never fitted'); + // A graph opens with its card titles readable: compact titles at least 11 px on screen, full cards + // only down to the compact zoom, 0.82, where their 12.5 px titles show at 10.25 px. (A graph with the + // library's own controls is drawn whole at any zoom, as Lineage's very large graphs are.) + for (const canvas of document.querySelectorAll('.graph-canvas:not(.unfitted)')) { + if (!canvas.querySelector('.fit-control')) continue; + const scale = /scale\(([\d.]+)\)/.exec(canvas.querySelector('.svelte-flow__viewport')?.style.transform ?? ''); + const compact = canvas.classList.contains('compact'); + const title = [...canvas.querySelectorAll(compact ? '.svelte-flow__node .cface .ctitle' : '.svelte-flow__node .name')].find((t) => t.offsetParent !== null); + if (!scale || !title) continue; + const px = parseFloat(getComputedStyle(title).fontSize) * Number(scale[1]); + if (px < (compact ? 10.95 : 10.2)) problems.push(`${compact ? 'compact' : 'card'} titles show at ${px.toFixed(1)} px`); + } + // Every scroll area has the themed scrollbar, not the browser's default. + for (const el of [document.documentElement, ...document.querySelectorAll('body *')]) { + const style = getComputedStyle(el); + const scrolls = (/(auto|scroll)/.test(style.overflowY) && el.scrollHeight > el.clientHeight + 1) || (/(auto|scroll)/.test(style.overflowX) && el.scrollWidth > el.clientWidth + 1); + if (scrolls && style.scrollbarColor === 'auto') { + problems.push(`${el.tagName.toLowerCase()}.${String(el.className).split(' ')[0]} scrolls with the default scrollbar`); + break; + } + } + for (const table of document.querySelectorAll('table.sb-table')) { + const rows = table.tBodies[0]?.rows ?? []; + if (rows.length > maxRows) problems.push(`a list draws ${rows.length} rows`); + // Header cells line up with the first full row's cells. + const heads = [...(table.tHead?.rows[0]?.cells ?? [])]; + const body = [...rows].find((r) => !r.classList.contains('spacer') && !r.classList.contains('load-row') && r.cells.length === heads.length); + if (body) { + heads.forEach((th, i) => { + const [h, d] = [th.getBoundingClientRect(), body.cells[i].getBoundingClientRect()]; + if (Math.abs(h.left - d.left) > 1 || Math.abs(h.width - d.width) > 1) problems.push(`header "${th.textContent?.trim()}" is off its column by ${Math.round(h.left - d.left)}px`); + }); + } + } + return problems; + }, MAX_ROWS); + found.push(...(await page.evaluate(statusLines))); + // A graph larger than its view opens with the card it starts on wholly inside the canvas. + found.push(...((await page.evaluate(`(${anchorOutside.toString()})()`)) as string[])); + expect(found, label).toEqual([]); +} + +/** + * Scrolls each list with a sticky header and checks that just above its header is the header or the + * top bar, never a row; then the same for sticky headers in each page body, such as the overview's + * timeline header, which must never show the page's content through a gap above it. + */ +async function checkStickyHeaders(page: Page, label: string) { + const found = await page.evaluate(async () => { + const problems: string[] = []; + const frame = () => new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(resolve))); + for (const body of document.querySelectorAll('.page-body')) { + if (body.scrollHeight <= body.clientHeight) continue; + const sticky = [...body.querySelectorAll('*')].filter((el) => getComputedStyle(el).position === 'sticky'); + if (!sticky.length) continue; + // Room below the page, so a header near its end can still scroll up to the top and stick. + const spacer = document.createElement('div'); + spacer.style.height = `${body.clientHeight}px`; + body.firstElementChild?.appendChild(spacer); + for (const el of sticky) { + // Scroll the header's own section up under it, so the header is stuck at the top of the body + // with its section's content passing beneath; a header whose section is gone isn't stuck. + body.scrollTop = 0; + await frame(); + const section = el.parentElement!.getBoundingClientRect(); + const natural = el.getBoundingClientRect().top - body.getBoundingClientRect().top; + const room = section.height - el.getBoundingClientRect().height - 4; + if (room < 8) continue; + body.scrollTop = natural + Math.min(120, room); + await frame(); + const box = el.getBoundingClientRect(); + const top = body.getBoundingClientRect().top + body.clientTop; + if (!box.width || !box.height) continue; + // Not stuck (the body couldn't scroll that far, or it sticks in a scroller of its own): nothing to check. + const resting = top + natural - body.scrollTop; + if (box.top <= resting + 1) continue; + // Every point between the body's top edge and the stuck header must be the header itself or + // another sticky header: anything else is content showing through a gap above it. + for (let y = top; y < box.top; y += 2) { + const hit = document.elementFromPoint(box.left + Math.min(box.width / 2, 200), y); + if (hit && body.contains(hit) && !sticky.some((s) => s.contains(hit))) { + problems.push(`page content shows ${Math.round(box.top - top)}px above a sticky ${el.tagName.toLowerCase()}.${String(el.className).split(' ')[0]}`); + break; + } + } + } + spacer.remove(); + body.scrollTop = 0; + } + for (const scroller of document.querySelectorAll('.sb-table-scroll:not(.embedded)')) { + if (scroller.scrollHeight <= scroller.clientHeight) continue; + scroller.scrollTop = 2000; + await new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(resolve))); + const head = scroller.querySelector('thead'); + if (!head) continue; + const box = head.getBoundingClientRect(); + const x = box.left + Math.min(box.width / 2, 200); + for (const y of [box.top - 1, box.top + 1]) { + const hit = document.elementFromPoint(x, y); + if (hit?.closest('tbody')) problems.push(`a row shows at y=${Math.round(y)}, around the sticky header`); + } + } + return problems; + }); + expect(found, label).toEqual([]); +} + +for (const width of WIDTHS) { + test.describe(`${width}px`, () => { + test.use({ viewport: { width, height: 1000 } }); + + for (const path of PAGES) { + test(`${path} fits, lines up and windows its lists`, async ({ page }) => { + await page.goto(awkward(path)); + await settle(page); + await checkLayout(page, path); + await checkStickyHeaders(page, path); + }); + } + + // Cards drawn in full, so their status lines show: a job's node cards, and a group's folder and + // job cards. Zoomed in until cards aren't compact, as a narrow window may open them compact. + for (const path of ['/jobs/core_build', '/jobs/warehouse_nightly']) { + test(`${path}'s cards keep their status line on one line`, async ({ page }) => { + await page.goto(awkward(path)); + await settle(page); + const canvas = page.locator('.graph-canvas').first(); + const pane = (await page.locator('.svelte-flow').first().boundingBox())!; + await page.mouse.move(pane.x + pane.width / 2, pane.y + pane.height / 2); + for (let i = 0; i < 20 && /\bcompact\b/.test((await canvas.getAttribute('class')) ?? ''); i++) { + await page.mouse.wheel(0, -60); + await page.waitForTimeout(300); + } + await expect(canvas).not.toHaveClass(/\bcompact\b/); + expect(await page.locator('.svelte-flow__node .status').count(), 'status lines').toBeGreaterThan(3); + expect(await page.evaluate(statusLines), path).toEqual([]); + }); + } + + test('a real-sized Lineage overview opens whole, readable, and Reset comes back to it', async ({ page }) => { + await page.setViewportSize({ width: 1900, height: 1060 }); + await page.goto('/lineage?mock&large'); + await settle(page); + await checkLayout(page, 'large Lineage'); + // About 13 folder cards: every one wholly in view, none cut by the canvas edge. + const cut = await page.evaluate(() => { + const flow = document.querySelector('.svelte-flow')!; + const pane = flow.getBoundingClientRect(); + const cards = [...flow.querySelectorAll('.svelte-flow__node')]; + return { cards: cards.length, cut: cards.filter((n) => { const r = n.getBoundingClientRect(); return r.left < pane.left || r.right > pane.right || r.top < pane.top || r.bottom > pane.bottom; }).map((n) => n.dataset.id) }; + }); + expect(cut.cards, 'folder cards drawn').toBeGreaterThanOrEqual(12); + expect(cut.cut, 'cards outside the canvas').toEqual([]); + await expectResetToInitialView(page); + }); + + test('the large mock group draws its direct waits only and opens on its cards', async ({ page }) => { + await page.goto(awkward(`/jobs/${BIG_GROUP}`)); + await settle(page); + await checkLayout(page, BIG_GROUP); + const drawn = () => + page.evaluate(() => { + const pane = document.querySelector('.svelte-flow')!.getBoundingClientRect(); + const rect = (s: string) => [...document.querySelectorAll(s)].map((n) => n.getBoundingClientRect()); + const cards = rect('.svelte-flow__node[data-id^="member:"], .svelte-flow__node[data-id^="jobfolder:"]'); + const inside = cards.filter((c) => c.width && c.left >= pane.left && c.right <= pane.right && c.top >= pane.top && c.bottom <= pane.bottom); + return { members: rect('.svelte-flow__node[data-id^="member:"]').length, folders: rect('.svelte-flow__node[data-id^="jobfolder:"]').length, cards: cards.length, edges: document.querySelectorAll('.svelte-flow__edge').length, inside: inside.length }; + }); + const opened = await viewOf(page); + // A group this large opens by folder, closed, with a line per pair of folders. + let found = await drawn(); + expect(found.folders, 'closed folders drawn').toBeGreaterThan(1); + expect(found.edges, 'lines between folders').toBeLessThanOrEqual(2 * found.cards); + expect(found.inside, 'cards wholly in view after load').toBeGreaterThan(0); + // Without folders: every member, and only the waits no other wait implies, at most two lines per member. + await page.getByRole('button', { name: 'Folders' }).click(); + await expect.poll(async () => (await drawn()).members).toBeGreaterThanOrEqual(40); + found = await drawn(); + expect(found.edges, 'lines drawn').toBeLessThanOrEqual(2 * found.members); + // Larger than the view now: the refit starts on a card wholly in view. + await page.waitForTimeout(500); + expect(await page.evaluate(`(${anchorOutside.toString()})()`), 'start card after Folders off').toEqual([]); + expect(await page.locator('.svelte-flow[data-fit-anchor]').getAttribute('data-fit-anchor')).toMatch(/^member:/); + // The shared toolbar: Expand all opens members in place, Reset closes them and brings folders back. + const boxes = () => page.locator('.svelte-flow__node-memberBox').count(); + await page.getByRole('button', { name: 'Expand all' }).click(); + await expect.poll(boxes).toBeGreaterThan(0); + await page.getByRole('button', { name: 'Reset' }).click(); + await expect.poll(boxes).toBe(0); + await expect.poll(async () => (await drawn()).folders).toBeGreaterThan(1); + // Back to the view it opened on, after all that, and again after zooming in. + await expect.poll(() => viewOf(page), { message: 'the view after Reset' }).toBe(opened); + await expectResetToInitialView(page, opened); + }); + + for (const { list, link } of ITEMS) { + test(`the first item of ${list} fits and lines up`, async ({ page }) => { + await page.goto(awkward(list)); + await settle(page); + const href = await page.locator(`table ${link}`).first().getAttribute('href'); + test.skip(!href, `${list} has no item to open`); + await page.goto(awkward(href!)); + await settle(page); + await checkLayout(page, href!); + await checkStickyHeaders(page, href!); + }); + } + }); +} diff --git a/ui/e2e/perf.spec.ts b/ui/e2e/perf.spec.ts new file mode 100644 index 0000000000..3a84d448ff --- /dev/null +++ b/ui/e2e/perf.spec.ts @@ -0,0 +1,189 @@ +import { expect, test, type Page } from '@playwright/test'; + +// Timing guards. They run after the layout suite, alone (see playwright.config.ts), so other tests +// competing for the machine don't blur the timings. + +async function settle(page: Page) { + await page.waitForLoadState('networkidle'); + await page.waitForTimeout(500); +} + +test.use({ viewport: { width: 1440, height: 1000 } }); +// One at a time: timings taken side by side would measure each other. +test.describe.configure({ mode: 'serial' }); + +test('Checks opens again quickly from the sidebar with thousands of checks', async ({ page }) => { + const nav = (name: string) => page.locator('nav a', { hasText: name }).first(); + const openChecks = async () => { + const started = Date.now(); + await nav('Checks').click(); + await page.locator('table.sb-table tbody tr:not(.spacer)').first().waitFor(); + await page.waitForLoadState('networkidle'); + return Date.now() - started; + }; + await page.goto('/?mock&awkward'); + await settle(page); + await openChecks(); + await nav('Overview').click(); + await settle(page); + // The first visit loads the list; the second took 12–19 s on a large project when filter options + // were counted by filtering the whole list once per option. + const second = await openChecks(); + console.info(`Checks, second visit: ${second} ms`); + expect(second, 'ms for the second visit to Checks to settle').toBeLessThan(2000); +}); + +// Panning and zooming a graph on a slow computer (the CPU slowed 6×): every gesture runs as a +// transform of the viewport's layer, so frames stay smooth. Wheel zoom with a mouse, a trackpad's +// two-finger scroll and its momentum tail (many small wheel events, which the canvas reads as zoom), +// pinch (ctrl + wheel), and drag-panning from empty canvas and from a card. Each gesture runs three +// times and the run with the median worst frame is judged, so a one-off stall of a shared machine +// doesn't decide it, while a cost every gesture pays does. + +type Frames = { p90: number; max: number }; +const P90_MS = 50; +const MAX_MS = 200; + +/** Frame times while `act` runs, from requestAnimationFrame. */ +async function frameTimes(page: Page, act: () => Promise): Promise { + await page.evaluate(() => { + const w = window as unknown as { __frames: number[]; __measuring: boolean }; + w.__frames = []; + w.__measuring = true; + let last = performance.now(); + const tick = (t: number) => { + w.__frames.push(t - last); + last = t; + if (w.__measuring) requestAnimationFrame(tick); + }; + requestAnimationFrame(tick); + }); + await act(); + await page.waitForTimeout(300); + const frames = await page.evaluate(() => { + const w = window as unknown as { __frames: number[]; __measuring: boolean }; + w.__measuring = false; + return w.__frames.slice(1); + }); + const sorted = [...frames].sort((a, b) => a - b); + return { p90: sorted[Math.floor(sorted.length * 0.9)] ?? 0, max: sorted[sorted.length - 1] ?? 0 }; +} + +async function medianRun(page: Page, act: () => Promise): Promise { + const runs: Frames[] = []; + for (let i = 0; i < 3; i++) runs.push(await frameTimes(page, act)); + return runs.sort((a, b) => a.max - b.max)[1]; +} + +/** A point on the canvas with no card under it, for panning from empty canvas. */ +async function emptyPoint(page: Page): Promise<{ x: number; y: number }> { + return page.evaluate(() => { + const r = document.querySelector('.svelte-flow')!.getBoundingClientRect(); + for (let y = r.top + 30; y < r.bottom - 30; y += 20) + for (let x = r.left + 80; x < r.right - 30; x += 20) if (!document.elementFromPoint(x, y)?.closest('.svelte-flow__node, .svelte-flow__controls, .svelte-flow__edge')) return { x, y }; + return { x: r.left + 80, y: r.top + 30 }; + }); +} + +const GRAPHS = [ + { name: 'the large group', path: '/jobs/warehouse_nightly?mock', panFromCard: true }, + // Lineage's cards are dragged, not panned from. + { name: 'the Lineage overview', path: '/lineage?mock&large', panFromCard: false } +]; + +for (const { name, path, panFromCard } of GRAPHS) { + test(`${name} pans and zooms smoothly on a slow computer`, async ({ page }) => { + test.setTimeout(240_000); + await page.setViewportSize({ width: 1600, height: 900 }); + await page.goto(path); + await page.locator('.graph-canvas:not(.unfitted) .svelte-flow__node').first().waitFor({ timeout: 20_000 }); + await page.waitForTimeout(800); + const box = (await page.locator('.svelte-flow').first().boundingBox())!; + const centre = { x: box.x + box.width / 2, y: box.y + box.height / 2 }; + const empty = await emptyPoint(page); + const card = (await page.locator('.svelte-flow__node').first().boundingBox())!; + const cdp = await page.context().newCDPSession(page); + await cdp.send('Emulation.setCPUThrottlingRate', { rate: 6 }); + + const wheel = async (steps: number, delta: (i: number) => number, gapMs: number) => { + await page.mouse.move(centre.x, centre.y); + for (let i = 0; i < steps; i++) { + await page.mouse.wheel(0, delta(i)); + await page.waitForTimeout(gapMs); + } + }; + const drag = async (from: { x: number; y: number }) => { + await page.mouse.move(from.x, from.y); + await page.mouse.down(); + for (let i = 1; i <= 30; i++) { + await page.mouse.move(from.x + i * 6, from.y + i * 2); + await page.waitForTimeout(16); + } + for (let i = 29; i >= 0; i--) await page.mouse.move(from.x + i * 6, from.y + i * 2); + await page.mouse.up(); + }; + const gestures: [string, () => Promise][] = [ + ['wheel zoom', () => wheel(60, (i) => (i < 30 ? 100 : -100), 16)], + ['trackpad momentum', () => wheel(60, (i) => Math.max(1, 40 * Math.pow(0.94, i)), 8)], + [ + 'pinch', + async () => { + await page.keyboard.down('Control'); + await wheel(30, (i) => (i < 15 ? 6 : -6), 8); + await page.keyboard.up('Control'); + } + ], + ['drag-pan from empty canvas', () => drag(empty)], + ...(panFromCard ? ([['drag-pan from a card', () => drag({ x: card.x + 40, y: card.y + 15 })]] as [string, () => Promise][]) : []) + ]; + const results: string[] = []; + const over: string[] = []; + for (const [gesture, act] of gestures) { + const f = await medianRun(page, act); + results.push(`${gesture}: p90 ${Math.round(f.p90)} ms, max ${Math.round(f.max)} ms`); + // Frames last whole multiples of about 16.7 ms: three frames (50.05 ms) is within 50 ms. + if (f.p90 > P90_MS + 1 || f.max > MAX_MS + 1) over.push(`${gesture}: p90 ${Math.round(f.p90)} ms, max ${Math.round(f.max)} ms`); + } + console.info(`${name}, CPU slowed 6×: ${results.join('; ')}`); + expect(over, `frames over budget (p90 ≤ ${P90_MS} ms, max ≤ ${MAX_MS} ms)`).toEqual([]); + }); +} + +// The plan page on a plan the size of a real `build --changed` (`?mock&large`), on a slow computer: the +// index switches section, and Show more adds a page, each with its rows on screen within a second. +const PLAN_STEP_MS = 1000; + +test('the plan page switches sections and shows more quickly on a slow computer', async ({ page }) => { + test.setTimeout(180_000); + await page.setViewportSize({ width: 1600, height: 1000 }); + await page.goto(`/plan?mock&large&cmd=${encodeURIComponent('sqb build --changed')}`); + await page.locator('.pane .plan-row').first().waitFor({ timeout: 30_000 }); + const cdp = await page.context().newCDPSession(page); + await cdp.send('Emulation.setCPUThrottlingRate', { rate: 6 }); + const timed = async (act: () => Promise, done: () => Promise) => { + const start = Date.now(); + await act(); + await expect.poll(done, { timeout: 10_000, intervals: [10] }).toBe(true); + return Date.now() - start; + }; + const steps: string[] = []; + const over: string[] = []; + for (const key of ['first_run', 'python', 'unchanged', 'warnings', 'changed']) { + const ms = await timed( + () => page.locator(`.index .entry[data-section="${key}"]`).click(), + async () => (await page.locator(`.pane[data-section="${key}"]`).count()) > 0 && (key === 'warnings' || (await page.locator('.pane .plan-row').count()) >= 50) + ); + steps.push(`${key} ${ms} ms`); + if (ms > PLAN_STEP_MS) over.push(`${key}: ${ms} ms`); + } + await page.locator('.index .entry[data-section="unchanged"]').click(); + await page.locator('.pane[data-section="unchanged"] .plan-row').nth(49).waitFor(); + const more = await timed( + () => page.getByRole('button', { name: /^Show more/ }).click(), + async () => (await page.locator('.pane .plan-row').count()) >= 100 + ); + steps.push(`show more ${more} ms`); + if (more > PLAN_STEP_MS) over.push(`show more: ${more} ms`); + console.info(`plan page, CPU slowed 6×: ${steps.join('; ')}`); + expect(over, `steps over ${PLAN_STEP_MS} ms`).toEqual([]); +}); diff --git a/ui/e2e/plan-browser.spec.ts b/ui/e2e/plan-browser.spec.ts new file mode 100644 index 0000000000..bc676121c5 --- /dev/null +++ b/ui/e2e/plan-browser.spec.ts @@ -0,0 +1,172 @@ +import { expect, test, type Page } from '@playwright/test'; + +// The plan page's list as an index and a pane, on a plan the size of a real `build --changed` +// (`?mock&large`): one section at a time, large sections' rows a page at a time. The mock makes no +// network requests, so the plan rows and diff reads are counted from its own record +// (`window.__sqbPlanRows`, `window.__sqbPlanDiffs`). +// With SHOTS set, each section is photographed: SHOTS=/some/dir npx playwright test e2e/plan-browser.spec.ts +const shots = process.env.SHOTS; +test.use({ viewport: { width: 1600, height: 1000 } }); + +const LARGE = `/plan?mock&large&cmd=${encodeURIComponent('sqb build --changed')}`; +type Call = { section: string; offset: number; limit: number; name?: string }; +const calls = (page: Page) => page.evaluate(() => (window as unknown as { __sqbPlanRows?: Call[] }).__sqbPlanRows ?? []); +const diffReads = (page: Page) => page.evaluate(() => (window as unknown as { __sqbPlanDiffs?: string[] }).__sqbPlanDiffs ?? []); +/** The large mock's model whose diff the server withholds, as without permission to read its SQL. */ +const WITHHELD = 'suppliers__stg__snapshot_79'; +const pane = (page: Page) => page.locator('.pane'); +const rows = (page: Page) => page.locator('.pane .plan-row'); + +async function openPlan(page: Page, path = LARGE) { + await page.goto(path); + await expect(page.locator('.plan-browser')).toBeVisible({ timeout: 30_000 }); +} +async function select(page: Page, key: string) { + await page.locator(`.index .entry[data-section="${key}"]`).click(); + await expect(pane(page)).toHaveAttribute('data-section', key); +} + +test('the index lists the sections in order, opens on the first, and the address keeps the one picked', async ({ page }) => { + await openPlan(page); + await expect(page).toHaveTitle('Plan · sqb build --changed'); + await expect(page.locator('.index .summary')).toHaveText('94 changed, 672 new, 121 renames, 64 loads, 433 Python, 40 held back · 1,519 unchanged, 300 skipped'); + expect(await page.locator('.index .entry').evaluateAll((es) => es.map((e) => (e as HTMLElement).dataset.section))).toEqual(['changed', 'renames', 'first_run', 'loads', 'python', 'held_back', 'warnings', 'unchanged', 'skipped']); + await expect(page.locator('.index .entry[aria-current="true"]')).toHaveAttribute('data-section', 'changed'); + await select(page, 'first_run'); + await expect(page).toHaveURL(/[?&]section=first_run/); + await expect(pane(page).locator('h3')).toContainText('First run'); + // A first run's range is its own bounds, in words. + await expect(pane(page).locator('.note', { hasText: 'from the last run → the latest data when the run starts' }).first()).toBeVisible(); + await expect(pane(page)).not.toContainText('__SQB_CURSOR'); + await page.goto(`${LARGE}§ion=python`); + await expect(pane(page)).toHaveAttribute('data-section', 'python', { timeout: 30_000 }); + await expect(page.locator('.index .entry[aria-current="true"]')).toHaveAttribute('data-section', 'python'); +}); + +test('a large section loads only its first page when opened, and Show more loads the next', async ({ page }) => { + await openPlan(page); + await expect(rows(page)).toHaveCount(50); + expect(await calls(page), 'the section opened reads its first page, and no other').toEqual([{ section: 'query_changed', offset: 0, limit: 50 }]); + await select(page, 'unchanged'); + await expect(rows(page)).toHaveCount(50); + expect((await calls(page)).slice(1)).toEqual([{ section: 'routine_models', offset: 0, limit: 50 }]); + // Every model of it runs for the same reason, by the server's count: said once, not on each row. + await expect(pane(page).locator('.subtitle')).toContainText('models: no completed record (all 1,507)'); + await expect(rows(page).locator('.detail', { hasText: 'no completed record' })).toHaveCount(0); + if (shots) await page.screenshot({ path: `${shots}/plan-b-unchanged.png` }); + await page.getByRole('button', { name: /^Show more/ }).click(); + await expect(rows(page)).toHaveCount(100); + expect((await calls(page)).slice(1)).toEqual([ + { section: 'routine_models', offset: 0, limit: 50 }, + { section: 'routine_models', offset: 50, limit: 50 } + ]); +}); + +test('a changed row shows its kind and line counts, and opens its diff a few lines at a time', async ({ page }) => { + await openPlan(page); + await expect(pane(page).locator('.subtitle')).toHaveText('87 query · 6 config · 1 function'); + // Whitespace-only diffs are labelled and lose their counts; a change of keyword case is a change. + await expect(rows(page).locator('.detail', { hasText: /^whitespace only/ })).toHaveCount(0); + await page.getByRole('button', { name: /^Show more/ }).click(); + await expect(rows(page)).toHaveCount(94); + await expect(rows(page).locator('.detail', { hasText: /^whitespace only/ })).toHaveCount(5); + await expect(rows(page).locator('.detail', { hasText: /^query \+1 −1/ })).toHaveCount(2); + await expect(rows(page).locator('.detail', { hasText: 'config · full_refresh true → false' })).toHaveCount(4); + await expect(rows(page).locator('.detail', { hasText: 'config + query · cursor observed_at → event_date' })).toHaveCount(1); + if (shots) await page.screenshot({ path: `${shots}/plan-b-changed.png` }); + // Rows carry counts, not diff text: only the small config changes shown inline have been read. + expect(await diffReads(page)).toHaveLength(5); + + const big = rows(page).filter({ hasText: 'query +71 −50' }).first(); + const bigName = await big.locator('.name').innerText(); + await big.getByRole('button', { name: 'diff' }).click(); + const diff = big.locator('pre.diff'); + await expect(diff).toBeVisible(); + expect((await diffReads(page)).filter((m) => m === bigName)).toHaveLength(1); + // Opened: the headers, a little context and the first 12 changed lines, then a link to the rest. + expect(await diff.locator('span.add, span.del').count()).toBe(12); + expect(await diff.locator('span').count()).toBeLessThanOrEqual(16); + const full = big.getByRole('button', { name: /^open the full diff \(\d+ more lines\)$/ }); + await expect(full).toHaveText('open the full diff (110 more lines)'); + if (shots) await big.scrollIntoViewIfNeeded().then(() => page.screenshot({ path: `${shots}/plan-b-changed-diff.png` })); + await full.click(); + expect(await diff.locator('span.add, span.del').count()).toBe(121); + expect(await diff.locator('span').count()).toBe(126); + await expect(full).toHaveCount(0); + // Closing returns the row to closed; opening it again starts from the short view. + await big.getByRole('button', { name: 'hide diff' }).click(); + await expect(diff).toHaveCount(0); + await big.getByRole('button', { name: 'diff' }).click(); + await expect(diff).toBeVisible(); + expect(await diff.locator('span.add, span.del').count()).toBe(12); + await big.getByRole('button', { name: 'hide diff' }).click(); + expect((await diffReads(page)).filter((m) => m === bigName), 'a diff read once is kept').toHaveLength(1); + // A config change of more than two keys opens as a diff too. + await expect(rows(page).locator('.detail', { hasText: /^config · diff$/ })).toHaveCount(1); + // A diff the server withholds says so, quietly. + const withheld = rows(page).filter({ has: page.locator('.name', { hasText: WITHHELD }) }); + await withheld.getByRole('button', { name: 'diff' }).click(); + await expect(withheld.locator('.diff-note')).toHaveText('diff not available'); + await expect(withheld.locator('pre.diff')).toHaveCount(0); +}); + +test('the filter asks the server for matching rows in the whole section, not just those loaded', async ({ page }) => { + await openPlan(page); + await select(page, 'python'); + await pane(page).getByLabel('Filter python').fill('orders_export_12'); + await expect(rows(page)).toHaveCount(2); + for (const name of await rows(page).locator('.name').allInnerTexts()) expect(name).toContain('orders_export_12'); + expect((await calls(page)).filter((c) => c.name)).toEqual([{ section: 'python_nodes', offset: 0, limit: 50, name: 'orders_export_12' }]); + // A large paged section: matches from beyond the first page, a page at a time. + await select(page, 'unchanged'); + await pane(page).getByLabel('Filter unchanged').fill('RETURNS__'); + await expect(rows(page)).toHaveCount(50); + await expect(pane(page)).not.toContainText('Filtering the'); + for (const name of await rows(page).locator('.name').allInnerTexts()) expect(name).toContain('returns__'); + const more = page.getByRole('button', { name: /^Show more · \d+ not shown$/ }); + await expect(more).toBeVisible(); + await more.click(); + await expect(rows(page)).toHaveCount(100); + expect((await calls(page)).filter((c) => c.name === 'RETURNS__').map((c) => [c.section, c.offset])).toEqual([ + ['routine_models', 0], + ['functions', 0], + ['routine_models', 50] + ]); + await pane(page).getByLabel('Filter unchanged').fill('no such name'); + await expect(pane(page).locator('.hint')).toHaveText('No unchanged match “no such name”.'); +}); + +test('warnings with one message about different names read as one row, opened to the list', async ({ page }) => { + await openPlan(page); + await select(page, 'warnings'); + const groups = pane(page).locator('.warning-group'); + await expect(groups).toHaveCount(2); + // Warnings group by message once read, so the pane reads them all (up to a cap) in large pages. + expect((await calls(page)).filter((c) => c.section === 'warnings')).toEqual([{ section: 'warnings', offset: 0, limit: 200 }]); + await expect(groups.first().locator('.template')).toHaveText('Scenario … provides fixture for unknown model …'); + await expect(groups.first()).toContainText('71 in 24 scenarios'); + await groups.first().getByRole('button', { name: 'show', exact: true }).click(); + await expect(groups.first().locator('li')).toHaveCount(20); + if (shots) await page.screenshot({ path: `${shots}/plan-b-warnings.png` }); + await groups.first().getByRole('button', { name: 'show all 71' }).click(); + await expect(groups.first().locator('li')).toHaveCount(71); + await expect(groups.first().getByRole('button', { name: 'show all 71' })).toHaveCount(0); +}); + +test('a small plan reads as a short index', async ({ page }) => { + await openPlan(page, `/plan?mock&cmd=${encodeURIComponent('sqb build -s path:models/marts')}`); + const entries = page.locator('.index .entry'); + expect(await entries.count()).toBeGreaterThanOrEqual(1); + expect(await entries.count()).toBeLessThanOrEqual(3); + if (shots) await page.screenshot({ path: `${shots}/plan-b-small.png` }); +}); + +test('photographs every section of the large plan', async ({ page }) => { + test.skip(!shots, 'SHOTS is not set'); + await openPlan(page); + for (const key of ['changed', 'renames', 'first_run', 'loads', 'python', 'held_back', 'skipped']) { + await select(page, key); + await expect(rows(page).first()).toBeVisible(); + await page.screenshot({ path: `${shots}/plan-b-${key}.png` }); + } +}); diff --git a/ui/e2e/plan-timings.spec.ts b/ui/e2e/plan-timings.spec.ts new file mode 100644 index 0000000000..e89050d910 --- /dev/null +++ b/ui/e2e/plan-timings.spec.ts @@ -0,0 +1,35 @@ +import { expect, test } from '@playwright/test'; + +// "Planned in 10.6 s" beside "Plan again" opens where the planning time went; a plan without timings shows no button. +const PLAN = `/plan?mock&cmd=${encodeURIComponent('sqb build --changed')}`; +const shots = process.env.SHOTS; +test.use({ viewport: { width: 1440, height: 900 } }); + +test('the plan time opens a breakdown of its steps', async ({ page }) => { + await page.goto(PLAN); + const button = page.getByTestId('plan-timings'); + await expect(button).toHaveText('Planned in 10.6 s', { timeout: 30_000 }); + // It sits just before "Plan again". + const again = page.getByRole('button', { name: 'Plan again' }); + const [b, a] = [await button.boundingBox(), await again.boundingBox()]; + expect(a!.x - (b!.x + b!.width)).toBeLessThan(24); + await button.click(); + const popover = page.locator('[data-slot="popover-content"]'); + await expect(popover.getByText('Where the planning time went')).toBeVisible(); + const rows = popover.locator('tbody tr'); + await expect(rows).toHaveCount(9); + await expect(rows.first()).toHaveText(/Reading warehouse state\s*3\.3\s*31%/); + await expect(rows.last()).toHaveText(/Other\s*0\.4\s*4%/); + await expect(popover.locator('tfoot')).toHaveText(/Total\s*10\.6/); + await expect(popover.locator('.foot')).toHaveText(/^On the location, from the synced warehouse state · \d+ nodes?$/); + if (shots) await page.screenshot({ path: `${shots}/plan-timings.png` }); + await page.keyboard.press('Escape'); + await expect(popover).toHaveCount(0); +}); + +test('a plan without timings shows no plan time', async ({ page }) => { + await page.addInitScript(() => ((window as unknown as { __sqbPlanWithoutTimings: boolean }).__sqbPlanWithoutTimings = true)); + await page.goto(PLAN); + await expect(page.getByRole('button', { name: 'Plan again' })).toBeVisible({ timeout: 30_000 }); + await expect(page.getByTestId('plan-timings')).toHaveCount(0); +}); diff --git a/ui/e2e/routes.spec.ts b/ui/e2e/routes.spec.ts new file mode 100644 index 0000000000..71471e1585 --- /dev/null +++ b/ui/e2e/routes.spec.ts @@ -0,0 +1,50 @@ +import { expect, test } from '@playwright/test'; + +// Edges run through the gaps between cards, never across a card they don't join: each graph's edge +// paths, sampled every few pixels, stay out of every card but their source and target. +test.use({ viewport: { width: 1600, height: 900 } }); + +/** Each edge that runs across a card it doesn't join, as problems. */ +function crossings(): string[] { + const problems: string[] = []; + const cards = [...document.querySelectorAll('.svelte-flow__node:not(.parent)')].map((n) => ({ id: n.dataset.id!, box: n.getBoundingClientRect() })); + const ids = new Set(cards.map((c) => c.id)); + for (const edge of document.querySelectorAll('.svelte-flow__edge')) { + const id = edge.dataset.id!; + // An edge's id is its source and target joined by '-'. + const ends = new Set([...ids].filter((c) => id.startsWith(`${c}-`) || id.endsWith(`-${c}`))); + const path = edge.querySelector('path.svelte-flow__edge-path')!; + const m = path.getScreenCTM()!; + const length = path.getTotalLength(); + for (const { id: card, box } of cards) { + if (ends.has(card)) continue; + for (let at = 0; at <= length; at += 3) { + const p = path.getPointAtLength(at); + const [x, y] = [m.a * p.x + m.c * p.y + m.e, m.b * p.x + m.d * p.y + m.f]; + if (x > box.left + 1 && x < box.right - 1 && y > box.top + 1 && y < box.bottom - 1) { + problems.push(`${id} runs across ${card}`); + break; + } + } + } + } + return problems; +} + +const GRAPHS: { name: string; path: string; click?: string }[] = [ + { name: 'the big group by folder', path: '/jobs/warehouse_nightly?mock' }, + { name: 'the big group by job', path: '/jobs/warehouse_nightly?mock', click: 'Folders' }, + { name: 'the Lineage overview', path: '/lineage?mock&large' }, + { name: 'Lineage', path: '/lineage?mock' } +]; + +for (const { name, path, click } of GRAPHS) { + test(`${name}: no edge runs across a card it doesn't join`, async ({ page }) => { + await page.goto(path); + await page.locator('.graph-canvas:not(.unfitted) .svelte-flow__edge').first().waitFor({ state: 'attached', timeout: 20_000 }); + if (click) await page.getByRole('button', { name: click }).click(); + await page.waitForTimeout(1200); + expect(await page.locator('.svelte-flow__edge').count(), 'edges drawn').toBeGreaterThan(3); + expect(await page.evaluate(crossings)).toEqual([]); + }); +} diff --git a/ui/e2e/screens.spec.ts b/ui/e2e/screens.spec.ts new file mode 100644 index 0000000000..4c8f956c33 --- /dev/null +++ b/ui/e2e/screens.spec.ts @@ -0,0 +1,32 @@ +import { test } from '@playwright/test'; + +// Screenshots for looking at pages by eye, not a check: skipped unless SHOTS names a directory. +// SHOTS=/some/dir npx playwright test e2e/screens.spec.ts +const dir = process.env.SHOTS; +const PAGES: { name: string; path: string; click?: string; size?: { width: number; height: number } }[] = [ + { name: 'lineage-overview', path: '/lineage?mock&large', size: { width: 1900, height: 1060 } }, + { name: 'group-page', path: '/jobs/warehouse_nightly?mock' }, + { name: 'group-page-members', path: '/jobs/warehouse_nightly?mock', click: 'Folders' }, + { name: 'lineage', path: '/lineage?mock' }, + { name: 'job-page', path: '/jobs/core_build?mock' } +]; + +test.describe('screenshots', () => { + test.skip(!dir, 'SHOTS is not set'); + test.use({ viewport: { width: 1600, height: 900 } }); + for (const { name, path, click, size } of PAGES) { + test(name, async ({ page }) => { + if (size) await page.setViewportSize(size); + await page.goto(path); + await page.waitForLoadState('networkidle'); + // Each page here draws a graph: wait for it to be drawn and fitted. + await page.locator('.graph-canvas:not(.unfitted)').first().waitFor({ timeout: 15_000 }); + if (click) { + await page.getByRole('button', { name: click }).click(); + await page.waitForTimeout(800); + } + await page.waitForTimeout(1000); + await page.screenshot({ path: `${dir}/${name}.png` }); + }); + } +}); diff --git a/ui/e2e/selection.spec.ts b/ui/e2e/selection.spec.ts new file mode 100644 index 0000000000..258fc3ed63 --- /dev/null +++ b/ui/e2e/selection.spec.ts @@ -0,0 +1,127 @@ +import { expect, test, type Page } from '@playwright/test'; + +// The selection bar on Lineage against the mock project: type a selector, pick a value, widen it +// upstream, read the count and the plain-words line, copy the command, leave nodes out, fix an +// unknown value, and open the plan. Set SELECTION_SHOT_DIR to save screenshots there. + +test.use({ viewport: { width: 1440, height: 900 }, permissions: ['clipboard-read', 'clipboard-write'] }); + +const shot = async (page: Page, name: string) => { + if (process.env.SELECTION_SHOT_DIR) await page.screenshot({ path: `${process.env.SELECTION_SHOT_DIR}/selection-real-${name}.png` }); +}; + +async function openLineage(page: Page, query = '') { + await page.goto(`/lineage?mock${query}`); + await page.waitForLoadState('networkidle'); + await expect(page.locator('.selection-bar')).toBeVisible(); +} + +const field = (page: Page, name = 'Selection') => page.getByRole('textbox', { name, exact: true }); +const chips = (page: Page, row = 0) => page.locator('.selection-bar .sel-field').nth(row).locator('.sel-chip'); + +test('a selection is typed, picked, widened, copied, narrowed and built', async ({ page }) => { + await openLineage(page); + await field(page).click(); + await field(page).pressSequentially('tag'); + const suggestions = page.getByRole('listbox', { name: 'Selection suggestions' }); + await expect(suggestions.getByRole('option', { name: /tag:\s*nodes with a tag/ })).toBeVisible(); + const finance = suggestions.getByRole('option').filter({ hasText: /tag:finance/ }); + await expect(finance).toContainText(/\d/); + await finance.click(); + + await expect(chips(page)).toHaveCount(1); + await expect(chips(page).first()).toHaveText(/tag:finance/); + await expect(field(page)).toHaveValue(''); + + // Upstream from the chip's picker: the + sits inside the chip. + await chips(page).first().locator('.chip-face').click(); + const panel = page.locator('.chip-panel'); + await expect(panel.getByRole('option', { name: /finance/ })).toHaveAttribute('aria-selected', 'true'); + await panel.getByRole('button', { name: 'Upstream' }).click(); + await page.keyboard.press('Escape'); + await expect(chips(page).first()).toHaveText(/^\+tag:finance/); + + const count = page.locator('.sb-count'); + await expect(count).toHaveText(/^\d+ nodes?$/); + await expect(page.locator('.sb-msg')).toContainText('Reads as: nodes tagged finance and everything upstream of them'); + expect(new URL(page.url()).searchParams.get('select')).toBe('+tag:finance'); + + await page.getByRole('button', { name: /Copy as/ }).click(); + expect(await page.evaluate(() => navigator.clipboard.readText())).toBe('sqb build -s +tag:finance'); + + // The except row. + await page.getByRole('button', { name: 'Exclude' }).click(); + await field(page, 'Except').click(); + await field(page, 'Except').pressSequentially('tag:revenue'); + await page.keyboard.press('Enter'); + await expect(chips(page, 1)).toHaveText([/tag:revenue/]); + await expect(page.locator('.sb-msg')).toContainText('minus nodes tagged revenue'); + await expect(count).toHaveText(/^\d+ nodes?$/); + expect(new URL(page.url()).searchParams.get('exclude')).toBe('tag:revenue'); + await page.getByRole('button', { name: /Copy as/ }).click(); + expect(await page.evaluate(() => navigator.clipboard.readText())).toBe('sqb build -s +tag:finance --exclude tag:revenue'); + + // Backspace turns the last chip back into its text, and Space makes it a chip again. + await field(page, 'Except').click(); + await page.keyboard.press('Backspace'); + await expect(field(page, 'Except')).toHaveValue('tag:revenue'); + await expect(chips(page, 1)).toHaveCount(0); + await page.keyboard.press(' '); + await expect(chips(page, 1)).toHaveCount(1); + await page.locator('.graph-panel').click({ position: { x: 5, y: 5 } }); + await shot(page, 'finished'); + + await page.getByRole('button', { name: 'Build these' }).click(); + await page.waitForURL(/\/plan\?/); + expect(new URL(page.url()).searchParams.get('cmd')).toBe('sqb build -s +tag:finance --exclude tag:revenue'); +}); + +test('an unknown value is a red chip with a fix, and Build these waits for it', async ({ page }) => { + await openLineage(page, '&select=%2Btag%3Afinanse'); + const bad = chips(page).first(); + await expect(bad).toHaveClass(/bad/); + const alert = page.getByRole('alert'); + await expect(alert).toContainText('No tag called finanse.'); + await expect(alert).toContainText('Did you mean'); + await expect(page.getByRole('button', { name: 'Build these' })).toBeDisabled(); + await expect(page.getByRole('button', { name: /Copy as/ })).toBeDisabled(); + await shot(page, 'error'); + + await field(page).click(); + await page.keyboard.press('Tab'); + await expect(chips(page).first()).toHaveText(/^\+tag:finance/); + await expect(chips(page).first()).not.toHaveClass(/bad/); + await expect(page.getByRole('alert')).toHaveCount(0); + await expect(page.getByRole('button', { name: 'Build these' })).toBeEnabled(); +}); + +test('a job page shows its selection as chips that open in Lineage', async ({ page }) => { + await page.goto('/jobs/revenue_refresh?mock'); + const chips = page.locator('.sel-chips'); + await expect(chips.locator('.sel-chip')).toHaveText([/daily_revenue/, /revenue_report/, /export_revenue_csv/]); + await chips.getByRole('link', { name: 'Open in Lineage' }).click(); + await page.waitForURL(/\/lineage\?/); + expect(new URL(page.url()).searchParams.get('select')).toBe('daily_revenue revenue_report export_revenue_csv'); + await expect(page.locator('.selection-bar .sel-chip')).toHaveCount(3); + await expect(page.locator('.sb-count')).toHaveText(/^\d+ nodes?$/); +}); + +test('a job selecting folders opens in Lineage without an error', async ({ page }) => { + await page.goto('/jobs/core_build?mock'); + await expect(page.locator('.sel-chips .sel-chip')).toHaveText([/path:models\/staging/, /path:models\/marts/]); + await page.locator('.sel-chips').getByRole('link', { name: 'Open in Lineage' }).click(); + await page.waitForURL(/\/lineage\?/); + await expect(page.locator('.sb-count')).toHaveText(/^\d+ nodes?$/); + await expect(page.getByRole('alert')).toHaveCount(0); +}); + +test('a folder on the Overview opens in the catalog as a path selection', async ({ page }) => { + await page.goto('/?mock'); + const marts = page.locator('a.group-row', { hasText: 'marts' }).first(); + await marts.click(); + await page.waitForURL(/\/catalog\?/); + expect(new URL(page.url()).searchParams.get('select')).toBe('path:models/marts'); + await expect(page.locator('.selection-bar .sel-chip')).toHaveText([/path:models\/marts/]); + await expect(page.getByRole('alert')).toHaveCount(0); + await expect(page.getByRole('button', { name: /^Folder/ })).toHaveCount(0); +}); diff --git a/ui/e2e/small-graph.spec.ts b/ui/e2e/small-graph.spec.ts new file mode 100644 index 0000000000..00aa1c42fd --- /dev/null +++ b/ui/e2e/small-graph.spec.ts @@ -0,0 +1,29 @@ +import { expect, test } from '@playwright/test'; + +// A small job graph opens in full form: whole, at a zoom where its cards are drawn in full, even when +// the fit's margin alone would have pushed it just under the compact zoom. +test.use({ viewport: { width: 1600, height: 900 } }); + +test('a small job graph opens with its cards in full', async ({ page }) => { + await page.goto('/jobs/core_build?mock'); + await page.locator('.graph-canvas:not(.unfitted) .svelte-flow__node').first().waitFor({ timeout: 20_000 }); + await page.waitForTimeout(800); + const view = await page.evaluate(() => { + const canvas = document.querySelector('.graph-canvas')!; + const pane = canvas.querySelector('.svelte-flow')!.getBoundingClientRect(); + const cut = [...canvas.querySelectorAll('.svelte-flow__node')].filter((n) => { + const r = n.getBoundingClientRect(); + return r.left < pane.left || r.right > pane.right || r.top < pane.top || r.bottom > pane.bottom; + }); + return { + zoom: Number(/scale\(([\d.]+)\)/.exec(canvas.querySelector('.svelte-flow__viewport')!.style.transform)?.[1]), + compact: canvas.classList.contains('compact'), + cards: canvas.querySelectorAll('.svelte-flow__node').length, + cut: cut.map((n) => n.dataset.id) + }; + }); + expect(view.cards).toBeGreaterThanOrEqual(5); + expect(view.compact, `cards compact at zoom ${view.zoom}`).toBe(false); + expect(view.zoom).toBeGreaterThanOrEqual(0.82); + expect(view.cut, 'cards outside the canvas').toEqual([]); +}); diff --git a/ui/e2e/zoom.ts b/ui/e2e/zoom.ts new file mode 100644 index 0000000000..a2dc24c883 --- /dev/null +++ b/ui/e2e/zoom.ts @@ -0,0 +1,22 @@ +import { expect, type Page } from '@playwright/test'; + +// Zooming a graph to an exact level with the mouse wheel, for the specs that look at it zoomed out. + +export const scaleOf = (page: Page) => + page.locator('.svelte-flow__viewport').first().evaluate((v) => Number(/scale\(([\d.]+)\)/.exec((v as HTMLElement).style.transform)?.[1] ?? 1)); + +/** Zooms with the wheel over the canvas until the zoom is within 0.02 of `target`, then lets the view settle. */ +export async function zoomTo(page: Page, target: number) { + const box = (await page.locator('.svelte-flow').first().boundingBox())!; + await page.mouse.move(box.x + box.width / 2, box.y + box.height / 2); + for (let i = 0; i < 200; i++) { + const zoom = await scaleOf(page); + if (Math.abs(zoom - target) <= 0.02) break; + // d3's wheel zoom: a factor of 2^(-deltaY / 500). + const delta = Math.max(-60, Math.min(60, Math.log2(zoom / target) * 500 * 0.8)); + await page.mouse.wheel(0, Math.abs(delta) < 4 ? Math.sign(delta) * 4 : delta); + await page.waitForTimeout(30); + } + expect(Math.abs((await scaleOf(page)) - target), `zoomed to ${target}`).toBeLessThanOrEqual(0.02); + await page.waitForTimeout(600); +} diff --git a/ui/package-lock.json b/ui/package-lock.json index 15b95eb3c5..1eb22adbd3 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -19,6 +19,7 @@ "@fontsource-variable/inter": "^5.2.8", "@internationalized/date": "^3.12.2", "@lucide/svelte": "^1.17.0", + "@playwright/test": "^1.64.1", "@sveltejs/adapter-static": "^3.0.10", "@sveltejs/kit": "^2.57.0", "@sveltejs/vite-plugin-svelte": "^7.0.0", @@ -239,6 +240,22 @@ "url": "https://github.com/sponsors/Boshen" } }, + "node_modules/@playwright/test": { + "version": "1.64.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.64.1.tgz", + "integrity": "sha512-IaYGml/nSOiZWROfPtuIL7neuu9qDOZ3TNvNljUIrhDMxiGTgPjzj/g9p9RxpS2tSepPDRY+UwAaEeiKtUvIAg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.64.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/@polka/url": { "version": "1.0.0-next.29", "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.29.tgz", @@ -1999,6 +2016,35 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/playwright": { + "version": "1.64.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.64.1.tgz", + "integrity": "sha512-HCrVhHr5QQuwYoswEUGTb7gGp522yflbv/ST8nF4sFWGuLjQCt17Q/EubpGGSNKYn1tsTCPZxqHhHCfVURu7UA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.64.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.64.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.64.1.tgz", + "integrity": "sha512-BC4s8y6SEVA6Q1rSxZTto8hI1KUI3xXlHGZgvGqP4IqPdI/a6FX9H29IDNz7vKo9l21b6LfYosScPL6M3xxyDQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/postcss": { "version": "8.5.15", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", diff --git a/ui/package.json b/ui/package.json index 685127b409..a3a4eaf7e3 100644 --- a/ui/package.json +++ b/ui/package.json @@ -13,12 +13,14 @@ "test": "vitest run", "gen:command-table": "node scripts/extract-command-table.mjs", "check:command-table": "node scripts/extract-command-table.mjs --check", - "check:checkboxes": "node scripts/check-checkboxes.mjs" + "check:checkboxes": "node scripts/check-checkboxes.mjs", + "test:e2e": "playwright test" }, "devDependencies": { "@fontsource-variable/inter": "^5.2.8", "@internationalized/date": "^3.12.2", "@lucide/svelte": "^1.17.0", + "@playwright/test": "^1.64.1", "@sveltejs/adapter-static": "^3.0.10", "@sveltejs/kit": "^2.57.0", "@sveltejs/vite-plugin-svelte": "^7.0.0", diff --git a/ui/playwright.config.ts b/ui/playwright.config.ts new file mode 100644 index 0000000000..feb0eec911 --- /dev/null +++ b/ui/playwright.config.ts @@ -0,0 +1,26 @@ +import { defineConfig } from '@playwright/test'; + +// The UI's browser suite against the mock client with awkward data; it is not part of `npm test`. +// CI builds the UI (`npm run build`) and runs the layout project as a required step and the perf +// project in its own job, which reports its timings but does not block merges. +export default defineConfig({ + testDir: './e2e', + timeout: 120_000, + fullyParallel: true, + // Timing guards run once the rest has finished, so they have the machine to themselves. + projects: [ + { name: 'layout', testIgnore: /perf\.spec\.ts/ }, + { name: 'perf', testMatch: /perf\.spec\.ts/, dependencies: ['layout'] } + ], + use: { + baseURL: process.env.UI_URL || 'http://127.0.0.1:4175', + launchOptions: process.env.CHROME_PATH ? { executablePath: process.env.CHROME_PATH } : {} + }, + webServer: process.env.UI_URL + ? undefined + : { + command: 'npx vite preview --host 127.0.0.1 --port 4175 --strictPort', + url: 'http://127.0.0.1:4175', + reuseExistingServer: false + } +}); diff --git a/ui/src/lib/api/client.ts b/ui/src/lib/api/client.ts index 7dbdfec6fd..ee46b7ee51 100644 --- a/ui/src/lib/api/client.ts +++ b/ui/src/lib/api/client.ts @@ -1,6 +1,9 @@ import type { AdhocRunCheck, PlanResult, + PlanPage, + PlanDiff, + PlanPageSection, AdhocRunRequest, Bootstrap, BulkRunResult, @@ -31,10 +34,14 @@ import type { RunCostRow, CostSummary, NodeBuildCost, + NodeStates, Snapshot, RunTabCounts, Scope, SnapshotChange, + SelectionEvaluation, + SelectionRequest, + SelectionVocabulary, SnapshotNode, StatementSql, Timeline, @@ -66,13 +73,18 @@ export interface ApiClient extends PlatformApi { * Health, attention items, metrics, build status by group and recent changes; null when the * person may view only part of the target, because its listings need view on every node. */ - overview(scope: Scope): Promise; - /** Every node in the current snapshot with its build state; also the lineage graph. */ + overview(scope: Scope, listing?: CatalogNode[]): Promise; + /** + * Every node in the current snapshot with its build state; also the lineage graph. Pages read it + * through the shared catalog store (`$lib/state/catalog.svelte`), which reads it once per snapshot. + */ listCatalogNodes(scope: Scope): Promise; + /** Every node's build state at the current status version, for applying `node_status` changes to a listing. */ + listNodeStates(scope: Scope): Promise; /** `listing` is a current `listCatalogNodes` result the caller already holds, so it isn't read again. */ getCatalogNode(scope: Scope, key: string, listing?: CatalogNode[]): Promise; - /** The snapshot's column lineage around one node. */ - getColumnLineage(scope: Scope, key: string): Promise; + /** The snapshot's column lineage around one node; `listing` as for `getCatalogNode`. */ + getColumnLineage(scope: Scope, key: string, listing?: CatalogNode[]): Promise; /** Every audit and check with its latest result. */ listChecks(scope: Scope): Promise; getCheck(scope: Scope, id: string): Promise; @@ -122,11 +134,20 @@ export interface ApiClient extends PlatformApi { rerunSelectedNodes(scope: Scope, runId: string, nodes: string[]): Promise; listSnapshotNodes(scope: Scope): Promise; + /** Resolves a selection exactly as `sqb build` would; a refused selection is a result with its error, not a throw. */ + evaluateSelection(scope: Scope, request: SelectionRequest): Promise; + /** Tags, folders, jobs, sources, seeds and kinds of the current publication, with node counts. */ + selectionVocabulary(scope: Scope): Promise; /** The server's pre-launch check: resolution, write set and authorization. */ checkAdhocRun(scope: Scope, request: AdhocRunRequest): Promise; launchAdhocRun(scope: Scope, request: AdhocRunRequest): Promise; /** Asks the location to run `sqb plan` on a build command against the warehouse, and waits for it. */ planBuild(scope: Scope, command: string[]): Promise; + /** One page of a large section of the plan `requestId` names, which its summary only counts. */ + /** One page of a plan section; `name` finds rows whose name contains it, ignoring case. */ + planRows(scope: Scope, requestId: string, section: PlanPageSection, offset: number, limit?: number, name?: string): Promise; + /** One planned model's diff text, read on demand; null without one. */ + planDiff(scope: Scope, requestId: string, model: string): Promise; listJobs(scope: Scope): Promise; getJob(scope: Scope, key: string): Promise; @@ -151,7 +172,7 @@ export interface ApiClient extends PlatformApi { /** Asks the location to read SQLBuild's state back from the warehouse now. `connection.test` */ requestWarehouseSync(scope: Scope): Promise; - /** Live "changed" signals (SSE in the server client); consumers refetch. */ + /** Live "changed" signals (SSE in the server client); consumers refetch or apply their payload. */ changes: ChangeSubscription; } diff --git a/ui/src/lib/api/http-catalog.test.ts b/ui/src/lib/api/http-catalog.test.ts index bba825a119..f39ca1fe70 100644 --- a/ui/src/lib/api/http-catalog.test.ts +++ b/ui/src/lib/api/http-catalog.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import type { components } from '$control/schema.d.ts'; -import { findNode, nodeDetailOf, nodeJobsOf, toCatalogNodes, toColumnLineage, toMicrobatchCoverage, toNodeRuns } from './http-catalog'; +import { findNode, nodeDetailOf, nodeJobsOf, toCatalogNodes, toColumnLineage, toMicrobatchCoverage, toNodeRuns, toNodeStates } from './http-catalog'; import { toCheck } from './http-checks'; type W = components['schemas']; @@ -30,6 +30,33 @@ describe('catalog node mapping', () => { ]); }); + it('lists Python nodes and functions with their edges, and leaves out audits, tests, scenarios, checks, macros and hooks', () => { + const nodes = toCatalogNodes([ + node('asset:order_feed'), + node('loader:raw_customers'), + node('task:refresh_inventory', { dependencies: ['model:orders'] }), + node('udf:cents_to_amount'), + node('table_fn:orders_between'), + node('model:orders', { materialization: 'table', dependencies: ['asset:order_feed', 'udf:cents_to_amount', 'audit:orders_positive'] }), + node('audit:orders_positive'), + node('sql_test:orders_total'), + node('sql_scenario:orders_backfill'), + node('check:orders_recent'), + node('macro:cents'), + node('hook:notify') + ]); + expect(nodes.map((n) => [n.key, n.kind, n.materialization])).toEqual([ + ['asset:order_feed', 'asset', null], + ['loader:raw_customers', 'loader', null], + ['task:refresh_inventory', 'task', null], + ['udf:cents_to_amount', 'udf', null], + ['table_fn:orders_between', 'table_fn', null], + ['model:orders', 'table', 'table'] + ]); + expect(nodes.find((n) => n.key === 'model:orders')?.upstream).toEqual(['asset:order_feed', 'udf:cents_to_amount']); + expect(nodes.find((n) => n.key === 'task:refresh_inventory')?.upstream).toEqual(['model:orders']); + }); + it('keeps only upstream nodes that are listed, so graphs never point at a hidden node', () => { const [orders] = toCatalogNodes([ node('model:orders', { dependencies: ['source:raw_orders', 'macro:cents', 'model:unpublished'] }), @@ -157,7 +184,7 @@ describe('catalog node detail', () => { describe('catalog node runs and jobs', () => { const snapshot = '01a1b000-0000-7000-8000-000000000001'; - const job = (key: string, kind: 'job' | 'group', selection: string[]): W['JobRecord'] => ({ job_key: key, kind, description: key, selection, concurrency_groups: [], snapshot_id: snapshot, overlap: 'queue' }); + const job = (key: string, kind: 'job' | 'group', selection: string[]): W['JobRecord'] => ({ job_key: key, kind, description: key, selection, pools: [], snapshot_id: snapshot, overlap: 'queue' }); const trigger = (id: string, jobKey: string) => ({ trigger_id: id, job_key: jobKey }) as W['TriggerRecord']; const run = (id: string, status: string, extra: Partial = {}): W['NodeRunRecord'] => ({ run_id: id, job_key: 'orders', queued_at: '2026-11-08T05:59:00Z', status, ...extra }); @@ -224,3 +251,14 @@ describe('column lineage mapping', () => { expect(toColumnLineage({ snapshot_id: 's', nodes: [], edges: [] }, nodes)).toBeNull(); }); }); + +describe('node states mapping', () => { + it('carries the changed build planned for the current snapshot, and none from an older server', () => { + const record = { snapshot_id: 'snap-1', version: 3, states: {} }; + const build = { generation: 2, reported_at: '2026-11-08T12:00:00Z', changed: ['model:orders'], downstream: ['model:customers'] }; + expect(toNodeStates({ ...record, changed_build: build }).changedBuild).toEqual({ generation: 2, changed: ['model:orders'], downstream: ['model:customers'], failing: null }); + const failed = { ...build, changed: [], downstream: [], failing: 'Model orders cannot plan a full rebuild' }; + expect(toNodeStates({ ...record, changed_build: failed }).changedBuild?.failing).toBe('Model orders cannot plan a full rebuild'); + expect(toNodeStates(record).changedBuild).toBeNull(); + }); +}); diff --git a/ui/src/lib/api/http-catalog.ts b/ui/src/lib/api/http-catalog.ts index 93e4c7df0c..7c673cf03e 100644 --- a/ui/src/lib/api/http-catalog.ts +++ b/ui/src/lib/api/http-catalog.ts @@ -1,14 +1,21 @@ import type { components } from '$control/schema.d.ts'; import { shortName } from '$lib/build-area/labels'; import { cardStatus } from '$lib/lineage/catalog-graph'; -import type { CatalogNode, CatalogNodeDetail, Check, CheckCounts, ColumnLens, ColumnLineage, MicrobatchCoverage, NodeJob, NodeOutcome, NodeRunEntry, StaleReason, WarehouseObject } from './types/build'; +import { withBehind } from './node-states'; +import type { CatalogNode, CatalogNodeDetail, Check, CheckCounts, ColumnLens, ColumnLineage, MicrobatchCoverage, NodeJob, NodeOutcome, NodeRunEntry, NodeState, NodeStates, NodeStatusDelta, WarehouseObject } from './types/build'; import type { NodeKind } from './types/operate'; type Wire = components['schemas']; -/** The catalog shows models, seeds and sources; audits, functions, macros and hooks are not catalog nodes. */ -function kindOf(wire: Wire['CatalogNodeRecord']): NodeKind | null { - if (wire.kind === 'source' || wire.kind === 'seed') return wire.kind; +/** Snapshot kinds that are graph nodes under their own name; models take theirs from the materialization. */ +const GRAPH_KINDS: ReadonlySet = new Set(['source', 'seed', 'asset', 'loader', 'task', 'udf', 'table_fn']); + +/** + * Every graph node kind is a catalog node: models, sources, seeds, Python nodes and functions. + * Audits, SQL tests, scenarios, checks, macros and hooks are not. + */ +export function kindOf(wire: Pick): NodeKind | null { + if (GRAPH_KINDS.has(wire.kind)) return wire.kind as NodeKind; if (wire.kind !== 'model') return null; if (wire.materialization === 'view') return 'view'; return wire.materialization?.startsWith('incremental') ? 'incremental' : 'table'; @@ -24,11 +31,11 @@ function kindOf(wire: Wire['CatalogNodeRecord']): NodeKind | null { export function toCatalogNode(wire: Wire['CatalogNodeRecord']): CatalogNode | null { const kind = kindOf(wire); if (!kind) return null; - const outcome = wire.last_outcome?.status; + const state = toNodeState(wire); return { key: wire.node_key, kind, - materialization: kind === 'source' || kind === 'seed' ? null : (wire.materialization ?? null), + materialization: kind === 'source' || kind === 'seed' || kind === 'udf' || kind === 'table_fn' ? null : (wire.materialization ?? null), group: null, tags: wire.tags, description: wire.description ?? '', @@ -36,57 +43,52 @@ export function toCatalogNode(wire: Wire['CatalogNodeRecord']): CatalogNode | nu upstream: wire.dependencies, relation: wire.relation ? { database: wire.relation.database ?? null, schema: wire.relation.schema, name: wire.relation.name } : null, columnCount: wire.column_count ?? 0, + lastBuild: state.lastBuild, + lastOutcome: state.lastOutcome, + stale: + state.definitionChanged && state.lastBuild + ? [{ reason: 'changed', change: 'definition', detail: 'The definition changed since it was last built here.', since: state.lastBuild.builtAt }] + : [], + sourceFreshness: null, + checks: state.checks + }; +} + +/** A node's build state, from its listing record or the node-states map, which carry the same fields. */ +export function toNodeState(wire: Wire['NodeState']): NodeState { + const outcome = wire.last_outcome?.status; + return { lastBuild: wire.last_build ? { fingerprintRunId: wire.last_build.run_id, runId: null, jobKey: null, builtAt: wire.last_build.observed_at, fingerprint: wire.last_build.version_hash } : null, lastOutcome: outcome === 'success' || outcome === 'warn' || outcome === 'failed' ? outcome : null, - stale: - wire.definition_changed && wire.last_build - ? [{ reason: 'changed', change: 'definition', detail: 'The definition changed since it was last built here.', since: wire.last_build.observed_at }] - : [], - sourceFreshness: null, + definitionChanged: wire.definition_changed, // A recording from an older server, such as the static demo's, has no counts: unknown, not zero. checks: wire.checks ?? null }; } -/** - * When the data a node reads last changed: its own last build, or for a view, which reads its - * upstream when queried, the newest data behind it. Null when nothing behind it is built. - */ -function dataTimes(listed: Map): (key: string) => number | null { - const memo = new Map(); - const at = (key: string): number | null => { - if (memo.has(key)) return memo.get(key) ?? null; - memo.set(key, null); - const node = listed.get(key); - let time = node?.lastBuild ? Date.parse(node.lastBuild.builtAt) : null; - if (node?.kind === 'view') { - const upstream = node.upstream.flatMap((k) => at(k) ?? []); - time = upstream.length ? Math.max(...upstream) : null; - } - memo.set(key, time); - return time; +export function toNodeStates(wire: Wire['NodeStatesRecord']): NodeStates { + const build = wire.changed_build; + return { + snapshotId: wire.snapshot_id ?? null, + version: wire.version, + states: Object.fromEntries(Object.entries(wire.states).map(([key, state]) => [key, toNodeState(state)])), + changedBuild: build ? { generation: build.generation, changed: build.changed, downstream: build.downstream, failing: build.failing ?? null } : null }; - return at; } /** - * Upstream nodes with newer data than this node's own last build. Views read their upstream when - * queried, so they are never behind, and an unbuilt node is not built yet rather than behind. + * The changed states a `node_status` notice carries, or null for a notice without them, as an + * older server sends. Malformed states are dropped rather than applied. */ -function behindOf(node: CatalogNode, dataAt: (key: string) => number | null): StaleReason[] { - const builtAt = node.lastBuild?.builtAt; - if (!builtAt || node.kind === 'view') return []; - const newer = node.upstream.flatMap((key) => { - const at = dataAt(key); - return at !== null && at > Date.parse(builtAt) ? [{ key, at }] : []; - }); - if (!newer.length) return []; - const since = new Date(Math.min(...newer.map((n) => n.at))).toISOString(); - const names = newer.map((n) => shortName(n.key)); - const detail = `${names.length === 1 ? names[0] : `${names.length} upstream nodes`} ${names.length === 1 ? 'has' : 'have'} newer data than this build.`; - return [{ reason: 'behind', upstream: newer.map((n) => n.key), detail, since }]; +export function toNodeStatusDelta(data: unknown): NodeStatusDelta | null { + if (!data || typeof data !== 'object') return null; + const { version, previous_version: previousVersion, states, full } = data as { version?: unknown; previous_version?: unknown; states?: unknown; full?: unknown }; + if (typeof version !== 'number' || typeof previousVersion !== 'number') return null; + if (full === true) return { version, previousVersion, full: true }; + if (!states || typeof states !== 'object') return null; + return { version, previousVersion, states: Object.fromEntries(Object.entries(states as Record).map(([key, state]) => [key, toNodeState(state)])) }; } /** Catalog nodes in listing order, keeping only dependencies that are themselves listed. */ @@ -94,8 +96,7 @@ export function toCatalogNodes(wire: Wire['CatalogNodeRecord'][]): CatalogNode[] const nodes = wire.flatMap((node) => toCatalogNode(node) ?? []); const keys = new Set(nodes.map((node) => node.key)); const linked = nodes.map((node) => ({ ...node, upstream: node.upstream.filter((key) => keys.has(key)) })); - const dataAt = dataTimes(new Map(linked.map((node) => [node.key, node]))); - return linked.map((node) => ({ ...node, stale: [...(node.stale ?? []), ...behindOf(node, dataAt)] })); + return withBehind(linked); } /** Passing, warning and failing by each check's newest result; checks without one count only towards the total. */ diff --git a/ui/src/lib/api/http-changes.ts b/ui/src/lib/api/http-changes.ts index 518a43ed28..1b50115883 100644 --- a/ui/src/lib/api/http-changes.ts +++ b/ui/src/lib/api/http-changes.ts @@ -7,7 +7,7 @@ type Wire = components['schemas']; /** How many runs of each deploy the feed lists. */ const FOLLOWING_RUNS = 8; -const NODE_KINDS: Record = { seed: 'seed', source: 'source' }; +const NODE_KINDS: Record = { seed: 'seed', source: 'source', asset: 'asset', loader: 'loader', task: 'task', udf: 'udf', table_fn: 'table_fn' }; function opOf(kind: string): ChangeOp { return kind === 'added' || kind === 'removed' ? kind : 'changed'; diff --git a/ui/src/lib/api/http-checks.ts b/ui/src/lib/api/http-checks.ts index af34c2e6df..ed3d34253f 100644 --- a/ui/src/lib/api/http-checks.ts +++ b/ui/src/lib/api/http-checks.ts @@ -31,7 +31,8 @@ export function toEvaluation(wire: Wire['CheckResult']): CheckEvaluation { violationCount: wire.violation_count ?? null, measuredValue: wire.measured_value == null ? null : Number(wire.measured_value), evidenceCount: wire.evidence_count ?? null, - evidenceTruncated: wire.evidence_truncated === 'true' + evidenceTruncated: wire.evidence_truncated === 'true', + ignored: wire.ignored ?? false }; } diff --git a/ui/src/lib/api/http-client.test.ts b/ui/src/lib/api/http-client.test.ts index c9bb4078ae..bbbb9f87c1 100644 --- a/ui/src/lib/api/http-client.test.ts +++ b/ui/src/lib/api/http-client.test.ts @@ -190,6 +190,78 @@ describe('HttpChangeFeed', () => { 'all' ]); }); + + it('passes on the node states a node_status notice carries, and none for one without', () => { + const { changes } = feed(async () => true); + const signals: ChangedSignal[] = []; + changes.subscribe((signal) => signals.push(signal)); + changes.follow(ORDERS); + const state = { last_build: { observed_at: '2026-01-02T00:00:00Z', run_id: 'r2', version_hash: 'h2' }, last_outcome: { at: '2026-01-02T00:00:00Z', status: 'failed' }, definition_changed: false, checks: { total: 1, passing: 0, warning: 0, failing: 1 } }; + const send = (notice: unknown) => latest().listeners.get('changed')!(new MessageEvent('changed', { data: JSON.stringify(notice) })); + send({ topic: 'node_status:development', version: 8, previous_version: 7, states: { 'model:orders': state } }); + send({ topic: 'node_status:development', version: 9, previous_version: 8, full: true }); + send({ topic: 'node_status:development' }); + expect(signals.map((s) => s.nodeStatus)).toEqual([ + { + version: 8, + previousVersion: 7, + states: { + 'model:orders': { + lastBuild: { fingerprintRunId: 'r2', runId: null, jobKey: null, builtAt: '2026-01-02T00:00:00Z', fingerprint: 'h2' }, + lastOutcome: 'failed', + definitionChanged: false, + checks: { total: 1, passing: 0, warning: 0, failing: 1 } + } + } + }, + { version: 9, previousVersion: 8, full: true }, + undefined + ]); + }); +}); + +describe('node listing and states', () => { + const record = { node_key: 'model:orders', name: 'orders', kind: 'model', materialization: 'table', snapshot_id: '0191f0cc-0000-0000-0000-000000000001', tags: [], dependencies: [], definition_changed: false, checks: { total: 0, passing: 0, warning: 0, failing: 0 } }; + + function server() { + const asked: (string | null)[] = []; + vi.stubGlobal( + 'fetch', + vi.fn(async (request: Request) => { + const path = new URL(request.url).pathname; + if (path.endsWith('/node-states')) { + return Response.json({ snapshot_id: record.snapshot_id, version: 3, states: { 'model:orders': { ...record, definition_changed: false, last_outcome: { at: '2026-01-02T00:00:00Z', status: 'success' } } } }); + } + const etag = request.headers.get('if-none-match'); + asked.push(etag); + if (etag === '"s1:3"') return new Response(null, { status: 304, headers: { etag: '"s1:3"' } }); + return Response.json([record], { headers: { etag: '"s1:3"' } }); + }) + ); + return { asked, client: createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} }) }; + } + + it('sends the held ETag and keeps the held listing on a 304', async () => { + const { asked, client } = server(); + const first = await client.listCatalogNodes(ORDERS); + const again = await client.listCatalogNodes(ORDERS); + expect(asked).toEqual([null, '"s1:3"']); + expect(again).toBe(first); + }); + + it('shares one listing read between callers asking at once', async () => { + const { asked, client } = server(); + const [a, b] = await Promise.all([client.listCatalogNodes(ORDERS), client.listCatalogNodes(ORDERS)]); + expect(asked).toEqual([null]); + expect(a).toBe(b); + }); + + it('reads every node state with its version', async () => { + const { client } = server(); + const states = await client.listNodeStates(ORDERS); + expect(states.version).toBe(3); + expect(states.states['model:orders'].lastOutcome).toBe('success'); + }); }); describe('feedAllowed', () => { diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index bbc74eb99a..423eccd7ee 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -4,7 +4,7 @@ import { isActiveRun } from '$lib/status'; import type { components, paths } from '$control/schema.d.ts'; import { ApiError, type ApiClient } from './client'; import { checkDetailOf, toCheck, toCheckEvidence, toCheckSql } from './http-checks'; -import { findNode, nodeDetailOf, toCatalogNodes, nodeJobsOf, toMicrobatchCoverage, toNodeRuns, toColumnLineage } from './http-catalog'; +import { findNode, nodeDetailOf, toCatalogNodes, nodeJobsOf, toMicrobatchCoverage, toNodeRuns, toColumnLineage, toNodeStates, toNodeStatusDelta } from './http-catalog'; import { auditMatches, toAuditEntry } from './http-audit'; import { toSnapshotChanges } from './http-changes'; import { manualConnectionTest, snapshotOf, toLocationDetail, toLocationSummary, toWarehouseConnection, warehouseSyncOf } from './http-locations'; @@ -17,11 +17,12 @@ import { roleDetailOf, toRoleDefinition } from './http-roles'; import { jobDetailOf, runWindowOf, toJob, toTick, toTrigger, toTriggerStateEvent, triggerDetailOf } from './http-jobs'; import { timelineOf } from './http-timeline'; import { RUN_WINDOW, groupRunNodes, parseRunCursor, planResultOf, queueEntriesOf, toAdhocCheck, runDetailOf, runRowPageOver, runsMatching, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunOutput, toRunStatements, toRunCostRows, toCostSummary, toNodeBuildCosts } from './http-runs'; +import { toSelectionEvaluation, toSelectionVocabulary } from './http-selection'; import type { ChangeSubscription, ChangedChannel, ChangedSignal, ConnectionState } from './signals'; import type { AuthMode, Bootstrap, Capability, Location, NavCounts, PlatformEvent, Scope, Snapshot, WarehouseSync } from './types'; import type { Account, AuditEntry, AuditQuery, ConnectionTest, LocationDetail, ServerSettings, WarehouseConnection, LocationSummary, PlatformEventQuery, PlatformEventRecord, PlatformHealth, MaintenanceHistory, ProviderConnection, RoleAssignment, RoleAssignmentRequest, RoleDefinition, RoleDetail } from './types/platform'; -import type { Acknowledgement, CatalogNode, CatalogNodeDetail, Check, CheckDetail, CheckEvidence, CheckSql, Overview, SnapshotChange, ColumnLineage } from './types/build'; -import type { AdhocRunCheck, AdhocRunRequest, BulkRunResult, PlanResult, QueueEntry, Timeline, Job, JobDetail, JobLaunchOptions, RerunMode, SnapshotNode, Run, RunDetail, RunListQuery, RunRowPage, RunRowQuery, RunTabCounts, Trigger, TriggerDetail, TriggerState, RunEventPage, RunNodeError, RunOutputLine, RunStatement, RunCostRow, CostSummary, NodeBuildCost, StatementSql } from './types/operate'; +import type { Acknowledgement, CatalogNode, CatalogNodeDetail, Check, CheckDetail, CheckEvidence, CheckSql, NodeStates, NodeStatusDelta, Overview, SnapshotChange, ColumnLineage } from './types/build'; +import type { AdhocRunCheck, AdhocRunRequest, BulkRunResult, SelectionEvaluation, SelectionRequest, SelectionVocabulary, PlanDiff, PlanPage, PlanPageSection, PlanResult, QueueEntry, Timeline, Job, JobDetail, JobLaunchOptions, RerunMode, SnapshotNode, Run, RunDetail, RunListQuery, RunRowPage, RunRowQuery, RunTabCounts, Trigger, TriggerDetail, TriggerState, RunEventPage, RunNodeError, RunOutputLine, RunStatement, RunCostRow, CostSummary, NodeBuildCost, StatementSql } from './types/operate'; type Wire = components['schemas']; @@ -66,6 +67,7 @@ const SUPPORTED: ReadonlySet = new Set([ 'getCheckSql', 'getCheckEvidence', 'listCatalogNodes', + 'listNodeStates', 'getCatalogNode', 'getColumnLineage', 'listSnapshotChanges', @@ -92,9 +94,13 @@ const SUPPORTED: ReadonlySet = new Set([ 'listProviders', 'serverSettings', 'listSnapshotNodes', + 'evaluateSelection', + 'selectionVocabulary', 'checkAdhocRun', 'launchAdhocRun', - 'planBuild' + 'planBuild', + 'planRows', + 'planDiff' ]); /** How often a connection test asks for the location's answer, and the longest it waits. */ @@ -103,6 +109,7 @@ const CONNECTION_TEST_WAIT_MS = 60_000; /** How often the plan page asks for the location's plan; the server expires a plan after ten minutes. */ const PLAN_POLL_MS = 750; +/** Rows of each paged plan section loaded with the summary, as many as the plan report shows before "Show more". */ /** The server's largest audit page, and how many the audit log reads before it stops looking. */ const AUDIT_PAGE = 100; @@ -231,12 +238,13 @@ export class HttpChangeFeed implements ChangeSubscription { }; source.addEventListener('changed', (event) => { const notice = JSON.parse((event as MessageEvent).data) as Wire['ChangeNotice']; - this.#emit(channelOf(notice.topic, scope)); + const channel = channelOf(notice.topic, scope); + this.#emit(channel, channel.startsWith('node_status:') ? toNodeStatusDelta(notice) : null); }); } - #emit(channel: ChangedChannel) { - const signal = { channel, at: new Date().toISOString() }; + #emit(channel: ChangedChannel, nodeStatus: NodeStatusDelta | null = null) { + const signal: ChangedSignal = { channel, at: new Date().toISOString(), ...(nodeStatus ? { nodeStatus } : {}) }; for (const handler of this.#handlers) handler(signal); } @@ -280,6 +288,8 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, const errorRefused = new Set(); // Targets whose health the person may not read; each refusal is audited, so Connections asks once per page load. const healthRefused = new Set(); + // The node listing per location with the server's ETag; the server answers a repeat with 304 until the snapshot or states change. + const listings = new Map(); // Statistics samples per relation, oldest first, with the cursor after the newest; a refresh reads only newer samples. const samples = new Map(); async function relationStatistics(scope: Scope, relation: { database: string | null; schema: string; name: string }): Promise { @@ -311,6 +321,29 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, samples.set(key, { records, after }); return records; } + /** One page of a plan section, its rows found by `name` when given, under the plan's authorization. */ + async function planRows(scope: Scope, requestId: string, section: PlanPageSection, offset: number, limit?: number, name?: string): Promise { + const { locationId, target } = parseLocationKey(scope.locationId); + const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/plans/{request_id}/rows', { + params: { path: { location_id: locationId, target, request_id: requestId }, query: { section, offset, limit, ...(name ? { name } : {}) } } + }); + if (response.status === 401) onUnauthorized(); + if (!response.ok) throw new ApiError(errorText(error, response), response.status); + if (!data) throw new ApiError('The plan request disappeared', 404); + return data as unknown as PlanPage; + } + /** One planned model's diff text; null without one, or without view.sql on the target. */ + async function planDiff(scope: Scope, requestId: string, model: string): Promise { + const { locationId, target } = parseLocationKey(scope.locationId); + const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/plans/{request_id}/diff', { + params: { path: { location_id: locationId, target, request_id: requestId }, query: { model } } + }); + if (response.status === 401) onUnauthorized(); + if (response.status === 404) return null; + if (!response.ok || !data) throw new ApiError(errorText(error, response), response.status); + return { model: data.model, ...(data.query_diff ? { query_diff: data.query_diff } : {}), ...(data.config_diff ? { config_diff: data.config_diff } : {}) }; + } + const implemented = { supports: (method: keyof ApiClient) => SUPPORTED.has(method), changes: changes ?? new HttpChangeFeed(baseUrl, (scope) => feedAllowed(baseUrl, scope, onUnauthorized)), @@ -353,7 +386,8 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, listQueue: (scope: Scope): Promise => queueEntries(scope), async navCounts(scope: Scope): Promise { // The nodes listing goes first: a person who can't list the whole target gets one audited refusal, not five. - const nodes = await implemented.listCatalogNodes(scope); + // A held listing has the count already; only a new snapshot changes it, and that reads the listing again. + const nodes = listings.get(scope.locationId)?.nodes ?? (await implemented.listCatalogNodes(scope)); const [checks, runs, triggers, events] = await Promise.all([implemented.listChecks(scope), listedRuns(scope), listedTriggers(scope), problems()]); return { catalogNodes: nodes.length, @@ -497,17 +531,33 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, launchJob: (scope: Scope, key: string, options?: JobLaunchOptions): Promise => startJob(scope, key, options, false), /** The server checks a full refresh against `run.rewrite_history` and refuses it with a window. */ launchJobFromScratch: (scope: Scope, key: string, options?: JobLaunchOptions): Promise => startJob(scope, key, options, true), - async listCatalogNodes(scope: Scope): Promise { + listCatalogNodes: (scope: Scope): Promise => shared(`nodes ${scope.locationId}`, () => catalogListing(scope)), + async listNodeStates(scope: Scope): Promise { const { locationId, target } = parseLocationKey(scope.locationId); - const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/nodes', { params: { path: { location_id: locationId, target } } }); + const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/node-states', { params: { path: { location_id: locationId, target } } }); if (response.status === 401) onUnauthorized(); if (!data) throw new ApiError(errorText(error, response), response.status); - return toCatalogNodes(data); + return toNodeStates(data); }, async listSnapshotNodes(scope: Scope): Promise { const nodes = await implemented.listCatalogNodes(scope); return nodes.map((n) => ({ key: n.key, kind: n.kind, group: n.group ?? '', tags: n.tags, path: n.path, cursor: null, upstream: n.upstream })); }, + async evaluateSelection(scope: Scope, request: SelectionRequest): Promise { + const { locationId, target } = parseLocationKey(scope.locationId); + const query = { select: request.select, exclude: request.exclude, offset: request.offset, limit: request.limit }; + const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/selection', { params: { path: { location_id: locationId, target }, query } }); + if (response.status === 401) onUnauthorized(); + if (!data) throw new ApiError(errorText(error, response), response.status); + return toSelectionEvaluation(data); + }, + async selectionVocabulary(scope: Scope): Promise { + const { locationId, target } = parseLocationKey(scope.locationId); + const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/selection/vocabulary', { params: { path: { location_id: locationId, target } } }); + if (response.status === 401) onUnauthorized(); + if (!data) throw new ApiError(errorText(error, response), response.status); + return toSelectionVocabulary(data); + }, async checkAdhocRun(scope: Scope, request: AdhocRunRequest): Promise { const { locationId, target } = parseLocationKey(scope.locationId); const body = { location_id: locationId, target, command: request.command, priority: request.priority }; @@ -525,7 +575,10 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, if (!response.ok) throw new ApiError(errorText(error, response), response.status); return requireRun(scope, body.run_id); }, - /** Queues the plan, then waits for the location's document; a plan the location could not make is a result, not a throw. */ + /** + * Queues the plan, then waits for the location's summary: changed models in full and the large sections counted. + * The first page of each large section loads with it; a plan the location could not make is a result, not a throw. + */ async planBuild(scope: Scope, command: string[]): Promise { const { locationId, target } = parseLocationKey(scope.locationId); const path = { location_id: locationId, target }; @@ -539,17 +592,25 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, if (!asked.response.ok) throw new ApiError(errorText(asked.error, asked.response), asked.response.status); for (;;) { await new Promise((resolve) => setTimeout(resolve, PLAN_POLL_MS)); - const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/plans/{request_id}', { + const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/plans/{request_id}/summary', { params: { path: { ...path, request_id: requestId } } }); if (response.status === 401) onUnauthorized(); if (!response.ok) throw new ApiError(errorText(error, response), response.status); if (!data) throw new ApiError('The plan request disappeared', 404); - const outcome = data.result_json?.outcome; - if (outcome) return planResultOf(outcome); + const outcome = data.outcome; + if (outcome) { + const result = planResultOf(outcome); + if (!result.plan) return result; + // The summary alone: the plan page loads a large section's rows a page at a time, when it is opened. + const plan = data.timings ? { ...result.plan, timings: data.timings } : result.plan; + return { plan, error: null, requestId }; + } if (Date.parse(data.expires_at) <= Date.now()) return { plan: null, error: 'The location did not plan the command in time; check that sqb location serve is running' }; } }, + planRows, + planDiff, async listSnapshotChanges(scope: Scope, query: { limit?: number } = {}): Promise { const { locationId, target } = parseLocationKey(scope.locationId); const [snapshots, runs] = await Promise.all([ @@ -683,10 +744,12 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, const read = await Promise.all( locations.map(async (location) => { const { locationId, target } = parseLocationKey(location.id); - const { data, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/health', { params: { path: { location_id: locationId, target } } }); - if (response.status === 401) onUnauthorized(); + const [[{ data, response }, snapshot], totals] = await Promise.all([ + locationHealth(location.id), + http.GET('/api/control/locations/{location_id}/targets/{target}/mirrored-totals', { params: { path: { location_id: locationId, target } } }) + ]); if (response.status === 403) healthRefused.add(location.id); - return data ? [toWarehouseConnection(location, data)] : []; + return data ? [toWarehouseConnection(location, data, snapshot, totals.data ?? null)] : []; }) ); return read.flat(); @@ -734,8 +797,8 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, }, disableAccount: (name: string): Promise => changePerson(name, 'disable'), enableAccount: (name: string): Promise => changePerson(name, 'enable'), - async getColumnLineage(scope: Scope, ref: string): Promise { - const nodes = await implemented.listCatalogNodes(scope); + async getColumnLineage(scope: Scope, ref: string, listing?: CatalogNode[]): Promise { + const nodes = listing ?? (await implemented.listCatalogNodes(scope)); const node = findNode(nodes, ref); if (!node) return null; const { locationId, target } = parseLocationKey(scope.locationId); @@ -780,12 +843,12 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, microbatch: coverage?.data ? toMicrobatchCoverage(coverage.data) : null }); }, - async overview(scope: Scope): Promise { + async overview(scope: Scope, listing?: CatalogNode[]): Promise { // A week of publications for the models metric; the server returns at most 100. const reads = Promise.all([ recentRuns(scope), implemented.listChecks(scope), - implemented.listCatalogNodes(scope), + listing ?? implemented.listCatalogNodes(scope), implemented.listSnapshotChanges(scope, { limit: 100 }), implemented.listPlatformEvents(), acknowledgementsOf(scope) @@ -895,6 +958,27 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, return started; } + /** Sends the held listing's ETag, so an unchanged listing comes back as a 304 and is not sent or converted again. */ + async function catalogListing(scope: Scope): Promise { + const { locationId, target } = parseLocationKey(scope.locationId); + const held = listings.get(scope.locationId); + const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/nodes', { + params: { path: { location_id: locationId, target } }, + headers: held ? { 'if-none-match': held.etag } : undefined + }); + if (response.status === 401) onUnauthorized(); + if (response.status === 304 && held) return held.nodes; + if (!data) { + if (isRefusalStatus(response.status)) listings.delete(scope.locationId); + throw new ApiError(errorText(error, response), response.status); + } + const nodes = toCatalogNodes(data); + const etag = response.headers.get('etag'); + if (etag) listings.set(scope.locationId, { etag, nodes }); + else listings.delete(scope.locationId); + return nodes; + } + const listedJobs = (scope: Scope) => shared(`jobs ${scope.locationId}`, () => fetchJobs(scope)); async function fetchJobs(scope: Scope) { @@ -1174,6 +1258,8 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, }) as unknown as ApiClient; } +const isRefusalStatus = (status: number) => status === 401 || status === 403 || status === 404; + /** The server's message: a JSON error body's text, or a 4xx plain-text body such as a request the route couldn't read; never an HTML page. */ function errorText(error: unknown, response: Response): string { // Plain text comes from the server's request rejections; a 5xx one can name internal types, so it shows as the status. diff --git a/ui/src/lib/api/http-jobs.test.ts b/ui/src/lib/api/http-jobs.test.ts index dbe843d7e0..acbe4d9211 100644 --- a/ui/src/lib/api/http-jobs.test.ts +++ b/ui/src/lib/api/http-jobs.test.ts @@ -114,7 +114,9 @@ describe('job mapping', () => { kind: 'job', description: 'Build orders', selection: ['model:customers', 'model:orders'], - concurrency_groups: ['warehouse_heavy'], + pools: [{ pool: 'warehouse_heavy', amount: 3, limit: 4, cap: 2, server_limit: 2, clamped: true, shared_with: ['nightly'] }], + resolved_concurrency: { limit: 4, source: 'job' }, + concurrency: 4, snapshot_id: SNAPSHOT, select: '+orders', declared_at: { path: 'orchestration/jobs.py', line: 12 }, @@ -124,15 +126,19 @@ describe('job mapping', () => { }; const triggers = [toTrigger(daily, [], null), toTrigger(onInventory, [], null)]; const job = toJob(wire, triggers, [run('r-inv', 'inventory', null), run('r-ord', 'orders', null)]); - expect(job).toMatchObject({ key: 'orders', kind: 'job', nodeCount: 2, triggerIds: ['t-daily'], concurrencyGroups: ['warehouse_heavy'], select: '+orders', sourceFile: 'orchestration/jobs.py:12', owner: 'Orders team', overlap: 'skip', onFailure: { retries: 2, retryDelaySeconds: 300 }, priority: null }); + expect(job).toMatchObject({ key: 'orders', kind: 'job', nodeCount: 2, triggerIds: ['t-daily'], concurrency: 4, select: '+orders', sourceFile: 'orchestration/jobs.py:12', owner: 'Orders team', overlap: 'skip', onFailure: { retries: 2, retryDelaySeconds: 300 }, priority: null }); expect(toJob({ ...wire, owner: ' ', overlap: 'queue', on_failure: null }, triggers, [])).toMatchObject({ owner: null, overlap: 'queue', onFailure: null }); expect(job.lastRun?.id).toBe('r-ord'); + expect(job.pools).toEqual([{ pool: 'warehouse_heavy', amount: 3, limit: 4, cap: 2, serverLimit: 2, clamped: true, sharedWith: ['nightly'] }]); + expect(job.resolvedConcurrency).toEqual({ limit: 4, source: 'job' }); + expect(toJob({ ...wire, resolved_concurrency: null }, triggers, []).resolvedConcurrency).toBeNull(); + expect(toJob({ ...wire, concurrency: null }, triggers, []).concurrency).toBeNull(); const detail = jobDetailOf(job, { ...wire, settings: { vars: '[REDACTED]', concurrency: 4, empty_ok: true, models: ['a', 'b'] }, members: [], member_of: ['nightly'], critical_nodes: ['model:orders'], exclude: ' stg_orders audit_orders', targets: ['production'], cursor_type: 'integer' }); expect(detail.resolvedNodes).toEqual(['model:customers', 'model:orders']); expect(detail.settings).toEqual({ vars: '[REDACTED]', concurrency: '4', empty_ok: true, models: '["a","b"]' }); expect([detail.groupMembers, detail.members, detail.memberOf, detail.criticalNodes]).toEqual([null, null, ['nightly'], ['model:orders']]); expect([detail.exclude, detail.targets, detail.cursorType]).toEqual([['stg_orders', 'audit_orders'], ['production'], 'integer']); - expect([detail.concurrency, detail.declaration]).toEqual([null, null]); + expect([detail.poolHolders, detail.declaration]).toEqual([null, null]); expect(toJob({ ...wire, job_key: 'nightly', kind: 'group' }, triggers, []).kind).toBe('group'); }); @@ -142,7 +148,7 @@ describe('job mapping', () => { kind: 'group', description: 'Everything nightly', selection: [], - concurrency_groups: [], + pools: [], snapshot_id: SNAPSHOT, settings: {}, member_of: [], @@ -154,10 +160,10 @@ describe('job mapping', () => { cursor_type: null, overlap: 'queue', members: [ - { job_key: 'stock', waits_for: [] }, - { job_key: 'orders', waits_for: [] }, - { job_key: 'customers', waits_for: ['orders'] }, - { job_key: 'audit', waits_for: ['customers', 'stock'] } + { job_key: 'stock', waits_for: [], reasons: [] }, + { job_key: 'orders', waits_for: [], reasons: [] }, + { job_key: 'customers', waits_for: ['orders'], reasons: [{ upstream_job: 'orders', from_node: 'model:orders', to_node: 'model:customers' }] }, + { job_key: 'audit', waits_for: ['customers', 'stock'], reasons: [] } ] }; const detail = jobDetailOf(toJob(wire, [], []), wire); @@ -168,6 +174,7 @@ describe('job mapping', () => { ['audit', 2, ['customers', 'stock']] ]); expect(detail.groupMembers?.flatMap((m) => m.upstream.map((u) => u.source))).toEqual([null, null, null]); + expect(detail.groupMembers?.[2].reasons).toEqual([{ upstreamJob: 'orders', fromNode: 'model:orders', toNode: 'model:customers' }]); expect(detail.members).toEqual(['stock', 'orders', 'customers', 'audit']); expect([detail.select, detail.sourceFile, detail.exclude, detail.targets, detail.cursorType]).toEqual([null, null, [], null, null]); }); diff --git a/ui/src/lib/api/http-jobs.ts b/ui/src/lib/api/http-jobs.ts index e424ad66c9..3c8aa1f5e7 100644 --- a/ui/src/lib/api/http-jobs.ts +++ b/ui/src/lib/api/http-jobs.ts @@ -2,6 +2,7 @@ import type { components } from '$control/schema.d.ts'; import { ApiError } from './client'; import type { Iso } from './types'; import type { CursorWindow, Job, JobDetail, JobGroupMember, Run, RunRef, SensorEvaluation, SensorEvaluationOutcome, Tick, Trigger, TriggerDetail, TriggerKind, TriggerStateEvent, TriggerStateEventKind } from './types/operate'; +import { plural } from '$lib/format/plural'; type Wire = components['schemas']; @@ -30,7 +31,7 @@ export function triggerKindOf(declaration: Wire['TriggerDeclaration']): TriggerK export function cadenceOf(declaration: Wire['TriggerDeclaration']): string { if (isSchedule(declaration)) { const s = declaration.schedule; - return s.kind === 'cron' ? `${s.expression} (${s.timezone})` : `${s.times.length} calendar time${s.times.length === 1 ? '' : 's'}`; + return s.kind === 'cron' ? `${s.expression} (${s.timezone})` : `${plural(s.times.length, 'calendar time')}`; } const w = declaration.when; if (w.kind === 'run_event') return `when ${w.job_key} ${w.kinds.join(' or ')}`; @@ -194,9 +195,13 @@ export function toJob(wire: Wire['JobRecord'], triggers: Trigger[], runs: Run[]) select: wire.select ?? null, members: null, nodeCount: wire.selection.length, + nodes: wire.selection, overlap: wire.overlap, onFailure: wire.on_failure ? { retries: wire.on_failure.retries, retryDelaySeconds: wire.on_failure.retry_delay_seconds } : null, - concurrencyGroups: wire.concurrency_groups, + pools: wire.pools.map((use) => ({ pool: use.pool, amount: use.amount, limit: use.limit, cap: use.cap ?? null, serverLimit: use.server_limit, clamped: use.clamped, sharedWith: use.shared_with })), + concurrency: wire.concurrency ?? null, + resolvedConcurrency: wire.resolved_concurrency ?? null, + ignoreAuditFailures: wire.ignore_audit_failures ?? null, priority: null, owner: wire.owner?.trim() || null, sourceFile: wire.declared_at ? `${wire.declared_at.path}:${wire.declared_at.line}` : null, @@ -215,7 +220,8 @@ function groupMembersOf(members: Wire['JobMember'][]): JobGroupMember[] { const stage = Math.max(-1, ...m.waits_for.map((w) => stageOf.get(w) ?? -1)) + 1; stageOf.set(m.job_key, stage); // The server reports the waits, not whether lineage or `after` caused each one. - return { jobKey: m.job_key, upstream: m.waits_for.map((jobKey) => ({ jobKey, source: null })), stage }; + const reasons = (m.reasons ?? []).map((r) => ({ upstreamJob: r.upstream_job, fromNode: r.from_node, toNode: r.to_node })); + return { jobKey: m.job_key, upstream: m.waits_for.map((jobKey) => ({ jobKey, source: null })), stage, reasons }; }); } @@ -228,7 +234,7 @@ export function jobDetailOf(job: Job, wire: Wire['JobDetailRecord']): JobDetail exclude: wire.exclude?.split(/\s+/).filter(Boolean) ?? [], settings: Object.fromEntries(Object.entries(wire.settings).map(([name, value]) => [name, shownSetting(value)])), resolvedNodes: wire.selection, - concurrency: null, + poolHolders: null, targets: wire.targets.length ? wire.targets : null, groupMembers: wire.kind === 'group' ? groupMembersOf(wire.members) : null, criticalNodes: wire.critical_nodes, diff --git a/ui/src/lib/api/http-locations.test.ts b/ui/src/lib/api/http-locations.test.ts index 943e50e401..2ead32cb87 100644 --- a/ui/src/lib/api/http-locations.test.ts +++ b/ui/src/lib/api/http-locations.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { connectionTestOf, healthOf, instancesOf, toLocationDetail, toWarehouseConnection, warehouseSyncOf } from './http-locations'; +import { connectionDetailsOf, connectionTestOf, firstSyncPending, healthOf, instancesOf, poolCapWarningsOf, toLocationDetail, toWarehouseConnection, warehouseSyncOf } from './http-locations'; const now = Date.parse('2026-11-08T12:00:00Z'); const at = (secondsAgo: number) => new Date(now - secondsAgo * 1000).toISOString(); @@ -14,6 +14,18 @@ describe('location health', () => { expect(healthOf([seen(91, { kind: 'heartbeat' })], now)).toEqual({ health: 'broken', liveInstances: 0, lastSeenAt: at(91) }); }); + it("reads the server's newest pool-cap warning, job by job", () => { + const job = { job_key: 'close_of_business', kind: 'group' as const, pool: 'warehouse_heavy', amount: 3, cap: 1, message: 'group close_of_business takes 3 of pool warehouse_heavy; this server caps it at 1, so it runs alone in that pool' }; + const capped = seen(5, { kind: 'pool_capped', generation: 2, jobs: [job] }, '00000000-0000-0000-0000-000000000000'); + expect(poolCapWarningsOf([capped, seen(10, { kind: 'heartbeat' })])).toEqual([{ jobKey: 'close_of_business', kind: 'group', pool: 'warehouse_heavy', amount: 3, cap: 1, message: job.message }]); + expect(poolCapWarningsOf([seen(10, { kind: 'heartbeat' })])).toEqual([]); + }); + + it("doesn't count the server's pool-cap warning as a location check-in", () => { + const capped = seen(5, { kind: 'pool_capped', generation: 2, jobs: [] }, '00000000-0000-0000-0000-000000000000'); + expect(healthOf([capped, seen(91, { kind: 'heartbeat' })], now)).toEqual({ health: 'broken', liveInstances: 0, lastSeenAt: at(91) }); + }); + it('counts each live instance once', () => { const observations = [seen(10, { kind: 'heartbeat' }), seen(20, { kind: 'connection', generation: 2, healthy: true }), seen(30, { kind: 'heartbeat' }, 's-2')]; expect(healthOf(observations, now)).toMatchObject({ health: 'healthy', liveInstances: 2, lastSeenAt: at(10) }); @@ -37,6 +49,16 @@ describe('location health', () => { expect(healthOf(observations, now).health).toBe('degraded'); expect(connectionTestOf(observations)).toMatchObject({ outcome: 'failed', at: at(5) }); }); + + it('records the startup check with its duration, and an older location\'s check as after publishing', () => { + expect(connectionTestOf([seen(5, { kind: 'connection', generation: 1, healthy: true, trigger: 'startup', elapsed_millis: 410 })])).toMatchObject({ + outcome: 'ok', + trigger: 'startup', + durationMs: 410, + message: 'Connected on startup' + }); + expect(connectionTestOf([seen(5, { kind: 'connection', generation: 2, healthy: false })])).toMatchObject({ trigger: 'publish', durationMs: 0, message: 'Could not connect after publishing' }); + }); }); describe('location processes', () => { @@ -46,6 +68,8 @@ describe('location processes', () => { ['s-1', at(5), 'live'], ['old', at(95), 'stale'] ]); + const capped = seen(1, { kind: 'pool_capped', generation: 2, jobs: [] }, '00000000-0000-0000-0000-000000000000'); + expect(instancesOf([capped, ...observations], now).map((i) => i.id)).toEqual(['s-1', 'old']); }); }); @@ -57,6 +81,12 @@ describe('toLocationDetail', () => { expect([detail.health, detail.failedDeploy, detail.lastLoadError, detail.snapshot]).toEqual(['falling_back', null, null, null]); }); + it('lists the rule findings that failed the load, as sqb compile prints them', () => { + const rules = { finding_count: 3, findings: [{ code: 'SQBRMODEL102', message: 'SELECT * is allowed only inside dependency import CTEs', path: 'models/orders.sql', line: 2 }] }; + const detail = toLocationDetail(location, [seen(20, { kind: 'compile_failed', generation: 3, rules })], null, now); + expect(detail.lastLoadError).toBe('Compile failed: 3 rule findings\nmodels/orders.sql:2 [SQBRMODEL102] SELECT * is allowed only inside dependency import CTEs\n… and 2 more'); + }); + it('is healthy again once a newer generation connects', () => { const detail = toLocationDetail(location, [seen(5, { kind: 'connection', generation: 4, healthy: true }), seen(20, { kind: 'compile_failed', generation: 3 })], null, now); expect([detail.health, detail.connectionTest?.outcome]).toEqual(['healthy', 'ok']); @@ -80,7 +110,7 @@ describe('warehouseSyncOf', () => { sync('2026-11-08T12:01:00Z', 'catalog', null), sync('2026-11-08T12:00:00Z', 'statistics', 'delivery') ]); - expect(result).toEqual({ lastSyncedAt: '2026-11-08T12:02:00Z', lastError: 'catalog: failed while reading the warehouse', lastReportedAt: '2026-11-08T12:03:00Z' }); + expect(result).toEqual({ lastSyncedAt: '2026-11-08T12:02:00Z', lastError: 'catalog: failed while reading the warehouse', lastReportedAt: '2026-11-08T12:03:00Z', syncing: null, completed: null }); }); it("gives each table its newest sync, the time it last synced cleanly, and where the state tables are read from", () => { @@ -98,11 +128,91 @@ describe('warehouseSyncOf', () => { ]); }); + it('counts the records mirrored of each table, and names the next sync from the report that completed one', () => { + const location = { id: 'loc:development', name: 'Orders', health: null, lastSeenAt: null, target: { name: 'development', adapter: null, connectionSummary: null }, warehouseSync: { lastSyncedAt: null, lastError: null } }; + const completed = sync('2026-11-08T12:03:00Z', 'statistics', null); + const observations = [{ ...completed, report: { ...completed.report, health: { ...completed.report.health, next_sync_at: '2026-11-08T12:33:00Z' } } }, sync('2026-11-08T12:02:00Z', 'catalog', null)]; + const connection = toWarehouseConnection(location, observations, null, [ + { table: 'catalog', records: 120 }, + { table: 'statistics', records: 4_800 } + ]); + expect(connection.tables.map((t) => [t.table, t.mirroredTotal])).toEqual([ + ['catalog', 120], + ['statistics', 4_800] + ]); + expect(connection.nextSyncAt).toBe('2026-11-08T12:33:00Z'); + const running = toWarehouseConnection(location, [sync('2026-11-08T12:40:00Z', 'catalog', null), ...observations]); + expect([running.nextSyncAt, running.tables[0].mirroredTotal]).toEqual([null, null]); + }); + + it('marks a sync that failed reads stopped, from the newest report only', () => { + const location = { id: 'loc:development', name: 'Orders', health: null, lastSeenAt: null, target: { name: 'development', adapter: null, connectionSummary: null }, warehouseSync: { lastSyncedAt: null, lastError: null } }; + const failed = sync('2026-11-08T12:03:00Z', 'catalog', 'read', 'permission denied for table _sqlbuild_catalog'); + const progress = { position: 2, kinds: 14, rows: 40, elapsed_millis: 900, completed: false, halted: true }; + const stopped = { ...failed, report: { ...failed.report, health: { ...failed.report.health, progress } } }; + const halted = toWarehouseConnection(location, [stopped, sync('2026-11-08T12:02:00Z', 'statistics', null)]); + expect([halted.halted, halted.nextSyncAt, halted.tables.find((t) => t.table === 'catalog')?.error]).toEqual([true, null, 'permission denied for table _sqlbuild_catalog']); + const resumed = toWarehouseConnection(location, [sync('2026-11-08T12:10:00Z', 'catalog', null), stopped]); + expect(resumed.halted).toBe(false); + expect(warehouseSyncOf([stopped])?.syncing).toBeNull(); + expect(warehouseSyncOf([{ ...stopped, report: { ...stopped.report, health: { ...stopped.report.health, progress: { ...progress, halted: false } } } }])?.syncing).toMatchObject({ position: 2, kinds: 14 }); + }); + + it('shows the connection settings published with the newest snapshot, and none before it publishes them', () => { + const location = { id: 'loc:development', name: 'Orders', health: null, lastSeenAt: null, target: { name: 'development', adapter: null, connectionSummary: null }, warehouseSync: { lastSyncedAt: null, lastError: null } }; + const published = { + snapshot_id: 'snap-1', + generation: 3, + git_dirty: false, + environment_hash: 'env', + sqb_version: '1.0.0', + published_at: '2026-11-08T12:00:00Z', + changes: [], + connection: { adapter: 'snowflake', account: 'orders-account', user: 'orders_loader', role: 'LOADER', warehouse: 'ORDERS_WH', database: 'ORDERS', auth_method: 'key_pair' } + }; + expect(toWarehouseConnection(location, [], published).details).toEqual({ + adapter: 'snowflake', + account: 'orders-account', + host: null, + user: 'orders_loader', + role: 'LOADER', + warehouse: 'ORDERS_WH', + database: 'ORDERS', + authMethod: 'key_pair' + }); + expect(connectionDetailsOf({ ...published, connection: null })).toBeNull(); + expect(toWarehouseConnection(location, []).details).toBeNull(); + }); + it('shows a build holding the warehouse file as a wait, not a failure', () => { const location = { id: 'loc:development', name: 'Orders', health: null, lastSeenAt: null, target: { name: 'development', adapter: null, connectionSummary: null }, warehouseSync: { lastSyncedAt: null, lastError: null } }; const observations = [sync('2026-11-08T12:03:00Z', 'catalog', 'busy'), sync('2026-11-08T12:01:00Z', 'catalog', null)]; expect(toWarehouseConnection(location, observations).tables.map((t) => [t.failure, t.waiting, t.lastSyncedAt])).toEqual([['Waiting for a build that holds the warehouse file', true, '2026-11-08T12:01:00Z']]); - expect(warehouseSyncOf(observations)).toEqual({ lastSyncedAt: '2026-11-08T12:01:00Z', lastError: null, lastReportedAt: '2026-11-08T12:03:00Z' }); + expect(warehouseSyncOf(observations)).toEqual({ lastSyncedAt: '2026-11-08T12:01:00Z', lastError: null, lastReportedAt: '2026-11-08T12:03:00Z', syncing: null, completed: null }); + }); + + it('says why a read failed and on what page size, without the warehouse message', () => { + const timedOut = sync('2026-11-08T12:03:00Z', 'statistics', 'read'); + const health = { ...timedOut.report.health, read_failure: { kind: 'timed_out' as const, seconds: 130 }, page_limit: 100 }; + expect(warehouseSyncOf([{ ...timedOut, report: { ...timedOut.report, health } }])?.lastError).toBe('statistics: failed while reading the warehouse: no reply within 130 s, on a page of 100 relations'); + const erroring = sync('2026-11-08T12:04:00Z', 'catalog', 'read'); + const error = { ...erroring.report.health, read_failure: { kind: 'read_error' as const }, page_limit: 2000 }; + expect(warehouseSyncOf([{ ...erroring, report: { ...erroring.report, health: error } }])?.lastError).toBe('catalog: failed while reading the warehouse: the warehouse returned an error; the location’s log has it, on a page of 2,000 rows'); + }); + + it('shows a running sync and the newest completed one, and a first sync as pending until it completes', () => { + const at = (time: string, table: 'catalog' | 'statistics', position: number, rows: number, completed: boolean) => { + const report = sync(time, table, null); + return { ...report, report: { ...report.report, health: { ...report.report.health, progress: { position, kinds: 14, rows, elapsed_millis: 12_800, completed } } } }; + }; + const now = Date.parse('2026-11-08T12:10:30Z'); + const first = warehouseSyncOf([at('2026-11-08T12:10:00Z', 'catalog', 13, 1640, false)]); + expect(first?.syncing).toEqual({ position: 13, kinds: 14, table: 'catalog', rows: 1640 }); + expect(firstSyncPending(first, now)).toBe(true); + expect(firstSyncPending(first, now + 60 * 60_000)).toBe(false); + const later = warehouseSyncOf([at('2026-11-08T12:10:00Z', 'catalog', 13, 10, false), at('2026-11-08T12:00:00Z', 'statistics', 14, 31_204, true)]); + expect(later?.completed).toEqual({ at: '2026-11-08T12:00:00Z', kinds: 14, rows: 31_204, elapsedMs: 12_800 }); + expect(firstSyncPending(later, now)).toBe(false); }); it("shows a failed read's warehouse error after its stage", () => { diff --git a/ui/src/lib/api/http-locations.ts b/ui/src/lib/api/http-locations.ts index 374fbbffb8..7e017abd87 100644 --- a/ui/src/lib/api/http-locations.ts +++ b/ui/src/lib/api/http-locations.ts @@ -1,6 +1,6 @@ import type { components } from '$control/schema.d.ts'; import type { Location, LocationHealth, Snapshot, WarehouseSync } from './types'; -import type { ConnectionTest, LocationDetail, LocationInstance, LocationSummary, WarehouseConnection, WarehouseTableSync } from './types/platform'; +import type { ConnectionDetails, ConnectionTest, LocationDetail, PoolCapWarning, LocationInstance, LocationSummary, WarehouseConnection, WarehouseTableSync } from './types/platform'; type Wire = components['schemas']; @@ -13,8 +13,10 @@ function latestOf(observations: Wire[' /** Health from the newest observations: down without a recent check-in, then load failures, then missing provider secrets, then connection failures. */ export function healthOf(observations: Wire['LocationHealthObservation'][], now: number): { health: LocationHealth; liveInstances: number; lastSeenAt: string | null } { - const live = new Set(observations.filter((o) => now - Date.parse(o.received_at) <= LIVE_WINDOW_MS).map((o) => o.report.session_id)); - const lastSeenAt = observations.reduce((latest, o) => (!latest || o.received_at > latest ? o.received_at : latest), null); + // The server's own pool-cap warning is not a location check-in. + const reported = observations.filter((o) => o.report.health.kind !== 'pool_capped'); + const live = new Set(reported.filter((o) => now - Date.parse(o.received_at) <= LIVE_WINDOW_MS).map((o) => o.report.session_id)); + const lastSeenAt = reported.reduce((latest, o) => (!latest || o.received_at > latest ? o.received_at : latest), null); const failed = latestOf(observations, 'compile_failed'); const connection = latestOf(observations, 'connection'); const secrets = latestOf(observations, 'provider_secrets'); @@ -30,12 +32,23 @@ export function healthOf(observations: Wire['LocationHealthObservation'][], now: return { health, liveInstances: live.size, lastSeenAt }; } -/** The connection check a location ran after publishing; the server keeps no message or duration. */ +/** + * The newest connection check a location ran on its own: on starting, or after publishing. Older + * locations report neither why nor how long, so their checks count as after publishing, with no duration. + */ export function connectionTestOf(observations: Wire['LocationHealthObservation'][]): ConnectionTest | null { const latest = latestOf(observations, 'connection'); if (!latest) return null; - const ok = latest.report.health.healthy; - return { outcome: ok ? 'ok' : 'failed', at: latest.received_at, durationMs: 0, trigger: 'publish', by: null, message: ok ? 'Connected after publishing' : 'Could not connect after publishing' }; + const { healthy: ok, trigger, elapsed_millis } = latest.report.health; + const when = trigger === 'startup' ? 'on startup' : 'after publishing'; + return { + outcome: ok ? 'ok' : 'failed', + at: latest.received_at, + durationMs: elapsed_millis ?? 0, + trigger: trigger ?? 'publish', + by: null, + message: ok ? `Connected ${when}` : `Could not connect ${when}` + }; } /** @@ -92,10 +105,11 @@ export function toLocationSummary( }; } -/** One process per reporting session, newest check-in first; the server keeps no image, commit or start time for them. */ +/** One process per reporting session, newest check-in first; the server keeps no image, commit or start time for them, and its own pool-cap warning is no process. */ export function instancesOf(observations: Wire['LocationHealthObservation'][], now: number): LocationInstance[] { const lastSeen = new Map(); for (const o of observations) { + if (o.report.health.kind === 'pool_capped') continue; const seen = lastSeen.get(o.report.session_id); if (!seen || o.received_at > seen) lastSeen.set(o.report.session_id, o.received_at); } @@ -104,7 +118,22 @@ export function instancesOf(observations: Wire['LocationHealthObservation'][], n .map(([id, lastSeenAt]) => ({ id, imageId: null, gitRevision: null, startedAt: null, lastSeenAt, state: now - Date.parse(lastSeenAt) <= LIVE_WINDOW_MS ? 'live' : 'stale' })); } +/** The server's newest pool-cap warning: each job or group asking a pool for more than this server's cap. */ +export function poolCapWarningsOf(observations: Wire['LocationHealthObservation'][]): PoolCapWarning[] { + return (latestOf(observations, 'pool_capped')?.report.health.jobs ?? []).map((w) => ({ jobKey: w.job_key, kind: w.kind === 'group' ? 'group' : 'job', pool: w.pool, amount: w.amount, cap: w.cap, message: w.message })); +} + /** A location's page from its health observations and newest snapshot; locations never send compile error text, so a failed reload shows only as its health. */ +/** The rule findings that failed the newest load, as `sqb compile` prints them; null when that load had none or a newer one connected. */ +export function loadErrorOf(observations: Wire['LocationHealthObservation'][], health: LocationHealth): string | null { + const rules = health === 'falling_back' ? latestOf(observations, 'compile_failed')?.report.health.rules : undefined; + if (!rules) return null; + const noun = rules.finding_count === 1 ? 'rule finding' : 'rule findings'; + const lines = rules.findings.map((f) => `${f.path ? `${f.path}${f.line ? `:${f.line}` : ''} ` : ''}[${f.code}] ${f.message}`); + const more = rules.finding_count - rules.findings.length; + return [`Compile failed: ${rules.finding_count} ${noun}`, ...lines, ...(more > 0 ? [`… and ${more} more`] : [])].join('\n'); +} + export function toLocationDetail(location: Location, observations: Wire['LocationHealthObservation'][], published: Wire['PublishedSnapshot'] | null, now: number): LocationDetail { const { health } = healthOf(observations, now); return { @@ -114,7 +143,8 @@ export function toLocationDetail(location: Location, observations: Wire['Locatio createdAt: null, brokenDeployPolicy: null, failedDeploy: null, - lastLoadError: null, + lastLoadError: loadErrorOf(observations, health), + poolCapWarnings: poolCapWarningsOf(observations), instances: instancesOf(observations, now), target: location.target, snapshot: snapshotOf(published), @@ -129,11 +159,28 @@ const SYNC_STAGE: Record, string> delivery: 'sending results to the server' }; -/** What held up a table's newest sync, with a failed read's warehouse error; a build holding the warehouse file is a wait, not a failure. */ -const syncProblemOf = (failure: Wire['WarehouseSyncFailure'], error?: string | null) => - failure === 'busy' ? 'Waiting for a build that holds the warehouse file' : `Failed while ${SYNC_STAGE[failure]}${error ? `: ${error}` : ''}`; - type Sync = Wire['LocationHealthObservation'] & { report: { health: Extract } }; +type SyncHealth = Sync['report']['health']; + +/** Why a read failed and on what page size, e.g. "no reply within 30 s, on a page of 100 relations"; empty for other failures. The warehouse's own error text, when reported, is shown beside it. */ +function readCauseOf(health: SyncHealth): string { + const cause = health.read_failure; + if (!cause) return ''; + const reason = + cause.kind === 'timed_out' + ? `no reply within ${cause.seconds} s` + : cause.kind === 'reader_start' + ? 'the metadata reader didn’t start' + : health.error + ? 'the warehouse returned an error' + : 'the warehouse returned an error; the location’s log has it'; + const page = health.page_limit ? `, on a page of ${health.page_limit.toLocaleString('en-US')} ${health.table === 'statistics' ? 'relations' : 'rows'}` : ''; + return `: ${reason}${page}`; +} + +/** What held up a table's newest sync; a build holding the warehouse file is a wait, not a failure. */ +const syncProblemOf = (health: SyncHealth & { failure: Wire['WarehouseSyncFailure'] }) => + health.failure === 'busy' ? 'Waiting for a build that holds the warehouse file' : `Failed while ${SYNC_STAGE[health.failure]}${readCauseOf(health)}`; /** Sync reports newest first, and the newest of each table. */ function syncsOf(observations: Wire['LocationHealthObservation'][]): { syncs: Sync[]; newest: Map } { @@ -143,8 +190,26 @@ function syncsOf(observations: Wire['LocationHealthObservation'][]): { syncs: Sy return { syncs, newest }; } -/** A location target's connection from its health reports: the newest connection check, where its state tables are read from, and each table's newest sync. */ -export function toWarehouseConnection(location: Location, observations: Wire['LocationHealthObservation'][]): WarehouseConnection { +/** The non-secret connection settings the location published with its snapshot; null when it published none. */ +export function connectionDetailsOf(published: Wire['PublishedSnapshot'] | null): ConnectionDetails | null { + const c = published?.connection; + return c + ? { adapter: c.adapter, account: c.account ?? null, host: c.host ?? null, user: c.user ?? null, role: c.role ?? null, warehouse: c.warehouse ?? null, database: c.database ?? null, authMethod: c.auth_method ?? null } + : null; +} + +/** + * A location target's connection from its health reports, newest snapshot and mirrored totals: the + * settings it published, the newest connection check, where its state tables are read from, each + * table's newest sync and how many records the server mirrors of it, and when the next sync is due. + * `totals` is null when the server doesn't report them. + */ +export function toWarehouseConnection( + location: Location, + observations: Wire['LocationHealthObservation'][], + published: Wire['PublishedSnapshot'] | null = null, + totals: Wire['MirroredTotal'][] | null = null +): WarehouseConnection { const { syncs, newest } = syncsOf(observations); const tables: WarehouseTableSync[] = [...newest.values()] .map(({ received_at, report: { health } }) => ({ @@ -152,10 +217,11 @@ export function toWarehouseConnection(location: Location, observations: Wire['Lo at: received_at, recordsSent: health.records_sent, elapsedMs: health.elapsed_millis, - failure: health.failure ? syncProblemOf(health.failure) : null, + failure: health.failure ? syncProblemOf({ ...health, failure: health.failure }) : null, error: health.error ?? null, waiting: health.failure === 'busy', - lastSyncedAt: syncs.find((s) => s.report.health.table === health.table && !s.report.health.failure)?.received_at ?? null + lastSyncedAt: syncs.find((s) => s.report.health.table === health.table && !s.report.health.failure)?.received_at ?? null, + mirroredTotal: totals ? (totals.find((t) => t.table === health.table)?.records ?? 0) : null })) .sort((a, b) => a.table.localeCompare(b.table)); const state = syncs[0]?.report.health.location; @@ -163,8 +229,12 @@ export function toWarehouseConnection(location: Location, observations: Wire['Lo id: location.id, location: location.name, target: location.target.name, + details: connectionDetailsOf(published), stateLocation: state ? { database: state.database ?? null, schema: state.schema } : null, lastTest: connectionTestOf(observations), + // The report that completes a sync names the next safety sync; one still running names none. + nextSyncAt: syncs[0]?.report.health.next_sync_at ?? null, + halted: syncs[0]?.report.health.progress?.halted ?? false, tables }; } @@ -173,10 +243,30 @@ export function toWarehouseConnection(location: Location, observations: Wire['Lo export function warehouseSyncOf(observations: Wire['LocationHealthObservation'][]): WarehouseSync | null { const { syncs, newest } = syncsOf(observations); if (!syncs.length) return null; - const failing = [...newest.values()].flatMap(({ report: { health } }) => (health.failure && health.failure !== 'busy' ? [`${health.table}: ${syncProblemOf(health.failure, health.error).replace(/^F/, 'f')}`] : [])); + const failing = [...newest.values()].flatMap(({ report: { health } }) => + health.failure && health.failure !== 'busy' ? [`${health.table}: failed while ${SYNC_STAGE[health.failure]}${readCauseOf(health)}${health.error ? `: ${health.error}` : ''}`] : [] + ); + const running = syncs[0].report.health; + const done = syncs.find((s) => s.report.health.progress?.completed); return { lastSyncedAt: syncs.find((s) => !s.report.health.failure)?.received_at ?? null, lastError: failing.length ? failing.sort().join('\n') : null, - lastReportedAt: syncs[0].received_at + lastReportedAt: syncs[0].received_at, + syncing: running.progress && !running.progress.completed && !running.progress.halted ? { position: running.progress.position, kinds: running.progress.kinds, table: running.table, rows: running.progress.rows } : null, + completed: done?.report.health.progress ? { at: done.received_at, kinds: done.report.health.progress.kinds, rows: done.report.health.progress.rows, elapsedMs: done.report.health.progress.elapsed_millis } : null }; } + +/** Older than this, a sync report that left a sync running no longer shows it running: the location has stopped reporting. */ +export const SYNC_REPORT_STALE_MS = 10 * 60_000; + +/** The running sync, unless its newest report is stale. */ +export function liveSyncOf(sync: WarehouseSync | null | undefined, now: number) { + if (!sync?.syncing || !sync.lastReportedAt) return null; + return now - Date.parse(sync.lastReportedAt) < SYNC_REPORT_STALE_MS ? sync.syncing : null; +} + +/** Plans, history and costs read the server's copy of the warehouse; until a first sync completes, it may be missing or partial. */ +export function firstSyncPending(sync: WarehouseSync | null | undefined, now: number): boolean { + return !!liveSyncOf(sync, now) && !sync?.completed; +} diff --git a/ui/src/lib/api/http-overview.ts b/ui/src/lib/api/http-overview.ts index 477e8177b8..56f25a02c4 100644 --- a/ui/src/lib/api/http-overview.ts +++ b/ui/src/lib/api/http-overview.ts @@ -1,4 +1,5 @@ import { topFolder } from '$lib/lineage/catalog-graph'; +import { recentDeploys } from '$lib/build-area/overview-panels'; import { checkCountsOf } from './http-catalog'; import type { Acknowledgement, AttentionItem, AttentionRun, CatalogNode, Check, GroupStatus, Overview, PlatformEvent, Run, SnapshotChange } from './types'; @@ -6,7 +7,6 @@ const DAY = 24 * 60 * 60 * 1000; const MODEL_KINDS = new Set(['view', 'table', 'incremental']); const PROBLEM_STATUSES = new Set(['failed', 'partial', 'presumed_failed']); const ACTIVE_STATUSES = new Set(['queued', 'launched', 'running', 'unresponsive']); -const RECENT_CHANGES = 4; export type OverviewInputs = { /** Listed runs, newest first; the listing holds the newest 1,000, so a busier day undercounts. */ @@ -105,7 +105,7 @@ export function overviewOf({ runs, checks, nodes, changes, events, acknowledgeme }, attention, groups: nodes.some((n) => n.stale !== null) ? groupStatusOf(nodes) : null, - recentChanges: changes.slice(0, RECENT_CHANGES), + recentChanges: recentDeploys(changes), latestRun: latest ? { id: latest.id, status: latest.status, endedAt: latest.endedAt } : null }; } diff --git a/ui/src/lib/api/http-people.test.ts b/ui/src/lib/api/http-people.test.ts index 6a3f7d34e2..a01a4a291d 100644 --- a/ui/src/lib/api/http-people.test.ts +++ b/ui/src/lib/api/http-people.test.ts @@ -44,7 +44,7 @@ describe('toServerSettings', () => { login_mode: 'password', scheduler: false, installation: { storage: 'sqlite', installed_at: '2026-11-01T00:00:00Z', schema_checksum: 'abc', schema_applied_at: '2026-11-01T00:00:00Z', store_bytes: 52428800 }, - retention: { audit_days: 400, run_log_days: 14, run_logs_kept: 200, snapshots_kept: 10, statistics_days: 90, platform_alert_days: 30, location_health_days: 7, sensor_evaluation_days: 6, heartbeat_days: 1 } + retention: { audit_days: 400, run_log_days: 14, run_logs_kept: 200, snapshots_kept: 10, state_history_versions: 5, statistics_days: 90, platform_alert_days: 30, location_health_days: 7, sensor_evaluation_days: 6, heartbeat_days: 1 } }); expect([settings.authMode, settings.schedulerEnabled, settings.storage, settings.storeBytes]).toEqual(['password', false, 'sqlite', 52428800]); expect(settings.retention.map((r) => [r.name, r.value])).toEqual([ @@ -52,6 +52,7 @@ describe('toServerSettings', () => { ['Run logs', '14 days'], ['Logged runs', '200'], ['Published versions', '10'], + ['Builds and check results', '5'], ['Warehouse row counts', '90 days'], ['Platform problems', '30 days'], ['Location check-ins', '7 days'], diff --git a/ui/src/lib/api/http-people.ts b/ui/src/lib/api/http-people.ts index 759f4129c5..4d4f0c4a56 100644 --- a/ui/src/lib/api/http-people.ts +++ b/ui/src/lib/api/http-people.ts @@ -76,6 +76,7 @@ export function toServerSettings(wire: Wire['ServerSettings']): ServerSettings { { name: 'Run logs', value: days(r.run_log_days), description: 'What finished runs logged. The runs themselves are kept.' }, { name: 'Logged runs', value: String(r.run_logs_kept), description: 'Newest finished runs per location target that keep their logs.' }, { name: 'Published versions', value: String(r.snapshots_kept), description: 'Kept for each location target; older versions are removed.' }, + { name: 'Builds and check results', value: String(r.state_history_versions), description: 'Newest builds of each node and results of each check copied from the warehouse.' }, { name: 'Warehouse row counts', value: days(r.statistics_days), description: 'Row counts and sizes sampled from the warehouse.' }, { name: 'Platform problems', value: days(r.platform_alert_days), description: 'Late schedules, failed launches and locations that stopped checking in.' }, { name: 'Location check-ins', value: days(r.location_health_days), description: 'Health reports from location processes.' }, diff --git a/ui/src/lib/api/http-providers.test.ts b/ui/src/lib/api/http-providers.test.ts index 4c40d63e74..fe00ecfc74 100644 --- a/ui/src/lib/api/http-providers.test.ts +++ b/ui/src/lib/api/http-providers.test.ts @@ -20,7 +20,9 @@ describe('toProviders', () => { { kind: 'sink', name: 'order_events' }, { kind: 'event_exporter', name: 'run_events' } ], - jobs: ['order_exports'] + nodes: ['task:publish_orders'], + jobs: ['order_exports'], + job_nodes: { order_exports: ['task:publish_orders'] } } ] }; @@ -29,5 +31,12 @@ describe('toProviders', () => { expect(provider.settings[0]).toEqual({ name: 'api_key', type: 'SecretStr', required: true, secret: true, env: 'FULFILLMENT_API_KEY', description: null, missing: true }); expect(provider.usedBy.map((c) => `${c.kind}:${c.name}`)).toEqual(['task:publish_orders', 'sink:order_events', 'event_exporter:run_events']); expect(provider.jobs).toEqual(['order_exports']); + expect(provider.nodes).toEqual(['task:publish_orders']); + expect(provider.jobNodes).toEqual({ order_exports: ['task:publish_orders'] }); + }); + + it('defaults the nodes to none when the server leaves them out', () => { + const [provider] = toProviders({ providers: [{ name: 'chat', class: 'providers.chat.Chat', description: '', path: 'providers/chat.py', settings: [], used_by: [] }] }); + expect([provider.nodes, provider.jobs, provider.jobNodes]).toEqual([[], [], {}]); }); }); diff --git a/ui/src/lib/api/http-providers.ts b/ui/src/lib/api/http-providers.ts index 1d343173bf..59bc5d419a 100644 --- a/ui/src/lib/api/http-providers.ts +++ b/ui/src/lib/api/http-providers.ts @@ -11,6 +11,8 @@ export function toProviders(wire: Wire['ProvidersRead']): ProviderConnection[] { sourceFile: p.path, settings: p.settings.map((s) => ({ name: s.name, type: s.type, required: s.required, secret: s.secret, env: s.env, description: s.description ?? null, missing: s.missing ?? false })), usedBy: p.used_by.map((c) => ({ kind: c.kind as ProviderConsumer['kind'], name: c.name })), - jobs: p.jobs ?? [] + nodes: p.nodes ?? [], + jobs: p.jobs ?? [], + jobNodes: p.job_nodes ?? {} })); } diff --git a/ui/src/lib/api/http-runs.test.ts b/ui/src/lib/api/http-runs.test.ts index 258a043059..25a8559f4e 100644 --- a/ui/src/lib/api/http-runs.test.ts +++ b/ui/src/lib/api/http-runs.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import type { components } from '$control/schema.d.ts'; -import { groupRunNodes, parseRunCursor, queueEntriesOf, planResultOf, runDetailOf, runRowPage, runRowPageOver, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunStatements, toRunOutput } from './http-runs'; +import { groupRunNodes, mergePlanPage, parseRunCursor, queueEntriesOf, planResultOf, runDetailOf, runRowPage, runRowPageOver, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunStatements, toRunOutput } from './http-runs'; type RunRecord = components['schemas']['RunRecord']; type W = components['schemas']; @@ -125,7 +125,7 @@ describe('run queue', () => { it('keeps claim order, drops runs that started meanwhile and never names a run elsewhere', () => { const entries = queueEntriesOf( [ - wire('urgent', { status: 'queued', queue: { position: 1, waits: [{ kind: 'concurrency_limit', limit: 'warehouse', holders: ['held'], elsewhere: 2 }] } }), + wire('urgent', { status: 'queued', queue: { position: 1, waits: [{ kind: 'pool_full', pool: 'warehouse', holders: ['held'] }] } }), wire('started', { status: 'queued' }), wire('blocked', { status: 'queued', @@ -133,8 +133,9 @@ describe('run queue', () => { position: 2, waits: [ { kind: 'write_claim', run_id: null, node_key: 'model.orders', other_nodes: 3 }, - { kind: 'earlier_run', run_id: 'urgent', limit: 'warehouse' }, + { kind: 'earlier_run', run_id: 'urgent', pool: 'warehouse' }, { kind: 'upstream_job', run_id: 'stage', job_key: 'staging' }, + { kind: 'upstream_job', run_id: 'retry', job_key: 'partner_export', attempt: 2 }, { kind: 'job_running', run_id: 'same' }, { kind: 'write_claim', run_id: 'same', node_key: 'model.customers', other_nodes: 0 } ] @@ -147,11 +148,12 @@ describe('run queue', () => { ['urgent', 1], ['blocked', 2] ]); - expect(entries[0].waits).toEqual([{ kind: 'concurrency_limit', limit: 'warehouse', holders: ['held'], elsewhere: 2 }]); + expect(entries[0].waits).toEqual([{ kind: 'pool_full', pool: 'warehouse', holders: ['held'] }]); expect(entries[1].waits).toEqual([ { kind: 'write_claim', runId: null, nodeKey: 'model.orders', otherNodes: 3 }, - { kind: 'earlier_run', runId: 'urgent', limit: 'warehouse' }, - { kind: 'upstream_job', runId: 'stage', jobKey: 'staging' }, + { kind: 'earlier_run', runId: 'urgent', pool: 'warehouse' }, + { kind: 'upstream_job', runId: 'stage', jobKey: 'staging', attempt: null }, + { kind: 'upstream_job', runId: 'retry', jobKey: 'partner_export', attempt: 2 }, { kind: 'job_running', runId: 'same' } ]); }); @@ -236,12 +238,16 @@ describe('run Recall reads', () => { event('statement_completed', { duration_ms: 1500, affected_rows: 1, resource_id: `["${inv}","model:orders"]`, statement_id: 's1' }), event('resource_attempt_failed', { severity: 'error', error_type: 'ExecutionFailed', error_code: 'R002', resource_id: `["${inv}","model:orders"]` }), event('operation_started'), - event('location_health', { invocation_id: 'scheduler/r1' }) + event('location_health', { invocation_id: 'scheduler/r1' }), + event('run_member_retrying', { invocation_id: 'scheduler/r1', message: 'partner_export_job failed, retrying in 15 min (attempt 2 of 3)' }), + event('run_member_recovered', { invocation_id: 'scheduler/r1', message: 'partner_export_job succeeded on attempt 2' }) ].flatMap((e) => toRunEvent(e, 'r1') ?? []); expect(mapped.map((e) => [e.type, e.producer, e.level, e.nodeKey, e.statementId, e.message])).toEqual([ ['claimed', 'scheduler', 'info', null, null, ''], ['statement_completed', 'sqb', 'info', 'model:orders', 's1', '1.5 s · 1 row'], - ['resource_attempt_failed', 'sqb', 'error', 'model:orders', null, 'ExecutionFailed: R002'] + ['resource_attempt_failed', 'sqb', 'error', 'model:orders', null, 'ExecutionFailed: R002'], + ['member_retrying', 'scheduler', 'info', null, null, 'partner_export_job failed, retrying in 15 min (attempt 2 of 3)'], + ['member_recovered', 'scheduler', 'info', null, null, 'partner_export_job succeeded on attempt 2'] ]); }); @@ -305,3 +311,45 @@ describe('planResultOf', () => { if (result.plan) expect(result.plan.warnings).toEqual([]); }); }); + +describe('mergePlanPage', () => { + const summary = planResultOf({ + kind: 'plan', + document: { + models: [{ name: 'orders', action: 'create_table', reason: 'query_changed' }], + pages: { routine_models: 2, unchanged: 3 }, + changed: { counts: ['definition'], reach: 'all', runs: [{ node: 'model:orders', reason: 'definition changed', downstream_of: null }], held_back: [], unchanged: [] } + } + } as never).plan!; + it.each([ + { + description: 'a routine page adds its models and their runs after the changed ones', + section: 'routine_models' as const, + page: { total: 2, rows: [{ name: 'customers', reason: 'no_change' }], runs: [{ node: 'model:customers', reason: 'upstream changed', downstream_of: 'model:orders' }] }, + expected: { models: ['orders', 'customers'], runs: ['model:orders', 'model:customers'], unchanged: [], pages: { routine_models: 2, unchanged: 3 } } + }, + { + description: 'an unchanged page adds its nodes and keeps the models', + section: 'unchanged' as const, + page: { total: 3, rows: ['model:payments'], runs: [] }, + expected: { models: ['orders'], runs: ['model:orders'], unchanged: ['model:payments'], pages: { routine_models: 2, unchanged: 3 } } + }, + { + description: 'a first-run page adds its models, counts what it loaded and keeps the section total, not the filter matches', + section: 'first_run' as const, + page: { total: 1, section_total: 672, rows: [{ name: 'daily_orders', reason: 'first_run' }], runs: [], warnings: [], cursor_window_starts: {} }, + expected: { models: ['orders', 'daily_orders'], runs: ['model:orders'], unchanged: [], pages: { routine_models: 2, unchanged: 3, first_run: 672 } } + } + ])('$description', ({ section, page, expected }) => { + const merged = mergePlanPage(summary, section, page as never); + expect({ models: merged.models.map((m) => m.name), runs: merged.changed?.runs.map((r) => r.node), unchanged: merged.changed?.unchanged, pages: merged.pages }).toEqual(expected); + expect(merged.loaded?.[section]).toBe(page.rows.length); + }); + it("keeps a warning once when a model's page and the warnings section both bring it, and adds the page's window starts", () => { + const warning = { severity: 'warning', message: 'orders: already-built dates keep their old results' }; + const withCopy = { ...summary, synced_copy: { synced_at: null, cursor_window_starts: {}, unknown_until_run: [] } }; + const page = { total: 1, section_total: 1, rows: [{ name: 'orders', reason: 'query_changed' }], runs: [], warnings: [warning], cursor_window_starts: { orders: '2026-11-01' } }; + const once = mergePlanPage(mergePlanPage(withCopy, 'query_changed', page as never), 'warnings', { ...page, rows: [warning], warnings: [], cursor_window_starts: {} } as never); + expect({ warnings: once.warnings, starts: once.synced_copy?.cursor_window_starts }).toEqual({ warnings: [warning], starts: { orders: '2026-11-01' } }); + }); +}); diff --git a/ui/src/lib/api/http-runs.ts b/ui/src/lib/api/http-runs.ts index 36edda94dd..ae9ea7f95e 100644 --- a/ui/src/lib/api/http-runs.ts +++ b/ui/src/lib/api/http-runs.ts @@ -1,7 +1,7 @@ import type { components } from '$control/schema.d.ts'; import { isActiveRun } from '$lib/status'; import type { Capability, Principal } from './types'; -import type { AdhocRunCheck, LaunchVerb, NodeCounts, PlanDocument, PlanResult, QueueEntry, QueueWait, Run, RunDetail, RunListQuery, RunNode, RunNodeStatus, RunRow, RunRowPage, RunRowQuery, RunTabCounts, RunEvent, RunEventLevel, RunOutputLine, RunStatement, RunCostRow, CostGroup, CostSummary, NodeBuildCost } from './types/operate'; +import type { AdhocRunCheck, LaunchVerb, NodeCounts, PlanChanged, PlanDocument, PlanPage, PlanPageSection, PlanResult, PlanWarning, QueueEntry, QueueWait, Run, RunDetail, RunListQuery, RunNode, RunNodeStatus, RunRow, RunRowPage, RunRowQuery, RunTabCounts, RunEvent, RunEventLevel, RunOutputLine, RunStatement, RunCostRow, CostGroup, CostSummary, NodeBuildCost } from './types/operate'; import { RECALL_EVENT_TYPES, SCHEDULER_FACTS } from './types/operate'; type Wire = components['schemas']; @@ -36,12 +36,69 @@ export function planResultOf(outcome: Wire['LocationWorkOutcome']): PlanResult { table_type_conversions: list(document.table_type_conversions), retention: list(document.retention), synced_copy: document.synced_copy ?? null, - changed: document.changed ?? null + changed: document.changed ?? null, + ...(document.ignore_audit_failures ? { ignore_audit_failures: true } : {}), + ...(document.pages ? { pages: document.pages } : {}), + ...(document.sections ? { sections: document.sections } : {}) }, error: null }; } +/** The plan list a section's rows join: models for model sections, `changed` lists for held-back and unchanged nodes. */ +const SECTION_LIST: Record = { + full_refresh: 'models', + query_changed: 'models', + function_changed: 'models', + config_changed: 'models', + schema_changed: 'models', + first_run: 'models', + renamed: 'models', + routine_models: 'models', + migrations: 'migrations', + column_migrations: 'column_migrations', + table_type_conversions: 'table_type_conversions', + retention: 'retention', + loader_reloads: 'source_loads', + loader_loads: 'source_loads', + source_reloads: 'source_loads', + source_loads: 'source_loads', + changed_functions: 'functions', + functions: 'functions', + seeds: 'seeds', + python_nodes: 'python_nodes', + held_back: 'held_back', + unchanged: 'unchanged', + warnings: 'warnings' +}; + +/** A warning once, however many pages bring it. */ +const warningKey = (w: PlanWarning) => `${w.model_name ?? ''}\u0000${w.message}`; + +/** + * `plan` with one more page of a section: its rows join their list, with the `--changed` runs, warnings and + * cursor window starts of its models, and the section's loaded count moves on. + */ +export function mergePlanPage(plan: PlanDocument, section: PlanPageSection, page: PlanPage): PlanDocument { + const pages = { ...plan.pages, [section]: page.section_total ?? page.total }; + const loaded = { ...plan.loaded, [section]: (plan.loaded?.[section] ?? 0) + page.rows.length }; + const seen = new Set(plan.warnings.map(warningKey)); + const extra = [...(page.warnings ?? []), ...(section === 'warnings' ? (page.rows as PlanWarning[]) : [])].filter((w) => !seen.has(warningKey(w)) && (seen.add(warningKey(w)), true)); + const starts = page.cursor_window_starts ?? {}; + const synced_copy = plan.synced_copy && Object.keys(starts).length ? { ...plan.synced_copy, cursor_window_starts: { ...plan.synced_copy.cursor_window_starts, ...starts } } : plan.synced_copy; + const runs = plan.changed && page.runs.length ? { ...plan.changed, runs: [...plan.changed.runs, ...page.runs] } : plan.changed; + const next: PlanDocument = { ...plan, pages, loaded, synced_copy, changed: runs, warnings: [...plan.warnings, ...extra] }; + const list = SECTION_LIST[section]; + if (list === 'warnings') return next; + if (list === 'held_back' || list === 'unchanged') { + if (!next.changed) return next; + return list === 'held_back' + ? { ...next, changed: { ...next.changed, held_back: [...next.changed.held_back, ...(page.rows as PlanChanged['held_back'])] } } + : { ...next, changed: { ...next.changed, unchanged: [...next.changed.unchanged, ...(page.rows as string[])] } }; + } + return { ...next, [list]: [...(next[list] as unknown[]), ...page.rows] }; +} + /** The server's dry run of an ad-hoc command. */ export function toAdhocCheck(wire: Wire['AdhocCheck']): AdhocRunCheck { return { @@ -97,6 +154,7 @@ export function toRun(wire: Wire['RunRecord'], locationId: string, childRunCount tickScheduledFor: null, runKey: null, rerunOfRunId: wire.rerun_of_run_id ?? null, + memberAttempt: wire.member_attempt ?? null, narrowing: wire.narrowing ?? null, window: wire.window ? { start: String(wire.window.start), end: String(wire.window.end) } : null, requestedBy: requesterOf(wire), @@ -111,7 +169,8 @@ export function toRun(wire: Wire['RunRecord'], locationId: string, childRunCount startedAt: wire.started_at ?? null, endedAt: wire.ended_at ?? null, nodeCounts: countsOf(wire), - cost: null + cost: null, + ignoredAuditFailures: wire.ignored_audit_failures ?? 0 }; } @@ -132,12 +191,12 @@ function toQueueWait(wait: Wire['QueueWait']): QueueWait { switch (wait.kind) { case 'write_claim': return { kind: wait.kind, runId: wait.run_id ?? null, nodeKey: wait.node_key, otherNodes: wait.other_nodes }; - case 'concurrency_limit': - return { kind: wait.kind, limit: wait.limit, holders: wait.holders, elsewhere: wait.elsewhere }; + case 'pool_full': + return { kind: wait.kind, pool: wait.pool, holders: wait.holders }; case 'earlier_run': - return { kind: wait.kind, runId: wait.run_id ?? null, limit: wait.limit }; + return { kind: wait.kind, runId: wait.run_id ?? null, pool: wait.pool }; case 'upstream_job': - return { kind: wait.kind, runId: wait.run_id, jobKey: wait.job_key }; + return { kind: wait.kind, runId: wait.run_id, jobKey: wait.job_key, attempt: wait.attempt ?? null }; case 'job_running': case 'group_parent': return { kind: wait.kind, runId: wait.run_id }; @@ -365,7 +424,7 @@ export function toRunEvent(wire: Wire['RunEventRecord'], runId: string): RunEven const rows = wire.affected_rows ?? wire.row_count; const failure = [wire.error_type, wire.error_code].filter(Boolean).join(': '); const facts = [durationText(wire.duration_ms), rows === null || rows === undefined ? '' : `${rows} ${rows === 1 ? 'row' : 'rows'}`].filter(Boolean).join(' · '); - const message = type === 'requested' ? 'Requested' : type === 'cancel_requested' ? 'Cancel requested' : wire.event_type.endsWith('_failed') ? failure || facts || 'no error code recorded' : type === 'audit_completed' ? (wire.outcome ?? '') : facts; + const message = wire.message ? wire.message : type === 'requested' ? 'Requested' : type === 'cancel_requested' ? 'Cancel requested' : wire.event_type.endsWith('_failed') ? failure || facts || 'no error code recorded' : type === 'audit_completed' ? (wire.outcome ?? '') : facts; return { id: wire.event_id, storageId: wire.storage_id, diff --git a/ui/src/lib/api/http-selection.test.ts b/ui/src/lib/api/http-selection.test.ts new file mode 100644 index 0000000000..b85b25d608 --- /dev/null +++ b/ui/src/lib/api/http-selection.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from 'vitest'; +import { toSelectionEvaluation, toSelectionVocabulary } from './http-selection'; + +describe('selection mapping', () => { + it('keeps a page of keys and the total, with no error', () => { + expect(toSelectionEvaluation({ snapshot_id: 'snap-1', count: 3, keys: ['model:customers', 'model:orders'], next_offset: 2, error: null })).toEqual({ + snapshotId: 'snap-1', + count: 3, + keys: ['model:customers', 'model:orders'], + nextOffset: 2, + error: null + }); + }); + + it('keeps where a refused selector is and what might have been meant', () => { + const evaluation = toSelectionEvaluation({ + snapshot_id: 'snap-1', + count: 0, + keys: [], + next_offset: null, + error: { + code: 'S000', + message: "no SQL resources or Python nodes found with tag 'ordrs'", + help: null, + location: { field: 'select', token: 'tag:ordrs', start: 10, end: 19 }, + suggestions: ['tag:orders'], + refused: { part: '+tag:ordrs', index: 1, type: 'tag', value: 'ordrs' } + } + }); + expect(evaluation.error).toEqual({ + code: 'S000', + message: "no SQL resources or Python nodes found with tag 'ordrs'", + help: null, + location: { field: 'select', token: 'tag:ordrs', start: 10, end: 19 }, + suggestions: ['tag:orders'], + refused: { part: 1, type: 'tag', value: 'ordrs' } + }); + }); + + it('reads a refused bare name or name pattern as an untyped part, and no refused part as null', () => { + const refusal = (refused: { part: string; index: number; type: string; value: string } | null) => + toSelectionEvaluation({ + snapshot_id: 'snap-1', + count: 0, + keys: [], + next_offset: null, + error: { code: 'S007', message: 'unknown selector', help: null, location: null, suggestions: [], refused } + }).error?.refused; + expect([ + refusal({ part: 'custmers', index: 0, type: 'name', value: 'custmers' }), + refusal({ part: 'missing_*', index: 0, type: 'glob', value: 'missing_*' }), + refusal(null) + ]).toEqual([{ part: 0, type: null, value: 'custmers' }, { part: 0, type: null, value: 'missing_*' }, null]); + }); + + it('maps each selector type to its values and counts', () => { + const entry = (value: string, count: number) => ({ value, count }); + expect( + toSelectionVocabulary({ + snapshot_id: null, + tags: [entry('orders', 4)], + paths: [entry('models/marts', 3)], + jobs: [entry('nightly_orders', 12)], + sources: [entry('shop.orders', 1)], + seeds: [entry('currencies', 1)], + kinds: [entry('model', 4)] + }) + ).toEqual({ + snapshotId: null, + tags: [entry('orders', 4)], + paths: [entry('models/marts', 3)], + jobs: [entry('nightly_orders', 12)], + sources: [entry('shop.orders', 1)], + seeds: [entry('currencies', 1)], + kinds: [entry('model', 4)] + }); + }); +}); diff --git a/ui/src/lib/api/http-selection.ts b/ui/src/lib/api/http-selection.ts new file mode 100644 index 0000000000..2b630ea296 --- /dev/null +++ b/ui/src/lib/api/http-selection.ts @@ -0,0 +1,43 @@ +// Server selection evaluations and selector vocabularies, in the UI's shapes. + +import type { components } from '$control/schema.d.ts'; +import type { SelectionEvaluation, SelectionVocabulary, VocabularyEntry } from './types/operate'; + +type Wire = components['schemas']; + +export function toSelectionEvaluation(wire: Wire['SelectionEvaluation']): SelectionEvaluation { + const problem = wire.error ?? null; + return { + snapshotId: wire.snapshot_id ?? null, + count: wire.count, + keys: wire.keys, + nextOffset: wire.next_offset ?? null, + error: problem && { + code: problem.code ?? null, + message: problem.message, + help: problem.help ?? null, + location: problem.location ?? null, + suggestions: problem.suggestions, + refused: problem.refused ? toRefusal(problem.refused) : null + } + }; +} + +/** The server's refused part in the UI's shape: its index as `part`, and a bare name or glob as type `null`. */ +function toRefusal(refused: Wire['SelectionRefusal']): NonNullable['refused'] { + const bare = refused.type === 'name' || refused.type === 'glob'; + return { part: refused.index, type: bare ? null : refused.type, value: refused.value }; +} + +export function toSelectionVocabulary(wire: Wire['SelectionVocabulary']): SelectionVocabulary { + const entries = (list: Wire['VocabularyEntry'][]): VocabularyEntry[] => list.map(({ value, count }) => ({ value, count })); + return { + snapshotId: wire.snapshot_id ?? null, + tags: entries(wire.tags), + paths: entries(wire.paths), + jobs: entries(wire.jobs), + sources: entries(wire.sources), + seeds: entries(wire.seeds), + kinds: entries(wire.kinds) + }; +} diff --git a/ui/src/lib/api/index.ts b/ui/src/lib/api/index.ts index 8b6c78282c..e86830c114 100644 --- a/ui/src/lib/api/index.ts +++ b/ui/src/lib/api/index.ts @@ -1,6 +1,8 @@ import type { ApiClient } from './client'; import { createHttpClient } from './http-client'; import { MockApiClient, type MockOptions } from './mock/mock-client'; +import { largeProject } from './mock/large-project'; +import { awkward } from './mock/awkward'; import { replayFetch, type Recording } from './replay'; import { LocalChangeBus } from './signals'; import { to } from '$lib/paths'; @@ -10,6 +12,7 @@ export type { ApiClient } from './client'; export { ApiError, isRefusal } from './client'; export * from './types'; export * from './signals'; +export { mergePlanPage } from './http-runs'; export { hasCapability, hasAnyCapability, ADMIN_CAPABILITIES } from './capabilities'; const ROLES: readonly BuiltInRole[] = ['viewer', 'operator', 'developer', 'admin']; @@ -32,7 +35,8 @@ async function loadRecording(): Promise { /** * The client the app uses: the server, or the mock with `?mock` or `VITE_SQB_MOCK=1`. * With the mock, `?as=`, `?events=none|all` and `?live=reconnecting|offline` - * adjust it for demos. + * adjust it for demos, `?awkward` stretches every list to thousands of rows with very long + * names, for the layout test, and `?large` swaps in a catalog the size of a real project. */ export function createClient(search: URLSearchParams, onUnauthorized: () => void = () => {}): ApiClient { if (DEMO) return createHttpClient({ onUnauthorized, fetch: replayFetch(loadRecording), changes: new LocalChangeBus() }); @@ -40,9 +44,11 @@ export function createClient(search: URLSearchParams, onUnauthorized: () => void const role = search.get('as'); const events = search.get('events'); const live = search.get('live'); - return new MockApiClient({ + const mock = new MockApiClient({ role: ROLES.find((r) => r === role), events: EVENTS.find((e) => e === events), connection: LIVE.find((l) => l === live) }); + const sized = search.has('large') ? largeProject(mock) : mock; + return search.has('awkward') ? awkward(sized) : sized; } diff --git a/ui/src/lib/api/mock/awkward.test.ts b/ui/src/lib/api/mock/awkward.test.ts new file mode 100644 index 0000000000..9c423bec89 --- /dev/null +++ b/ui/src/lib/api/mock/awkward.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from 'vitest'; +import { stretch } from './awkward'; + +describe('awkward mock data', () => { + it('stretches a list with unique long names, keeping the originals reachable and enums untouched', () => { + const rows = stretch([{ key: 'orders', status: 'success', description: 'Daily orders' }], 5); + expect(rows).toHaveLength(5); + expect(rows[0].key).toBe('orders'); + expect(new Set(rows.map((r) => r.key)).size).toBe(5); + expect(rows.every((r) => r.status === 'success')).toBe(true); + expect(rows[3].description.length).toBeGreaterThan(80); + }); + + it('points copies at a bounded set of long names, shared by several rows', () => { + const rows = stretch([{ key: 'a', node: 'orders' }, { key: 'b', node: 'customers' }, { key: 'c', node: 'payments' }], 5000); + expect(new Set(rows.map((r) => r.node)).size).toBe(2000); + expect(rows[0].node).toBe('orders'); + }); + + it('leaves lists of plain values alone', () => { + expect(stretch(['a', 'b'], 10)).toEqual(['a', 'b']); + }); +}); diff --git a/ui/src/lib/api/mock/awkward.ts b/ui/src/lib/api/mock/awkward.ts new file mode 100644 index 0000000000..f32e0a222f --- /dev/null +++ b/ui/src/lib/api/mock/awkward.ts @@ -0,0 +1,44 @@ +// Awkward data for layout tests: every list the mock returns is stretched to thousands of rows, and +// every name, description and selection made very long. Turned on with `?mock&awkward`; the layout +// test opens each page this way. Pure apart from wrapping the client it is given. + +import type { ApiClient } from '../client'; + +/** Fields that name or identify an item: copies get a long, unique value. */ +const NAMES = ['id', 'key', 'name', 'jobKey', 'runId', 'eventId', 'nodeKey', 'migrationEventId', 'invocationId', 'runKey', 'table', 'group', 'displayName']; +/** Fields that point at another item, such as a check's node: copies point at one of `REFERENCED` long names, shared by a few rows each, so a filter over them has thousands of options. */ +const REFS = ['node']; +const REFERENCED = 2000; +/** Fields of free text: made very long. */ +const TEXTS = ['description', 'select', 'summary', 'message', 'title', 'text', 'cadence', 'className', 'sourceFile', 'path']; +const FILLER = ' with a deliberately long tail of words that keeps going well past any column width a list could give it'; +const LONG_ID = '_with_an_unusually_long_identifier_that_no_column_is_wide_enough_to_show'; + +/** `items` stretched to `count` rows: the originals first, unchanged in their names, then copies with long, unique names; text made long throughout. */ +export function stretch(items: readonly T[], count: number): T[] { + if (!items.length || typeof items[0] !== 'object' || items[0] === null) return [...items]; + return Array.from({ length: Math.max(count, items.length) }, (_, i) => { + const copy = { ...(items[i % items.length] as Record) }; + for (const field of NAMES) if (i >= items.length && typeof copy[field] === 'string') copy[field] = `${copy[field]}${LONG_ID}_${i}`; + // The same reference number always names the same item, so there are at most `REFERENCED` of them. + const ref = i % REFERENCED; + const base = items[ref % items.length] as Record; + for (const field of REFS) if (i >= items.length && typeof copy[field] === 'string' && typeof base[field] === 'string') copy[field] = ref < items.length ? base[field] : `${base[field]}${LONG_ID}_${ref}`; + for (const field of TEXTS) if (typeof copy[field] === 'string' && copy[field]) copy[field] = `${copy[field]}${FILLER}`; + return copy as T; + }); +} + +/** The client with every `list…` read that answers an array stretched to `count` rows. */ +export function awkward(client: ApiClient, count = 5000): ApiClient { + return new Proxy(client, { + get(target, prop, receiver) { + const value = Reflect.get(target, prop, receiver); + if (typeof value !== 'function' || typeof prop !== 'string' || !prop.startsWith('list')) return typeof value === 'function' ? value.bind(target) : value; + return async (...args: unknown[]) => { + const answer = await value.apply(target, args); + return Array.isArray(answer) ? stretch(answer, count) : answer; + }; + } + }); +} diff --git a/ui/src/lib/api/mock/big-group.ts b/ui/src/lib/api/mock/big-group.ts new file mode 100644 index 0000000000..7f4473fa7b --- /dev/null +++ b/ui/src/lib/api/mock/big-group.ts @@ -0,0 +1,72 @@ +// A large job group, for layout tests and for looking at how a big group draws: members in stages, +// each waiting for two members of the stage before it, and a few members on their own. The server +// lists every wait, the implied ones too, so each member's waits are its whole upstream. Pure. + +export const BIG_GROUP = 'warehouse_nightly'; + +const STAGES = ['load', 'stage', 'clean', 'join', 'enrich', 'aggregate', 'publish']; +// One loader feeds ten staging jobs, so the first column's card sits halfway down beside a second +// column that starts at the top: the shape where a fit starting from the card nearest the corner left +// the first column off screen. +const SIZES = [1, 10, 5, 5, 5, 5, 5]; +const ALONE = 6; + +export type BigGroupShape = { + /** Members in declaration order. */ + members: string[]; + /** What each member waits for directly. */ + direct: Map; + /** What the server lists: every member upstream of each one. */ + waits: Map; + /** Members with no waits either way. */ + alone: string[]; +}; + +/** + * The mock catalog's node each stage's members build: a lineage chain, so most waits have lineage + * behind them. activity_report doesn't read daily_activity_rollup, so publish's waits on aggregate + * have none and draw dashed, as an ordering-only wait does. + */ +const NODES: Record = { + load: 'raw__orders', + stage: 'stg_orders', + clean: 'fact_orders', + join: 'hourly_order_activity', + enrich: 'hourly_activity_ctx', + aggregate: 'daily_activity_rollup', + publish: 'activity_report', + housekeeping: 'country_codes' +}; + +/** + * Where a member is declared: a folder per stage under `jobs/warehouse/`, and one export job alone in + * its own folder, which the group page draws as the job's card rather than a folder. + */ +export const bigGroupFile = (member: string): string => `jobs/warehouse/${member === 'publish_e' ? 'exports' : member.split('_')[0]}/${member}.py`; + +/** The node a member builds, as a selector. */ +export const bigGroupSelect = (member: string): string => NODES[member.split('_')[0]]; + +export function bigGroupShape(): BigGroupShape { + const stages = STAGES.map((name, s) => Array.from({ length: SIZES[s] }, (_, i) => `${name}_${String.fromCharCode(97 + i)}`)); + const alone = Array.from({ length: ALONE }, (_, i) => `housekeeping_${i + 1}`); + const direct = new Map(); + stages.forEach((stage, s) => + stage.forEach((m, i) => { + const prev = stages[s - 1] ?? []; + direct.set(m, prev.length ? [...new Set([prev[i % prev.length], prev[(i + 1) % prev.length]])] : []); + }) + ); + for (const m of alone) direct.set(m, []); + // The export also waits directly for a staging job none of its other waits come through, an edge + // across four columns of jobs. + direct.get('publish_e')!.push('stage_j'); + const upstreamOf = (m: string): Set => new Set((direct.get(m) ?? []).flatMap((u) => [u, ...upstreamOf(u)])); + const members = [...alone.slice(0, 3), ...stages.flat(), ...alone.slice(3)]; + return { + members, + direct, + waits: new Map(members.map((m) => [m, [...upstreamOf(m)]])), + alone + }; +} diff --git a/ui/src/lib/api/mock/build-fixtures.ts b/ui/src/lib/api/mock/build-fixtures.ts index bcc0817ff8..b6e014eea5 100644 --- a/ui/src/lib/api/mock/build-fixtures.ts +++ b/ui/src/lib/api/mock/build-fixtures.ts @@ -306,6 +306,8 @@ export type CheckSpec = { history: { failEvery?: number; warnEvery?: number; measuredBase?: number }; /** A failed build of the node failed in this audit's query (an execution error), not before audits ran. */ errorOnFailure?: boolean; + /** The scheduler run whose build let this error-severity audit's latest failure through (`ignore_audit_failures`). */ + ignoredIn?: string; /** Compiled audit query, with `{relation}` for the attached object. */ sql: string; evidenceColumns?: string[]; @@ -313,12 +315,12 @@ export type CheckSpec = { export const CHECK_SPECS: CheckSpec[] = [ { id: 'chk-fo-not-null', name: 'not_null(order_id)', definition: 'not_null', kind: 'audit', attachment: 'model', node: 'fact_orders', column: 'order_id', severity: 'error', runScopePhase: 'delta_and_final', evaluationMode: 'violations', thresholds: [], description: null, latest: { outcome: 'pass', violations: 0 }, history: {}, sql: 'SELECT order_id FROM {relation} WHERE order_id IS NULL' }, - { id: 'chk-fo-unique', name: 'unique(order_id)', definition: 'unique', kind: 'audit', attachment: 'model', node: 'fact_orders', column: 'order_id', severity: 'error', runScopePhase: 'delta_and_final', evaluationMode: 'violations', thresholds: [], description: null, latest: { outcome: 'pass', violations: 0 }, history: { failEvery: 17 }, sql: 'SELECT order_id, count(*) AS n FROM {relation} GROUP BY order_id HAVING count(*) > 1', evidenceColumns: ['order_id', 'n'] }, + { id: 'chk-fo-unique', name: 'unique(order_id)', definition: 'unique', kind: 'audit', attachment: 'model', node: 'fact_orders', column: 'order_id', severity: 'error', runScopePhase: 'delta_and_final', evaluationMode: 'violations', thresholds: [], description: null, latest: { outcome: 'fail', violations: 3 }, history: { failEvery: 17 }, ignoredIn: 'run-2b70', sql: 'SELECT order_id, count(*) AS n FROM {relation} GROUP BY order_id HAVING count(*) > 1', evidenceColumns: ['order_id', 'n'] }, { id: 'chk-fo-rel', name: 'relationships(customer_id → dim_customers)', definition: 'relationships', kind: 'audit', attachment: 'model', node: 'fact_orders', column: 'customer_id', severity: 'warn', runScopePhase: 'final', evaluationMode: 'violations', thresholds: [], description: 'Every order belongs to a known customer.', latest: { outcome: 'pass', violations: 0 }, history: { warnEvery: 9 }, sql: 'SELECT f.customer_id FROM {relation} f LEFT JOIN "WAFFLE_SHOP"."MARTS"."DIM_CUSTOMERS" d USING (customer_id) WHERE d.customer_id IS NULL', evidenceColumns: ['customer_id'] }, { id: 'chk-sc-not-null', name: 'not_null(customer_id)', definition: 'not_null', kind: 'audit', attachment: 'model', node: 'stg_customers', column: 'customer_id', severity: 'error', runScopePhase: 'final', evaluationMode: 'violations', thresholds: [], description: null, latest: { outcome: 'pass', violations: 0 }, history: {}, sql: 'SELECT customer_id FROM {relation} WHERE customer_id IS NULL' }, { id: 'chk-sp-not-null', name: 'not_null(payment_id)', definition: 'not_null', kind: 'audit', attachment: 'model', node: 'stg_payments', column: 'payment_id', severity: 'warn', runScopePhase: 'final', evaluationMode: 'violations', thresholds: [], description: null, latest: { outcome: 'warn', violations: 2 }, history: { warnEvery: 3 }, sql: 'SELECT order_id, payment_method, amount_cents FROM {relation} WHERE payment_id IS NULL', evidenceColumns: ['order_id', 'payment_method', 'amount_cents'] }, { id: 'chk-so-values', name: 'accepted_values(order_status)', definition: 'accepted_values', kind: 'audit', attachment: 'model', node: 'stg_orders', column: 'order_status', severity: 'error', runScopePhase: 'final', evaluationMode: 'violations', thresholds: [], description: null, latest: { outcome: 'pass', violations: 0 }, history: {}, sql: "SELECT order_status, count(*) AS n FROM {relation} WHERE order_status NOT IN ('completed', 'pending', 'cancelled') GROUP BY 1" }, - { id: 'chk-dc-unique', name: 'unique(customer_id)', definition: 'unique', kind: 'audit', attachment: 'model', node: 'dim_customers', column: 'customer_id', severity: 'error', runScopePhase: 'delta_and_final', evaluationMode: 'violations', thresholds: [], description: null, latest: { outcome: 'pass', violations: 0 }, history: {}, sql: 'SELECT customer_id, count(*) AS n FROM {relation} GROUP BY customer_id HAVING count(*) > 1' }, + { id: 'chk-dc-unique', name: 'unique(customer_id)', definition: 'unique', kind: 'audit', attachment: 'model', node: 'dim_customers', column: 'customer_id', severity: 'error', runScopePhase: 'delta_and_final', evaluationMode: 'violations', thresholds: [], description: null, latest: { outcome: 'fail', violations: 1 }, history: {}, ignoredIn: 'run-2b70', sql: 'SELECT customer_id, count(*) AS n FROM {relation} GROUP BY customer_id HAVING count(*) > 1' }, { id: 'chk-dr-positive', name: 'expression_is_true(total_revenue_cents >= 0)', definition: 'expression_is_true', kind: 'audit', attachment: 'model', node: 'daily_revenue', column: 'total_revenue_cents', severity: 'error', runScopePhase: 'delta_and_final', evaluationMode: 'violations', thresholds: [], description: 'Revenue is never negative.', latest: { outcome: 'pass', violations: 0 }, history: {}, errorOnFailure: true, sql: 'SELECT revenue_date, total_revenue_cents FROM {relation} WHERE NOT (total_revenue_cents >= 0)', evidenceColumns: ['revenue_date', 'total_revenue_cents'] }, { id: 'chk-dr-unique', name: 'unique(revenue_date)', definition: 'unique', kind: 'audit', attachment: 'model', node: 'daily_revenue', column: 'revenue_date', severity: 'error', runScopePhase: 'final', evaluationMode: 'violations', thresholds: [], description: null, latest: { outcome: 'pass', violations: 0 }, history: {}, sql: 'SELECT revenue_date, count(*) AS n FROM {relation} GROUP BY 1 HAVING count(*) > 1' }, { id: 'chk-hoa-rows', name: 'row_count(activity_hour)', definition: 'row_count', kind: 'audit', attachment: 'model', node: 'hourly_order_activity', column: null, severity: 'warn', runScopePhase: 'delta_and_final', evaluationMode: 'measurement', thresholds: [{ level: 'warn', operator: 'below', value: 500 }, { level: 'error', operator: 'below', value: 100 }], description: 'Each batch should hold a normal hour of orders.', latest: { outcome: 'warn', measured: 412 }, history: { measuredBase: 640, warnEvery: 6 }, sql: 'SELECT count(*) AS measured_value FROM {relation} WHERE activity_hour >= :batch_start AND activity_hour < :batch_end' }, diff --git a/ui/src/lib/api/mock/build.ts b/ui/src/lib/api/mock/build.ts index 278da0c8bc..b5b5958afe 100644 --- a/ui/src/lib/api/mock/build.ts +++ b/ui/src/lib/api/mock/build.ts @@ -5,6 +5,7 @@ import { ApiError } from '../client'; import { groupStatusOf, problemRunsOf } from '../http-overview'; +import { recentDeploys } from '$lib/build-area/overview-panels'; import type { AttentionItem, BuiltDefinition, @@ -279,7 +280,7 @@ export class MockBuild { }, attention, groups: groupStatusOf(nodes), - recentChanges: this.#changes(scope).slice(0, 4), + recentChanges: recentDeploys(this.#changes(scope)), latestRun: latest ? { id: latest.id, status: latest.status, endedAt: latest.endedAt } : null }); } @@ -550,7 +551,8 @@ export class MockBuild { violationCount: violations, measuredValue: isViolations || !counted ? null : (extra.measured ?? (outcome === 'pass' ? measured(index) : breach)), evidenceCount: violations, - evidenceTruncated: (violations ?? 0) > 10 + evidenceTruncated: (violations ?? 0) > 10, + ignored: false }; }; // Audits run inside the build, so the node's outcome bounds the check's. @@ -571,6 +573,10 @@ export class MockBuild { const outcome: CheckOutcome = !out.length && i === 0 ? spec.latest.outcome : fail ? (spec.severity === 'error' ? 'fail' : 'warn') : warn ? 'warn' : 'pass'; out.push(at(seed, i, outcome, { evaluatedAt: iso(oldest - (i - entries.length + 1) * 4 * HOUR), ...(i === 0 ? { violations: spec.latest.violations, measured: spec.latest.measured } : {}) })); } + // The latest result of an audit a build let fail: from that run, failed, and marked ignored. + if (spec.ignoredIn && out.length) { + out[0] = { ...at(spec.ignoredIn, 0, 'fail', { runId: spec.ignoredIn, evaluatedAt: out[0].evaluatedAt, violations: spec.latest.violations }), ignored: true }; + } return out.slice(0, limit); } diff --git a/ui/src/lib/api/mock/large-plan.ts b/ui/src/lib/api/mock/large-plan.ts new file mode 100644 index 0000000000..c56677d97b --- /dev/null +++ b/ui/src/lib/api/mock/large-plan.ts @@ -0,0 +1,211 @@ +// A plan the size of a real project's `sqb build --changed`, for the plan page with `?mock&large`: about +// 2,400 nodes, with changed models carrying diffs (some whitespace-only, some changing keyword case), +// first runs, renames, loads, Python nodes, repeated warnings and 1,507 routine models. Served as the +// server does: a summary, with routine models, held-back and skipped nodes a page at a time. Pure. + +import type { PlanDiff, PlanDiffStats, PlanDocument, PlanModel, PlanPage, PlanPageSection, PlanSectionCount, PlanSectionNote, PlanWarning } from '../types'; + +const DOMAINS = ['orders', 'customers', 'inventory', 'payments', 'shipping', 'returns', 'pricing', 'suppliers', 'regions', 'finance', 'marketing', 'support']; +const LAYERS = ['stg', 'int', 'mart']; +const NOUNS = ['daily_rollup', 'events', 'snapshot', 'curated', 'enriched', 'mapped', 'totals', 'history', 'latest', 'summary']; +/** A neutral model name, distinct for each `i`. */ +const name = (i: number) => `${DOMAINS[i % DOMAINS.length]}__${LAYERS[Math.floor(i / DOMAINS.length) % LAYERS.length]}__${NOUNS[Math.floor(i / (DOMAINS.length * LAYERS.length)) % NOUNS.length]}_${i}`; + +const HEAD = ['--- previous', '+++ current']; +/** A query diff of `added` and `removed` lines. */ +function queryDiff(i: number, added: number, removed: number): string[] { + return [...HEAD, `@@ -1,${removed + 2} +1,${added + 2} @@`, ' select', ...Array.from({ length: removed }, (_, k) => `- old_column_${k},`), ...Array.from({ length: added }, (_, k) => `+ new_column_${k}_${i},`), ` from ${name(i + 1)}`]; +} +const model = (i: number, reason: string, extra: Partial = {}): PlanModel => ({ + name: name(i), + materialization_type: 'table', + action: 'create_table', + reason, + backfill: { action: 'forward', duration: null }, + qualified_name: `analytics.${name(i)}`, + ...extra +}); +const incremental = (extra: Partial = {}): Partial => ({ + materialization_type: 'incremental', + action: 'incremental_merge', + incremental_strategy: 'merge', + cursor_column: 'updated_at', + cursor_type: 'timestamp', + cursor: { requested_bounds: { start: null, end: null }, resolved_bounds: { start: '__SQB_CURSOR_START__', end: '__SQB_CURSOR_END__' } }, + ...extra +}); + +/** The whole plan, as `sqb plan --json` prints it. */ +export function largePlanDocument(changed: boolean): PlanDocument { + let i = 0; + const models: PlanModel[] = []; + // 94 changed: 80 query changes of every size, 5 whitespace-only, 2 changing keyword case, 6 config, 1 function. + for (let k = 0; k < 80; k++) models.push(model(i++, 'query_changed', { action: 'create_table', backfill: { action: k % 3 ? 'forward' : 'full', duration: null }, query_diff: queryDiff(i, 1 + ((k * 7) % 90), (k * 5) % 60) })); + for (let k = 0; k < 5; k++) models.push(model(i++, 'query_changed', { query_diff: [...HEAD, '@@ -1,2 +1,3 @@', '-select id, amount', '-from orders', '+select id,', '+ amount', '+from orders'] })); + for (let k = 0; k < 2; k++) models.push(model(i++, 'query_changed', { query_diff: [...HEAD, '@@ -1,2 +1,2 @@', '-select id from orders', '+SELECT id FROM orders'] })); + for (let k = 0; k < 4; k++) models.push(model(i++, 'config_changed', { config_diff: [...HEAD, '@@ -1,3 +1,3 @@', ' "materialized": "table",', '- "full_refresh": true,', '+ "full_refresh": false,'] })); + models.push(model(i++, 'config_changed', { ...incremental(), config_diff: ['- "cursor": "observed_at",', '+ "cursor": "event_date",'], query_diff: queryDiff(i, 3, 2) })); + models.push(model(i++, 'config_changed', { config_diff: ['- "grain": "hour",', '+ "grain": "day",', '- "lookback": 2,', '+ "lookback": 3,', '- "partition_by": "event_date",', '+ "partition_by": "event_hour",'] })); + models.push(model(i++, 'function_changed', { changed_functions: ['parse_orders_html'] })); + // 672 first runs; a third are incremental, their range the run's own bounds. + for (let k = 0; k < 672; k++) models.push(model(i++, 'first_run', k % 3 === 0 ? incremental() : {})); + // 1,507 routine models. + const routineFrom = i; + for (let k = 0; k < 1507; k++) models.push(model(i++, k % 4 === 0 ? 'normal_incremental' : 'no_change', k % 4 === 0 ? incremental({ cursor: null }) : {})); + const keys = models.map((m) => `model:${m.name}`); + return { + selected_count: 2400, + models, + seeds: Array.from({ length: 10 }, (_, k) => ({ name: `reference_codes_${k}`, reason: 'no_change' })), + source_loads: Array.from({ length: 64 }, (_, k) => ({ name: `${DOMAINS[k % DOMAINS.length]}_feed_${k}`, kind: 'source', write_strategy: 'append', cursor_column: 'loaded_at', is_reload: false })), + functions: Array.from({ length: 12 }, (_, k) => ({ name: `format_${DOMAINS[k % DOMAINS.length]}_code`, language: 'sql', reason: 'no_change' })), + python_nodes: Array.from({ length: 433 }, (_, k) => ({ name: `${DOMAINS[k % DOMAINS.length]}_export_${k}`, kind: 'asset', phase: 'after_models', identity_status: 'known' })), + warnings: [ + // 71 alike, about 24 scenarios' unused mocks, and 3 of another kind. + ...Array.from({ length: 71 }, (_, k) => ({ severity: 'warning', message: `Scenario 'test_${DOMAINS[k % 12]}_${k % 24}' provides fixture for unknown model '${name(k + 3000)}'.` })), + ...Array.from({ length: 3 }, (_, k) => ({ severity: 'warning', message: `Scenario 'test_returns_${k}' requires source 'raw.returns_${k}', but no fixture was provided. Add a CTE named __source__raw__returns_${k} AS (...).` })) + ], + migrations: Array.from({ length: 121 }, (_, k) => ({ model: name(5000 + k), origin_model: name(6000 + k), decision: 'renamed', completed_at: null })), + column_migrations: [], + table_type_conversions: [], + retention: [], + synced_copy: null, + changed: changed + ? { + counts: ['definition'], + reach: 'all', + runs: keys.map((node, k) => ({ node, reason: k >= routineFrom ? 'no completed record' : 'definition changed', downstream_of: null })), + held_back: Array.from({ length: 40 }, (_, k) => ({ node: `model:${name(7000 + k)}`, refreshed_by: ['nightly_all'] })), + unchanged: Array.from({ length: 300 }, (_, k) => `model:${name(8000 + k)}`) + } + : null + }; +} + +const MODEL_SECTIONS: PlanPageSection[] = ['full_refresh', 'query_changed', 'function_changed', 'config_changed', 'schema_changed', 'first_run', 'renamed']; +/** Every section in plan order, as the server counts them. */ +const SECTIONS: PlanPageSection[] = [...MODEL_SECTIONS, 'routine_models', 'migrations', 'column_migrations', 'table_type_conversions', 'retention', 'loader_reloads', 'loader_loads', 'source_reloads', 'source_loads', 'changed_functions', 'functions', 'seeds', 'python_nodes', 'held_back', 'unchanged', 'warnings']; +const ROW_LISTS = ['models', 'seeds', 'source_loads', 'functions', 'python_nodes', 'warnings', 'migrations', 'column_migrations', 'table_type_conversions', 'retention'] as const; +const keyName = (key: string) => key.slice(key.indexOf(':') + 1); + +/** The section's rows in plan order, as the server's plan rows read finds them. */ +function sectionRows(doc: PlanDocument, section: PlanPageSection): { rows: unknown[]; name: (row: unknown) => string } { + const runs = doc.changed && new Set(doc.changed.runs.map((r) => r.node)); + const runsNode = (key: string) => !runs || runs.has(key); + const named = (rows: { name: string }[]) => ({ rows, name: (row: unknown) => (row as { name: string }).name }); + const byModel = (rows: { model: string }[]) => ({ rows, name: (row: unknown) => (row as { model: string }).model }); + if (MODEL_SECTIONS.includes(section) || section === 'routine_models') { + const built = doc.models.filter((m) => m.action !== 'skip' && runsNode(`model:${m.name}`)); + return named(built.filter((m) => (MODEL_SECTIONS.includes(m.reason as PlanPageSection) ? m.reason === section : section === 'routine_models'))); + } + const loads = (kind: string, reload: boolean) => named(doc.source_loads.filter((l) => l.kind === kind && l.is_reload === reload)); + const changedFunction = (f: PlanDocument['functions'][number]) => !!f.reason && f.reason !== 'no_change'; + switch (section) { + case 'migrations': + return byModel(doc.migrations); + case 'column_migrations': + return byModel(doc.column_migrations); + case 'table_type_conversions': + return byModel(doc.table_type_conversions); + case 'retention': + return byModel(doc.retention); + case 'loader_reloads': + return loads('loader', true); + case 'loader_loads': + return loads('loader', false); + case 'source_reloads': + return loads('source', true); + case 'source_loads': + return loads('source', false); + case 'changed_functions': + return named(doc.functions.filter(changedFunction)); + case 'functions': + return named(doc.functions.filter((f) => !changedFunction(f))); + case 'seeds': + return named(doc.seeds.filter((s) => runsNode(`seed:${s.name}`))); + case 'python_nodes': + return named(doc.python_nodes); + case 'held_back': + return { rows: doc.changed?.held_back ?? [], name: (row) => keyName((row as { node: string }).node) }; + case 'unchanged': + return { rows: doc.changed?.unchanged ?? [], name: (row) => keyName(row as string) }; + default: + return { rows: doc.warnings, name: (row) => `${(row as PlanWarning).model_name ?? ''} ${(row as PlanWarning).message}` }; + } +} + +/** The notes of a section's model rows the mock counts: why `--changed` runs each. */ +function runNotes(doc: PlanDocument, rows: unknown[]): PlanSectionNote[] { + const runs = new Map((doc.changed?.runs ?? []).map((r) => [r.node, r])); + const counts = new Map(); + for (const row of rows) { + const run = runs.get(`model:${(row as { name: string }).name}`); + if (!run) continue; + const [kind, value] = run.downstream_of ? ['reads', keyName(run.downstream_of)] : ['run_reason', run.reason]; + const note = counts.get(`${kind}\u0000${value}`) ?? { kind, value, count: 0 }; + note.count++; + counts.set(`${kind}\u0000${value}`, note); + } + return [...counts.values()].sort((a, b) => b.count - a.count); +} + +/** The server's summary of `doc`: its facts, every row list empty, and each section counted. */ +export function planSummary(doc: PlanDocument): PlanDocument { + const sections: PlanSectionCount[] = SECTIONS.flatMap((section) => { + const { rows } = sectionRows(doc, section); + const models = MODEL_SECTIONS.includes(section) || section === 'routine_models'; + return rows.length ? [{ section, total: rows.length, notes: models ? runNotes(doc, rows) : [] }] : []; + }); + const empty = Object.fromEntries(ROW_LISTS.map((list) => [list, []])); + return { + ...doc, + ...empty, + sections, + pages: Object.fromEntries(sections.map((s) => [s.section, s.total])), + changed: doc.changed && { ...doc.changed, runs: [], held_back: [], unchanged: [] }, + synced_copy: doc.synced_copy && { ...doc.synced_copy, cursor_window_starts: {} } + }; +} + +/** One page of a section's rows whose name contains `name`, as the server's plan rows read returns it. */ +export function planPage(doc: PlanDocument, section: PlanPageSection, offset: number, limit = 50, name?: string): PlanPage { + const { rows: all, name: nameOf } = sectionRows(doc, section); + const wanted = name?.toLowerCase(); + const matching = wanted ? all.filter((row) => nameOf(row).toLowerCase().includes(wanted)) : all; + const rows = matching.slice(offset, offset + limit); + const models = MODEL_SECTIONS.includes(section) || section === 'routine_models'; + const names = new Set(models ? rows.map((row) => (row as { name: string }).name) : []); + const starts = doc.synced_copy?.cursor_window_starts ?? {}; + return { + total: matching.length, + section_total: all.length, + rows: models ? (rows as PlanModel[]).map(withDiffStats) : rows, + runs: (doc.changed?.runs ?? []).filter((r) => names.has(keyName(r.node))), + warnings: doc.warnings.filter((w) => names.has(w.model_name ?? w.message.split(':', 1)[0])), + cursor_window_starts: Object.fromEntries(Object.entries(starts).filter(([model]) => names.has(model))) + }; +} + +/** A diff's changed lines, and whether they differ only in whitespace, as the server counts them. */ +function stats(diff: string[]): PlanDiffStats { + const side = (sign: string) => diff.filter((l) => l.startsWith(sign) && !/^(---|\+\+\+)/.test(l)); + const text = (lines: string[]) => lines.map((l) => l.slice(1)).join('').replace(/\s+/g, ''); + const [added, removed] = [side('+'), side('-')]; + return { added: added.length, removed: removed.length, whitespace_only: text(added) === text(removed) }; +} + +/** A model row as a page carries it: its diff counted, the text left for the diff read. */ +function withDiffStats({ query_diff, config_diff, ...m }: PlanModel): PlanModel { + if (!query_diff && !config_diff) return m; + return { ...m, diff_stats: { ...(query_diff ? { query: stats(query_diff) } : {}), ...(config_diff ? { config: stats(config_diff) } : {}) } }; +} + +/** A model whose diff the server won't give, as for a user without permission to read its SQL. */ +export const DIFF_WITHHELD = name(79); + +/** A model's diff text from the whole document, as the server's plan diff read returns it. */ +export function planModelDiff(doc: PlanDocument, model: string): PlanDiff | null { + const found = doc.models.find((m) => m.name === model); + if (!found || model === DIFF_WITHHELD || !(found.query_diff || found.config_diff)) return null; + return { model, ...(found.query_diff ? { query_diff: found.query_diff } : {}), ...(found.config_diff ? { config_diff: found.config_diff } : {}) }; +} diff --git a/ui/src/lib/api/mock/large-project.ts b/ui/src/lib/api/mock/large-project.ts new file mode 100644 index 0000000000..02104efa43 --- /dev/null +++ b/ui/src/lib/api/mock/large-project.ts @@ -0,0 +1,87 @@ +// A project the size of a real one, for looking at Lineage's overview and for the layout test: 13 +// top-level folders seven layers deep, too wide to show whole at a readable zoom, each with staging and marts +// subfolders, about 400 nodes, more than Lineage draws one by one, so it opens by folder. Turned on +// with `?mock&large`. Pure apart from wrapping the client it is given. + +import type { ApiClient } from '../client'; +import type { PlanDocument, PlanPage, PlanResult } from '../types'; +import { largePlanDocument, planModelDiff, planPage, planSummary } from './large-plan'; +import { recordPlanDiff, recordPlanRows } from './plan-rows-log'; +import type { CatalogNode } from '../types'; + +const LAYERS = [['sources'], ['seeds', 'orders'], ['customers', 'inventory'], ['payments', 'shipping'], ['returns', 'pricing'], ['suppliers', 'regions'], ['finance', 'marketing']]; +const PER_SUBFOLDER = 15; + +/** The models' last name parts, one per model of a subfolder. */ +const NOUNS = ['events', 'daily_totals', 'snapshot', 'latest', 'history', 'summary', 'mapped', 'enriched', 'curated', 'keys', 'lookup', 'changes', 'facts', 'rollup', 'audit_trail']; + +/** The large project's nodes, each a copy of `template` with its own key, folder and lineage. */ +export function largeProjectNodes(template: CatalogNode): CatalogNode[] { + const nodes: CatalogNode[] = []; + const marts = new Map(); + LAYERS.forEach((layer, depth) => { + layer.forEach((folder, f) => { + // Each folder reads the marts of two folders in the layer before it. + const before = LAYERS[depth - 1] ?? []; + const feeds = before.length ? [before[f % before.length], before[(f + 1) % before.length]].flatMap((b) => marts.get(b) ?? []) : []; + // Names as projects write them, layered with `__`: the same endings recur across layers and + // folders, so a zoomed-out card needs more of its name to tell it apart. + const staging = Array.from({ length: PER_SUBFOLDER }, (_, i) => `${folder}__stg__${NOUNS[i % NOUNS.length]}`); + const mart = Array.from({ length: PER_SUBFOLDER }, (_, i) => `${folder}__mart__${NOUNS[i % NOUNS.length]}`); + staging.forEach((key, i) => + nodes.push({ ...template, key, group: `${folder}/staging`, path: `models/${folder}/staging/${key}.sql`, upstream: feeds.length ? [feeds[i % feeds.length]] : [], description: `Staged ${folder} data` }) + ); + mart.forEach((key, i) => nodes.push({ ...template, key, group: `${folder}/marts`, path: `models/${folder}/marts/${key}.sql`, upstream: [staging[i], staging[(i + 1) % staging.length]], description: `${folder} mart` })); + marts.set(folder, mart); + // Finance also reconciles against the orders marts directly, a read across five layers. + if (folder === 'finance') nodes.find((n) => n.key === staging[0])!.upstream.push(marts.get('orders')![0]); + }); + }); + return nodes; +} + +/** The large plans planned so far, by request, for their pages. */ +const plans = new Map(); + +/** The client with its catalog replaced by the large project, and a build planned as the large plan. */ +export function largeProject(client: ApiClient): ApiClient { + return new Proxy(client, { + get(target, prop, receiver) { + const value = Reflect.get(target, prop, receiver); + if (prop === 'planBuild') { + return async (...args: Parameters): Promise => { + const result = await target.planBuild(...args); + if (!result.plan) return result; + const doc = largePlanDocument(args[1].some((t) => t === '--changed' || t.startsWith('--changed='))); + const requestId = crypto.randomUUID(); + plans.set(requestId, doc); + return { plan: { ...planSummary(doc), timings: result.plan.timings }, error: null, requestId }; + }; + } + if (prop === 'planRows') { + return async (...[scope, requestId, section, offset, limit, name]: Parameters): Promise => { + const doc = plans.get(requestId); + if (!doc) return target.planRows(scope, requestId, section, offset, limit, name); + recordPlanRows(section, offset, limit, name); + await new Promise((resolve) => setTimeout(resolve, 120)); + return planPage(doc, section, offset, limit, name); + }; + } + if (prop === 'planDiff') { + return async (...[scope, requestId, model]: Parameters) => { + const doc = plans.get(requestId); + if (!doc) return target.planDiff(scope, requestId, model); + recordPlanDiff(model); + await new Promise((resolve) => setTimeout(resolve, 80)); + return planModelDiff(doc, model); + }; + } + if (prop !== 'listCatalogNodes') return typeof value === 'function' ? value.bind(target) : value; + return async (...args: Parameters) => { + const real = await target.listCatalogNodes(...args); + const template = real.find((n) => n.kind === 'table') ?? real[0]; + return template ? largeProjectNodes(template) : real; + }; + } + }); +} diff --git a/ui/src/lib/api/mock/mock-client.ts b/ui/src/lib/api/mock/mock-client.ts index 2a3abcc862..8799610cec 100644 --- a/ui/src/lib/api/mock/mock-client.ts +++ b/ui/src/lib/api/mock/mock-client.ts @@ -1,9 +1,17 @@ +import { recordPlanRows } from './plan-rows-log'; import type { ApiClient } from '../client'; import { ApiError } from '../client'; import { LocalChangeBus, type ConnectionState } from '../signals'; +import { stateOf } from '../node-states'; import type { AdhocRunCheck, + SelectionEvaluation, + SelectionRequest, + SelectionVocabulary, PlanResult, + PlanPage, + PlanDiff, + PlanPageSection, AdhocRunRequest, Bootstrap, BuiltInRole, @@ -26,6 +34,8 @@ import type { RunCostRow, CostSummary, NodeBuildCost, + NodeState, + NodeStates, Scope, Snapshot, SnapshotNode, @@ -66,6 +76,9 @@ export class MockApiClient implements ApiClient { readonly #syncs = new Map(); readonly #platform: MockPlatformApi; readonly #build: MockBuild; + /** Node states as last read or announced, by location, so a finished run announces only what changed. */ + readonly #nodeStates = new Map }>(); + #announcing: Promise = Promise.resolve(); constructor(options: MockOptions = {}) { this.#options = { @@ -105,6 +118,26 @@ export class MockApiClient implements ApiClient { platform: this.#platform, platformEvents: () => this.#platformEvents() }); + // As the server does: when runs or checks change, the node states that changed go out on `node_status`. + this.changes.subscribe((signal) => { + if (signal.channel === 'runs' || signal.channel === 'checks') this.#announcing = this.#announcing.then(() => this.#announceNodeStates()); + }); + } + + async #currentStates(scope: Scope): Promise> { + const nodes = await this.#build.listCatalogNodes(scope); + return Object.fromEntries(nodes.map((n) => [n.key, stateOf(n)])); + } + + async #announceNodeStates(): Promise { + for (const [locationId, held] of this.#nodeStates) { + const current = await this.#currentStates({ locationId }); + const changed = Object.fromEntries(Object.entries(current).filter(([key, state]) => JSON.stringify(held.states[key]) !== JSON.stringify(state))); + if (!Object.keys(changed).length) continue; + const previousVersion = held.version; + this.#nodeStates.set(locationId, { version: previousVersion + 1, states: current }); + this.changes.emit(`node_status:${locationId}`, { version: previousVersion + 1, previousVersion, states: changed }); + } } async bootstrap(): Promise { @@ -136,19 +169,33 @@ export class MockApiClient implements ApiClient { overview(...args: Parameters) { this.#location(args[0]); - return this.#build.overview(...args); + return this.#build.overview(args[0]); } listCatalogNodes(...args: Parameters) { this.#location(args[0]); return this.#build.listCatalogNodes(...args); } + /** The first read starts the location's versions; later runs announce changes against it. */ + async listNodeStates(scope: Scope): Promise { + const location = this.#location(scope); + await this.#announcing; + const states = await this.#currentStates(scope); + const held = this.#nodeStates.get(scope.locationId); + const version = held && JSON.stringify(held.states) === JSON.stringify(states) ? held.version : (held?.version ?? 0) + 1; + this.#nodeStates.set(scope.locationId, { version, states }); + // The demo's location plans what changed: each changed node, and every node reading one of them. + const nodes = await this.#build.listCatalogNodes(scope); + const changed = nodes.filter((n) => states[n.key]?.definitionChanged).map((n) => n.key); + const downstream = nodes.filter((n) => !changed.includes(n.key) && n.upstream.some((u) => changed.includes(u))).map((n) => n.key); + return { snapshotId: location.snapshot?.id ?? 'unpublished', version, states, changedBuild: { generation: 1, changed, downstream, failing: null } }; + } getCatalogNode(...args: Parameters) { this.#location(args[0]); return this.#build.getCatalogNode(...args); } getColumnLineage(...args: Parameters) { this.#location(args[0]); - return this.#build.getColumnLineage(...args); + return this.#build.getColumnLineage(args[0], args[1]); } listChecks(...args: Parameters) { this.#location(args[0]); @@ -281,6 +328,16 @@ export class MockApiClient implements ApiClient { return this.#operate.listSnapshotNodes(scope); } + async evaluateSelection(scope: Scope, request: SelectionRequest): Promise { + this.#location(scope); + return this.#operate.evaluateSelection(scope, request); + } + + async selectionVocabulary(scope: Scope): Promise { + this.#location(scope); + return this.#operate.selectionVocabulary(scope); + } + async checkAdhocRun(scope: Scope, request: AdhocRunRequest): Promise { this.#location(scope); return this.#operate.checkAdhocRun(scope, request); @@ -300,6 +357,22 @@ export class MockApiClient implements ApiClient { return this.#operate.planBuild(scope, command, changed); } + /** The mock plans whole documents, so no section is paged; `?large` serves a paged plan (large-project.ts). */ + async planRows(scope: Scope, requestId: string, section: PlanPageSection, offset: number, limit?: number): Promise { + this.#location(scope); + recordPlanRows(section, offset, limit); + void requestId; + return { total: 0, section_total: 0, rows: [], runs: [], warnings: [], cursor_window_starts: {} }; + } + + /** The mock's plans carry their diffs whole, so none is read apart. */ + async planDiff(scope: Scope, requestId: string, model: string): Promise { + this.#location(scope); + void requestId; + void model; + return null; + } + async listJobs(scope: Scope): Promise { this.#location(scope); return this.#operate.listJobs(scope); diff --git a/ui/src/lib/api/mock/mock-node-states.test.ts b/ui/src/lib/api/mock/mock-node-states.test.ts new file mode 100644 index 0000000000..dd8b5a8d05 --- /dev/null +++ b/ui/src/lib/api/mock/mock-node-states.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from 'vitest'; +import { stateOf } from '../node-states'; +import type { ChangedSignal } from '../signals'; +import { MockApiClient } from './mock-client'; +import { LOCATION_SHOP } from './fixtures'; + +const SCOPE = { locationId: LOCATION_SHOP }; + +describe('mock node states', () => { + it('answers the node-states read with the listing states at a stable version', async () => { + const client = new MockApiClient({ latencyMs: 0 }); + const [nodes, first] = await Promise.all([client.listCatalogNodes(SCOPE), client.listNodeStates(SCOPE)]); + expect(first.states).toEqual(Object.fromEntries(nodes.map((n) => [n.key, stateOf(n)]))); + expect((await client.listNodeStates(SCOPE)).version).toBe(first.version); + }); + + it('announces nothing on node_status when a runs change leaves every node state as it was', async () => { + const client = new MockApiClient({ latencyMs: 0 }); + await client.listNodeStates(SCOPE); + const seen: ChangedSignal[] = []; + client.changes.subscribe((signal) => seen.push(signal)); + client.changes.emit('runs'); + await client.listNodeStates(SCOPE); + expect(seen.map((s) => s.channel)).toEqual(['runs']); + }); +}); diff --git a/ui/src/lib/api/mock/operate-graph-selection.test.ts b/ui/src/lib/api/mock/operate-graph-selection.test.ts new file mode 100644 index 0000000000..f3f76c21a7 --- /dev/null +++ b/ui/src/lib/api/mock/operate-graph-selection.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest'; +import { evaluateMockSelection, mockVocabulary } from './operate-graph'; + +describe('mock selection evaluation', () => { + it('expands + on its own comma part before intersecting', () => { + expect(evaluateMockSelection('+fact_orders,kind:source', '')).toEqual({ keys: ['raw__customers', 'raw__payments'], error: null }); + }); + + it('locates an unknown tag in the field it was typed in and suggests a close one', () => { + expect(evaluateMockSelection('tag:core', 'tag:fin').error).toEqual({ + code: 'S007', + message: "unknown selector name 'tag:fin'", + help: null, + location: { field: 'exclude', token: 'tag:fin', start: 0, end: 7 }, + suggestions: ['tag:finance'], + refused: { part: 0, type: 'tag', value: 'fin' } + }); + }); + + it('keeps the + of the part it fixes in a suggestion', () => { + expect(evaluateMockSelection('+tag:finanse,kind:model', '').error?.suggestions).toEqual(['+tag:finance,kind:model']); + }); + + it('counts models of every materialization as one kind', () => { + expect(mockVocabulary().kinds).toEqual([ + { value: 'loader', count: 1 }, + { value: 'model', count: 12 }, + { value: 'seed', count: 1 }, + { value: 'source', count: 2 }, + { value: 'task', count: 1 } + ]); + }); +}); diff --git a/ui/src/lib/api/mock/operate-graph.ts b/ui/src/lib/api/mock/operate-graph.ts index cdeb98711d..4bc3120723 100644 --- a/ui/src/lib/api/mock/operate-graph.ts +++ b/ui/src/lib/api/mock/operate-graph.ts @@ -2,6 +2,7 @@ // resolves selectors against the real snapshot; this stands in for it. import type { NodeKind, SnapshotNode } from '../types'; +import type { SelectionProblem, SelectionVocabulary, VocabularyEntry } from '../types/operate'; function node( key: string, @@ -136,3 +137,81 @@ export function resolveSelection(select: readonly string[], exclude: readonly st export function writeSetOf(select: readonly string[], exclude: readonly string[] = []): string[] { return resolveSelection(select, exclude).nodes.filter(isWritable); } + +const MODEL_KINDS: ReadonlySet = new Set(['view', 'table', 'incremental']); + +/** The kind `kind:` matches: models of every materialization are `model`, as the server names them. */ +function selectorKind(n: SnapshotNode): string { + return MODEL_KINDS.has(n.kind) ? 'model' : n.kind; +} + +/** The folder a node's file is in, such as `models/marts`. */ +function folderOf(n: SnapshotNode): string { + return n.path.includes('/') ? n.path.slice(0, n.path.lastIndexOf('/')) : ''; +} + +/** One comma part of a selector, with its own `+`; `null` when it matches nothing. */ +function matchPart(part: string): Set | null { + const up = part.startsWith('+'); + const down = part.endsWith('+') && part.length > 1; + const body = part.replace(/^\+/, '').replace(/\+$/, ''); + const [type, value] = body.includes(':') ? [body.slice(0, body.indexOf(':')), body.slice(body.indexOf(':') + 1)] : ['name', body]; + const base = SNAPSHOT_NODES.filter((n) => + type === 'tag' ? n.tags.includes(value) : type === 'kind' ? selectorKind(n) === value : type === 'path' ? folderOf(n) === value || folderOf(n).startsWith(`${value}/`) : n.key === value + ); + if (!base.length) return null; + const out = new Set(base.map((n) => n.key)); + for (const n of base) { + if (up) for (const a of ancestors(n.key)) out.add(a); + if (down) for (const d of descendants(n.key)) out.add(d); + } + return out; +} + +/** The mock's stand-in for the server's evaluation: tokens are unioned, comma parts intersected, `+` expands its own part. */ +export function evaluateMockSelection(select: string, exclude: string): { keys: string[]; error: SelectionProblem | null } { + const resolve = (field: 'select' | 'exclude', text: string): Set | SelectionProblem => { + const chosen = new Set(); + for (const match of text.matchAll(/\S+/g)) { + const token = match[0]; + let hits = null as Set | null; + for (const [index, part] of token.split(',').entries()) { + const found = matchPart(part); + if (!found) { + const bare = part.replace(/^\+/, '').replace(/\+$/, ''); + const suggestions = [...new Set(SNAPSHOT_NODES.flatMap((n) => n.tags))].filter((tag) => bare.startsWith('tag:') && tag.startsWith(bare.slice(4, 6))).map((tag) => token.replace(part, part.replace(bare, `tag:${tag}`))); + const colon = bare.indexOf(':'); + const refused = { part: index, type: colon > 0 ? bare.slice(0, colon) : null, value: colon > 0 ? bare.slice(colon + 1) : bare }; + return { code: 'S007', message: `unknown selector name '${part}'`, help: null, location: { field, token, start: match.index, end: match.index + token.length }, suggestions, refused }; + } + hits = hits ? new Set([...hits].filter((key) => found.has(key))) : found; + } + for (const key of hits ?? []) chosen.add(key); + } + return chosen; + }; + const selected = select.trim() ? resolve('select', select) : new Set(SNAPSHOT_NODES.map((n) => n.key)); + if (!(selected instanceof Set)) return { keys: [], error: selected }; + const excluded = resolve('exclude', exclude); + if (!(excluded instanceof Set)) return { keys: [], error: excluded }; + const keys = [...selected].filter((key) => !excluded.has(key)).sort(); + if (!keys.length) return { keys, error: { code: null, message: 'the selectors match no nodes in the current publication', help: null, location: null, suggestions: [], refused: null } }; + return { keys, error: null }; +} + +/** Tags, folders, sources, seeds and kinds of the mock project, with node counts. */ +export function mockVocabulary(): Omit { + const counted = (values: string[]): VocabularyEntry[] => { + const counts = new Map(); + for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1); + return [...counts].sort(([a], [b]) => a.localeCompare(b)).map(([value, count]) => ({ value, count })); + }; + const folders = (n: SnapshotNode) => folderOf(n).split('/').filter(Boolean).map((_, i, parts) => parts.slice(0, i + 1).join('/')); + return { + tags: counted(SNAPSHOT_NODES.flatMap((n) => n.tags)), + paths: counted(SNAPSHOT_NODES.filter((n) => n.kind !== 'source' && n.kind !== 'seed').flatMap(folders)), + sources: counted(SNAPSHOT_NODES.filter((n) => n.kind === 'source').map((n) => n.key)), + seeds: counted(SNAPSHOT_NODES.filter((n) => n.kind === 'seed').map((n) => n.key)), + kinds: counted(SNAPSHOT_NODES.map(selectorKind)) + }; +} diff --git a/ui/src/lib/api/mock/operate-store.ts b/ui/src/lib/api/mock/operate-store.ts index 27b08bed6e..b861e1875a 100644 --- a/ui/src/lib/api/mock/operate-store.ts +++ b/ui/src/lib/api/mock/operate-store.ts @@ -6,12 +6,19 @@ import { parseCommand } from '$lib/run/command'; import { flagSpec, VERBS } from '$lib/run/command-table'; import { ApiError } from '../client'; import type { LocalChangeBus } from '../signals'; +import { memberLinks, pairKey } from '$lib/lineage/member-graph'; import type { Acknowledgement, + CatalogNode, + WaitReason, AdhocPermission, AdhocRunCheck, + SelectionEvaluation, + SelectionRequest, + SelectionVocabulary, PlanChanged, PlanResult, + PlanTimings, AdhocRunRequest, BuiltInRole, BulkRunResult, @@ -51,12 +58,16 @@ import type { TriggerStateEvent } from '../types'; import { - CONCURRENCY_LIMITS, + serverLimitOf, IMAGE_ID, JOB_DEFS, + DECLARED_WAITS, SNAPSHOT_GENERATION, TRIGGER_DEFS, - concurrencyGroupsOf, + poolHoldersOf, + poolUsesOf, + resolvedConcurrencyOf, + runConcurrencyOf, createOperateFixtures, jobDef, jobNodes, @@ -69,7 +80,7 @@ import { type TriggerDef } from './operate'; import { SCHEDULER } from './principals'; -import { ancestors, isWritable, resolveSelection, SNAPSHOT_NODES, snapshotNode, sortTopologically, edgesWithin, writeSetOf } from './operate-graph'; +import { ancestors, evaluateMockSelection, isWritable, mockVocabulary, resolveSelection, SNAPSHOT_NODES, snapshotNode, sortTopologically, edgesWithin, writeSetOf } from './operate-graph'; import { commandOf, computeNodes, @@ -123,6 +134,26 @@ export type OperateContext = { reply: (value: T) => Promise; }; +/** The end-to-end tests' stand-in for a plan recorded without timings: `window.__sqbPlanWithoutTimings`. */ +const withoutTimings = () => typeof window !== 'undefined' && !!(window as unknown as { __sqbPlanWithoutTimings?: boolean }).__sqbPlanWithoutTimings; + +/** A location preview's planning steps, as the plan summary reports them. */ +const MOCK_PLAN_TIMINGS: PlanTimings = { + total_seconds: 10.6, + steps: [ + { name: 'Reading warehouse state', seconds: 3.3 }, + { name: 'Detecting changes', seconds: 2.4 }, + { name: 'Checking source freshness', seconds: 0.2 }, + { name: 'Building plan entries', seconds: 2.9 }, + { name: 'Preparing test fixtures', seconds: 0.1 }, + { name: 'Assembling the plan', seconds: 0.6 }, + { name: 'Reading build history', seconds: 0.3 }, + { name: 'Writing the plan', seconds: 0.4 }, + { name: 'Other', seconds: 0.4 } + ], + planned_from: 'synced_state' +}; + export class MockOperate { readonly #ctx: OperateContext; readonly #fx: OperateFixtures; @@ -518,6 +549,21 @@ export class MockOperate { return this.#ctx.reply(structuredClone(SNAPSHOT_NODES)); } + evaluateSelection(scope: Scope, request: SelectionRequest): Promise { + void scope; + const { keys, error } = evaluateMockSelection(request.select, request.exclude); + const offset = request.offset ?? 0; + const limit = Math.min(Math.max(request.limit ?? 1000, 1), 10_000); + const page = keys.slice(offset, offset + limit); + return this.#ctx.reply({ snapshotId: 'mock-snapshot', count: keys.length, keys: page, nextOffset: offset + limit < keys.length ? offset + limit : null, error }); + } + + selectionVocabulary(scope: Scope): Promise { + void scope; + const jobs = JOB_DEFS.map((j) => ({ value: j.key, count: jobNodes(j.key).length })).sort((a, b) => a.value.localeCompare(b.value)); + return this.#ctx.reply({ snapshotId: 'mock-snapshot', jobs, ...mockVocabulary() }); + } + // ---- actions: runs ---- cancelRun(scope: Scope, runId: string): Promise { @@ -681,10 +727,10 @@ export class MockOperate { const synced_copy = { synced_at: new Date(Date.now() - 300_000).toISOString(), cursor_window_starts: Object.fromEntries(models.filter((m) => m.materialization_type === 'incremental' && reason !== 'full_refresh').map((m) => [m.name, windowStart])), - unknown_until_run: ['source freshness', 'retention', 'column rename support', 'sources outside mirrored schemas'] + unknown_until_run: ['source freshness', 'retention', 'column rename support', 'sources outside mirrored schemas', 'recorded migrations'] }; return this.#ctx.reply({ - plan: { selected_count: check.selection.length, models, seeds: [], source_loads: [], functions: [], python_nodes: [], warnings: [], migrations: [], column_migrations: [], table_type_conversions: [], retention: [], synced_copy, changed: decision }, + plan: { selected_count: check.selection.length, models, seeds: [], source_loads: [], functions: [], python_nodes: [], warnings: [], migrations: [], column_migrations: [], table_type_conversions: [], retention: [], synced_copy, changed: decision, ...(withoutTimings() ? {} : { timings: MOCK_PLAN_TIMINGS }), ...(command.includes('--ignore-audit-failures') ? { ignore_audit_failures: true } : {}) }, error: null }); } @@ -975,20 +1021,19 @@ export class MockOperate { if (overlap.length) waits.push({ kind: 'write_claim', runId: h.run.id, nodeKey: overlap[0], otherNodes: overlap.length - 1 }); } - const fullGroups: string[] = []; - for (const group of rec.script.concurrencyGroups) { - const inGroup = holders.filter((h) => h.script.concurrencyGroups.includes(group)).map((h) => h.run.id); - const limit = CONCURRENCY_LIMITS[group] ?? 1; - if (inGroup.length >= limit) { - waits.push({ kind: 'concurrency_limit', limit: group, holders: inGroup, elsewhere: 0 }); - fullGroups.push(group); + const fullPools: string[] = []; + for (const [pool, amount] of Object.entries(rec.script.pools)) { + const inPool = holders.filter((h) => pool in h.script.pools); + const limit = serverLimitOf(pool); + const held = inPool.reduce((sum, h) => sum + h.script.pools[pool], 0); + if (held + Math.min(amount, limit) > limit) { + waits.push({ kind: 'pool_full', pool, holders: inPool.map((h) => h.run.id) }); + fullPools.push(pool); } } - if (fullGroups.length) { - for (const group of fullGroups) { - const first = ahead.find((a) => a.script.concurrencyGroups.includes(group)); - if (first) waits.push({ kind: 'earlier_run', runId: first.run.id, limit: group }); - } + for (const pool of fullPools) { + const first = ahead.find((a) => pool in a.script.pools); + if (first) waits.push({ kind: 'earlier_run', runId: first.run.id, pool }); } if (rec.run.parentRunId) { const parent = this.#byId(rec.run.parentRunId); @@ -998,7 +1043,7 @@ export class MockOperate { const me = members.find((m) => m.jobKey === rec.run.jobKey); for (const up of me?.upstream ?? []) { const sibling = this.#children(parent).find((c) => c.run.jobKey === up.jobKey); - if (sibling && isActive(sibling.run.status)) waits.push({ kind: 'upstream_job', runId: sibling.run.id, jobKey: up.jobKey }); + if (sibling && isActive(sibling.run.status)) waits.push({ kind: 'upstream_job', runId: sibling.run.id, jobKey: up.jobKey, attempt: sibling.run.memberAttempt }); } } if (parent && parent.run.status === 'queued') waits.push({ kind: 'group_parent', runId: parent.run.id }); @@ -1050,6 +1095,7 @@ export class MockOperate { tickScheduledFor: null, runKey: null, rerunOfRunId: spec.rerunOf ?? null, + memberAttempt: null, narrowing: spec.narrowing ?? null, window: null, requestedBy: spec.by, @@ -1064,7 +1110,8 @@ export class MockOperate { startedAt: null, endedAt: null, nodeCounts: { success: 0, warn: 0, failed: 0, skipped: 0, running: 0, pending: 0 }, - cost: null + cost: null, + ignoredAuditFailures: 0 }, script: { verb: spec.verb ?? 'build', @@ -1082,7 +1129,7 @@ export class MockOperate { lastHeartbeatAt: null, imageId: IMAGE_ID, snapshotGeneration: SNAPSHOT_GENERATION, - concurrencyGroups: job?.concurrencyGroups ?? [] + pools: job?.pools ?? {} } }; this.#records.unshift(rec); @@ -1164,7 +1211,8 @@ export class MockOperate { if (!job || job.kind !== 'group') return []; const nodesOf = new Map(job.members.map((m) => [m, new Set(jobNodes(m))])); const upstream = new Map(); - for (const b of job.members) { + const declared = DECLARED_WAITS[key]; + for (const b of declared ? [] : job.members) { const ups: JobGroupMember['upstream'] = []; const needs = new Set([...nodesOf.get(b)!].flatMap((n) => [...ancestors(n)])); for (const a of job.members) { @@ -1173,6 +1221,7 @@ export class MockOperate { } upstream.set(b, ups); } + for (const [b, ups] of declared ?? []) upstream.set(b, ups.map((a) => ({ jobKey: a, source: 'lineage' as const }))); const stage = new Map(); const stageOf = (m: string): number => { if (stage.has(m)) return stage.get(m)!; @@ -1180,7 +1229,16 @@ export class MockOperate { stage.set(m, s); return s; }; - return job.members.map((m) => ({ jobKey: m, upstream: upstream.get(m) ?? [], stage: stageOf(m) })); + const members = job.members.map((m) => ({ jobKey: m, upstream: upstream.get(m) ?? [], stage: stageOf(m), reasons: [] as WaitReason[] })); + // Published as the server publishes them: node by node, from the snapshot's lineage. + const info = new Map(job.members.map((m) => [m, { nodeCount: null, lastRun: null, nodes: jobNodes(m) }])); + const links = memberLinks(members, info, SNAPSHOT_NODES as unknown as CatalogNode[]); + for (const m of members) { + for (const { jobKey: up } of m.upstream) { + for (const link of links.get(pairKey(up, m.jobKey)) ?? []) m.reasons.push({ upstreamJob: up, fromNode: link.from, toNode: link.to }); + } + } + return members; } #jobView(key: string, scope: Scope): JobDetail { @@ -1194,10 +1252,15 @@ export class MockOperate { return structuredClone({ ...job, nodeCount: resolved.filter((n) => snapshotNode(n)?.kind !== 'source').length, + nodes: resolved, resolvedNodes: resolved, lastRun: view ? { id: view.id, status: view.status, startedAt: view.startedAt, endedAt: view.endedAt } : null, triggerIds: TRIGGER_DEFS.filter((t) => t.jobKey === key).map((t) => triggerId(t.name)), - concurrency: concurrencyGroupsOf(job).map((g) => ({ ...g, holders: holders.filter((h) => h.script.concurrencyGroups.includes(g.key)).map((h) => h.run.id) })), + pools: poolUsesOf(job), + concurrency: runConcurrencyOf(job), + ignoreAuditFailures: job.ignoreAuditFailures ?? null, + resolvedConcurrency: resolvedConcurrencyOf(job), + poolHolders: poolHoldersOf(job).map((g) => ({ ...g, holders: holders.filter((h) => g.pool in h.script.pools).map((h) => h.run.id) })), memberOf: JOB_DEFS.filter((g) => g.members.includes(key)).map((g) => g.key), cursorType: resolved.map((n) => snapshotNode(n)?.cursor).find((c) => c) ?? null, groupMembers: this.#groupMembers(key), diff --git a/ui/src/lib/api/mock/operate.ts b/ui/src/lib/api/mock/operate.ts index 168130a8fb..d788bd5f14 100644 --- a/ui/src/lib/api/mock/operate.ts +++ b/ui/src/lib/api/mock/operate.ts @@ -6,9 +6,11 @@ import type { CancelProgress, CatchUpPolicy, - ConcurrencyGroup, JobDetail, LaunchVerb, + PoolHolders, + PoolUse, + ResolvedConcurrency, Principal, Run, RunOrigin, @@ -21,6 +23,7 @@ import type { TriggerStateEvent } from '../types'; import { CI_ACCOUNT, DEMO_USER, OPS_USER, SCHEDULER, SENSOR_DAEMON } from './principals'; +import { BIG_GROUP, bigGroupFile, bigGroupSelect, bigGroupShape } from './big-group'; import { resolveSelection, writeSetOf } from './operate-graph'; import type { RunEnd, RunRecord, RunScript } from './run-detail'; @@ -35,11 +38,27 @@ export const SNAPSHOT_GENERATION = 42; type Known = Omit & { [P in K]-?: NonNullable }; type JobDef = Known< - Omit, + Omit, 'members' | 'overlap' | 'priority' | 'sourceFile' | 'declaration' | 'settings' ->; +> & { + /** Each pool the job takes, with its amount. */ + pools: Record; + /** The job's own `ignore_audit_failures`, when it sets one. */ + ignoreAuditFailures?: boolean; +}; + +/** Each pool's limit in the mock project, and this mock server's `pool_caps`, which lower it. */ +export const POOL_LIMITS: Record = { warehouse_heavy: 3, supplier_api: 2 }; +export const POOL_CAPS: Record = { warehouse_heavy: 1 }; +/** A pool's limit on this server: the project's, lowered to the cap. */ +export const serverLimitOf = (pool: string) => Math.min(POOL_LIMITS[pool] ?? 1, POOL_CAPS[pool] ?? Infinity); -export const CONCURRENCY_LIMITS: Record = { warehouse_heavy: 1, 'target:prod': 4 }; +/** Where each job's run concurrency comes from, when not its own setting: one of every layer. */ +const CONCURRENCY_FROM: Record = { + marts_only: { limit: 6, source: 'target' }, + activity_microbatch: { limit: 2, source: 'local' }, + load_sources: { limit: 4, source: 'default' } +}; const decl = (sourceFile: string, line: number, code: string) => ({ sourceFile, line, code }); @@ -53,7 +72,7 @@ export const JOB_DEFS: JobDef[] = [ members: [], overlap: 'queue', onFailure: { retries: 1, retryDelaySeconds: 300 }, - concurrencyGroups: [], + pools: { supplier_api: 1 }, priority: 5, sourceFile: 'orchestration/jobs.py', settings: {}, @@ -76,29 +95,31 @@ export const JOB_DEFS: JobDef[] = [ key: 'core_build', kind: 'job', description: 'Staging and marts models', - select: 'group:staging group:marts', + select: 'path:models/staging path:models/marts', exclude: [], members: [], overlap: 'skip', onFailure: { retries: 2, retryDelaySeconds: 600 }, - concurrencyGroups: ['warehouse_heavy'], + pools: { warehouse_heavy: 1 }, priority: 0, sourceFile: 'orchestration/jobs.py', settings: { '--concurrency': '8' }, targets: null, owner: 'data-platform', criticalNodes: ['fact_orders'], + ignoreAuditFailures: true, declaration: decl( 'orchestration/jobs.py', 24, `core_build = job( "core_build", description="Staging and marts models", - select=["group:staging", "group:marts"], + select=["path:models/staging", "path:models/marts"], overlap="skip", on_failure=on_failure(retries=2, retry_delay_seconds=600), - concurrency_groups=[warehouse_heavy], + pools={"warehouse_heavy": 1}, settings={"--concurrency": 8}, + ignore_audit_failures=True, owner="data-platform", schedule=schedule(cron="{cron:core_build}", timezone="UTC"), )` @@ -108,12 +129,12 @@ export const JOB_DEFS: JobDef[] = [ key: 'marts_only', kind: 'job', description: 'Marts models only, reading the current staging layer', - select: 'group:marts', + select: 'path:models/marts', exclude: [], members: [], overlap: 'queue', onFailure: null, - concurrencyGroups: [], + pools: {}, priority: 0, sourceFile: 'orchestration/jobs.py', settings: {}, @@ -126,7 +147,7 @@ export const JOB_DEFS: JobDef[] = [ `marts_only = job( "marts_only", description="Marts models only, reading the current staging layer", - select="group:marts", + select="path:models/marts", targets=["prod", "staging"], owner="analytics", )` @@ -136,12 +157,12 @@ export const JOB_DEFS: JobDef[] = [ key: 'activity_microbatch', kind: 'job', description: 'Incremental activity models, four microbatches at a time', - select: 'group:activity', + select: 'path:models/activity', exclude: [], members: [], overlap: 'queue', onFailure: null, - concurrencyGroups: [], + pools: {}, priority: 0, sourceFile: 'orchestration/activity.py', settings: { '--max-microbatches': '4' }, @@ -154,7 +175,7 @@ export const JOB_DEFS: JobDef[] = [ `activity_microbatch = job( "activity_microbatch", description="Incremental activity models, four microbatches at a time", - select="group:activity", + select="path:models/activity", settings={"--max-microbatches": 4}, owner="analytics", schedule=schedule(cron="{cron:activity_microbatch}", timezone="UTC", @@ -171,7 +192,7 @@ export const JOB_DEFS: JobDef[] = [ members: [], overlap: 'queue', onFailure: null, - concurrencyGroups: [], + pools: {}, priority: 0, sourceFile: 'orchestration/finance.py', settings: { '--fail-fast': true }, @@ -201,7 +222,7 @@ export const JOB_DEFS: JobDef[] = [ members: [], overlap: 'queue', onFailure: null, - concurrencyGroups: [], + pools: {}, priority: 0, sourceFile: 'orchestration/finance.py', settings: {}, @@ -223,12 +244,12 @@ export const JOB_DEFS: JobDef[] = [ key: 'reporting_exports', kind: 'job', description: 'Reporting tables and the revenue export', - select: 'group:reporting group:exports', + select: 'path:models/reporting export_revenue_csv', exclude: [], members: [], overlap: 'queue', onFailure: null, - concurrencyGroups: ['warehouse_heavy'], + pools: { warehouse_heavy: 1, supplier_api: 1 }, priority: 10, sourceFile: 'orchestration/reporting.py', settings: {}, @@ -241,8 +262,8 @@ export const JOB_DEFS: JobDef[] = [ `reporting_exports = job( "reporting_exports", description="Reporting tables and the revenue export", - select=["group:reporting", "group:exports"], - concurrency_groups=[warehouse_heavy], + select=["path:models/reporting", "export_revenue_csv"], + pools={"warehouse_heavy": 1}, priority=10, targets=["prod"], )` @@ -257,7 +278,7 @@ export const JOB_DEFS: JobDef[] = [ members: ['load_sources', 'core_build', 'activity_microbatch', 'reporting_exports'], overlap: 'queue', onFailure: null, - concurrencyGroups: [], + pools: {}, priority: 0, sourceFile: 'orchestration/groups.py', settings: {}, @@ -286,7 +307,7 @@ export const JOB_DEFS: JobDef[] = [ members: ['revenue_refresh', 'activity_microbatch', 'customer_segments'], overlap: 'queue', onFailure: { retries: 1, retryDelaySeconds: 900 }, - concurrencyGroups: ['warehouse_heavy'], + pools: { warehouse_heavy: 3 }, priority: 20, sourceFile: 'orchestration/groups.py', settings: {}, @@ -302,7 +323,7 @@ export const JOB_DEFS: JobDef[] = [ jobs=[revenue_refresh, activity_microbatch, customer_segments], on_failure=on_failure(retries=1, retry_delay_seconds=900), critical=["daily_revenue"], - concurrency_groups=[warehouse_heavy], + pools={"warehouse_heavy": 1}, priority=20, targets=["prod"], )` @@ -310,6 +331,52 @@ export const JOB_DEFS: JobDef[] = [ } ]; +// A large group of small jobs, to see how a big group draws (see big-group.ts). +const BIG = bigGroupShape(); +JOB_DEFS.push( + ...BIG.members.map( + (key): JobDef => ({ + key, + kind: 'job', + description: `Warehouse step ${key.replace('_', ' ')}`, + select: bigGroupSelect(key), + exclude: [], + members: [], + overlap: 'queue', + onFailure: null, + pools: {}, + priority: 0, + sourceFile: bigGroupFile(key), + settings: {}, + targets: null, + owner: 'data-platform', + criticalNodes: [], + declaration: decl(bigGroupFile(key), 1, `${key} = job("${key}", select="${bigGroupSelect(key)}")`) + }) + ), + { + key: BIG_GROUP, + kind: 'group', + description: 'Every warehouse step, in stages', + select: null, + exclude: [], + members: BIG.members, + overlap: 'queue', + onFailure: null, + pools: {}, + priority: 0, + sourceFile: 'orchestration/warehouse.py', + settings: {}, + targets: null, + owner: 'data-platform', + criticalNodes: [], + declaration: decl('orchestration/warehouse.py', 60, `${BIG_GROUP} = job_group("${BIG_GROUP}", jobs=[...])`) + } +); + +/** Waits a group declares instead of those its members' lineage gives: the big group's whole upstreams. */ +export const DECLARED_WAITS: Record> = { [BIG_GROUP]: BIG.waits }; + export function jobDef(key: string | null): JobDef | undefined { return key ? JOB_DEFS.find((j) => j.key === key) : undefined; } @@ -326,8 +393,45 @@ export function jobNodes(key: string): string[] { return writeSetOf(selectorTerms(job), job.exclude); } -export function concurrencyGroupsOf(job: JobDef | undefined): ConcurrencyGroup[] { - return (job?.concurrencyGroups ?? []).map((key) => ({ key, limit: CONCURRENCY_LIMITS[key] ?? 1, holders: [] })); +/** Each pool the job takes with its amount, limit and the other jobs taking it, as the server publishes them. */ +export function poolUsesOf(job: JobDef | undefined): PoolUse[] { + return Object.entries(job?.pools ?? {}).map(([pool, amount]) => ({ + pool, + amount, + limit: POOL_LIMITS[pool] ?? 1, + cap: POOL_CAPS[pool] ?? null, + serverLimit: serverLimitOf(pool), + clamped: amount > serverLimitOf(pool), + sharedWith: JOB_DEFS.filter((other) => other.key !== job?.key && pool in other.pools).map((other) => other.key) + })); +} + +export function poolHoldersOf(job: JobDef | undefined): PoolHolders[] { + return Object.keys(job?.pools ?? {}).map((pool) => ({ pool, limit: serverLimitOf(pool), holders: [] })); +} + +/** What a run of the job uses: its own setting, else the layer the mock gives it, else the mock project's 4. */ +export function resolvedConcurrencyOf(job: JobDef | undefined): ResolvedConcurrency { + const own = runConcurrencyOf(job); + return own ? { limit: own, source: 'job' } : (CONCURRENCY_FROM[job?.key ?? ''] ?? { limit: 4, source: 'project' }); +} + +/** The jobs and groups asking a pool for more than this server allows, as the server's pool_capped warning names them. */ +export function poolCapWarnings(): { jobKey: string; kind: 'job' | 'group'; pool: string; amount: number; cap: number; message: string }[] { + return JOB_DEFS.flatMap((job) => + Object.entries(job.pools) + .filter(([pool, amount]) => amount > serverLimitOf(pool)) + .map(([pool, amount]) => { + const cap = serverLimitOf(pool); + return { jobKey: job.key, kind: job.kind, pool, amount, cap, message: `${job.kind} ${job.key} takes ${amount} of pool ${pool}; this server caps it at ${cap}, so it runs alone in that pool` }; + }) + ); +} + +/** The job's own `concurrency` setting, as the server reports it. */ +export function runConcurrencyOf(job: JobDef | undefined): number | null { + const value = Number(job?.settings?.concurrency ?? job?.settings?.['--concurrency']); + return Number.isInteger(value) && value > 0 ? value : null; } // ---- triggers ---- @@ -654,6 +758,8 @@ type Seed = { image?: string; generation?: number; reason?: string; + /** Error-severity audit failures a successful run let through. */ + ignored?: number; }; function rng(seed: number) { @@ -743,7 +849,7 @@ export function createOperateFixtures(now: number): OperateFixtures { lastHeartbeatAt: seed.lastHeartbeatAgoSec != null ? now - seed.lastHeartbeatAgoSec * 1000 : null, imageId: seed.image ?? IMAGE_ID, snapshotGeneration: seed.generation ?? SNAPSHOT_GENERATION, - concurrencyGroups: job?.concurrencyGroups ?? [] + pools: job?.pools ?? {} }; const run: Run = { id: seed.id, @@ -762,6 +868,7 @@ export function createOperateFixtures(now: number): OperateFixtures { tickScheduledFor: tick != null ? new Date(tick).toISOString() : null, runKey: seed.runKey ?? null, rerunOfRunId: seed.rerunOf ?? null, + memberAttempt: null, narrowing: null, window: null, requestedBy: seed.by, @@ -776,7 +883,8 @@ export function createOperateFixtures(now: number): OperateFixtures { startedAt: started != null ? new Date(started).toISOString() : null, endedAt: ended ? new Date(ended).toISOString() : null, nodeCounts: { success: 0, warn: 0, failed: 0, skipped: 0, running: 0, pending: 0 }, - cost: null + cost: null, + ignoredAuditFailures: seed.ignored ?? 0 }; const rec = { run, script }; records.push(rec); @@ -837,7 +945,7 @@ export function createOperateFixtures(now: number): OperateFixtures { queuedAgo: at('core_build', 0, 0), startedAgo: at('core_build', 0, 10), durationMin: 6, warn: ['stg_payments'], fail: ['dim_customers'], errors: { dim_customers: 'Numeric value out of range in lifetime_value' } }); - record({ id: 'run-2b70', origin: 'schedule', job: 'core_build', trigger: 'core_build', tick: 1, by: SCHEDULER, queuedAgo: at('core_build', 1, 0), startedAgo: at('core_build', 1, 9), durationMin: 7 }); + record({ id: 'run-2b70', origin: 'schedule', job: 'core_build', trigger: 'core_build', tick: 1, by: SCHEDULER, queuedAgo: at('core_build', 1, 0), startedAgo: at('core_build', 1, 9), durationMin: 7, ignored: 2 }); record({ id: 'run-2b71', origin: 'sensor', job: 'reporting_exports', trigger: 'after_core_build', runKey: 'run-2b70', by: SENSOR_DAEMON, queuedAgo: at('core_build', 1, 9) - 7.2, startedAgo: at('core_build', 1, 9) - 7.4, durationMin: 3 }); record({ id: 'run-0c7d', origin: 'rerun', job: 'core_build', by: CI_ACCOUNT, rerunOf: 'run-2b70', queuedAgo: 600, startedAgo: 599.8, durationMin: 3, end: 'cancelled', cancel: { requestedAgoSec: 75, by: CI_ACCOUNT, sigtermAgoSec: 74, done: true }, reason: 'Cancelled by svc-ci' }); record({ id: 'run-91cd', origin: 'manual', job: 'marts_only', by: DEMO_USER, queuedAgo: 97, startedAgo: 96.8, durationMin: 5 }); diff --git a/ui/src/lib/api/mock/plan-rows-log.ts b/ui/src/lib/api/mock/plan-rows-log.ts new file mode 100644 index 0000000000..3308d53d60 --- /dev/null +++ b/ui/src/lib/api/mock/plan-rows-log.ts @@ -0,0 +1,20 @@ +// The mock's record of the plan pages and diffs the page asked for, for the end-to-end tests: the mock +// makes no network requests, so this stands in for the server's plan reads. Read as +// `window.__sqbPlanRows` and `window.__sqbPlanDiffs`. + +import type { PlanPageSection } from '../types'; + +export type PlanRowsCall = { section: PlanPageSection; offset: number; limit: number | undefined; name?: string }; + +type Log = { __sqbPlanRows?: PlanRowsCall[]; __sqbPlanDiffs?: string[] }; +const log = () => (typeof window === 'undefined' ? null : (window as unknown as Log)); + +export function recordPlanRows(section: PlanPageSection, offset: number, limit: number | undefined, name?: string): void { + const w = log(); + if (w) (w.__sqbPlanRows ??= []).push({ section, offset, limit, ...(name ? { name } : {}) }); +} + +export function recordPlanDiff(model: string): void { + const w = log(); + if (w) (w.__sqbPlanDiffs ??= []).push(model); +} diff --git a/ui/src/lib/api/mock/platform.ts b/ui/src/lib/api/mock/platform.ts index 3bd6c052fe..b8f668a3f9 100644 --- a/ui/src/lib/api/mock/platform.ts +++ b/ui/src/lib/api/mock/platform.ts @@ -15,6 +15,7 @@ import type { AuditQuery, BuiltInRole, Capability, + ConnectionDetails, ConnectionTest, EffectiveAccess, IssuedToken, @@ -59,6 +60,25 @@ type MockWarehouse = { values: Record; lastTest: ConnectionTest | null; }; + +/** The non-secret settings a mock connection would publish: values as set, defaults otherwise, and the sign-in method by name. */ +function mockConnectionDetails(c: MockWarehouse): ConnectionDetails { + const value = (key: string): string | null => { + const v = c.values[key] ?? c.settings.find((f) => f.key === key)?.default; + return v === undefined ? null : String(v); + }; + const secret = c.settings.find((f) => f.type === 'secret')?.key; + return { + adapter: c.adapter, + account: value('account'), + host: value('host') ?? value('path'), + user: value('user'), + role: value('role'), + warehouse: value('warehouse'), + database: value('database') ?? value('dbname'), + authMethod: secret === 'private_key' ? 'key_pair' : secret === 'password' ? 'password' : 'none' + }; +} import { ADMIN_ONLY, baseGrantsOf, @@ -73,7 +93,7 @@ import { type AccessData } from './access'; import { LOCATION_MARKETING, LOCATION_SHOP, LOCATION_SHOP_DEV, LOCATION_SHOP_STAGING, ROLE_CAPABILITIES, SHOP_LOCATIONS } from './fixtures'; -import { triggerId } from './operate'; +import { poolCapWarnings, triggerId } from './operate'; import { ALEX, CI_ACCOUNT, DEMO_USER, JORDAN, OPS_USER, REPORTING, RILEY, SAM, SYSTEM } from './principals'; const MIN = 60_000; @@ -250,7 +270,7 @@ function createPlatformFixtures(now: number, role: BuiltInRole) { }); const loadError = "ImportError: cannot import name 'campaign_touches' from 'models.attribution' (orchestration/jobs.py, line 12)"; - type LocationExtras = Pick; + type LocationExtras = Pick; const locationExtras: Record = { [LOCATION_SHOP]: { createdAt: ago(days(210)), @@ -259,6 +279,8 @@ function createPlatformFixtures(now: number, role: BuiltInRole) { lastLoadError: null, imageId: 'sha256:4be1c07d9a52', warehouseConnection: 'prod', + // This mock server caps a pool below what a group asks for. + poolCapWarnings: poolCapWarnings(), connectionTest: publishTest(190, 820, 'Connected as TRANSFORMER to WAFFLE_SHOP; warehouse TRANSFORMING_WH available'), instances: [ { id: 'location-waffle-shop-7c2d', imageId: 'sha256:4be1c07d9a52', gitRevision: '7d2e41a', startedAt: ago(192), lastSeenAt: ago(0.2), state: 'live' }, @@ -273,6 +295,7 @@ function createPlatformFixtures(now: number, role: BuiltInRole) { lastLoadError: null, imageId: 'sha256:9f02b6e4c1a8', warehouseConnection: 'staging', + poolCapWarnings: [], connectionTest: publishTest(35, 760, 'Connected as TRANSFORMER to WAFFLE_SHOP_STAGING'), instances: [ { id: 'location-waffle-shop-staging-c7e3', imageId: 'sha256:61d0a4f7b2e9', gitRevision: 'e09a6c3', startedAt: ago(1), lastSeenAt: ago(0.1), state: 'live' }, @@ -286,6 +309,7 @@ function createPlatformFixtures(now: number, role: BuiltInRole) { lastLoadError: null, imageId: null, warehouseConnection: 'dev', + poolCapWarnings: [], connectionTest: publishTest(2, 12, 'Opened warehouse.duckdb'), instances: [{ id: 'location-waffle-shop-dev-local', imageId: 'local', gitRevision: 'b81c09e', startedAt: ago(30), lastSeenAt: ago(0.3), state: 'live' }] }, @@ -296,6 +320,7 @@ function createPlatformFixtures(now: number, role: BuiltInRole) { lastLoadError: loadError, imageId: 'sha256:3e5f7aa20c19', warehouseConnection: 'prod', + poolCapWarnings: [], connectionTest: publishTest(1440, 140, 'Connected to analytics as sqb_builder'), instances: [ { id: 'location-marketing-analytics-03f6', imageId: 'sha256:3e5f7aa20c19', gitRevision: '3e5f7aa', startedAt: ago(days(1) + 30), lastSeenAt: ago(0.5), state: 'live' }, @@ -392,7 +417,9 @@ function createPlatformFixtures(now: number, role: BuiltInRole) { setting('api_key', 'SecretStr', 'ORDERS_API_KEY', 'API key', { required: true }) ], usedBy: [{ kind: 'loader', name: 'raw__orders' }], - jobs: ['load_sources', 'nightly_all'] + nodes: ['raw__orders'], + jobs: ['load_sources', 'nightly_all'], + jobNodes: { load_sources: ['raw__orders'], nightly_all: ['raw__orders'] } }, { name: 'export_bucket', @@ -410,7 +437,9 @@ function createPlatformFixtures(now: number, role: BuiltInRole) { { kind: 'task', name: 'export_revenue_csv' }, { kind: 'sink', name: 'run_log_archive' } ], - jobs: ['revenue_refresh', 'reporting_exports'] + nodes: ['export_revenue_csv'], + jobs: ['revenue_refresh', 'reporting_exports'], + jobNodes: { revenue_refresh: ['export_revenue_csv'], reporting_exports: ['export_revenue_csv'] } }, { name: 'support_desk', @@ -423,7 +452,9 @@ function createPlatformFixtures(now: number, role: BuiltInRole) { setting('token', 'SecretStr', 'SUPPORT_DESK_TOKEN', 'Access token', { required: true }) ], usedBy: [{ kind: 'loader', name: 'raw__support_tickets' }], - jobs: ['load_sources'] + nodes: ['raw__support_tickets'], + jobs: ['load_sources'], + jobNodes: { load_sources: ['raw__support_tickets'] } }, { name: 'chat_webhook', @@ -435,7 +466,9 @@ function createPlatformFixtures(now: number, role: BuiltInRole) { setting('timeout_seconds', 'int', 'CHAT_WEBHOOK_TIMEOUT_SECONDS', 'Request timeout') ], usedBy: [], - jobs: [] + nodes: [], + jobs: [], + jobNodes: {} } ]; @@ -714,9 +747,13 @@ export class MockPlatformApi implements PlatformApi { id: l.id, location: l.name, target: c.target, + // Shows the page as it will look once locations publish their settings and the sync schedule. + details: mockConnectionDetails(c), stateLocation: { database: typeof c.values.database === 'string' ? c.values.database : null, schema: 'sqlbuild' }, lastTest: this.#connectionTests.get(`${l.id}/${c.name}`) ?? c.lastTest, - tables: ['audit_results', 'catalog', 'fingerprints', 'statistics'].map((table, i) => ({ table, at: synced, recordsSent: 12 * (i + 1), elapsedMs: 180 + 40 * i, failure: null, error: null, waiting: false, lastSyncedAt: synced })) + nextSyncAt: new Date(Date.now() + 26 * 60_000).toISOString(), + halted: false, + tables: ['audit_results', 'catalog', 'fingerprints', 'statistics'].map((table, i) => ({ table, at: synced, recordsSent: 12 * (i + 1), elapsedMs: 180 + 40 * i, failure: null, error: null, waiting: false, lastSyncedAt: synced, mirroredTotal: 4_800 * (i + 1) })) })) ); return this.#ctx.reply(structuredClone(connections)); diff --git a/ui/src/lib/api/mock/run-detail.ts b/ui/src/lib/api/mock/run-detail.ts index 3556bd268c..9fc5fb9d11 100644 --- a/ui/src/lib/api/mock/run-detail.ts +++ b/ui/src/lib/api/mock/run-detail.ts @@ -43,8 +43,8 @@ export type RunScript = { lastHeartbeatAt: number | null; imageId: string; snapshotGeneration: number; - /** Concurrency groups the run takes a slot in. */ - concurrencyGroups: string[]; + /** Each pool the run takes, with how many of its slots. */ + pools: Record; }; export type RunRecord = { run: Run; script: RunScript }; @@ -277,7 +277,9 @@ const LEVEL: Record = { launched: 'info', cancel_requested: 'warn', presumed_failed: 'error', - terminated: 'info' + terminated: 'info', + member_retrying: 'warn', + member_recovered: 'info' }; export function invocationId(run: Run, attempt = 1): string { @@ -365,7 +367,7 @@ export function execution(record: RunRecord, nodes: RunNode[], now?: number): Ru fact('queued', queued, `Queued at priority ${run.priority}`); if (run.launchedAt) { const launched = Date.parse(run.launchedAt); - fact('claimed', launched - 300, script.nodes.length ? `Claimed ${script.nodes.length} write-set nodes${script.concurrencyGroups.length ? ` and a slot in ${script.concurrencyGroups.join(', ')}` : ''}` : 'Claimed'); + fact('claimed', launched - 300, script.nodes.length ? `Claimed ${script.nodes.length} write-set nodes${Object.keys(script.pools).length ? ` and slots in pool ${Object.keys(script.pools).join(', ')}` : ''}` : 'Claimed'); fact('launched', launched, script.nodes.length ? `Container started from image ${script.imageId.slice(0, 19)}` : `Started ${run.childRunCount} child runs in order`); } if (script.cancel) fact('cancel_requested', script.cancel.requestedAt, `Cancel requested by ${script.cancel.requestedBy.displayName}`); diff --git a/ui/src/lib/api/node-states.test.ts b/ui/src/lib/api/node-states.test.ts new file mode 100644 index 0000000000..452bd68d4e --- /dev/null +++ b/ui/src/lib/api/node-states.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from 'vitest'; +import { applyNodeStates, deltaAction, stateOf, withBehind } from './node-states'; +import type { CatalogNode, NodeState } from './types/build'; + +function node(key: string, upstream: string[] = [], builtAt: string | null = '2026-01-01T00:00:00.000Z', kind: CatalogNode['kind'] = 'table'): CatalogNode { + return { + key, + kind, + materialization: kind, + group: null, + tags: [], + description: '', + path: '', + upstream, + relation: null, + columnCount: 0, + lastBuild: builtAt ? { fingerprintRunId: `r-${key}`, runId: null, jobKey: null, builtAt, fingerprint: 'f' } : null, + lastOutcome: builtAt ? 'success' : null, + stale: [], + sourceFreshness: null, + checks: { total: 0, passing: 0, warning: 0, failing: 0 } + }; +} + +const built = (key: string, builtAt: string, extra: Partial = {}): NodeState => ({ + lastBuild: { fingerprintRunId: `r2-${key}`, runId: null, jobKey: null, builtAt, fingerprint: 'g' }, + lastOutcome: 'success', + definitionChanged: false, + checks: { total: 1, passing: 1, warning: 0, failing: 0 }, + ...extra +}); + +describe('applyNodeStates', () => { + const listing = withBehind([node('model:orders'), node('model:order_items', ['model:orders']), node('model:customers')]); + + it('replaces the state of named nodes and leaves the rest as they were', () => { + const applied = applyNodeStates(listing, { 'model:customers': built('model:customers', '2026-01-02T00:00:00.000Z') }); + expect(applied[2].lastBuild?.fingerprintRunId).toBe('r2-model:customers'); + expect(applied[2].checks).toEqual({ total: 1, passing: 1, warning: 0, failing: 0 }); + expect(applied[0]).toBe(listing[0]); + expect(applied[1]).toBe(listing[1]); + }); + + it('marks downstream nodes behind when an upstream build is newer', () => { + const applied = applyNodeStates(listing, { 'model:orders': built('model:orders', '2026-01-02T00:00:00.000Z') }); + expect(applied[1].stale).toEqual([expect.objectContaining({ reason: 'behind', upstream: ['model:orders'] })]); + expect(applied[2]).toBe(listing[2]); + }); + + it('adds and clears the changed reason with the definition', () => { + const changed = applyNodeStates(listing, { 'model:customers': built('model:customers', '2026-01-01T00:00:00.000Z', { definitionChanged: true }) }); + expect(changed[2].stale?.map((r) => r.reason)).toEqual(['changed']); + expect(stateOf(changed[2]).definitionChanged).toBe(true); + const cleared = applyNodeStates(changed, { 'model:customers': built('model:customers', '2026-01-03T00:00:00.000Z') }); + expect(cleared[2].stale).toEqual([]); + }); + + it('ignores states of nodes the listing does not hold', () => { + expect(applyNodeStates(listing, { 'model:unknown': built('model:unknown', '2026-01-02T00:00:00.000Z') })).toBe(listing); + }); + + it('keeps a node state round-trip unchanged', () => { + const states = Object.fromEntries(listing.map((n) => [n.key, stateOf(n)])); + expect(applyNodeStates(listing, states)).toEqual(listing); + }); +}); + +describe('deltaAction', () => { + const delta = (previousVersion: number, version: number) => ({ previousVersion, version, states: {} }); + + it('applies the next version', () => expect(deltaAction(4, delta(4, 5))).toBe('apply')); + it('reads again after a gap', () => expect(deltaAction(4, delta(5, 6))).toBe('refetch')); + it('ignores a version already held', () => expect(deltaAction(6, delta(4, 5))).toBe('ignore')); + it('reads again before any version is held', () => expect(deltaAction(null, delta(4, 5))).toBe('refetch')); + it('reads again on a full change', () => expect(deltaAction(4, { previousVersion: 4, version: 5, full: true })).toBe('refetch')); +}); diff --git a/ui/src/lib/api/node-states.ts b/ui/src/lib/api/node-states.ts new file mode 100644 index 0000000000..7930854153 --- /dev/null +++ b/ui/src/lib/api/node-states.ts @@ -0,0 +1,105 @@ +// Node build state apart from the node listing: the listing is read once per snapshot, and the +// states that change as runs finish are applied to it. Pure; no side effects. + +import { shortName } from '$lib/build-area/labels'; +import type { CatalogNode, NodeState, NodeStatusDelta, StaleReason } from './types/build'; + +/** The state part of a listed node. */ +export function stateOf(node: CatalogNode): NodeState { + return { + lastBuild: node.lastBuild, + lastOutcome: node.lastOutcome, + definitionChanged: !!node.stale?.some((r) => r.reason === 'changed'), + checks: node.checks + }; +} + +/** The reason a node whose definition changed since its last build is stale, keeping a fuller one already held. */ +function changedReason(node: CatalogNode, state: NodeState): StaleReason[] { + if (!state.definitionChanged || !state.lastBuild) return []; + const held = node.stale?.find((r) => r.reason === 'changed'); + return [held ?? { reason: 'changed', change: 'definition', detail: 'The definition changed since it was last built here.', since: state.lastBuild.builtAt }]; +} + +/** A node with `state` in place of its own; behind is left to `withBehind`, which needs every node. */ +function withState(node: CatalogNode, state: NodeState): CatalogNode { + const kept = (node.stale ?? []).filter((r) => r.reason !== 'changed'); + return { ...node, lastBuild: state.lastBuild, lastOutcome: state.lastOutcome, checks: state.checks, stale: [...changedReason(node, state), ...kept] }; +} + +/** + * When the data a node reads last changed: its own last build, or for a view, which reads its + * upstream when queried, the newest data behind it. Null when nothing behind it is built. + */ +function dataTimes(listed: Map): (key: string) => number | null { + const memo = new Map(); + const at = (key: string): number | null => { + if (memo.has(key)) return memo.get(key) ?? null; + memo.set(key, null); + const node = listed.get(key); + let time = node?.lastBuild ? Date.parse(node.lastBuild.builtAt) : null; + if (node?.kind === 'view') { + const upstream = node.upstream.flatMap((k) => at(k) ?? []); + time = upstream.length ? Math.max(...upstream) : null; + } + memo.set(key, time); + return time; + }; + return at; +} + +/** + * Upstream nodes with newer data than this node's own last build. Views read their upstream when + * queried, so they are never behind, and an unbuilt node is not built yet rather than behind. + */ +function behindOf(node: CatalogNode, dataAt: (key: string) => number | null): StaleReason[] { + const builtAt = node.lastBuild?.builtAt; + if (!builtAt || node.kind === 'view') return []; + const newer = node.upstream.flatMap((key) => { + const at = dataAt(key); + return at !== null && at > Date.parse(builtAt) ? [{ key, at }] : []; + }); + if (!newer.length) return []; + const since = new Date(Math.min(...newer.map((n) => n.at))).toISOString(); + const names = newer.map((n) => shortName(n.key)); + const detail = `${names.length === 1 ? names[0] : `${names.length} upstream nodes`} ${names.length === 1 ? 'has' : 'have'} newer data than this build.`; + return [{ reason: 'behind', upstream: newer.map((n) => n.key), detail, since }]; +} + +const behindKey = (reasons: StaleReason[]) => JSON.stringify(reasons.filter((r) => r.reason === 'behind')); + +/** + * Nodes with "behind" worked out again from upstream build times. A node whose behind reasons + * did not change is returned as it was, so views keyed on it do not redraw. + */ +export function withBehind(nodes: CatalogNode[]): CatalogNode[] { + const dataAt = dataTimes(new Map(nodes.map((node) => [node.key, node]))); + return nodes.map((node) => { + const behind = behindOf(node, dataAt); + const own = node.stale ?? []; + if (behindKey(own) === JSON.stringify(behind)) return node; + return { ...node, stale: [...own.filter((r) => r.reason !== 'behind'), ...behind] }; + }); +} + +/** The listing with `states` applied; nodes without a state keep theirs, and states of unlisted nodes are ignored. */ +export function applyNodeStates(nodes: CatalogNode[], states: Record): CatalogNode[] { + let changed = false; + const applied = nodes.map((node) => { + const state = states[node.key]; + if (!state) return node; + changed = true; + return withState(node, state); + }); + return changed ? withBehind(applied) : nodes; +} + +/** + * What to do with a `node_status` change given the version held: apply it, ignore one already + * covered, or read every state again after a gap, a `full` change or before any version is held. + */ +export function deltaAction(held: number | null, delta: NodeStatusDelta): 'apply' | 'ignore' | 'refetch' { + if (held !== null && delta.version <= held) return 'ignore'; + if (held === null || delta.full || delta.previousVersion !== held) return 'refetch'; + return 'apply'; +} diff --git a/ui/src/lib/api/signals.ts b/ui/src/lib/api/signals.ts index 744ac5b84c..7d71d6859b 100644 --- a/ui/src/lib/api/signals.ts +++ b/ui/src/lib/api/signals.ts @@ -1,7 +1,8 @@ -import type { Scope } from './types'; +import type { NodeStatusDelta, Scope } from './types'; -// "Changed" signals: payload-free notifications that something changed on the -// server. Clients refetch through the normal endpoints when one arrives. +// "Changed" signals: notifications that something changed on the server. Clients refetch +// through the normal endpoints when one arrives; a `node_status` signal also carries the +// node states that changed, so the node listing is updated in place instead. export type ChangedChannel = /** Refetch everything: sent on connect and whenever notices may have been missed. */ @@ -17,7 +18,7 @@ export type ChangedChannel = | `node_status:${string}` | `warehouse_sync:${string}`; -export type ChangedSignal = { channel: ChangedChannel; at: string }; +export type ChangedSignal = { channel: ChangedChannel; at: string; nodeStatus?: NodeStatusDelta }; export type ConnectionState = 'connecting' | 'live' | 'reconnecting' | 'offline'; @@ -57,8 +58,8 @@ export class LocalChangeBus implements ChangeSubscription { // Every mock signal belongs to the one in-process store. } - emit(channel: ChangedChannel): void { - const signal = { channel, at: new Date().toISOString() }; + emit(channel: ChangedChannel, nodeStatus?: NodeStatusDelta): void { + const signal: ChangedSignal = { channel, at: new Date().toISOString(), ...(nodeStatus ? { nodeStatus } : {}) }; for (const handler of this.#handlers) handler(signal); } diff --git a/ui/src/lib/api/types.ts b/ui/src/lib/api/types.ts index 03ceda96b3..57b6e679d5 100644 --- a/ui/src/lib/api/types.ts +++ b/ui/src/lib/api/types.ts @@ -70,6 +70,10 @@ export type WarehouseSync = { lastError: string | null; /** When the newest sync report arrived, failed or not; a new one means a requested sync has finished. */ lastReportedAt?: Iso | null; + /** The sync now running: which kind of how many, and rows so far; null when none is. */ + syncing?: { position: number; kinds: number; table: string; rows: number } | null; + /** The newest completed sync among recent reports; null when none completed in them. */ + completed?: { at: Iso; kinds: number; rows: number; elapsedMs: number } | null; }; /** The one target a location is deployed against, e.g. `prod` on Snowflake. */ diff --git a/ui/src/lib/api/types/build.ts b/ui/src/lib/api/types/build.ts index 5f9e8024d7..6346f6636c 100644 --- a/ui/src/lib/api/types/build.ts +++ b/ui/src/lib/api/types/build.ts @@ -79,6 +79,34 @@ export type CatalogNode = { checks: CheckCounts | null; }; +// ---- node states ---- + +/** A node's build state, the part of a catalog node that changes between publications. */ +export type NodeState = { + lastBuild: NodeBuild | null; + lastOutcome: NodeOutcome | null; + /** The definition differs from the one the last build recorded. */ + definitionChanged: boolean; + checks: CheckCounts | null; +}; + +/** Every node's state at one status version of a snapshot. */ +/** + * What `sqb build --changed` with default options runs, as the location planned it from the synced copy + * after its last publication or sync: nodes changed themselves, and nodes downstream of them, as keys. + * `failing` is why that plan failed, with both lists empty; a failed plan is never shown as a count. + */ +export type ChangedBuild = { generation: number; changed: string[]; downstream: string[]; failing: string | null }; + +/** `changedBuild` is null until the location plans the current snapshot, and from an older server. */ +export type NodeStates = { snapshotId: string | null; version: number; states: Record; changedBuild: ChangedBuild | null }; + +/** + * A `node_status` change: the states that changed between two versions, or `full` when too many + * changed to list, so the whole map is read again. + */ +export type NodeStatusDelta = { version: number; previousVersion: number; states: Record; full?: false } | { version: number; previousVersion: number; full: true }; + // ---- node detail ---- export type ColumnDef = { @@ -229,6 +257,8 @@ export type CheckEvaluation = { measuredValue: number | null; evidenceCount: number | null; evidenceTruncated: boolean; + /** An error-severity failure its build let through (`ignore_audit_failures`) instead of stopping. */ + ignored: boolean; }; /** One audit or check with its latest result. */ @@ -369,6 +399,7 @@ export type Overview = { attention: AttentionItem[]; /** Null when the server reports no per-group build state. */ groups: GroupStatus[] | null; + /** Newest first, up to the fourth deploy that changed something, with the deploys without changes between. */ recentChanges: SnapshotChange[]; latestRun: (Pick & { status: RunStatus }) | null; }; diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index a9e851e32c..206d09e924 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -18,6 +18,7 @@ export const RUN_STATUSES = [ 'partial', 'failed', 'cancelled', + 'skipped', 'presumed_failed' ] as const; export type RunStatus = (typeof RUN_STATUSES)[number]; @@ -67,6 +68,8 @@ export type Run = { /** Sensor run requests are de-duplicated by run key. */ runKey: string | null; rerunOfRunId: string | null; + /** A group member's attempt once its group retried it, from 2; null for a first attempt and every other run. */ + memberAttempt: number | null; /** How a person narrowed this run of a job, as typed; null when it runs the whole job. */ narrowing: { select: string[]; exclude: string[] } | null; /** The cursor window a backfill processes, inclusive; null for every other run. */ @@ -86,6 +89,8 @@ export type Run = { endedAt: Iso | null; nodeCounts: NodeCounts; cost: RunCost | null; + /** Error-severity audit failures the run let through instead of stopping (`ignore_audit_failures`); a group sums its members'. */ + ignoredAuditFailures: number; }; export type RunListQuery = { @@ -191,7 +196,8 @@ export type RunNodeStatus = /** The run ended (lost worker, never started) before this node ran. */ | 'not_run'; -export type NodeKind = 'source' | 'loader' | 'seed' | 'view' | 'table' | 'incremental' | 'task'; +/** Graph node kinds: SQL models by materialization, sources and seeds, Python nodes (asset, loader, task) and functions (udf, table_fn). */ +export type NodeKind = 'source' | 'loader' | 'seed' | 'view' | 'table' | 'incremental' | 'task' | 'asset' | 'udf' | 'table_fn'; export type SkippedBecauseOf = { nodeKey: string; @@ -258,7 +264,9 @@ export const SCHEDULER_FACTS = [ 'launched', 'cancel_requested', 'presumed_failed', - 'terminated' + 'terminated', + 'member_retrying', + 'member_recovered' ] as const; export type SchedulerFact = (typeof SCHEDULER_FACTS)[number]; @@ -471,15 +479,16 @@ export type RunDetail = { /** Why the scheduler can't claim a queued run yet; a run in another location or target is never named (`runId` null). */ export type QueueWait = | { kind: 'write_claim'; runId: string | null; nodeKey: string; otherNodes: number } - /** Running runs hold slots of a concurrency limit this run takes; `elsewhere` counts holders in other locations or targets. */ - | { kind: 'concurrency_limit'; limit: string; holders: string[]; elsewhere: number } - /** An earlier queued run takes the same limit, so it goes first. */ - | { kind: 'earlier_run'; runId: string | null; limit: string } + /** Running runs of the same location and target hold the slots of a pool this run takes. */ + | { kind: 'pool_full'; pool: string; holders: string[] } + /** An earlier queued run takes the same pool, so it goes first. */ + | { kind: 'earlier_run'; runId: string | null; pool: string } | { kind: 'job_running'; runId: string } - | { kind: 'upstream_job'; runId: string; jobKey: string } + /** `attempt` is set while the group retries that job: its attempt number, from 2. */ + | { kind: 'upstream_job'; runId: string; jobKey: string; attempt: number | null } /** A child run waits for its group run to start. */ | { kind: 'group_parent'; runId: string } - /** A retry of a failed run waits out the job's `on_failure` delay. */ + /** A retry of a failed run waits out the job's `[retry]` delay. */ | { kind: 'retry_delay'; attempt: number; dueAt: Iso }; export type QueueEntry = { @@ -535,6 +544,62 @@ export type SnapshotNode = { upstream: string[]; }; +// ---- selections (the selection bar) ---- + +/** A selection as typed: each text holds space-separated selectors, as `sqb build -s … --exclude …` takes them. */ +export type SelectionRequest = { + select: string; + exclude: string; + /** Sorted keys to skip. */ + offset?: number; + /** Page size; the server defaults to 1,000 and allows up to 10,000. */ + limit?: number; +}; + +/** The token a refused selection comes from: its field and its range there, in JavaScript string indexes. */ +export type SelectionLocation = { field: 'select' | 'exclude'; token: string; start: number; end: number }; + +/** Why `sqb` refuses a selection; `code` is null when the selection as a whole is refused. */ +export type SelectionProblem = { + code: string | null; + message: string; + /** `sqb`'s own help line, such as "did you mean 'orders'?". */ + help: string | null; + location: SelectionLocation | null; + /** Whole-token replacements, closest first. */ + suggestions: string[]; + /** + * The comma part of the located token `sqb` refuses: its index among the token's parts, its selector + * type (`null` for a bare name or name pattern) and its value without type prefix or `+` markers. + * Null when the selection is refused as a whole or the part is a path between two nodes. + */ + refused: { part: number; type: string | null; value: string } | null; +}; + +/** What `sqb build --select … --exclude …` selects in the current publication: one page of keys and the total. */ +export type SelectionEvaluation = { + snapshotId: string | null; + count: number; + keys: string[]; + nextOffset: number | null; + error: SelectionProblem | null; +}; + +/** One selector value and how many nodes it selects. */ +export type VocabularyEntry = { value: string; count: number }; + +/** The values each selector type takes in the current publication, for autocomplete. */ +export type SelectionVocabulary = { + snapshotId: string | null; + tags: VocabularyEntry[]; + /** Folders such as `models/staging`, counting the nodes below each at any depth. */ + paths: VocabularyEntry[]; + jobs: VocabularyEntry[]; + sources: VocabularyEntry[]; + seeds: VocabularyEntry[]; + kinds: VocabularyEntry[]; +}; + // ---- ad-hoc runs (the plan page) ---- export type LaunchVerb = 'build' | 'test' | 'seed' | 'audit' | 'freshness' | 'load'; @@ -583,8 +648,16 @@ export type PlanModel = { /** Unified diff lines of the model's config since its last build. */ config_diff?: string[]; schema_findings?: { kind: string; column: string; expected_type: string | null; actual_type: string | null }[]; + /** A paged row's diff counts in place of its diff text, which `planDiff` reads on demand. */ + diff_stats?: { query?: PlanDiffStats; config?: PlanDiffStats }; }; +/** A diff's changed lines; `whitespace_only` when they read the same with all whitespace taken out (a case change is a change). */ +export type PlanDiffStats = { added: number; removed: number; whitespace_only: boolean }; + +/** One model's diff text, read on demand. */ +export type PlanDiff = { model: string; query_diff?: string[]; config_diff?: string[] }; + export type PlanWarning = { severity: string; message: string; model_name?: string; code?: string }; /** `sqb plan --json` as the location printed it; sections the page doesn't show are kept as they are. */ @@ -604,6 +677,61 @@ export type PlanDocument = { synced_copy: PlanSyncedCopy | null; /** What `--changed` runs, holds back and skips; `null` without `--changed`. */ changed: PlanChanged | null; + /** How long the location took to plan, step by step; absent for plans recorded without timings. */ + timings?: PlanTimings | null; + /** Set when audit failures won't stop the build this plan previews; absent otherwise. */ + ignore_audit_failures?: boolean; + /** A server summary's sections, by size; their rows load a page at a time. */ + pages?: Partial>; + /** A server summary's nonempty sections in plan order, each with the notes its rows share most. */ + sections?: PlanSectionCount[]; + /** Rows of each paged section loaded so far, which is the offset of its next page. */ + loaded?: Partial>; +}; + +/** Every plan section a server summary counts and pages, in plan order. */ +export type PlanPageSection = + | 'full_refresh' + | 'query_changed' + | 'function_changed' + | 'config_changed' + | 'schema_changed' + | 'first_run' + | 'renamed' + | 'routine_models' + | 'migrations' + | 'column_migrations' + | 'table_type_conversions' + | 'retention' + | 'loader_reloads' + | 'loader_loads' + | 'source_reloads' + | 'source_loads' + | 'changed_functions' + | 'functions' + | 'seeds' + | 'python_nodes' + | 'held_back' + | 'unchanged' + | 'warnings'; + +/** A note rows of a section carry, such as `run_reason` "no completed record", and how many rows carry it. */ +export type PlanSectionNote = { kind: string; value: string | null; count: number }; + +/** A section's size and its most frequent notes, so the page can say a note applies to all rows, or N of M. */ +export type PlanSectionCount = { section: PlanPageSection; total: number; notes: PlanSectionNote[] }; + +/** + * One page of a plan section's rows matching a name filter: `total` matches of the section's `section_total` + * rows, and the `--changed` runs, warnings and cursor window starts of the page's models. + */ +export type PlanPage = { + total: number; + section_total: number; + rows: unknown[]; + runs: PlanChanged['runs']; + warnings: PlanWarning[]; + cursor_window_starts: Record; }; /** `sqb plan --changed`'s decision. Nodes are keys such as `model:orders`. */ @@ -618,6 +746,14 @@ export type PlanChanged = { unchanged: string[]; }; +/** How long one plan took: steps in pipeline order with `Other` last, and what it read. */ +export type PlanTimings = { + total_seconds: number; + steps: { name: string; seconds: number }[]; + /** `synced_state` for a location preview over the synced warehouse state; `live` when it read the warehouse. */ + planned_from: string; +}; + /** The server's synced copy of warehouse metadata a location's plan preview planned from. */ export type PlanSyncedCopy = { /** When every table the preview reads last synced; `null` until all of them have. */ @@ -629,7 +765,7 @@ export type PlanSyncedCopy = { }; /** The location's plan of a build command, or why it could not plan it. */ -export type PlanResult = { plan: PlanDocument; error: null } | { plan: null; error: string }; +export type PlanResult = { plan: PlanDocument; error: null; requestId?: string } | { plan: null; error: string }; /** Cursor window for a launch, typed from the selected nodes' cursor type. */ export type CursorWindow = { @@ -653,7 +789,7 @@ export type RerunMode = 'all' | 'from_failure' | 'selected'; /** What a schedule tick or sensor request does while a run of the job is active; people's runs always queue. */ export type OverlapPolicy = 'queue' | 'skip'; -/** A failed or partial run reruns from failure up to `retries` times, `retryDelaySeconds` after the last ended. */ +/** A job's `[retry]`: a failed or partial run reruns from failure up to `retries` times (`attempts` is one more, counting the first run), `retryDelaySeconds` (`delay_seconds`) after the last ended. */ export type OnFailure = { retries: number; retryDelaySeconds: number }; export type JobKind = 'job' | 'group'; @@ -668,10 +804,19 @@ export type Job = { /** Member jobs for groups; null when the server doesn't say. */ members: string[] | null; nodeCount: number; + /** The nodes the job selects, by catalog key; a group's are its members' together. */ + nodes: string[]; /** Null where the server doesn't report it (also `onFailure`, `priority`, `sourceFile`). */ overlap: OverlapPolicy | null; onFailure: OnFailure | null; - concurrencyGroups: string[]; + /** Each pool the job takes: its slots, the pool's limit and the other jobs and groups taking it, as published. */ + pools: PoolUse[]; + /** The job's own `concurrency` setting, models building at once in a run; null uses the project's or target's. */ + concurrency: number | null; + /** What a run uses on the target and which layer set it; null for a snapshot published before it was recorded. */ + resolvedConcurrency: ResolvedConcurrency | null; + /** The job's own `ignore_audit_failures` setting: failing audits don't stop its runs; null uses the target's, then the project's. */ + ignoreAuditFailures: boolean | null; priority: number | null; owner: string | null; sourceFile: string | null; @@ -679,10 +824,32 @@ export type Job = { lastRun: RunRef | null; }; -export type ConcurrencyGroup = { - key: string; +/** A pool a job takes: `amount` of its `limit` slots, shared with the other jobs and groups taking it. */ +export type PoolUse = { + pool: string; + /** The slots the job asks for. */ + amount: number; + /** The project's limit on the target. */ + limit: number; + /** This server's `pool_caps` entry; null when it sets none. */ + cap: number | null; + /** The limit on this server: `limit` lowered to `cap`. */ + serverLimit: number; + /** `amount` is above `serverLimit`, so on this server the job takes the whole pool. */ + clamped: boolean; + sharedWith: string[]; +}; + +/** Models a run builds at once, and the layer that set it. */ +export type ResolvedConcurrency = { limit: number; source: 'job' | 'target' | 'local' | 'project' | 'default' }; + +/** One reason a member waits: `fromNode` of `upstreamJob` feeds its `toNode`, directly or through nodes no member selects. */ +export type WaitReason = { upstreamJob: string; fromNode: string; toNode: string }; + +export type PoolHolders = { + pool: string; limit: number; - /** Active runs holding a slot. */ + /** Active runs holding its slots. */ holders: string[]; }; @@ -692,16 +859,18 @@ export type JobGroupMember = { upstream: { jobKey: string; source: 'lineage' | null }[]; /** Position in the derived order; members in the same stage run side by side. */ stage: number; + /** Node-level reasons for its waits, as the snapshot records them; every member's is empty in a snapshot published before they were recorded. */ + reasons: WaitReason[]; }; export type JobDetail = Job & { - /** Null where the server doesn't report it, as for `declaration`, `concurrency` and `targets`. */ + /** Null where the server doesn't report it, as for `declaration`, `poolHolders` and `targets`. */ declaration: { sourceFile: string; line: number; code: string } | null; exclude: string[]; /** Settings are CLI flags validated against the command table. */ settings: Record | null; resolvedNodes: string[]; - concurrency: ConcurrencyGroup[] | null; + poolHolders: PoolHolders[] | null; /** Null when the job applies to every target. */ targets: string[] | null; groupMembers: JobGroupMember[] | null; diff --git a/ui/src/lib/api/types/platform.ts b/ui/src/lib/api/types/platform.ts index 7affa8ff1f..77d935ad2a 100644 --- a/ui/src/lib/api/types/platform.ts +++ b/ui/src/lib/api/types/platform.ts @@ -132,8 +132,11 @@ export type ConnectionTest = { outcome: TestOutcome; at: Iso; durationMs: number; - /** `publish` when the location ran it after publishing a snapshot. */ - trigger: 'publish' | 'manual'; + /** + * `publish` when the location ran it after publishing a snapshot; `startup` when the location ran it on + * starting (`sqb dev` or `sqb location serve`); `manual` for Test connection. + */ + trigger: 'publish' | 'startup' | 'manual'; by: Principal | null; message: string; }; @@ -168,8 +171,13 @@ export type LocationDetail = { /** Null when the server doesn't name it. */ warehouseConnection: string | null; connectionTest: ConnectionTest | null; + /** Jobs and groups this server's pool caps make run alone in a pool: the server's own warning, not a check-in. */ + poolCapWarnings: PoolCapWarning[]; }; +/** A job or group asking a pool for more than this server's cap, so it takes the whole pool. */ +export type PoolCapWarning = { jobKey: string; kind: 'job' | 'group'; pool: string; amount: number; cap: number; message: string }; + export type LocationSummary = Pick & { /** Null when the server can't report it to this person. */ health: LocationHealth | null; @@ -215,15 +223,48 @@ export type SettingField = { secretSource?: string; }; -/** A location target's warehouse connection as its checks and syncs report it; locations publish no connection settings. */ +/** + * How a location signs in to its warehouse, by name only; never the secret. Known names have labels; + * an unknown name is shown as it is. + */ +export type ConnectionAuthMethod = 'key_pair' | 'password' | 'oauth' | 'external_browser' | 'token' | 'none' | (string & {}); + +/** + * The non-secret connection settings a location publishes with each snapshot. Visible to everyone who + * can view the location. Fields the adapter does not have are null. + */ +export type ConnectionDetails = { + /** e.g. `snowflake`, `postgres`, `duckdb`. */ + adapter: string; + /** The account identifier, for adapters that have one. */ + account: string | null; + /** The server host, or for a file database its path. */ + host: string | null; + user: string | null; + role: string | null; + /** The compute warehouse, for adapters that have one. */ + warehouse: string | null; + /** The database the target builds into by default. */ + database: string | null; + authMethod: ConnectionAuthMethod | null; +}; + +/** A location target's warehouse connection as its checks and syncs report it. */ export type WarehouseConnection = { /** The location key, `location_id:target`. */ id: string; location: string; target: string; + /** Published connection settings; null until the location publishes them with a snapshot. */ + details: ConnectionDetails | null; /** Where SQLBuild's own tables are read back from; null before a sync reports it. */ stateLocation: { database: string | null; schema: string } | null; + /** The newest recorded connection check: on startup, after a deploy, or a manual test. */ lastTest: ConnectionTest | null; + /** When the next background state history sync is due; null while a sync runs or when not reported. */ + nextSyncAt: Iso | null; + /** Failed reads in a row stopped the newest sync: it resumes at `nextSyncAt` when set, otherwise after the next run, a sync request or a deploy. */ + halted: boolean; /** The newest sync of each SQLBuild table, in table order. */ tables: WarehouseTableSync[]; }; @@ -241,6 +282,8 @@ export type WarehouseTableSync = { waiting: boolean; /** The newest sync of the table that succeeded. */ lastSyncedAt: Iso | null; + /** Records the server mirrors of the table in total; null when not reported. */ + mirroredTotal: number | null; }; /** One setting a provider declares. Its value stays in the location's environment and never reaches the server. */ @@ -272,8 +315,12 @@ export type ProviderConnection = { sourceFile: string; settings: ProviderSetting[]; usedBy: ProviderConsumer[]; + /** Keys of the snapshot nodes that run the provider's consumers, e.g. `source:orders`. */ + nodes: string[]; /** Jobs whose selection includes a node that uses the provider. */ jobs: string[]; + /** Each of those jobs with the provider's nodes it runs. */ + jobNodes: Record; }; // ---- maintenance (synced warehouse state tables) ---- diff --git a/ui/src/lib/build-area/attention.test.ts b/ui/src/lib/build-area/attention.test.ts index 0cc0402f25..e4de69c642 100644 --- a/ui/src/lib/build-area/attention.test.ts +++ b/ui/src/lib/build-area/attention.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import type { AttentionItem, AttentionRun, PlatformEvent } from '$lib/api'; -import { attentionViews } from './attention'; +import { attentionViews, staleSentence } from './attention'; import { CHANGED_PLAN_HREF, planHref } from '$lib/run/command'; const now = Date.parse('2026-01-01T12:00:00Z'); @@ -70,18 +70,57 @@ describe('overview alerts', () => { it('offers one build: what changed when anything changed, else a rebuild of the nodes behind', () => { const build = (changed: string[], behind: string[]) => attentionViews([{ kind: 'stale_models', changed, behind }], now)[0].view.build; - expect(build(['model:a'], ['model:a', 'model:b'])).toEqual({ label: 'Build what changed', href: CHANGED_PLAN_HREF }); + expect(build(['model:a'], ['model:a', 'model:b'])).toEqual({ + label: 'Build 1 changed', + extra: null, + tooltip: 'Plan a build of the 1 changed node and the downstream nodes it reaches. You review the plan before anything runs.', + href: CHANGED_PLAN_HREF, + variant: 'stale' + }); // `--changed` would build nothing when nodes are only behind, so the rebuild names them. - expect(build([], ['model:b', 'model:c'])).toEqual({ label: 'Rebuild 2', href: planHref(['model:b', 'model:c']) }); + expect(build([], ['model:b', 'model:c'])).toEqual({ label: 'Rebuild 2', extra: null, tooltip: null, href: planHref(['model:b', 'model:c']), variant: 'stale-outline' }); + }); + + it('labels the build with the location\'s plan of what changed and how much it reaches downstream', () => { + const [{ view }] = attentionViews([{ kind: 'stale_models', changed: ['model:a'], behind: ['model:b'] }], now, { generation: 2, changed: ['model:a', 'model:c'], downstream: ['model:b'], failing: null }); + expect([view.build?.label, view.build?.extra]).toEqual(['Build 2 changed', ' (+1 downstream)']); }); it('states how many nodes are out of date, counting a node both changed and behind once', () => { - const detail = (changed: string[], behind: string[]) => attentionViews([{ kind: 'stale_models', changed, behind }], now)[0].view; - expect(detail(['model:a', 'model:b'], ['model:b']).title).toBe('2 nodes out of date'); - expect(detail(['model:a', 'model:b'], ['model:b']).detail).toBe('2 changed since their last build, 1 of them also behind an upstream rebuild: a, b'); - expect(detail(['model:a', 'model:b'], ['model:b', 'model:c']).detail).toBe('2 changed since their last build, 1 of them also behind an upstream rebuild, and 1 more behind an upstream rebuild: a, b, c'); - expect(detail(['model:a'], ['model:c']).detail).toBe('1 changed since its last build and 1 behind an upstream rebuild: a, c'); - expect(detail(['model:a'], ['model:a']).detail).toBe('1 changed since its last build, also behind an upstream rebuild: a'); - expect(detail([], ['model:c', 'model:d']).detail).toBe('2 behind an upstream rebuild: c, d'); + const view = (changed: string[], behind: string[]) => attentionViews([{ kind: 'stale_models', changed, behind }], now)[0].view; + expect(view(['model:a', 'model:b'], ['model:b']).title).toBe('2 nodes out of date'); + expect(view(['model:a', 'model:b'], ['model:b', 'model:c']).detail).toBe('2 changed since their last build, and 1 more is behind an upstream rebuild: a, b and c.'); + }); +}); + +describe('the out-of-date sentence', () => { + const keys = (prefix: string, n: number) => Array.from({ length: n }, (_, i) => `model:${prefix}${i + 1}`); + + it('reads as plain English, with counts that add up to the total and at most three names', () => { + // 30 changed, 9 of them also behind, and 82 only behind: 112 in all. + const changed = keys('orders_', 30); + const behind = [...changed.slice(0, 9), ...keys('customers_', 82)]; + expect(staleSentence(changed, behind)).toBe( + '30 changed since their last build, and 82 more are behind an upstream rebuild. Including orders_1, orders_2, orders_3 and 109 more.' + ); + }); + + it('drops the overlap: nodes changed and also behind count only as changed', () => { + expect(staleSentence(['model:orders', 'model:customers'], ['model:customers'])).toBe('2 changed since their last build: orders and customers.'); + expect(staleSentence(['model:orders'], ['model:orders'])).toBe('1 changed since its last build: orders.'); + }); + + it('handles singular and plural for each part', () => { + expect(staleSentence(['model:orders'], ['model:customers'])).toBe('1 changed since its last build, and 1 more is behind an upstream rebuild: orders and customers.'); + expect(staleSentence(keys('orders_', 2), keys('customers_', 2))).toBe( + '2 changed since their last build, and 2 more are behind an upstream rebuild. Including orders_1, orders_2, customers_1 and 1 more.' + ); + }); + + it('words only-changed and only-behind nodes naturally', () => { + expect(staleSentence(keys('orders_', 5), [])).toBe('5 changed since their last build. Including orders_1, orders_2, orders_3 and 2 more.'); + expect(staleSentence([], ['model:inventory'])).toBe('1 is behind an upstream rebuild: inventory.'); + expect(staleSentence([], ['model:inventory', 'model:orders', 'model:customers'])).toBe('3 are behind an upstream rebuild: inventory, orders and customers.'); + expect(staleSentence([], keys('inventory_', 4))).toBe('4 are behind an upstream rebuild. Including inventory_1, inventory_2, inventory_3 and 1 more.'); }); }); diff --git a/ui/src/lib/build-area/attention.ts b/ui/src/lib/build-area/attention.ts index 07dc7066ed..5bd97bc743 100644 --- a/ui/src/lib/build-area/attention.ts +++ b/ui/src/lib/build-area/attention.ts @@ -1,12 +1,13 @@ // Presentation of the overview's attention items. Pure; no side effects. -import type { AttentionItem, QueueWait } from '$lib/api'; +import type { AttentionItem, ChangedBuild, QueueWait } from '$lib/api'; import { describeWait, outcomeCounts } from '$lib/operate/labels'; import { eventKindLabel, problemHref } from '$lib/platform/labels'; import { formatAgo } from '$lib/format/time'; import { runStatusLabel } from '$lib/status'; import { shortName } from '$lib/build-area/labels'; -import { CHANGED_PLAN_HREF, planHref } from '$lib/run/command'; +import { CHANGED_PLAN_HREF, changedBuildCounts, changedBuildLabel, planHref } from '$lib/run/command'; +import { plural } from '$lib/format/plural'; export type AttentionTone = 'error' | 'warning' | 'stale' | 'info'; @@ -23,18 +24,46 @@ export type AttentionView = { href: string; action: string; consequences: Consequence[]; - /** The one build an out-of-date alert offers: `sqb build --changed` when anything changed, else a rebuild of the nodes behind. */ - build?: { label: string; href: string }; + /** + * The one build an out-of-date alert offers: `sqb build --changed` when anything changed, else a rebuild of the nodes behind. + * Purple means "needs building": solid for changed nodes, outlined for nodes only behind. + */ + build?: { label: string; extra: string | null; tooltip: string | null; href: string; variant: 'stale' | 'stale-outline' }; }; /** A heartbeat gap shorter than this reads as a pause, not an outage: the run carries on when its worker reports again. */ export const UNRESPONSIVE_OUTAGE_MS = 5 * 60_000; -const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`; const minutes = (seconds: number | null) => (seconds === null ? '—' : seconds % 3600 === 0 ? `${seconds / 3600}h` : `${Math.round(seconds / 60)}m`); const waitText = (waits: QueueWait[]) => waits.map((w) => describeWait(w).map((p) => p.text).join('')).join(' · '); -export function attentionView(item: AttentionItem, now: number): AttentionView { +/** How many names the out-of-date sentence lists before "and N more". */ +export const STALE_NAMES_SHOWN = 3; + +/** "a", "a and b", "a, b and c". */ +const andList = (parts: string[]) => (parts.length < 2 ? parts.join('') : `${parts.slice(0, -1).join(', ')} and ${parts[parts.length - 1]}`); + +/** + * The out-of-date alert's sentence, e.g. "30 changed since their last build, and 82 more are behind an upstream + * rebuild. Including a, b, c and 109 more." A node both changed and behind counts once, as changed, so the two + * counts add up to the alert's total. Up to three names; a complete list follows a colon instead. + */ +export function staleSentence(changed: string[], behind: string[]): string { + const changedSet = new Set(changed); + const changedCount = changedSet.size; + const onlyBehind = new Set(behind.filter((key) => !changedSet.has(key))).size; + const all = [...new Set([...changed, ...behind])].map(shortName); + const behindText = (n: number, more: string) => `${n}${more} ${n === 1 ? 'is' : 'are'} behind an upstream rebuild`; + const counts = changedCount + ? `${changedCount} changed since ${changedCount === 1 ? 'its' : 'their'} last build${onlyBehind ? `, and ${behindText(onlyBehind, ' more')}` : ''}` + : behindText(onlyBehind, ''); + if (all.length === 0) return `${counts}.`; + if (all.length <= STALE_NAMES_SHOWN) return `${counts}: ${andList(all)}.`; + return `${counts}. Including ${all.slice(0, STALE_NAMES_SHOWN).join(', ')} and ${all.length - STALE_NAMES_SHOWN} more.`; +} + +/** `changedBuild` is what the location planned `sqb build --changed` to run, when it has. */ +export function attentionView(item: AttentionItem, now: number, changedBuild: ChangedBuild | null = null): AttentionView { switch (item.kind) { case 'run_problem': return { @@ -67,25 +96,22 @@ export function attentionView(item: AttentionItem, now: number): AttentionView { }; } case 'stale_models': { - const changed = new Set(item.changed); - const both = item.behind.filter((key) => changed.has(key)).length; const all = [...new Set([...item.changed, ...item.behind])]; - const only = item.behind.length - both; - const alsoBehind = both ? `, ${item.changed.length === 1 ? 'also' : `${both} of them also`} behind an upstream rebuild` : ''; - const onlyBehind = only ? `${both ? `, and ${only} more` : ` and ${only}`} behind an upstream rebuild` : ''; - const counts = item.changed.length - ? `${item.changed.length} changed since ${item.changed.length === 1 ? 'its' : 'their'} last build${alsoBehind}${onlyBehind}` - : `${only} behind an upstream rebuild`; return { key: 'stale', tone: 'stale', title: `${plural(all.length, 'node')} out of date`, - detail: `${counts}: ${all.slice(0, 4).map(shortName).join(', ')}${all.length > 4 ? ` and ${all.length - 4} more` : ''}`, + detail: staleSentence(item.changed, item.behind), since: '', href: '/catalog?stale=1', action: 'View in catalog', consequences: [], - build: item.changed.length ? { label: 'Build what changed', href: CHANGED_PLAN_HREF } : { label: `Rebuild ${all.length}`, href: planHref(all) } + build: item.changed.length + ? (() => { + const counts = changedBuildCounts(new Set(item.changed).size, changedBuild); + return { ...changedBuildLabel(counts.changed, counts.downstream, 'banner', counts.failing), href: CHANGED_PLAN_HREF, variant: 'stale' as const }; + })() + : { label: `Rebuild ${all.length}`, extra: null, tooltip: null, href: planHref(all), variant: 'stale-outline' } }; } case 'stale_sources': @@ -169,8 +195,8 @@ function consequenceOf(item: AttentionItem, view: AttentionView): Consequence { } /** The items as alerts, errors first: an item that follows from another alert's cause is listed under it, not on its own. */ -export function attentionViews(items: AttentionItem[], now: number): { item: AttentionItem; view: AttentionView }[] { - const viewed = items.map((item) => ({ item, view: attentionView(item, now) })); +export function attentionViews(items: AttentionItem[], now: number, changedBuild: ChangedBuild | null = null): { item: AttentionItem; view: AttentionView }[] { + const viewed = items.map((item) => ({ item, view: attentionView(item, now, changedBuild) })); const byKey = new Map(viewed.map((v) => [v.view.key, v])); const shown = viewed.filter(({ item, view }) => { const cause = byKey.get(causeOf(item, items) ?? ''); diff --git a/ui/src/lib/build-area/changes.test.ts b/ui/src/lib/build-area/changes.test.ts index e0fcdafd73..cbc60a4136 100644 --- a/ui/src/lib/build-area/changes.test.ts +++ b/ui/src/lib/build-area/changes.test.ts @@ -9,13 +9,13 @@ describe('node change summaries', () => { it('counts a first deploy by change and kind, largest first, in plain nouns', () => { const nodes = [...many(3, 'model'), ...many(1200, 'audit'), change('udf:margin'), ...many(2, 'model', 'removed'), change('widget:x')]; expect(nodeChangeGroups(nodes)).toEqual([ - { op: 'added', prefix: 'audit', count: 1200, noun: 'checks' }, + { op: 'added', prefix: 'audit', count: 1200, noun: 'audits' }, { op: 'added', prefix: 'model', count: 3, noun: 'models' }, { op: 'removed', prefix: 'model', count: 2, noun: 'models' }, { op: 'added', prefix: 'udf', count: 1, noun: 'function' }, { op: 'added', prefix: 'widget', count: 1, noun: 'widget' } ]); - expect(nodeChangeLine(nodes)).toBe('added 1,200 checks · added 3 models · removed 2 models · 2 more kinds'); + expect(nodeChangeLine(nodes)).toBe('added 1,200 audits · added 3 models · removed 2 models · 2 more kinds'); }); it('keeps SQL and Python hooks apart, so each kind is its own group', () => { diff --git a/ui/src/lib/build-area/changes.ts b/ui/src/lib/build-area/changes.ts index f7da3d04ca..248ddf1090 100644 --- a/ui/src/lib/build-area/changes.ts +++ b/ui/src/lib/build-area/changes.ts @@ -1,16 +1,16 @@ // Summaries of change-feed entries. Pure; no side effects. import type { ChangeOp, NodeChange, SnapshotChange } from '$lib/api'; +import { nodeNoun, plural } from '$lib/format/plural'; -const count = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`; /** e.g. "3 nodes · 1 trigger · 1 role · dependencies". */ export function summarizeChange(change: SnapshotChange): string { const parts: string[] = []; - if (change.nodes.length) parts.push(`${change.nodes.length.toLocaleString('en')} ${change.nodes.length === 1 ? 'node' : 'nodes'}`); + if (change.nodes.length) parts.push(plural(change.nodes.length, 'node')); const orchestration = change.orchestration.length; - if (orchestration) parts.push(count(orchestration, 'job or trigger', 'jobs and triggers')); - if (change.roles.length) parts.push(count(change.roles.length, 'role')); + if (orchestration) parts.push(plural(orchestration, 'job or trigger', 'jobs and triggers')); + if (change.roles.length) parts.push(plural(change.roles.length, 'role')); if (change.environment) parts.push('environment'); return parts.length ? parts.join(' · ') : 'No definition changes'; } @@ -18,24 +18,6 @@ export function summarizeChange(change: SnapshotChange): string { /** A deploy lists its node changes one by one up to this many; more are counted by kind. */ export const NODE_LIST_LIMIT = 20; -/** Readable names for the node key prefixes a snapshot uses; an unknown prefix is shown as it is. */ -const NOUN: Record = { - model: ['model', 'models'], - audit: ['check', 'checks'], - sql_test: ['SQL test', 'SQL tests'], - sql_scenario: ['scenario', 'scenarios'], - source: ['source', 'sources'], - loader: ['loader', 'loaders'], - macro: ['macro', 'macros'], - seed: ['seed', 'seeds'], - snapshot: ['snapshot', 'snapshots'], - udf: ['function', 'functions'], - table_fn: ['table function', 'table functions'], - sql_hook: ['SQL hook', 'SQL hooks'], - hook: ['Python hook', 'Python hooks'], - materialization: ['materialization', 'materializations'] -}; - export type NodeChangeGroup = { op: ChangeOp; prefix: string; count: number; noun: string }; /** A deploy's node changes counted by change and kind, largest first, e.g. 252 added models. */ @@ -51,8 +33,7 @@ export function nodeChangeGroups(nodes: readonly NodeChange[]): NodeChangeGroup[ return [...groups.values()] .sort((a, b) => b.count - a.count || a.prefix.localeCompare(b.prefix)) .map(({ op, prefix, count }) => { - const [one, many] = NOUN[prefix] ?? [prefix, prefix]; - return { op, prefix, count, noun: count === 1 ? one : many }; + return { op, prefix, count, noun: nodeNoun(prefix, count) }; }); } diff --git a/ui/src/lib/build-area/kind-tabs.test.ts b/ui/src/lib/build-area/kind-tabs.test.ts new file mode 100644 index 0000000000..f25b0f028d --- /dev/null +++ b/ui/src/lib/build-area/kind-tabs.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from 'vitest'; +import { NODE_KINDS, kindTabOf, nodeIcon, nodeIconKey, parseKindTab } from './labels'; + +describe('catalog kind tabs', () => { + it('puts Python nodes and functions in their own tabs', () => { + expect(['asset', 'loader', 'task'].map((k) => kindTabOf(k as never))).toEqual(['python', 'python', 'python']); + expect(['udf', 'table_fn'].map((k) => kindTabOf(k as never))).toEqual(['functions', 'functions']); + expect(kindTabOf('view')).toBe('view'); + }); + + it('reads a tab or a kind from a link, and nothing else', () => { + expect(parseKindTab('python')).toBe('python'); + expect(parseKindTab('loader')).toBe('python'); + expect(parseKindTab('table_fn')).toBe('functions'); + expect(parseKindTab('audit')).toBeNull(); + expect(parseKindTab(null)).toBeNull(); + }); + + it('has an icon for every node kind', () => { + for (const kind of NODE_KINDS) expect(nodeIcon[nodeIconKey({ kind, materialization: null })], kind).toBeDefined(); + }); +}); diff --git a/ui/src/lib/build-area/labels.ts b/ui/src/lib/build-area/labels.ts index c6e5747701..2c6d569801 100644 --- a/ui/src/lib/build-area/labels.ts +++ b/ui/src/lib/build-area/labels.ts @@ -5,6 +5,9 @@ import Download from '@lucide/svelte/icons/download'; import Eye from '@lucide/svelte/icons/eye'; import GitMerge from '@lucide/svelte/icons/git-merge'; import Layers from '@lucide/svelte/icons/layers'; +import Package from '@lucide/svelte/icons/package'; +import SquareFunction from '@lucide/svelte/icons/square-function'; +import TableProperties from '@lucide/svelte/icons/table-properties'; import Sprout from '@lucide/svelte/icons/sprout'; import Table2 from '@lucide/svelte/icons/table-2'; import Workflow from '@lucide/svelte/icons/workflow'; @@ -30,7 +33,10 @@ export const nodeKindLabel: Record = { view: 'View', table: 'Table', incremental: 'Incremental', - task: 'Task' + task: 'Task', + asset: 'Asset', + udf: 'Function', + table_fn: 'Table function' }; export const nodeKindPlural: Record = { @@ -40,10 +46,44 @@ export const nodeKindPlural: Record = { view: 'Views', table: 'Tables', incremental: 'Incremental', - task: 'Tasks' + task: 'Tasks', + asset: 'Assets', + udf: 'Functions', + table_fn: 'Table functions' }; -export const NODE_KINDS: NodeKind[] = ['source', 'loader', 'seed', 'view', 'table', 'incremental', 'task']; +export const NODE_KINDS: NodeKind[] = ['source', 'loader', 'seed', 'view', 'table', 'incremental', 'task', 'asset', 'udf', 'table_fn']; + +/** The Catalog's kind tabs: SQL kinds one by one, Python nodes together, and functions together. */ +export type KindTab = 'source' | 'seed' | 'view' | 'table' | 'incremental' | 'python' | 'functions'; + +export const KIND_TABS: KindTab[] = ['source', 'seed', 'view', 'table', 'incremental', 'python', 'functions']; + +export const PYTHON_KINDS: readonly NodeKind[] = ['asset', 'loader', 'task']; +export const FUNCTION_KINDS: readonly NodeKind[] = ['udf', 'table_fn']; + +export function kindTabOf(kind: NodeKind): KindTab { + if (PYTHON_KINDS.includes(kind)) return 'python'; + if (FUNCTION_KINDS.includes(kind)) return 'functions'; + return kind as KindTab; +} + +export const kindTabLabel: Record = { + source: 'Sources', + seed: 'Seeds', + view: 'Views', + table: 'Tables', + incremental: 'Incremental', + python: 'Python', + functions: 'Functions' +}; + +/** A tab from a link, which may name a kind, such as `loader`, or a tab, such as `python`. */ +export function parseKindTab(value: string | null): KindTab | null { + if (!value) return null; + if ((KIND_TABS as string[]).includes(value)) return value as KindTab; + return (NODE_KINDS as string[]).includes(value) ? kindTabOf(value as NodeKind) : null; +} /** Icon key used by the lineage cards, from the node kind and materialization. */ export function nodeIconKey(node: Pick): string { @@ -62,7 +102,10 @@ export const nodeIcon: Record = { merge: GitMerge, delete_insert: Layers, microbatch: Layers, - task: Workflow + task: Workflow, + asset: Package, + udf: SquareFunction, + table_fn: TableProperties }; /** `incremental · merge` style kind line, falling back to the node kind. */ @@ -70,14 +113,26 @@ export function kindLine(node: Pick): s return node.materialization ?? nodeKindLabel[node.kind].toLowerCase(); } +/** What a check looks at, as a short fact; `runScopePhaseHelp` says when. */ export const runScopePhaseLabel: Record = { - delta_and_final: 'Each new batch before it is written, then the whole table', - final: 'Whole table, after writing' + delta_and_final: 'Each batch + full table', + final: 'Full table' }; +export const runScopePhaseHelp: Record = { + delta_and_final: "Each new batch is checked before it's written, then the whole table", + final: 'Checked against the whole table after the build writes it' +}; + +/** How a check decides its result, as a short fact; `evaluationModeHelp` explains it. */ export const evaluationModeLabel: Record = { - violations: 'Fails if any rows break the rule', - measurement: 'Measures a value' + violations: 'Any failing row', + measurement: 'Measured value' +}; + +export const evaluationModeHelp: Record = { + violations: 'Fails when any row breaks the rule', + measurement: 'Compares a measured value with thresholds' }; export const attachmentLabel: Record = { @@ -128,14 +183,14 @@ export function shortRunId(id: string): string { /** "run-4f2a" for scheduler builds, "from the command line · 0f9a3c27" otherwise. */ export function buildSourceLabel(build: Pick): string { - return build.runId ?? `from the command line · ${shortRunId(build.fingerprintRunId)}`; + return build.runId ?? `CLI · ${shortRunId(build.fingerprintRunId)}`; } /** Versions for the SQL viewer's drift mode, newest first. */ export function sqlVersions(builds: BuiltDefinition[], now: number): SqlVersion[] { return builds.map((b, i) => ({ id: i === 0 ? 'built' : b.build.fingerprintRunId, - label: i === 0 ? 'Last build' : b.build.runId ? `Build ${b.build.runId}` : 'Build from the command line', + label: i === 0 ? 'Last build' : b.build.runId ? `Build ${b.build.runId}` : 'CLI build', detail: `${i === 0 || !b.build.runId ? `${buildSourceLabel(b.build)} · ` : ''}${formatAgo(b.build.builtAt, now)} · ${formatStamp(b.build.builtAt)}`, raw: b.raw, compiled: b.compiled diff --git a/ui/src/lib/build-area/overview-panels.test.ts b/ui/src/lib/build-area/overview-panels.test.ts new file mode 100644 index 0000000000..944fb8553d --- /dev/null +++ b/ui/src/lib/build-area/overview-panels.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from 'vitest'; +import type { GroupStatus, SnapshotChange } from '$lib/api'; +import { quietDeployLine, recentDeployRows, recentDeploys, sortFolders, visibleFolders } from './overview-panels'; + +const HOUR = 60 * 60 * 1000; +const now = Date.parse('2026-11-16T12:00:00Z'); +const at = (hoursAgo: number) => new Date(now - hoursAgo * HOUR).toISOString(); + +const deploy = (id: string, hoursAgo: number, changed: boolean): SnapshotChange => ({ + snapshotId: id, + generation: 0, + previousGeneration: null, + gitRevision: null, + gitDirty: false, + gitAuthor: null, + publishedAt: at(hoursAgo), + publishedBy: null, + nodes: changed ? [{ key: `model:orders_${id}`, kind: null, op: 'changed', formerKey: null, compiledOnly: false, detail: '', columns: [] }] : [], + orchestration: [], + roles: [], + environment: null, + followingRuns: [], + costChanges: null +}); + +describe('recent deploys', () => { + // Newest first: two quiet, A, B, three quiet, C, D, then older ones the panel never needs. + const feed = [ + deploy('q1', 1, false), + deploy('q2', 2, false), + deploy('a', 3, true), + deploy('b', 4, true), + deploy('q3', 7, false), + deploy('q4', 8, false), + deploy('q5', 10, false), + deploy('c', 11, true), + deploy('d', 12, true), + deploy('q6', 13, false), + deploy('e', 14, true) + ]; + + it('takes deploys up to the fourth that changed something', () => { + expect(recentDeploys(feed).map((c) => c.snapshotId)).toEqual(['q1', 'q2', 'a', 'b', 'q3', 'q4', 'q5', 'c', 'd']); + }); + + it('merges each run of deploys without changes into one row', () => { + const rows = recentDeployRows(recentDeploys(feed)); + expect(rows.map((r) => (r.kind === 'deploy' ? r.change.snapshotId : `${r.count} quiet`))).toEqual(['2 quiet', 'a', 'b', '3 quiet', 'c', 'd']); + }); + + it('words a merged row with the span of its deploys', () => { + const [, , , quiet] = recentDeployRows(recentDeploys(feed)); + expect(quiet.kind === 'quiet' && quietDeployLine(quiet, now)).toBe('3 deploys without changes · 7h–10h ago'); + const [single] = recentDeployRows([deploy('q', 2, false)]); + expect(single.kind === 'quiet' && quietDeployLine(single, now)).toBe('1 deploy without changes · 2h ago'); + }); + + it('counts a role or environment change as a change', () => { + const roleOnly = { ...deploy('r', 1, false), roles: [{}] } as unknown as SnapshotChange; + expect(recentDeployRows([roleOnly]).map((r) => r.kind)).toEqual(['deploy']); + }); +}); + +const folder = (group: string, total: number, failed = 0, changed = 0, behind = 0): GroupStatus => ({ group, total, current: total - failed - changed, changed, behind, failed }); + +describe('build status by folder', () => { + it('sorts by failed, then changed, then behind, then size, then name', () => { + const sorted = sortFolders([folder('orders', 5), folder('customers', 9), folder('inventory', 3, 0, 0, 2), folder('billing', 4, 0, 1), folder('returns', 2, 1), folder('alpha', 9)]); + expect(sorted.map((g) => g.group)).toEqual(['returns', 'billing', 'inventory', 'alpha', 'customers', 'orders']); + }); + + it('shows eight folders, plus any beyond that need attention, until expanded', () => { + const calm = Array.from({ length: 12 }, (_, i) => folder(`calm_${String(i).padStart(2, '0')}`, 20 - i)); + const attention = Array.from({ length: 10 }, (_, i) => folder(`busy_${i}`, 1, 0, 0, 1)); + expect(visibleFolders(calm, false)).toMatchObject({ hidden: 4 }); + expect(visibleFolders(calm, false).rows.map((g) => g.group)).toEqual(calm.slice(0, 8).map((g) => g.group)); + const mixed = visibleFolders([...calm, ...attention], false); + expect(mixed.rows).toHaveLength(10); + expect(mixed.rows.every((g) => g.behind)).toBe(true); + expect(mixed.hidden).toBe(12); + expect(visibleFolders([...calm, ...attention], true)).toMatchObject({ hidden: 0 }); + expect(visibleFolders([...calm, ...attention], true).rows).toHaveLength(22); + }); +}); diff --git a/ui/src/lib/build-area/overview-panels.ts b/ui/src/lib/build-area/overview-panels.ts new file mode 100644 index 0000000000..e18de01a36 --- /dev/null +++ b/ui/src/lib/build-area/overview-panels.ts @@ -0,0 +1,85 @@ +// The Overview's lower panels: recent deploys with quiet runs merged, and folders by what needs attention. Pure. + +import type { GroupStatus, SnapshotChange } from '$lib/api'; +import { formatAgo } from '$lib/format/time'; +import { plural } from '$lib/format/plural'; + +/** Deploys that changed something the Recent changes panel shows. */ +export const RECENT_DEPLOYS = 4; +/** Folders the Build status panel shows before "Show all"; folders that need attention always show. */ +export const FOLDER_ROWS = 8; + +/** Whether a deploy changed any definition: a node, a job or trigger, a role or the environment. */ +export function changedSomething(change: SnapshotChange): boolean { + return change.nodes.length > 0 || change.orchestration.length > 0 || change.roles.length > 0 || change.environment !== null; +} + +/** + * The newest deploys, newest first, up to and including the `limit`th one that changed something, so + * deploys without changes between them are kept to be merged and never crowd out a real change. + */ +export function recentDeploys(changes: readonly SnapshotChange[], limit = RECENT_DEPLOYS): SnapshotChange[] { + const taken: SnapshotChange[] = []; + let changed = 0; + for (const change of changes) { + if (changed >= limit) break; + taken.push(change); + if (changedSomething(change)) changed += 1; + } + return taken; +} + +export type RecentDeployRow = + | { kind: 'deploy'; key: string; change: SnapshotChange } + | { kind: 'quiet'; key: string; count: number; newestAt: string; oldestAt: string; changes: SnapshotChange[] }; + +/** Deploys newest first, with each run of deploys that changed nothing merged into one row. */ +export function recentDeployRows(changes: readonly SnapshotChange[]): RecentDeployRow[] { + const rows: RecentDeployRow[] = []; + for (const change of changes) { + if (changedSomething(change)) { + rows.push({ kind: 'deploy', key: change.snapshotId, change }); + continue; + } + const last = rows.at(-1); + if (last?.kind === 'quiet') { + last.count += 1; + last.oldestAt = change.publishedAt; + last.changes.push(change); + } else { + rows.push({ kind: 'quiet', key: `quiet-${change.snapshotId}`, count: 1, newestAt: change.publishedAt, oldestAt: change.publishedAt, changes: [change] }); + } + } + return rows; +} + +/** e.g. "3 deploys without changes · 7h–10h ago", or "1 deploy without changes · 2d ago". */ +export function quietDeployLine(row: Extract, now: number): string { + const newest = formatAgo(row.newestAt, now); + const oldest = formatAgo(row.oldestAt, now); + const when = newest === oldest ? newest : `${newest.replace(/ ago$/, '')}–${oldest}`; + return `${plural(row.count, 'deploy')} without changes · ${when}`; +} + +/** Whether a folder has failed, changed or behind nodes. */ +export function needsAttention(g: GroupStatus): boolean { + return g.failed > 0 || g.changed > 0 || g.behind > 0; +} + +/** Folders by what needs attention: most failed, then most changed, then most behind, then largest, then by name. */ +export function sortFolders(groups: readonly GroupStatus[]): GroupStatus[] { + return [...groups].sort( + (a, b) => b.failed - a.failed || b.changed - a.changed || b.behind - a.behind || b.total - a.total || a.group.localeCompare(b.group) + ); +} + +/** + * The folder rows to show, sorted: the first `limit`, plus every folder beyond that needs attention, + * or all of them when expanded. `hidden` counts the rest, for "Show all N folders". + */ +export function visibleFolders(groups: readonly GroupStatus[], expanded: boolean, limit = FOLDER_ROWS): { rows: GroupStatus[]; hidden: number } { + const sorted = sortFolders(groups); + if (expanded) return { rows: sorted, hidden: 0 }; + const rows = sorted.filter((g, i) => i < limit || needsAttention(g)); + return { rows, hidden: sorted.length - rows.length }; +} diff --git a/ui/src/lib/components/admin/access-checker.svelte b/ui/src/lib/components/admin/access-checker.svelte index 1233d68b55..509822c659 100644 --- a/ui/src/lib/components/admin/access-checker.svelte +++ b/ui/src/lib/components/admin/access-checker.svelte @@ -103,16 +103,16 @@ Denial ID (optional; from a “permission denied” message) e.key === 'Enter' && lookup()} /> - + {#if sampleDecision && !decisionId} - + {/if} {#if decision}

{decision.decisionId}: {whoName(decision.query.principal)} was denied {formatAgo(decision.at, app.now)} · “{decision.summary}”. Checked again below with their access today. - +

{/if} @@ -150,7 +150,7 @@ On (optional) e.key === 'Enter' && check()} /> - + {#if failure}

{failure}

{/if} @@ -177,7 +177,7 @@ {#if g.via === 'base' && g.baseRole && g.baseRole !== g.role}which includes {builtInLabel(g.baseRole)}{:else if g.via === 'base'}a built-in role{:else}an addition{/if} - {g.selector ? 'on' : 'on every node'} + {g.selector ? 'on' : 'all nodes'} {#if g.selector}{g.selector}{/if} @@ -195,7 +195,7 @@ {/if} {#if result.wouldBeGrantedBy.length} -
Roles that would allow it here
+
Would allow

{#each result.wouldBeGrantedBy as r, i (r)}{#if i}, {/if}{#if canSeeRoles}{r}{:else}{r}{/if}{/each}

@@ -308,13 +308,9 @@ .chains :global(svg) { color: var(--sb-text-faint); } - a, - .link { + a { color: var(--primary); } - .link { - font-size: 12px; - } b { font-weight: 600; } diff --git a/ui/src/lib/components/admin/assignment-list.svelte b/ui/src/lib/components/admin/assignment-list.svelte index 66af05649f..392b9bb32c 100644 --- a/ui/src/lib/components/admin/assignment-list.svelte +++ b/ui/src/lib/components/admin/assignment-list.svelte @@ -1,4 +1,6 @@
- - - - - - - - - - - - {#each mine as a (a.id)} + a.id} empty="No roles yet."> + {#snippet row(a)} {@const role = roleOf(a.role)} {@const hint = lockedHint(a)} @@ -107,7 +106,7 @@ - {:else} - - {/each} - -
RoleWhereFromSince
{#if role}{a.role}{:else}{a.role}{/if} {#if role}
{roleSummary(role)}
{/if} - {#if a.stale}
This role no longer exists in the code, so it gives nothing
{/if} + {#if a.stale}
Removed from code
{/if} {#if isOwnLastAdmin(a)}
Your last admin role: removing it locks you out of administration
{/if}
{scopeWords(a.scope.locationId, locations)} {#if a.source === 'manual'} - remove(a)}>Remove + remove(a)}>Remove {:else if a.source === 'sso'} See SSO mapping {:else} @@ -115,11 +114,8 @@ {/if}
No roles yet.
+ {/snippet} + {#if app.can('role.assign')}
@@ -138,12 +134,6 @@
diff --git a/ui/src/lib/components/code/sql-block.svelte b/ui/src/lib/components/code/sql-block.svelte index 54a2a1c5e1..4e4bfed13e 100644 --- a/ui/src/lib/components/code/sql-block.svelte +++ b/ui/src/lib/components/code/sql-block.svelte @@ -1,29 +1,17 @@
{#if caption}{caption}{/if} - +
{@html html}
@@ -32,7 +20,7 @@ .sql-block { overflow: hidden; border: 1px solid var(--border); - border-radius: 4px; + border-radius: 2px; } .bar { display: flex; @@ -42,6 +30,9 @@ border-bottom: 1px solid var(--border); background: var(--sb-surface-low); } + .copy { + margin-left: auto; + } .caption { overflow: hidden; color: var(--sb-text-faint); @@ -50,21 +41,6 @@ text-overflow: ellipsis; white-space: nowrap; } - button { - display: inline-flex; - align-items: center; - gap: 4px; - margin-left: auto; - padding: 2px 6px; - border-radius: 3px; - color: var(--muted-foreground); - font-family: var(--font-mono); - font-size: 10.5px; - } - button:hover { - background: var(--sb-hover); - color: var(--foreground); - } pre { margin: 0; overflow: auto; diff --git a/ui/src/lib/components/code/sql-viewer.svelte b/ui/src/lib/components/code/sql-viewer.svelte index 1cea2293e6..7da2b7be6b 100644 --- a/ui/src/lib/components/code/sql-viewer.svelte +++ b/ui/src/lib/components/code/sql-viewer.svelte @@ -1,4 +1,8 @@ - diff --git a/ui/src/lib/components/disclosure-button.svelte b/ui/src/lib/components/disclosure-button.svelte new file mode 100644 index 0000000000..6b4d11e184 --- /dev/null +++ b/ui/src/lib/components/disclosure-button.svelte @@ -0,0 +1,56 @@ + + + + + diff --git a/ui/src/lib/components/graph/compact-face.svelte b/ui/src/lib/components/graph/compact-face.svelte new file mode 100644 index 0000000000..893ad44ae7 --- /dev/null +++ b/ui/src/lib/components/graph/compact-face.svelte @@ -0,0 +1,134 @@ + + + + + diff --git a/ui/src/lib/components/graph/compact-tails.test.ts b/ui/src/lib/components/graph/compact-tails.test.ts new file mode 100644 index 0000000000..c10f0c8f71 --- /dev/null +++ b/ui/src/lib/components/graph/compact-tails.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from 'vitest'; +import { compactTails, compactTitleOf, siblingTails } from './compact-tails'; + +describe('compact pill names', () => { + it('show the shortest ending of `__` parts no other card shares', () => { + const tails = compactTails(['orders__stg__events', 'customers__stg__events', 'orders__mart__daily_totals', 'regions', 'finance__regions']); + expect(Object.fromEntries(tails)).toEqual({ + orders__stg__events: 'orders__stg__events', + customers__stg__events: 'customers__stg__events', + orders__mart__daily_totals: 'daily_totals', + // A name that is another's ending keeps itself; the other needs more of its parts. + regions: 'regions', + finance__regions: 'finance__regions' + }); + }); + + it('takes as many parts as it needs, and never the same tail twice', () => { + const names = ['a__x__y__z', 'b__x__y__z', 'c__w__y__z', 'd__q']; + const tails = compactTails(names); + expect(names.map((n) => tails.get(n))).toEqual(['a__x__y__z', 'b__x__y__z', 'w__y__z', 'q']); + expect(new Set(tails.values()).size).toBe(names.length); + }); + + it('tells cards apart only from their siblings in the same box', () => { + const tails = siblingTails([ + { id: 'a', parent: 'orders/staging', title: 'orders__stg__events' }, + { id: 'b', parent: 'orders/staging', title: 'orders__stg__daily' }, + { id: 'c', parent: 'customers/staging', title: 'customers__stg__events' }, + { id: 'd', parent: '', title: 'orders__mart__events' }, + { id: 'e', parent: '', title: 'customers__mart__events' } + ]); + expect(Object.fromEntries(tails)).toEqual({ a: 'events', b: 'daily', c: 'events', d: 'orders__mart__events', e: 'customers__mart__events' }); + }); + + it('reads a card’s title from its label, name, job or folder', () => { + expect(compactTitleOf({ label: 'Marts', name: 'x' })).toBe('Marts'); + expect(compactTitleOf({ jobKey: 'core_build' })).toBe('core_build'); + expect(compactTitleOf({ group: 'staging' })).toBe('staging'); + expect(compactTitleOf({})).toBeNull(); + }); +}); diff --git a/ui/src/lib/components/graph/compact-tails.ts b/ui/src/lib/components/graph/compact-tails.ts new file mode 100644 index 0000000000..e85d2c1898 --- /dev/null +++ b/ui/src/lib/components/graph/compact-tails.ts @@ -0,0 +1,50 @@ +// The part of each card's name a zoomed-out pill shows: the shortest run of its last `__` parts that no +// sibling card (in the same box, or loose on the graph) ends with, so `orders__stg__events` reads +// `events` unless a sibling shares that ending. Pure. + +/** The title a card shows: its label, else its name, its job's key or its folder. */ +export function compactTitleOf(data: Record | undefined): string | null { + const title = data?.label ?? data?.name ?? data?.jobKey ?? data?.group; + return typeof title === 'string' ? title : null; +} + +/** Each name's tail: the shortest suffix of its `__` parts that is unique among `names`. */ +export function compactTails(names: Iterable): Map { + const unique = [...new Set(names)]; + const split = unique.map((name) => name.split('__')); + // How many names end with each run of parts; a name counts once per run. + const ends = new Map(); + for (const parts of split) for (let k = 1; k <= parts.length; k++) { + const tail = parts.slice(-k).join('__'); + ends.set(tail, (ends.get(tail) ?? 0) + 1); + } + const tails = new Map(); + unique.forEach((name, i) => { + const parts = split[i]; + let tail = name; + for (let k = 1; k < parts.length; k++) { + const run = parts.slice(-k).join('__'); + if (run && ends.get(run) === 1) { + tail = run; + break; + } + } + tails.set(name, tail); + }); + return tails; +} + +/** Each card's pill name, by card id: its title's tail among the cards with the same parent. */ +export function siblingTails(cards: { id: string; parent: string; title: string }[]): Map { + const byId = new Map(); + for (const siblings of Map.groupBy(cards, (c) => c.parent).values()) { + const tails = compactTails(siblings.map((c) => c.title)); + for (const c of siblings) byId.set(c.id, tails.get(c.title) ?? c.title); + } + return byId; +} + +/** The context key a graph canvas gives its cards their pill names under. */ +export const COMPACT_TAILS = Symbol('compact-tails'); +/** A card's pill name and its parent box, by card id; undefined outside a graph canvas. */ +export type CompactTails = { tail: (id: string) => string | undefined; parent: (id: string) => string | undefined }; diff --git a/ui/src/lib/components/graph/compact-title.ts b/ui/src/lib/components/graph/compact-title.ts new file mode 100644 index 0000000000..82d0ae3853 --- /dev/null +++ b/ui/src/lib/components/graph/compact-title.ts @@ -0,0 +1,59 @@ +// Text measured to fit a card: the status line's parts, and a zoomed-out pill's name. A pill's name is +// sized on screen, as large as fits between `PILL.minFont` and `PILL.maxFont`, then cut with an +// ellipsis at its end. Worked out once per compact level, never per zoom step. + +import { CARD_WIDTH } from '$lib/lineage/fit'; + +/** The width of `text` at `font` px (and `weight`), in px. */ +export type Measure = (text: string, font: number, weight?: number) => number; + +/** A zoomed-out pill's parts, in px on screen. */ +export const PILL = { height: 30, ring: 20, padLeft: 5, gap: 8, padRight: 14, border: 1, minFont: 12, maxFont: 14, weight: 500 } as const; + +/** Room for a pill's name on screen at `zoom`: the card's width less the pill's ring and padding. */ +export function pillRoom(zoom: number): number { + return CARD_WIDTH * zoom - 2 * PILL.border - PILL.padLeft - PILL.ring - PILL.gap - PILL.padRight; +} + +/** The largest size, in quarter px from `PILL.maxFont` down to `PILL.minFont`, at which `text` fits `room`. */ +export function pillFont(text: string, room: number, measure: Measure): number { + const { minFont, maxFont, weight } = PILL; + const widest = measure(text, maxFont, weight); + if (widest <= room) return maxFont; + // Width grows with the size, so start near where it fits and step down to be sure. + let font = Math.max(minFont, Math.min(maxFont, Math.floor(((maxFont * room) / widest) * 4) / 4)); + while (font > minFont && measure(text, font, weight) > room) font -= 0.25; + return font; +} + +let context: CanvasRenderingContext2D | null | undefined; +const loading = new Set(); +/** A measurer in `family` (or the family `family()` names when it is measured), estimating generously until the font has loaded. */ +function measureIn(family: () => string, fallback: number): Measure { + return (text, font, weight = 600) => { + const spec = `${weight} ${font}px ${family()}`; + if (context === undefined) context = typeof document === 'undefined' ? null : document.createElement('canvas').getContext('2d'); + if (!context) return text.length * font * fallback; + if (!document.fonts?.check(spec)) { + // Ask for the font, so it loads (and the text is fitted again) even before it first shows. + const face = `${weight} 16px ${family()}`; + if (!loading.has(face)) { + loading.add(face); + void document.fonts?.load(face).catch(() => {}); + } + return text.length * font * fallback; + } + context.font = spec; + return context.measureText(text).width; + }; +} + +/** The cards' own font, Inter, as the status line measures it. */ +export const measureText: Measure = measureIn(() => '"Inter Variable", sans-serif', 0.64); + +/** The pills' name font: whatever the page sets `--font-node-compact` to. */ +function compactFamily(): string { + if (typeof document === 'undefined') return 'sans-serif'; + return getComputedStyle(document.documentElement).getPropertyValue('--font-node-compact').trim() || 'sans-serif'; +} +export const measureCompact: Measure = measureIn(compactFamily, 0.66); diff --git a/ui/src/lib/components/graph/fonts.svelte.ts b/ui/src/lib/components/graph/fonts.svelte.ts new file mode 100644 index 0000000000..520b2415cc --- /dev/null +++ b/ui/src/lib/components/graph/fonts.svelte.ts @@ -0,0 +1,9 @@ +// Whether the cards' fonts have loaded. Text fitted to a card (compact names, the status line) is +// measured with the font's real widths once it has, so it is fitted once more then; never per zoom. +// `version` counts font loads after the first, such as the compact names' font, which loads only once +// a compact name first shows. +export const fonts = $state({ ready: false, version: 0 }); +if (typeof document !== 'undefined') { + void document.fonts?.ready.then(() => (fonts.ready = true)); + document.fonts?.addEventListener?.('loadingdone', () => fonts.version++); +} diff --git a/ui/src/lib/components/graph/graph-canvas.svelte b/ui/src/lib/components/graph/graph-canvas.svelte new file mode 100644 index 0000000000..f03a07ad59 --- /dev/null +++ b/ui/src/lib/components/graph/graph-canvas.svelte @@ -0,0 +1,212 @@ + + +
0} data-compact={level || undefined} style:--sb-title-unzoom={titleUnzoom} style:--sb-unzoom={controlScale} bind:this={element} onpointerdowncapture={onpointerdown} onpointermovecapture={onpointermove} onclickcapture={onclick}> + + + + +
+
+ {#if controls === 'fit'} fit?.()} {showLock} />{:else if controls === 'full'}{/if} + {@render children?.()} +
+
+ + diff --git a/ui/src/lib/components/graph/graph-toolbar.svelte b/ui/src/lib/components/graph/graph-toolbar.svelte new file mode 100644 index 0000000000..93d65c3f89 --- /dev/null +++ b/ui/src/lib/components/graph/graph-toolbar.svelte @@ -0,0 +1,91 @@ + + +
+ {#if start} + {@render start()} + + {/if} + {#if folders} + + {/if} + {#if expand} + + + {#if expand.note}{expand.note}{/if} + {/if} + {@render middle?.()} + {#if focus} + + {/if} + +
{@render end?.()}
+ {#if details} + + {/if} +
+ + diff --git a/ui/src/lib/components/graph/no-raw-graph.test.ts b/ui/src/lib/components/graph/no-raw-graph.test.ts new file mode 100644 index 0000000000..79167228b9 --- /dev/null +++ b/ui/src/lib/components/graph/no-raw-graph.test.ts @@ -0,0 +1,38 @@ +// Keeps every graph the same graph: the library is drawn only by GraphCanvas (graph-canvas.svelte), +// which brings the shared background, fit controls, readable fit and first-fit hiding, and pages put +// GraphToolbar over it for Lineage's controls. A page that draws itself, or imports it, +// fails here. +import { readdirSync, readFileSync, statSync } from 'node:fs'; +import { join, relative } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const SRC = new URL('../../../', import.meta.url).pathname; +const CANVAS = 'lib/components/graph/graph-canvas.svelte'; + +/** Lines that draw or import the graph library's canvas. */ +export function rawGraph(source: string): number[] { + const lines = source.split('\n'); + return lines.flatMap((line, i) => (/ { + const path = join(dir, name); + return statSync(path).isDirectory() ? svelteFiles(path) : path.endsWith('.svelte') ? [path] : []; + }); +} + +describe('graphs', () => { + it('spots the library drawn or imported directly', () => { + expect(rawGraph("import { SvelteFlow, type Node } from '@xyflow/svelte';\n")).toEqual([1, 2]); + expect(rawGraph("import type { Node } from '@xyflow/svelte';\n")).toEqual([]); + }); + + it('are drawn only by GraphCanvas', () => { + const found = svelteFiles(SRC).flatMap((path) => { + const file = relative(SRC, path); + return file === CANVAS ? [] : rawGraph(readFileSync(path, 'utf8')).map((line) => `${file}:${line}`); + }); + expect(found, 'Draw graphs with GraphCanvas (lib/components/graph/graph-canvas.svelte), and GraphToolbar for the toolbar').toEqual([]); + }); +}); diff --git a/ui/src/lib/components/help-tip.svelte b/ui/src/lib/components/help-tip.svelte new file mode 100644 index 0000000000..92336486f3 --- /dev/null +++ b/ui/src/lib/components/help-tip.svelte @@ -0,0 +1,62 @@ + + + + {#if text || children} + + + + + {#if children}{@render children()}{:else}{text}{/if} + + + + {/if} + {#if docs}Docs ↗{/if} + + + diff --git a/ui/src/lib/components/lineage/asset-node.svelte b/ui/src/lib/components/lineage/asset-node.svelte index fff66c582a..3534b6e52a 100644 --- a/ui/src/lib/components/lineage/asset-node.svelte +++ b/ui/src/lib/components/lineage/asset-node.svelte @@ -13,11 +13,16 @@ Repeat, Columns3, Workflow, + Package, + TableProperties, type Icon as IconType } from '@lucide/svelte'; import { nodeFields } from '$lib/lineage/node-fields.svelte'; import { formatAgo } from '$lib/format/time'; import { app } from '$lib/state/app.svelte'; + import CompactFace from '$lib/components/graph/compact-face.svelte'; + import { fonts } from '$lib/components/graph/fonts.svelte'; + import { fitStatus } from '$lib/lineage/status-line'; type AssetData = { name: string; @@ -39,7 +44,7 @@ columnCount?: number; }; - let { data, selected }: NodeProps = $props(); + let { id, data, selected }: NodeProps = $props(); const d = $derived(data as AssetData); const f = $derived(nodeFields.value); @@ -76,6 +81,9 @@ delete_insert: Repeat, microbatch: Layers, task: Workflow, + asset: Package, + udf: FunctionSquare, + table_fn: TableProperties, fn: FunctionSquare }; const Glyph = $derived(icons[d.icon ?? ''] ?? Table2); @@ -83,6 +91,23 @@ const kindShort = $derived(d.kind.split(' · ')[0]); const when = $derived(d.lastBuiltAt ? (d.status === 'src' ? `loaded ${formatAgo(d.lastBuiltAt, app.now)}` : formatAgo(d.lastBuiltAt, app.now)) : ''); + // Zoomed out, the card's state is its colour: a failure or a run first, then staleness, then the outcome. + const faceTone = $derived( + d.status === 'fail' || d.status === 'run' ? tone : d.stale === 'behind' ? 'var(--sb-behind)' : d.stale ? 'var(--sb-stale)' : tone + ); + // The status line stays one line: worked out once per card, the drift chip and then the status word + // shorten when its parts don't fit; the time never does. + const fit = $derived( + (void fonts.ready, + fitStatus({ + label: f.status ? statusLabel[d.status] : undefined, + when: f.lastBuilt && when ? when : undefined, + drift: d.stale ? staleLabel[d.stale] : undefined, + checks: f.checks && d.checks ? d.checks : undefined + })) + ); + const showStatus = $derived(f.status || (f.checks && d.checks) || (f.lastBuilt && when) || d.stale); + const hasDetail = $derived( !!( (f.description && d.description) || @@ -109,15 +134,15 @@
{/if} - {#if f.status || (f.checks && d.checks) || (f.lastBuilt && when) || d.stale} + {#if showStatus}
{#if f.status} - - {statusLabel[d.status]} + + {#if fit.label}{statusLabel[d.status]}{/if} {/if} - {#if f.lastBuilt && when}{when}{/if} + {#if f.lastBuilt && when}{when}{/if} {#if d.stale} - {staleLabel[d.stale]} + {#if fit.driftWord}{staleLabel[d.stale]}{/if} {/if} {#if f.checks && d.checks} {d.checks} @@ -125,11 +150,13 @@
{/if} + diff --git a/ui/src/lib/components/lineage/inspector-section.svelte b/ui/src/lib/components/lineage/inspector-section.svelte index 2254e45172..53a0807ee7 100644 --- a/ui/src/lib/components/lineage/inspector-section.svelte +++ b/ui/src/lib/components/lineage/inspector-section.svelte @@ -1,5 +1,5 @@ - - Node fields - {count} + + Node fields + {count}
@@ -21,6 +22,7 @@
{#each NODE_FIELDS as field (field.key)} +
- +
diff --git a/ui/src/lib/components/lineage/stub-node.svelte b/ui/src/lib/components/lineage/stub-node.svelte new file mode 100644 index 0000000000..a82ba1daae --- /dev/null +++ b/ui/src/lib/components/lineage/stub-node.svelte @@ -0,0 +1,56 @@ + + + + + + + diff --git a/ui/src/lib/components/lineage/too-many-card.svelte b/ui/src/lib/components/lineage/too-many-card.svelte new file mode 100644 index 0000000000..9864810349 --- /dev/null +++ b/ui/src/lib/components/lineage/too-many-card.svelte @@ -0,0 +1,56 @@ + + +
+
+ Too many nodes to draw at once +

{count.toLocaleString('en')} cards · limit {MAX_DRAWN.toLocaleString('en')}

+ {#if actions}
{@render actions()}
{/if} +
+
+ + diff --git a/ui/src/lib/components/lineage/view-summary.svelte b/ui/src/lib/components/lineage/view-summary.svelte new file mode 100644 index 0000000000..9f334049c9 --- /dev/null +++ b/ui/src/lib/components/lineage/view-summary.svelte @@ -0,0 +1,103 @@ + + + + {#snippet meta()}{plural(summary.total, 'node')}{/snippet} +
+ {#each summary.byKind as k (k.kind)}
{nodeKindPlural[k.kind]}
{k.count.toLocaleString('en')}
{/each} +
+

+ {#if summary.changed || summary.behind} + Needs building: + {#if summary.changed}{summary.changed.toLocaleString('en')} changed{/if} + {#if summary.behind}{summary.changed ? '+' : ''}{summary.behind.toLocaleString('en')} behind{/if} + {:else} + Nothing in view needs building. + {/if} + {#if summary.failed}· {summary.failed.toLocaleString('en')} failed{/if} +

+
+{#if filters.length} + +
    + {#each filters as f (f)}
  • {f}
  • {/each} +
+
+{/if} + +
    + {#each key as k (k.label)}
  • {k.label}
  • {/each} +
+
+

Select a node to see its details.

+ + diff --git a/ui/src/lib/components/list/choice-filter.svelte b/ui/src/lib/components/list/choice-filter.svelte index 529c562503..e575b1c61f 100644 --- a/ui/src/lib/components/list/choice-filter.svelte +++ b/ui/src/lib/components/list/choice-filter.svelte @@ -1,42 +1,66 @@ - - - {label}{#if current}{current}{/if} - - - - (value = v === '__all' ? '' : (v as T))}> - {all} - {#each options as o (o.value)} - - {o.label} - {#if o.count != null}{o.count}{/if} - - {/each} - - - + + + + {label}{#if summary}:{summary}{/if} + + + e.preventDefault()}> + (open = false)} /> + + + {#if chosen.length}{/if} + diff --git a/ui/src/lib/components/list/choice-list.svelte b/ui/src/lib/components/list/choice-list.svelte new file mode 100644 index 0000000000..cee8a68962 --- /dev/null +++ b/ui/src/lib/components/list/choice-list.svelte @@ -0,0 +1,255 @@ + + + +
+ {#if searchable} + + {/if} +
+ {#if multiple} + + + {:else} + + {/if} + {query.trim() ? `${shown.length.toLocaleString()} of ${options.length.toLocaleString()}` : chosen.length ? `${chosen.length} selected` : ''} +
+
+ {#if windowed && rows.before}
{/if} + {#each shown.slice(first, last) as o, j (o.value)} + {@const i = first + j} + {@const on = chosenSet.has(o.value)} + {@const parts = splitMiddle(o.label)} +
e.preventDefault()} + onclick={() => pick(o)} + onkeydown={() => {}} + onpointermove={() => (active = i)} + > + {#if multiple}{:else}{#if on}{/if}{/if} + {parts.head}{#if parts.tail}{parts.tail}{/if} + {#if o.count != null}{o.count.toLocaleString()}{/if} +
+ {:else} +
No {label.toLowerCase()} matches “{query.trim()}”.
+ {/each} + {#if windowed && rows.after}
{/if} +
+
+ + diff --git a/ui/src/lib/components/list/data-table.svelte b/ui/src/lib/components/list/data-table.svelte new file mode 100644 index 0000000000..7f5a7b67c4 --- /dev/null +++ b/ui/src/lib/components/list/data-table.svelte @@ -0,0 +1,178 @@ + + +
+ + + {#each columns as c (c.id)}{/each} + + + + {#each columns as c (c.id)} + + {/each} + + + + {#if first > 0}{/if} + {#each shown as item, i (key(item))} + {@render row(item, first + i)} + {:else} + {#if emptyRow}{@render emptyRow()}{:else}{/if} + {/each} + {#if isWindowed && last < rows.length}{/if} + {#if after && rows.length}{@render after()}{/if} + +
{#if head && !c.label}{@render head(c)}{:else}{c.label}{/if}
+
+ + diff --git a/ui/src/lib/components/list/list-toolbar.svelte b/ui/src/lib/components/list/list-toolbar.svelte index 1193d8b66c..da1e7b3d92 100644 --- a/ui/src/lib/components/list/list-toolbar.svelte +++ b/ui/src/lib/components/list/list-toolbar.svelte @@ -1,9 +1,24 @@ -
+
{#each options as o (o.value)} - {/each}
diff --git a/ui/src/lib/components/page-body.test.ts b/ui/src/lib/components/page-body.test.ts new file mode 100644 index 0000000000..6c82b03143 --- /dev/null +++ b/ui/src/lib/components/page-body.test.ts @@ -0,0 +1,27 @@ +// Every page scrolls in the shared PageBody (or a list's DataTable), whose scroller has no padding, so +// sticky headers stick to its very top. A page building its own padded scroller fails here. +import { readdirSync, readFileSync, statSync } from 'node:fs'; +import { join, relative } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const ROUTES = new URL('../../routes/', import.meta.url).pathname; + +function svelteFiles(dir: string): string[] { + return readdirSync(dir).flatMap((name) => { + const path = join(dir, name); + return statSync(path).isDirectory() ? svelteFiles(path) : path.endsWith('.svelte') ? [path] : []; + }); +} + +/** A page-sized scroller written by hand: a flex-1 box that scrolls, as a class list. */ +const HAND_SCROLLER = /class="[^"]*\bflex-1\b[^"]*\boverflow-(?:y-)?auto\b|class="[^"]*\boverflow-(?:y-)?auto\b[^"]*\bflex-1\b/; + +describe('page bodies', () => { + it('are the shared PageBody, not a hand-built scroller', () => { + const found = svelteFiles(ROUTES).flatMap((path) => { + const source = readFileSync(path, 'utf8'); + return source.split('\n').flatMap((text, i) => (HAND_SCROLLER.test(text) ? [`${relative(ROUTES, path)}:${i + 1}`] : [])); + }); + expect(found, 'Use PageBody from $lib/components/page-body.svelte').toEqual([]); + }); +}); diff --git a/ui/src/lib/components/platform/connection-test.svelte b/ui/src/lib/components/platform/connection-test.svelte index dc0c6c9920..16dc4f86cf 100644 --- a/ui/src/lib/components/platform/connection-test.svelte +++ b/ui/src/lib/components/platform/connection-test.svelte @@ -6,14 +6,17 @@ import { app } from '$lib/state/app.svelte'; import GatedButton from './gated-button.svelte'; - /** `compact` shows one line, e.g. "Warehouse ● connected 54s ago", with the message in the tooltip. */ - type Props = { scope: Scope; connection: string; last: ConnectionTest | null; compact?: boolean; ontested?: () => void }; - let { scope, connection, last, compact = false, ontested }: Props = $props(); + /** + * `compact` shows one line, e.g. "Warehouse ● connected 54s ago", with the message in the tooltip. + * `showLast` false shows only a test run here, for a page that shows the last recorded check itself. + */ + type Props = { scope: Scope; connection: string; last: ConnectionTest | null; compact?: boolean; showLast?: boolean; ontested?: () => void }; + let { scope, connection, last, compact = false, showLast = true, ontested }: Props = $props(); let testing = $state(false); let result = $state(null); let failure = $state(null); - const shown = $derived(result ?? last); + const shown = $derived(result ?? (showLast ? last : null)); async function test() { testing = true; @@ -29,7 +32,7 @@ } function origin(t: ConnectionTest): string { - return t.trigger === 'publish' ? 'after the last deploy' : t.by ? `by ${t.by.name}` : 'manually'; + return t.trigger === 'publish' ? 'on deploy' : t.trigger === 'startup' ? 'on startup' : t.by ? `by ${t.by.name}` : 'manual'; } @@ -54,7 +57,7 @@
{shown.message}
- {:else} + {:else if showLast} {/if} {#if failure}
{failure}
{/if} diff --git a/ui/src/lib/components/platform/gated-button.svelte b/ui/src/lib/components/platform/gated-button.svelte index cf9255d871..94eeb0f5d5 100644 --- a/ui/src/lib/components/platform/gated-button.svelte +++ b/ui/src/lib/components/platform/gated-button.svelte @@ -16,7 +16,7 @@ size?: ButtonSize; children: Snippet; }; - let { requires, onclick, disabled = false, busy = false, title, variant = 'outline', size = 'sm', children }: Props = $props(); + let { requires, onclick, disabled = false, busy = false, title, variant = 'outline', size = 'bar', children }: Props = $props(); const allowed = $derived(app.can(requires)); diff --git a/ui/src/lib/components/platform/issued-secret.svelte b/ui/src/lib/components/platform/issued-secret.svelte index b81d050385..6f3190a1b8 100644 --- a/ui/src/lib/components/platform/issued-secret.svelte +++ b/ui/src/lib/components/platform/issued-secret.svelte @@ -1,4 +1,5 @@
{#each rows as row (row.label)} -
{row.label}
+
{row.label}{#if row.help}{/if}
{#if row.content}{@render row.content()}{:else}{row.value ?? '—'}{/if}
@@ -23,6 +25,9 @@ font-size: 12.5px; } dt { + display: inline-flex; + align-items: center; + gap: 4px; color: var(--muted-foreground); } dd { diff --git a/ui/src/lib/components/platform/location-tokens.svelte b/ui/src/lib/components/platform/location-tokens.svelte index 3e0c64295a..2ee0c6b1de 100644 --- a/ui/src/lib/components/platform/location-tokens.svelte +++ b/ui/src/lib/components/platform/location-tokens.svelte @@ -73,8 +73,8 @@ {#if t.state === 'active'} - rotate(t)}>Rotate - revoke(t)}>Revoke + rotate(t)}>Rotate + revoke(t)}>Revoke {/if} diff --git a/ui/src/lib/components/platform/page-tabs.svelte b/ui/src/lib/components/platform/page-tabs.svelte index 612042cd1c..92ef270a60 100644 --- a/ui/src/lib/components/platform/page-tabs.svelte +++ b/ui/src/lib/components/platform/page-tabs.svelte @@ -1,16 +1,23 @@ -
+
{#each tabs as t (t.value)} - {/each} @@ -25,8 +32,13 @@ padding: 0 16px; border-bottom: 1px solid var(--border); } + .page-tabs.flush { + padding: 0; + border-bottom: 0; + } button { display: inline-flex; + margin-bottom: -1px; align-items: center; gap: 6px; padding: 10px 8px; diff --git a/ui/src/lib/components/platform/panel.svelte b/ui/src/lib/components/platform/panel.svelte index 7cd85f86ac..400cbc1023 100644 --- a/ui/src/lib/components/platform/panel.svelte +++ b/ui/src/lib/components/platform/panel.svelte @@ -1,14 +1,22 @@

{title}

+ {#if help || docs}{/if} {#if note}{note}{/if} + {#if audited}{/if} {#if actions}
{@render actions()}
{/if}
{@render children()} @@ -33,6 +41,10 @@ color: var(--muted-foreground); font-size: 12px; } + .audited { + display: inline-flex; + color: var(--sb-text-faint); + } .actions { display: flex; align-items: center; diff --git a/ui/src/lib/components/platform/problem-list.svelte b/ui/src/lib/components/platform/problem-list.svelte index c00848258f..b3c6ecf574 100644 --- a/ui/src/lib/components/platform/problem-list.svelte +++ b/ui/src/lib/components/platform/problem-list.svelte @@ -1,5 +1,6 @@ - - - - - - {#if showAffected}{/if} - {#if showReactions}{/if} - - - - {#each events as e (e.id)} + e.id} {resource} {empty}> + {#snippet row(e)} {/if} - {:else} - - - - {/each} - -
ProblemWhenAffectedSensors that reacted
@@ -103,21 +101,14 @@
- {#if resource.error}Couldn't load problems: {resource.error}{:else if resource.data === undefined}{:else}{empty}{/if} -
+ {/snippet} + diff --git a/ui/src/lib/components/platform/syncing-note.svelte b/ui/src/lib/components/platform/syncing-note.svelte new file mode 100644 index 0000000000..bd2f407182 --- /dev/null +++ b/ui/src/lib/components/platform/syncing-note.svelte @@ -0,0 +1,29 @@ + + +{#if running} +

+ Syncing warehouse history · {running.position}/{running.kinds} +

+{/if} + + diff --git a/ui/src/lib/components/platform/warehouse-sync-now.svelte b/ui/src/lib/components/platform/warehouse-sync-now.svelte index 84872ef1c3..aa5706764a 100644 --- a/ui/src/lib/components/platform/warehouse-sync-now.svelte +++ b/ui/src/lib/components/platform/warehouse-sync-now.svelte @@ -1,6 +1,7 @@ + + + + + + + Refresh now + + diff --git a/ui/src/lib/components/selection/selection-bar.svelte b/ui/src/lib/components/selection/selection-bar.svelte new file mode 100644 index 0000000000..e2d35463fd --- /dev/null +++ b/ui/src/lib/components/selection/selection-bar.svelte @@ -0,0 +1,253 @@ + + +
+
+ {@render leading?.()} + { + selection.select = r.select; + selection.exclude = r.exclude; + }} + onleave={() => selection.remember()} + /> + {#if !showExcept}{/if} + + {#if !selection.active}All {total.toLocaleString('en')} nodes + {:else if problem}0 nodes + {:else if selection.evaluation}{selection.evaluation.count.toLocaleString('en')} {noun(selection.evaluation.count, 'node')} + {:else}—{/if} + + + {#if build && canBuild} + Build these + {/if} +
+ {#if showExcept || view} +
+ {#if showExcept} + Except + + + {/if} + {#if view} +
+ View + {@render view()} +
+ {/if} +
+ {/if} + {#if problem} + + {:else if selection.failed} + + {:else if selection.active} +
+ + Reads as: {#each reading as s, i (i)}{#if s.strong}{s.text}{:else}{s.text}{/if}{/each} +
+ {/if} +
+ + diff --git a/ui/src/lib/components/selection/selection-chip.svelte b/ui/src/lib/components/selection/selection-chip.svelte new file mode 100644 index 0000000000..2e733565be --- /dev/null +++ b/ui/src/lib/components/selection/selection-chip.svelte @@ -0,0 +1,188 @@ + + + + {#if readonly} + {@render face()} + {:else} + + {@render face()} + e.preventDefault()}> +
+ {typeName} + {partText(part)} +
+ {#if options.length} + picked, setPicked} /> + {/if} +
+ Include + + +
+
+ + ⌫ in the field edits the last chip as text +
+
+
+ + {/if} +
+ +{#snippet face()} + {#if part.up}+{/if}{#if part.type}{part.type}:{/if}{part.value}{#if part.down}+{/if} +{/snippet} + + diff --git a/ui/src/lib/components/selection/selection-chips.svelte b/ui/src/lib/components/selection/selection-chips.svelte new file mode 100644 index 0000000000..ff4207595f --- /dev/null +++ b/ui/src/lib/components/selection/selection-chips.svelte @@ -0,0 +1,72 @@ + + +
+ {#if chosen.length} + {#each chosen as t, ti (ti)} + 1}> + {#each t.parts as p, pi (pi)}{#if pi}and{/if}{/each} + + {/each} + {:else}Every node{/if} + {#if dropped.length} + except + {#each dropped as t, ti (ti)} + 1}> + {#each t.parts as p, pi (pi)}{#if pi}and{/if}{/each} + + {/each} + {/if} + {#if lineage}{/if} +
+ + diff --git a/ui/src/lib/components/selection/selection-field.svelte b/ui/src/lib/components/selection/selection-field.svelte new file mode 100644 index 0000000000..627c4f3813 --- /dev/null +++ b/ui/src/lib/components/selection/selection-field.svelte @@ -0,0 +1,380 @@ + + + +
e.target === e.currentTarget && input?.focus()}> + {#if tone === 'select'}{:else}{/if} + {#each tokens as t, ti (ti)} + 1}> + {#each t.parts as p, pi (pi)} + {#if pi}and{/if} + + {/each} + + {/each} + 0}> + {#each draftParts as p, pi (pi)} + + and + {/each} + (focused = true)} + onblur={leave} + placeholder={field.tokens.length || field.parts.length ? '' : placeholder} + aria-label={label} + autocomplete="off" + spellcheck="false" + class="syntax" + /> + + {#if focused && groups.length} +
e.preventDefault()}> + {#each groups as g (g.name)} +
+
{g.name}{#if g.counts}nodes{/if}
+ {#each g.items as item (item.insert)} + {@const i = items.indexOf(item)} + {@const [a, m, b] = marked(item.label)} +
apply(item, g.name)} onpointermove={() => (active = i)}> + + + {#if item.type && item.complete}{item.insert.startsWith('+') ? '+' : ''}{item.type}:{:else if item.insert.startsWith('+') && item.complete}+{/if}{a}{#if m}{m}{/if}{b} + + {#if item.hint}{item.hint}{/if} + {#if item.count != null}{item.count.toLocaleString('en')}{/if} + {#if i === active && field.text}Tab{/if} +
+ {/each} +
+ {/each} +
+ +x upstream + x+ downstream + a~b path between + a b either + a,b both + * glob + Tab make chip · ⌫ edit chip as text +
+
+ {/if} +
+ + diff --git a/ui/src/lib/components/selection/selector-icon.svelte b/ui/src/lib/components/selection/selector-icon.svelte new file mode 100644 index 0000000000..64f16d641d --- /dev/null +++ b/ui/src/lib/components/selection/selector-icon.svelte @@ -0,0 +1,28 @@ + + + + + diff --git a/ui/src/lib/components/shell/platform-event-banner.svelte b/ui/src/lib/components/shell/platform-event-banner.svelte index 0cb9660bb2..ba2f766ee2 100644 --- a/ui/src/lib/components/shell/platform-event-banner.svelte +++ b/ui/src/lib/components/shell/platform-event-banner.svelte @@ -6,6 +6,7 @@ import { formatAgo } from '$lib/format/time'; import { eventKindLabel, problemHref } from '$lib/platform/labels'; import { app } from '$lib/state/app.svelte'; + import { plural } from '$lib/format/plural'; let { only = () => true }: { only?: (event: PlatformEvent) => boolean } = $props(); @@ -14,6 +15,8 @@ app.platformEvents.filter((e) => e.active && e.severity !== 'info' && only(e)).sort((a, b) => rank(b) - rank(a)) ); const critical = $derived(events.some((e) => e.severity === 'critical')); + /** The banner shows the worst few, so many open problems don't push the page out of the window. */ + const SHOWN = 3; /** Location problems link to the location's page if it exists, else the Locations list; server problems only to server status. */ function linkOf(event: PlatformEvent): { href: string; text: string } | null { if (event.locationId) return { href: app.client.supports('getLocation') ? problemHref(event) : '/locations', text: 'View location →' }; @@ -27,7 +30,7 @@ {#if critical}{:else}{/if}
{/if} diff --git a/ui/src/lib/components/shell/warehouse-sync-status.svelte b/ui/src/lib/components/shell/warehouse-sync-status.svelte index 8951d8d87c..21447e7cc4 100644 --- a/ui/src/lib/components/shell/warehouse-sync-status.svelte +++ b/ui/src/lib/components/shell/warehouse-sync-status.svelte @@ -8,8 +8,8 @@ const text = $derived.by(() => { if (!sync) return ''; - if (sync.lastError) return `Warehouse sync failing · last synced ${formatAgo(sync.lastSyncedAt, app.now)}`; - return sync.lastSyncedAt ? `Warehouse last synced ${formatAgo(sync.lastSyncedAt, app.now)}` : 'Warehouse not synced yet'; + if (sync.lastError) return `Sync failing · ${formatAgo(sync.lastSyncedAt, app.now)}`; + return sync.lastSyncedAt ? `Synced ${formatAgo(sync.lastSyncedAt, app.now)}` : 'Not synced yet'; }); const title = $derived.by(() => { @@ -20,7 +20,7 @@ -{sync ? text : 'Warehouse last synced just now'} +{sync ? text : 'Synced just now'} diff --git a/ui/src/lib/lineage/fit.test.ts b/ui/src/lib/lineage/fit.test.ts new file mode 100644 index 0000000000..97dbcb5125 --- /dev/null +++ b/ui/src/lib/lineage/fit.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, it } from 'vitest'; +import { ANCHOR_MARGIN, CARD_TITLE_PX, COMPACT_LEVELS, COMPACT_ZOOM, LABEL_MIN_ZOOM, MIN_TITLE_PX, READABLE_ZOOM, compactLevel, absolutePosition, anchoredViewport, firstCard, fitViewport, largestBox } from './fit'; + +describe('fitting a graph to the view', () => { + it('centres a graph that fits, never closer than full size', () => { + expect(fitViewport({ x: 0, y: 0, width: 400, height: 200 }, 1000, 500)).toEqual({ zoom: 1, x: 300, y: 150 }); + }); + + it('shrinks a larger graph to fit while it stays readable', () => { + const v = fitViewport({ x: 0, y: 0, width: 1100, height: 300 }, 1000, 500, { padding: 0 }); + expect(v.zoom).toBeCloseTo(1000 / 1100); + expect(v.x).toBeCloseTo(0); + }); + + it('stops at the readable zoom and starts at the top-left of a graph too large to show whole', () => { + const v = fitViewport({ x: 100, y: 50, width: 10_000, height: 4000 }, 1000, 500, { padding: 0 }); + expect(v).toEqual({ zoom: READABLE_ZOOM, x: -100 * READABLE_ZOOM, y: -50 * READABLE_ZOOM }); + }); + + it('centres on the largest cluster, without panning past the graph’s edges', () => { + const bounds = { x: 0, y: 0, width: 10_000, height: 4000 }; + const middle = fitViewport(bounds, 1000, 500, { padding: 0, focus: { x: 5000, y: 2000, width: 400, height: 200 } }); + expect(middle).toEqual({ zoom: READABLE_ZOOM, x: 500 - 5200 * READABLE_ZOOM, y: 250 - 2100 * READABLE_ZOOM }); + // A cluster in the corner keeps the graph's corner near the view's corner, the cluster a margin inside. + const corner = fitViewport(bounds, 1000, 500, { padding: 0, focus: { x: 0, y: 0, width: 100, height: 100 } }); + expect(corner).toEqual({ zoom: READABLE_ZOOM, x: ANCHOR_MARGIN, y: ANCHOR_MARGIN }); + }); + + it('centres the axis along which a too-large graph still fits', () => { + const v = fitViewport({ x: 0, y: 0, width: 10_000, height: 100 }, 1000, 500, { padding: 0 }); + expect(v.y).toBeCloseTo(250 - 50 * READABLE_ZOOM); + expect(v.x).toBe(0); + }); + + it('picks the largest box as the cluster, and none when there are only cards', () => { + const card = { x: 0, y: 0, width: 236, height: 61, box: false }; + expect(largestBox([card, { x: 10, y: 10, width: 500, height: 400, box: true }, { x: 0, y: 0, width: 300, height: 300, box: true }])).toEqual({ x: 10, y: 10, width: 500, height: 400 }); + expect(largestBox([card])).toBeNull(); + }); +}); + +describe('keeping the view steady when a folder opens or closes', () => { + it('pans by the clicked box’s layout shift, at the same zoom, so it stays put on screen', () => { + const view = { x: 40, y: -20, zoom: 0.8 }; + const before = { x: 300, y: 120 }; + const after = { x: 360, y: 90 }; + const next = anchoredViewport(view, before, after); + expect(next.zoom).toBe(0.8); + // The pan moves by the shift, scaled to the screen, and the box's screen position is unchanged. + expect(next.x - view.x).toBeCloseTo(-(after.x - before.x) * view.zoom); + expect(next.y - view.y).toBeCloseTo(-(after.y - before.y) * view.zoom); + expect(after.x * next.zoom + next.x).toBeCloseTo(before.x * view.zoom + view.x); + expect(after.y * next.zoom + next.y).toBeCloseTo(before.y * view.zoom + view.y); + }); + + it('finds a card’s place on the graph through the boxes around it', () => { + const nodes = [ + { id: 'folder:orders', position: { x: 100, y: 50 } }, + { id: 'folder:orders/staging', position: { x: 14, y: 44 }, parentId: 'folder:orders' }, + { id: 'model:orders', position: { x: 14, y: 44 }, parentId: 'folder:orders/staging' } + ]; + expect(absolutePosition(nodes, 'model:orders')).toEqual({ x: 128, y: 138 }); + expect(absolutePosition(nodes, 'group:missing')).toBeNull(); + }); +}); + +describe('a large graph without a box', () => { + it('starts on the card nearest its top-left, not on an empty corner', () => { + // A layered layout: the first column centred low, a later column reaching the top. + const cards = [ + { x: 0, y: 1800, width: 236, height: 82 }, + { x: 0, y: 1900, width: 236, height: 82 }, + { x: 3000, y: 0, width: 236, height: 82 } + ]; + const bounds = { x: 0, y: 0, width: 3236, height: 1982 }; + const inside = (r: (typeof cards)[number], view: { x: number; y: number; zoom: number }) => + r.x * view.zoom + view.x >= 0 && (r.x + r.width) * view.zoom + view.x <= 1000 && r.y * view.zoom + view.y >= 0 && (r.y + r.height) * view.zoom + view.y <= 500; + const start = firstCard(cards)!; + expect(start).toEqual(cards[0]); + const v = fitViewport(bounds, 1000, 500, { padding: 0, minZoom: 0.6, start }); + // The start card's left edge is a margin inside the view's left. + expect(v.x + start.x * v.zoom).toBe(ANCHOR_MARGIN); + expect(inside(start, v)).toBe(true); + // The bare corner would have shown no card at all. + const corner = fitViewport(bounds, 1000, 500, { padding: 0, minZoom: 0.6 }); + expect(cards.some((c) => inside(c, corner))).toBe(false); + }); +}); + +describe('the readable zoom', () => { + it('keeps card titles at least 11 px on screen', () => { + expect(MIN_TITLE_PX).toBeGreaterThanOrEqual(11); + expect(CARD_TITLE_PX * READABLE_ZOOM).toBeGreaterThanOrEqual(MIN_TITLE_PX - 1e-9); + }); +}); + +describe('the card a large graph opens on', () => { + it('is the top of the leftmost column, wholly in view with a margin, even when a later column starts higher', () => { + // Folder cards: the leftmost column sits lower than a card in the second column. + const cards = [ + { x: 0, y: 900, width: 236, height: 61 }, + { x: 0, y: 1000, width: 236, height: 61 }, + { x: 300, y: 0, width: 236, height: 61 }, + { x: 3000, y: 500, width: 236, height: 61 } + ]; + const start = firstCard(cards)!; + expect(start).toBe(cards[0]); + const view = { width: 1376, height: 723 }; + const v = fitViewport({ x: 0, y: 0, width: 3236, height: 1061 }, view.width, view.height, { start }); + const [left, top] = [start.x * v.zoom + v.x, start.y * v.zoom + v.y]; + expect(left).toBeGreaterThanOrEqual(ANCHOR_MARGIN); + expect(top).toBeGreaterThanOrEqual(ANCHOR_MARGIN); + expect(left + start.width * v.zoom).toBeLessThanOrEqual(view.width - ANCHOR_MARGIN); + expect(top + start.height * v.zoom).toBeLessThanOrEqual(view.height - ANCHOR_MARGIN); + }); +}); + +describe('a small graph opens in full form', () => { + // A job's graph as the job page draws it: 1136 × 294 in a 1054 × 635 canvas. + const bounds = { x: 0, y: 0, width: 1136, height: 294 }; + it("opens at the compact zoom, not below it for the sake of the margin, when it fits whole there", () => { + const v = fitViewport(bounds, 1054, 635, { minZoom: LABEL_MIN_ZOOM }); + expect(v.zoom).toBe(COMPACT_ZOOM); + expect(v.x).toBeGreaterThanOrEqual(ANCHOR_MARGIN); + expect(v.x + bounds.width * v.zoom).toBeLessThanOrEqual(1054 - ANCHOR_MARGIN); + }); + it('still goes compact for a graph too wide for that', () => { + expect(fitViewport({ ...bounds, width: 1600 }, 1054, 635, { minZoom: LABEL_MIN_ZOOM }).zoom).toBeLessThan(COMPACT_ZOOM); + }); +}); + +describe('semantic zoom', () => { + it('draws cards in full down to the compact zoom, well below the readable fit zoom', () => { + expect(compactLevel(1)).toBe(0); + expect(compactLevel(COMPACT_ZOOM)).toBe(0); + expect(compactLevel(COMPACT_ZOOM - 0.01)).toBe(1); + expect(compactLevel(0.6)).toBe(1); + expect(compactLevel(0.59)).toBe(2); + expect(compactLevel(0.44)).toBe(3); + expect(compactLevel(0.1)).toBe(3); + }); + it('switches where full cards still show their 11 px lines at 9 px', () => { + expect(11 * COMPACT_ZOOM).toBeGreaterThanOrEqual(9); + expect(COMPACT_LEVELS.at(-1)).toBe(LABEL_MIN_ZOOM); + }); +}); diff --git a/ui/src/lib/lineage/fit.ts b/ui/src/lib/lineage/fit.ts new file mode 100644 index 0000000000..f44a1fb846 --- /dev/null +++ b/ui/src/lib/lineage/fit.ts @@ -0,0 +1,143 @@ +// Where a graph's view goes when it is fitted. Pure; no side effects. + +export type Rect = { x: number; y: number; width: number; height: number }; +export type Viewport = { x: number; y: number; zoom: number }; + +/** A card's title size on the canvas (`.name` in the node cards), and the smallest it may appear on screen. */ +export const CARD_TITLE_PX = 12.5; +export const MIN_TITLE_PX = 11; +/** Every card's width on the canvas. */ +export const CARD_WIDTH = 236; + +/** Space kept between the card a large graph's view starts on and the edge of the view. */ +export const ANCHOR_MARGIN = 24; + +/** + * The smallest zoom an automatic fit goes to, where card titles are still at least `MIN_TITLE_PX` on + * screen. Zooming out by hand can go further. + */ +export const READABLE_ZOOM = MIN_TITLE_PX / CARD_TITLE_PX; + +/** + * A graph of up to this many cards opens, and resets, showing all of it; only a larger one opens at + * `READABLE_ZOOM` on its start card. Zoomed out further, cards take their compact form (`compactLevel`). + */ +export const WHOLE_FIT_CARDS = 150; + +/** The lowest zoom down to which titles are counter-scaled to `MIN_TITLE_PX`, and the lowest a whole-graph fit goes. */ +export const LABEL_MIN_ZOOM = 0.35; + +/** + * Below this zoom cards take their compact form, a one-line pill (CompactFace), since the cards' detail + * lines (11 px) would show under 9 px. Above it cards are drawn in full. + */ +export const COMPACT_ZOOM = 0.82; +/** + * The compact levels' lowest zooms. A pill's name is fitted once per level, at that level's lowest zoom, + * and keeps its size on screen through the level (`--sb-title-unzoom`); the last level runs on down, + * the pill shrinking with the card below `LABEL_MIN_ZOOM`. + */ +export const COMPACT_LEVELS = [0.6, 0.45, LABEL_MIN_ZOOM] as const; + +/** How cards are drawn at `zoom`: 0 in full, else the compact level, from 1. */ +export function compactLevel(zoom: number): number { + if (zoom >= COMPACT_ZOOM) return 0; + const level = COMPACT_LEVELS.findIndex((low) => zoom >= low); + return level < 0 ? COMPACT_LEVELS.length : level + 1; +} + +type FitOptions = { + /** Share of the view left empty on each side. */ + padding?: number; + maxZoom?: number; + minZoom?: number; + /** What to centre on when the graph doesn't fit at `minZoom`, such as its largest box. */ + focus?: Rect | null; + /** + * Without a focus, the card to start from, put at the view's top-left: the card nearest the + * graph's top-left (see `firstCard`). The corner itself can be empty, as a layered layout + * centres its columns, and a view on it shows only lines passing by. + */ + start?: Rect | null; +}; + +/** + * Fits `bounds` into a `width` by `height` view: the whole graph centred, no closer than `maxZoom`. + * A graph too large to show whole at `minZoom` is shown at `minZoom` instead of shrinking past it, + * centred on `focus` when given, else from its top-left, and never panned past its own edges, so + * the view always starts on cards and the person pans or zooms from there. + */ +export function fitViewport(bounds: Rect, width: number, height: number, { padding = 0.08, maxZoom = 1, minZoom = READABLE_ZOOM, focus = null, start = null }: FitOptions = {}): Viewport { + const [padX, padY] = [width * padding, height * padding]; + const fits = (marginX: number, marginY: number) => Math.min(maxZoom, (width - 2 * marginX) / Math.max(1, bounds.width), (height - 2 * marginY) / Math.max(1, bounds.height)); + let whole = fits(padX, padY); + // The margin alone mustn't tip a graph into compact cards: one that fits whole at the compact zoom + // with `ANCHOR_MARGIN` around it opens there, its cards drawn in full. + if (whole < COMPACT_ZOOM && fits(ANCHOR_MARGIN, ANCHOR_MARGIN) >= COMPACT_ZOOM) whole = COMPACT_ZOOM; + if (whole >= minZoom) { + return { zoom: whole, x: width / 2 - (bounds.x + bounds.width / 2) * whole, y: height / 2 - (bounds.y + bounds.height / 2) * whole }; + } + const zoom = minZoom; + /** The offset along one axis: centred on the focus, else the start card at the view's start, kept within the graph's edges where the graph is larger than the view. */ + const axis = (start: number, size: number, view: number, pad: number, centre: number | null, first: number | null) => { + const wanted = centre !== null ? view / 2 - centre * zoom : pad - (first ?? start) * zoom; + if (size * zoom + 2 * pad <= view) return view / 2 - (start + size / 2) * zoom; + const most = pad - start * zoom; // the graph's start at the view's start + const least = view - pad - (start + size) * zoom; // the graph's end at the view's end + return Math.min(most, Math.max(least, wanted)); + }; + // The card the view starts on (the focus, else the start card) stays wholly in view, `ANCHOR_MARGIN` + // from the edges; one larger than the view shows its top-left. + const anchor = focus ?? start; + const keep = (offset: number, from: number, size: number, view: number) => { + const [lo, hi] = [from * zoom + offset, (from + size) * zoom + offset]; + if (hi - lo > view - 2 * ANCHOR_MARGIN) return ANCHOR_MARGIN - from * zoom; + if (lo < ANCHOR_MARGIN) return offset + ANCHOR_MARGIN - lo; + if (hi > view - ANCHOR_MARGIN) return offset - (hi - (view - ANCHOR_MARGIN)); + return offset; + }; + const x = axis(bounds.x, bounds.width, width, padX, focus ? focus.x + focus.width / 2 : null, start?.x ?? null); + const y = axis(bounds.y, bounds.height, height, padY, focus ? focus.y + focus.height / 2 : null, start?.y ?? null); + return anchor ? { zoom, x: keep(x, anchor.x, anchor.width, width), y: keep(y, anchor.y, anchor.height, height) } : { zoom, x, y }; +} + +/** The largest box among `items`, the cluster a large graph's view starts on; null when there is no box. */ +export function largestBox(items: readonly (Rect & { box: boolean })[]): Rect | null { + let best: Rect | null = null; + for (const item of items) if (item.box && (!best || item.width * item.height > best.width * best.height)) best = item; + return best && { x: best.x, y: best.y, width: best.width, height: best.height }; +} + +/** The card nearest the top-left of `items`' bounds, where a large graph's view starts without a box to centre on. */ +export function firstCard(items: readonly R[]): R | null { + if (!items.length) return null; + // The leftmost column: cards starting within half a card of the leftmost edge. Its top card, not the + // card nearest the corner, which can sit in a later column and leave the first one off screen. + const left = Math.min(...items.map((r) => r.x)); + const reach = Math.min(...items.map((r) => r.width)) / 2; + const column = items.filter((r) => r.x <= left + reach); + return column.reduce((best, r) => (r.y < best.y ? r : best)); +} + +type Placed = { id: string; position: { x: number; y: number }; parentId?: string }; + +/** A node's position on the graph, adding up the boxes around it, or null when it isn't drawn. */ +export function absolutePosition(nodes: readonly Placed[], id: string): { x: number; y: number } | null { + const byId = new Map(nodes.map((n) => [n.id, n])); + let at = byId.get(id); + if (!at) return null; + let { x, y } = at.position; + while (at.parentId && (at = byId.get(at.parentId))) { + x += at.position.x; + y += at.position.y; + } + return { x, y }; +} + +/** + * The view after a re-layout that keeps a box where it was on screen: the same zoom, panned by the + * box's layout shift, so its top-left stays under the pointer that opened or closed it. + */ +export function anchoredViewport(view: Viewport, before: { x: number; y: number }, after: { x: number; y: number }): Viewport { + return { zoom: view.zoom, x: view.x - (after.x - before.x) * view.zoom, y: view.y - (after.y - before.y) * view.zoom }; +} diff --git a/ui/src/lib/lineage/flow-controller.svelte b/ui/src/lib/lineage/flow-controller.svelte index 5e5a52f654..864a26d696 100644 --- a/ui/src/lib/lineage/flow-controller.svelte +++ b/ui/src/lib/lineage/flow-controller.svelte @@ -1,49 +1,93 @@ -
+
+ {#if folder} + + + + + + + {:else}
{#if d.ontoggle} + {/if} {d.jobKey} - {d.nodeCount} node{d.nodeCount === 1 ? '' : 's'} + {plural(d.nodeCount, 'node')} {#if d.lastRun} @@ -31,11 +49,13 @@ {/if}
+ {/if}
diff --git a/ui/src/lib/lineage/member-expand.ts b/ui/src/lib/lineage/member-expand.ts new file mode 100644 index 0000000000..5feb204eaf --- /dev/null +++ b/ui/src/lib/lineage/member-expand.ts @@ -0,0 +1,13 @@ +// Which member jobs are open in place on a group's page. Pure; no side effects. + +/** `open` with the member opened, or closed when it was open. */ +export function toggled(open: ReadonlySet, jobKey: string): Set { + const next = new Set(open); + if (!next.delete(jobKey)) next.add(jobKey); + return next; +} + +/** `open` with the member open, as selecting it from elsewhere does. */ +export function withOpen(open: ReadonlySet, jobKey: string): Set { + return new Set([...open, jobKey]); +} diff --git a/ui/src/lib/lineage/member-folders.test.ts b/ui/src/lib/lineage/member-folders.test.ts new file mode 100644 index 0000000000..d0704e06ce --- /dev/null +++ b/ui/src/lib/lineage/member-folders.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from 'vitest'; +import { declarationFolder, memberFolderTree, membersIn, memberStandIn } from './member-folders'; + +const files: Record = { + load_orders: 'jobs/load/orders.py:3', + load_customers: 'jobs/load/customers.py:3', + build_orders: 'jobs/build/orders/daily.py:1', + build_returns: 'jobs/build/orders/returns.py:1', + build_regions: 'jobs/build/regions.py:1', + export_inventory: 'jobs/exports/inventory.py:9', + cleanup: 'jobs/cleanup.py:1' +}; +const members = Object.keys(files); +const tree = memberFolderTree(members, (m) => declarationFolder(files[m])); + +describe('member folders', () => { + it('takes the folder from a declaration', () => { + expect(declarationFolder('jobs/orders/daily.py:12')).toBe('jobs/orders'); + expect(declarationFolder('daily.py:12')).toBe(''); + expect(declarationFolder(null)).toBe(''); + }); + + it('opens the folders every member shares, and keeps the rest as Lineage does', () => { + expect(tree.roots).toEqual(['jobs/load', 'jobs/build']); + expect(tree.folders.get('jobs/load')!.label).toBe('load'); + expect(tree.folders.get('jobs/build')!.folders).toEqual(['jobs/build/orders']); + expect(tree.folders.get('jobs/build/orders')!.label).toBe('orders'); + expect(membersIn(tree, 'jobs/build')).toEqual(['build_regions', 'build_orders', 'build_returns']); + }); + + it('draws a folder holding one member as that member’s card', () => { + expect(tree.folders.has('jobs/exports')).toBe(false); + expect(tree.loose).toEqual(['export_inventory', 'cleanup']); + expect(tree.folders.get('jobs/build')!.members).toEqual(['build_regions']); + }); + + it('stands a member in for its outermost closed folder', () => { + const standIn = memberStandIn(tree, new Set(['jobs/build'])); + expect(standIn('build_orders')).toBe('jobs/build/orders'); + expect(standIn('build_regions')).toBe(null); + expect(standIn('load_orders')).toBe('jobs/load'); + expect(standIn('cleanup')).toBe(null); + }); +}); diff --git a/ui/src/lib/lineage/member-folders.ts b/ui/src/lib/lineage/member-folders.ts new file mode 100644 index 0000000000..195842f8a9 --- /dev/null +++ b/ui/src/lib/lineage/member-folders.ts @@ -0,0 +1,113 @@ +// A job group's members by the folder of the file each is declared in, drawn as Lineage draws folders: +// a closed folder is one card, an open one a box around its members and subfolders. Pure. + +/** A folder of members after merging: a folder whose only content is one subfolder is drawn as one with it. */ +export type MemberFolder = { + /** The full path, such as `jobs/orders/daily`. */ + id: string; + parent: string | null; + /** The path below the parent folder. */ + label: string; + folders: string[]; + /** Members declared directly in the folder, in group order. */ + members: string[]; +}; + +export type MemberFolderTree = { + folders: Map; + /** What the group draws at its top: folders, and members in no folder. */ + roots: string[]; + loose: string[]; + /** Each member's folders, outermost first; empty for a member drawn at the top. */ + chains: Map; +}; + +/** A group of more members than this opens with its folders closed, as Lineage opens a large scope by folder. */ +export const MEMBER_FOLDERS_CLOSED_ABOVE = 24; + +/** The folder of a declaration such as `jobs/orders/daily.py:12`; empty for a file at the top or no file. */ +export function declarationFolder(sourceFile: string | null | undefined): string { + const path = (sourceFile ?? '').replace(/:\d+$/, ''); + const slash = path.lastIndexOf('/'); + return slash > 0 ? path.slice(0, slash) : ''; +} + +type Trie = { direct: string[]; children: Map }; + +const size = (t: Trie): number => t.direct.length + [...t.children.values()].reduce((n, c) => n + size(c), 0); + +/** + * The members' folders. As on Lineage, a folder whose only content is one subfolder merges into it. + * Folders every member shares are opened (a group declared under `jobs/` draws what is inside it), and + * a folder holding a single member draws that member's card in its place, rather than a box or card + * around one job. + */ +export function memberFolderTree(members: readonly string[], folderOf: (member: string) => string): MemberFolderTree { + const root: Trie = { direct: [], children: new Map() }; + for (const m of members) { + let at = root; + for (const part of folderOf(m).split('/').filter(Boolean)) { + if (!at.children.has(part)) at.children.set(part, { direct: [], children: new Map() }); + at = at.children.get(part)!; + } + at.direct.push(m); + } + // Open the folders every member shares. + let top = root; + let prefix = ''; + while (!top.direct.length && top.children.size === 1) { + const [[part, only]] = top.children; + prefix = prefix ? `${prefix}/${part}` : part; + top = only; + } + const folders = new Map(); + const chains = new Map(); + const visit = (name: string, trie: Trie, parent: MemberFolder | null, base: string, chain: string[], into: { folders: string[]; members: string[] }) => { + // One member: its card in the parent's place, without a folder around it. + if (size(trie) === 1) { + const only = (function first(t: Trie): string { + return t.direct[0] ?? first([...t.children.values()][0]); + })(trie); + into.members.push(only); + chains.set(only, chain); + return; + } + let path = base ? `${base}/${name}` : name; + while (!trie.direct.length && trie.children.size === 1) { + const [[part, only]] = trie.children; + path = `${path}/${part}`; + trie = only; + } + const folder: MemberFolder = { id: path, parent: parent?.id ?? null, label: parent ? path.slice(parent.id.length + 1) : path.slice(prefix ? prefix.length + 1 : 0), folders: [], members: [] }; + folders.set(path, folder); + into.folders.push(path); + const own = [...chain, path]; + for (const m of trie.direct) { + folder.members.push(m); + chains.set(m, own); + } + for (const [part, child] of trie.children) visit(part, child, folder, path, own, folder); + }; + const level = { folders: [] as string[], members: [] as string[] }; + for (const m of top.direct) { + level.members.push(m); + chains.set(m, []); + } + for (const [part, child] of top.children) visit(part, child, null, prefix, [], level); + // Members keep the group's order within each level. + const order = new Map(members.map((m, i) => [m, i])); + const byOrder = (a: string, b: string) => order.get(a)! - order.get(b)!; + for (const f of folders.values()) f.members.sort(byOrder); + return { folders, roots: level.folders, loose: level.members.sort(byOrder), chains }; +} + +/** What stands for a member: itself when every folder around it is open, else its outermost closed folder. */ +export function memberStandIn(tree: MemberFolderTree, open: ReadonlySet): (member: string) => string | null { + return (member) => (tree.chains.get(member) ?? []).find((f) => !open.has(f)) ?? null; +} + +/** Every member in the folder and the folders inside it. */ +export function membersIn(tree: MemberFolderTree, folder: string): string[] { + const f = tree.folders.get(folder); + return f ? [...f.members, ...f.folders.flatMap((sub) => membersIn(tree, sub))] : []; +} diff --git a/ui/src/lib/lineage/member-graph.svelte b/ui/src/lib/lineage/member-graph.svelte new file mode 100644 index 0000000000..47d1e50321 --- /dev/null +++ b/ui/src/lib/lineage/member-graph.svelte @@ -0,0 +1,316 @@ + + +
+ 0 || (foldersOn && openMemberFolders.size > 0), allTitle: 'Open every job in place, up to the graph’s card limit', collapseTitle: 'Close every job', note: expandNote }} + focus={{ on: !!focusOn, can: !!selection, toggle: toggleFocus, title: 'Show only the selected job and the jobs it waits for or that wait for it' }} + onreset={reset} + {details} + /> + (selection = null)} + onnodepointerenter={({ node }) => (hoveredId = node.id)} + onnodepointerleave={() => (hoveredId = null)} + /> + {#if tooMany} + + {#snippet actions()} + {#if expanded.size}{/if} + {/snippet} + + {/if} +
+ + diff --git a/ui/src/lib/lineage/member-graph.test.ts b/ui/src/lib/lineage/member-graph.test.ts new file mode 100644 index 0000000000..d485a43cb6 --- /dev/null +++ b/ui/src/lib/lineage/member-graph.test.ts @@ -0,0 +1,289 @@ +import { describe, expect, it } from 'vitest'; +import type { CatalogNode, JobGroupMember, StaleReason } from '$lib/api'; +import { layoutFolderGraph } from './folder-graph'; +import { toggled, withOpen } from './member-expand'; +import { bigGroupFile, bigGroupShape } from '$lib/api/mock/big-group'; +import { declarationFolder, memberFolderTree } from './member-folders'; +import { + directWaits, + edgeTitle, + edgeWidth, + innerId, + layoutMemberGraph, + litEdges, + memberFolderBoxId, + memberFolderCardId, + memberGraph, + memberLinks, + memberNodeId, + publishedLinks, + memberReach, + needsBuilding, + pairKey, + transitiveReduction, + waitReasons, + type MemberEdgeData, + type MemberInfo +} from './member-graph'; + +const member = (jobKey: string, stage: number, waitsFor: string[] = []): JobGroupMember => ({ + jobKey, + stage, + upstream: waitsFor.map((k) => ({ jobKey: k, source: null })), + reasons: [] +}); + +function node(key: string, upstream: string[] = [], stale: StaleReason['reason'][] = []): CatalogNode { + const reasons = stale.map((reason) => ({ reason, upstream: [], detail: '', since: '', change: 'definition' }) as unknown as StaleReason); + return { key, kind: 'table', path: `models/${key.split(':')[1]}.sql`, upstream, group: null, tags: [], stale: reasons, lastOutcome: null, lastBuild: null } as unknown as CatalogNode; +} + +// customers builds two staging models; orders reads both, through a source no member selects for one; +// shipments reads orders and stock; inventory has no lineage to shipments but the server orders it first. +const catalog = [ + node('model:stg_customers'), + node('model:customer_dim', ['model:stg_customers'], ['changed']), + node('source:raw_orders', ['model:customer_dim']), + node('model:orders', ['model:stg_customers', 'source:raw_orders'], ['behind']), + node('model:order_lines', ['model:orders', 'model:customer_dim'], ['changed']), + node('model:shipments', ['model:order_lines']) +]; +const members = [member('customers', 0), member('inventory', 0), member('orders', 1, ['customers']), member('shipments', 2, ['orders', 'inventory'])]; +const nodesOf: Record = { + customers: ['model:stg_customers', 'model:customer_dim'], + inventory: [], + orders: ['model:orders', 'model:order_lines'], + shipments: ['model:shipments'] +}; +const info = new Map(Object.entries(nodesOf).map(([k, nodes]) => [k, { nodeCount: nodes.length, lastRun: null, nodes }])); +const links = memberLinks(members, info, catalog); +const pairs = (edges: { source: string; target: string }[]) => edges.map((e) => `${e.source} > ${e.target}`); + +describe('published wait reasons', () => { + // The server's reasons for the same members: shipments waits for inventory with none, an ordering. + const published = members.map((m) => + m.jobKey === 'orders' + ? { ...m, reasons: [{ upstreamJob: 'customers', fromNode: 'model:customer_dim', toNode: 'model:orders' }] } + : m.jobKey === 'shipments' + ? { ...m, reasons: [{ upstreamJob: 'orders', fromNode: 'model:order_lines', toNode: 'model:shipments' }] } + : m + ); + + it("uses the server's reasons, each wait keyed as the lineage derivation keys it", () => { + const links = publishedLinks(published)!; + expect(links.get(pairKey('customers', 'orders'))).toEqual([{ from: 'model:customer_dim', to: 'model:orders' }]); + expect(links.get(pairKey('orders', 'shipments'))).toEqual([{ from: 'model:order_lines', to: 'model:shipments' }]); + expect(links.get(pairKey('inventory', 'shipments'))).toEqual([]); + }); + + it('draws a direct wait the server gives no reasons for dashed', () => { + const { edges } = memberGraph(published, info, { catalog, links: publishedLinks(published)! }); + const style = (from: string, to: string) => String(edges.find((e) => e.source === memberNodeId(from) && e.target === memberNodeId(to))!.style); + expect(style('inventory', 'shipments')).toContain('stroke-dasharray'); + expect(style('orders', 'shipments')).not.toContain('stroke-dasharray'); + }); + + it('has none to give for a snapshot that predates them, so the page works them out from lineage', () => { + expect(publishedLinks(members)).toBeNull(); + }); +}); + +describe('member links', () => { + it('finds the node-level reasons for each wait, through nodes no member selects', () => { + expect(links.get(pairKey('customers', 'orders'))).toEqual([ + { from: 'model:stg_customers', to: 'model:orders' }, + { from: 'model:customer_dim', to: 'model:orders' }, + { from: 'model:customer_dim', to: 'model:order_lines' } + ]); + expect(links.get(pairKey('orders', 'shipments'))).toEqual([{ from: 'model:order_lines', to: 'model:shipments' }]); + }); + + it('leaves a wait with no lineage between the members empty, and never counts lineage through another member', () => { + expect(links.get(pairKey('inventory', 'shipments'))).toEqual([]); + // shipments reads customer_dim only through orders' order_lines, so customers is not a reason for shipments. + expect(links.has(pairKey('customers', 'shipments'))).toBe(false); + }); + + it('caps the reasons listed for a wait, counting the rest', () => { + expect(waitReasons('orders', members, links, 2)).toEqual([{ upstream: 'customers', reasons: links.get(pairKey('customers', 'orders'))!.slice(0, 2), more: 1 }]); + }); +}); + +describe('member graph edges at the visible level', () => { + it('draws one edge per pair of collapsed members, thicker and counted when many node links join them', () => { + const { nodes, edges } = memberGraph(members, info, { catalog, links }); + expect(nodes.map((n) => n.type)).toEqual(['member', 'member', 'member', 'member']); + expect(pairs(edges)).toEqual([ + `${memberNodeId('customers')} > ${memberNodeId('orders')}`, + `${memberNodeId('orders')} > ${memberNodeId('shipments')}`, + `${memberNodeId('inventory')} > ${memberNodeId('shipments')}` + ]); + const [customers, , inventory] = edges.map((e) => e.data as MemberEdgeData); + expect(customers.count).toBe(3); + expect(String(edges[0].style)).toContain(`stroke-width: ${edgeWidth(3)}`); + expect(edgeTitle(customers)).toBe('orders waits for customers: 3 node links'); + // A wait the lineage doesn't show stays a dashed edge between the cards. + expect(inventory.count).toBe(0); + expect(String(edges[2].style)).toContain('dasharray'); + }); + + it('ends edges at the nodes inside an expanded member, still starting from collapsed members', () => { + const { nodes, edges } = memberGraph(members, info, { catalog, links, expanded: new Set(['orders']) }); + const box = nodes.find((n) => n.id === memberNodeId('orders'))!; + expect(box.type).toBe('memberBox'); + expect(nodes.filter((n) => n.parentId === box.id).map((n) => n.id)).toEqual([innerId('orders', 'model:orders'), innerId('orders', 'model:order_lines')]); + // customers also feeds order_lines directly, but that edge follows from customers > orders > + // order_lines, so only the first is drawn. + expect(pairs(edges)).toEqual([ + `${memberNodeId('customers')} > ${innerId('orders', 'model:orders')}`, + `${innerId('orders', 'model:order_lines')} > ${memberNodeId('shipments')}`, + `${memberNodeId('inventory')} > ${memberNodeId('shipments')}`, + // Lineage inside the expanded member. + `${innerId('orders', 'model:orders')} > ${innerId('orders', 'model:order_lines')}` + ]); + expect((edges[0].data as MemberEdgeData).count).toBe(2); + }); + + it('draws node to node between two expanded members', () => { + const { edges } = memberGraph(members, info, { catalog, links, expanded: new Set(['customers', 'orders']) }); + expect(pairs(edges).filter((p) => p.startsWith(`in:customers`) && p.includes('in:orders'))).toEqual([ + `${innerId('customers', 'model:stg_customers')} > ${innerId('orders', 'model:orders')}`, + `${innerId('customers', 'model:customer_dim')} > ${innerId('orders', 'model:orders')}`, + `${innerId('customers', 'model:customer_dim')} > ${innerId('orders', 'model:order_lines')}` + ]); + }); + + it('keeps a member with no known nodes as a card, even when asked to expand it', () => { + const { nodes } = memberGraph(members, info, { catalog, links, expanded: new Set(['inventory']) }); + expect(nodes.find((n) => n.id === memberNodeId('inventory'))).toMatchObject({ type: 'member', data: { expandable: false, expanded: false } }); + }); + + it('draws every wait between cards without links, and nothing for a group without members', () => { + expect(pairs(memberGraph(members, info).edges)).toHaveLength(3); + expect(memberGraph([], new Map())).toEqual({ nodes: [], edges: [] }); + }); + + it('lays an expanded member out as a box around its nodes, upstream members to its left', () => { + const { nodes, edges } = memberGraph(members, info, { catalog, links, expanded: new Set(['orders']) }); + const placed = layoutFolderGraph(nodes, edges); + const at = (id: string) => placed.find((n) => n.id === id)!; + const box = at(memberNodeId('orders')); + const inner = at(innerId('orders', 'model:order_lines')); + expect(box.width).toBeGreaterThanOrEqual(inner.position.x + 236); + expect(box.height).toBeGreaterThan(inner.position.y); + expect(at(memberNodeId('customers')).position.x).toBeLessThan(box.position.x); + expect(at(memberNodeId('shipments')).position.x).toBeGreaterThan(box.position.x + box.width!); + }); +}); + +describe('member cards', () => { + it('count the nodes needing a build: changed, and only behind', () => { + const byKey = new Map(catalog.map((n) => [n.key, n])); + expect(needsBuilding(nodesOf.orders, byKey)).toEqual({ changed: 1, behind: 1 }); + const { nodes } = memberGraph(members, info, { catalog, links }); + expect(nodes.find((n) => n.id === memberNodeId('customers'))!.data).toMatchObject({ jobKey: 'customers', nodeCount: 2, waitsFor: 0, needs: { changed: 1, behind: 0 } }); + }); +}); + +describe('expand state', () => { + it('opens a member, closes it again, and leaves others as they were', () => { + const open = toggled(new Set(['customers']), 'orders'); + expect([...open]).toEqual(['customers', 'orders']); + expect([...toggled(open, 'customers')]).toEqual(['orders']); + expect([...withOpen(open, 'orders')]).toEqual(['customers', 'orders']); + }); +}); + +describe('a large group whose waits list every member upstream', () => { + const shape = bigGroupShape(); + const big = shape.members.map((m) => member(m, 0, shape.waits.get(m))); + const closure = [...shape.waits.values()].reduce((n, w) => n + w.length, 0); + const reduction = [...shape.direct.values()].reduce((n, w) => n + w.length, 0); + const ends = (e: { data?: unknown }) => e.data as MemberEdgeData; + + it('draws only the direct waits: the transitive reduction, far fewer than the waits listed', () => { + const { edges } = memberGraph(big, new Map()); + expect(shape.members.length).toBeGreaterThanOrEqual(40); + expect(edges).toHaveLength(reduction); + expect(closure).toBeGreaterThan(4 * reduction); + expect(edges.length).toBeLessThanOrEqual(2 * shape.members.length); + expect(new Set(edges.map((e) => `${ends(e).from}>${ends(e).to}`))).toEqual(new Set([...shape.direct].flatMap(([m, ups]) => ups.map((u) => `${u}>${m}`)))); + expect(directWaits(big).direct).toEqual(shape.direct); + }); + + it('lights a member’s whole upstream and downstream, implied waits included', () => { + const { nodes, edges } = memberGraph(big, new Map()); + const reach = memberReach(big); + const focus = 'join_c'; + const lit = litEdges(memberNodeId(focus), edges, nodes, reach); + const related = new Set([focus, ...reach.up.get(focus)!, ...reach.down.get(focus)!]); + expect(reach.up.get(focus)).toEqual(new Set(shape.waits.get(focus))); + for (const e of edges.filter((e) => lit.has(e.id))) expect(related.has(ends(e).from) && related.has(ends(e).to)).toBe(true); + // Every member upstream is reached through lit edges, and the rest of the graph is dimmed. + const litFrom = new Set(edges.filter((e) => lit.has(e.id)).map((e) => ends(e).from)); + for (const u of reach.up.get(focus)!) expect(litFrom.has(u)).toBe(true); + expect(lit.size).toBeLessThan(edges.length); + }); + + it('lists a member’s direct waits first, then each implied one with the wait it comes through', () => { + const waits = waitReasons('clean_a', big, new Map()); + const direct = shape.direct.get('clean_a')!; + expect(waits.filter((w) => w.through === undefined).map((w) => w.upstream)).toEqual(direct); + const implied = waits.filter((w) => w.through !== undefined); + expect(implied.map((w) => w.upstream).sort()).toEqual(shape.waits.get('clean_a')!.filter((u) => !direct.includes(u)).sort()); + for (const w of implied) expect(direct).toContain(w.through); + expect(waits.findIndex((w) => w.through !== undefined)).toBe(direct.length); + }); + + it('puts members with no waits either way in a compact block under the graph', () => { + const { nodes, edges } = memberGraph(big, new Map()); + const placed = layoutMemberGraph(nodes, edges); + const at = (m: string) => placed.find((n) => n.id === memberNodeId(m))!.position; + const graphBottom = Math.max(...shape.members.filter((m) => !shape.alone.includes(m)).map((m) => at(m).y)); + const block = shape.alone.map(at); + for (const p of block) expect(p.y).toBeGreaterThan(graphBottom); + expect(new Set(block.map((p) => p.y)).size).toBe(1); + }); +}); + +describe('transitive reduction', () => { + const e = (source: string, target: string) => ({ source, target }); + it('drops an edge a longer path implies, keeps what only it joins, and survives a cycle', () => { + expect(transitiveReduction([e('a', 'c'), e('a', 'b'), e('b', 'c')])).toEqual([e('a', 'b'), e('b', 'c')]); + expect(transitiveReduction([e('a', 'b'), e('c', 'b')])).toHaveLength(2); + expect(transitiveReduction([e('a', 'b'), e('b', 'a'), e('a', 'c'), e('b', 'c')])).toHaveLength(3); + expect(transitiveReduction([e('a', 'c'), e('a', 'b'), e('b', 'c')], (x) => x.source !== 'a' || x.target !== 'c')).toHaveLength(3); + }); +}); + +describe('a large group drawn by the folders its members are declared in', () => { + const shape = bigGroupShape(); + const big = shape.members.map((m) => member(m, 0, shape.waits.get(m))); + const tree = memberFolderTree(shape.members, (m) => declarationFolder(bigGroupFile(m))); + const stages = ['stage', 'clean', 'join', 'enrich', 'aggregate', 'publish', 'housekeeping'].map((s) => `jobs/warehouse/${s}`); + + it('draws each closed folder as one card, a lone member as its own card, and one line per pair of cards', () => { + const { nodes, edges } = memberGraph(big, new Map(), { folders: { tree, open: new Set() } }); + expect(nodes.filter((n) => n.type === 'folder').map((n) => n.id).sort()).toEqual(stages.map(memberFolderCardId).sort()); + // load_a and publish_e are each alone in their folder, so they are drawn as themselves. + expect(nodes.filter((n) => n.type === 'member').map((n) => n.id).sort()).toEqual([memberNodeId('load_a'), memberNodeId('publish_e')].sort()); + const pairs = edges.map((e) => `${e.source} > ${e.target}`).sort(); + const card = (s: string) => memberFolderCardId(`jobs/warehouse/${s}`); + expect(pairs).toEqual( + [ + `${memberNodeId('load_a')} > ${card('stage')}`, + ...['stage', 'clean', 'join', 'enrich', 'aggregate'].map((s, i, all) => `${card(s)} > ${card(all[i + 1] ?? 'publish')}`), + `${card('aggregate')} > ${memberNodeId('publish_e')}` + ].sort() + ); + // The line between two folders stands for every wait between their members, and lights with them. + const stageToClean = edges.find((e) => e.source === card('stage') && e.target === card('clean'))!; + expect((stageToClean.data as MemberEdgeData).pairs).toHaveLength(10); + }); + + it('draws an open folder as a box around its members, with the lines between members again', () => { + const { nodes, edges } = memberGraph(big, new Map(), { folders: { tree, open: new Set(tree.folders.keys()) } }); + expect(nodes.filter((n) => n.parentId === memberFolderBoxId('jobs/warehouse/clean')).map((n) => n.id)).toEqual(['clean_a', 'clean_b', 'clean_c', 'clean_d', 'clean_e'].map(memberNodeId)); + expect(edges).toHaveLength([...shape.direct.values()].reduce((n, w) => n + w.length, 0)); + }); +}); diff --git a/ui/src/lib/lineage/member-graph.ts b/ui/src/lib/lineage/member-graph.ts new file mode 100644 index 0000000000..c40fd1b52f --- /dev/null +++ b/ui/src/lib/lineage/member-graph.ts @@ -0,0 +1,443 @@ +// A job group's member jobs as a graph, drawn as Lineage draws folders: a collapsed member is one +// card, an expanded member a box around its nodes, and edges join whatever is visible at each end. +// Pure; no side effects. + +import type { Edge, Node } from '@xyflow/svelte'; +import type { CatalogNode, JobGroupMember, RunRef } from '$lib/api'; +import { folderTree, layoutFolderGraph, scopeGraph, type FolderTree } from './folder-graph'; +import { needsBuilding, type NeedsBuilding } from './view-summary'; +import { edgeStyle, edgesTouching, edgeWidth, visibleEdges, type NodeLink } from './visible-edges'; +import { plural } from '$lib/format/plural'; +import { membersIn, memberStandIn, type MemberFolderTree } from './member-folders'; +import type { FolderBoxData } from './folder-graph'; +import type { GroupCard } from './catalog-graph'; + +export { needsBuilding }; + +export const MEMBER_WIDTH = 236; +export const MEMBER_HEIGHT = 82; + +/** What the jobs list reports for a member; null where it isn't known. */ +export type MemberInfo = { + nodeCount: number | null; + lastRun: RunRef | null; + description?: string; + /** The member's nodes, by catalog key. */ + nodes?: string[] | null; + /** Where the member is declared, such as `jobs/orders/daily.py:12`; its folder groups it on the graph. */ + sourceFile?: string | null; +}; + +export type MemberCardData = { + jobKey: string; + nodeCount: number | null; + lastRun: RunRef | null; + /** How many members this one waits for. */ + waitsFor: number; + needs: NeedsBuilding; + /** Whether the member can open in place: its nodes are known. */ + expandable: boolean; + expanded: boolean; + ontoggle?: (jobKey: string) => void; +}; + +/** What is selected on a group's graph: a member job, or one of its nodes drawn inside it. */ +export type MemberSelection = { member: string; node?: string }; + +/** One node-level reason a member waits for another: `from` in the upstream member feeds `to` in the waiting one. */ +export type { NodeLink }; +export { edgeWidth }; + +export type MemberEdgeData = { + /** How many node-level links the edge stands for; 0 for a wait with no lineage between drawn nodes. */ + count: number; + /** The members it joins, upstream first. */ + from: string; + to: string; + /** The tooltip. */ + title: string; + /** Every member wait the edge stands for, when a closed folder of members stands at an end. */ + pairs?: { from: string; to: string }[]; +}; + + + +export const memberNodeId = (jobKey: string) => `member:${jobKey}`; +/** A node, folder card or folder box drawn inside an expanded member. Members may share nodes, so ids carry the member. */ +export const innerId = (jobKey: string, id: string) => `in:${jobKey}:${id}`; +export const pairKey = (upstream: string, member: string) => `${upstream}>${member}`; + +/** Each member's waits within the group, without repeats or waits on itself. */ +function waitsWithin(members: readonly JobGroupMember[]): Map { + const known = new Set(members.map((m) => m.jobKey)); + return new Map(members.map((m) => [m.jobKey, [...new Set(m.upstream.map((u) => u.jobKey))].filter((k) => known.has(k) && k !== m.jobKey)])); +} + +/** + * The edges no path through the other edges implies, in their order. Each implied edge is dropped in + * turn, so what stays still joins everything the input joined, even round a cycle; `droppable` says + * which edges may go at all. On a graph without cycles this is its transitive reduction. + */ +export function transitiveReduction(edges: readonly E[], droppable: (e: E) => boolean = () => true): E[] { + const kept = new Set(edges.map((_, i) => i)); + const out = Map.groupBy(edges.keys(), (i) => edges[i].source); + const reaches = (from: string, to: string, skip: number) => { + const seen = new Set([from]); + const stack = [from]; + while (stack.length) { + for (const i of out.get(stack.pop()!) ?? []) { + if (i === skip || !kept.has(i)) continue; + const next = edges[i].target; + if (next === to) return true; + if (!seen.has(next)) { + seen.add(next); + stack.push(next); + } + } + } + return false; + }; + for (const [i, e] of edges.entries()) if (e.source !== e.target && droppable(e) && reaches(e.source, e.target, i)) kept.delete(i); + return edges.filter((_, i) => kept.has(i)); +} + +/** + * Each member's waits split in two: `direct`, the waits drawn, and `implied`, the waits that follow + * from them through other members (A waits for B and B for C, so A's wait for C is implied). The + * server lists every wait, implied ones too; drawing those turns a large group into a tangle. + */ +export function directWaits(members: readonly JobGroupMember[]): { direct: Map; implied: Map } { + const waits = waitsWithin(members); + const all = [...waits].flatMap(([member, ups]) => ups.map((up) => ({ source: up, target: member }))); + const kept = new Set(transitiveReduction(all).map((e) => pairKey(e.source, e.target))); + const direct = new Map(); + const implied = new Map(); + for (const [member, ups] of waits) { + direct.set(member, ups.filter((up) => kept.has(pairKey(up, member)))); + implied.set(member, ups.filter((up) => !kept.has(pairKey(up, member)))); + } + return { direct, implied }; +} + +/** Every member each member waits for, directly or through others, and every member waiting for it. */ +export function memberReach(members: readonly JobGroupMember[]): { up: Map>; down: Map> } { + const waits = waitsWithin(members); + const after = new Map(); + for (const [member, ups] of waits) for (const up of ups) after.set(up, [...(after.get(up) ?? []), member]); + const closure = (start: string, next: (k: string) => string[]) => { + const seen = new Set(); + const stack = [...next(start)]; + while (stack.length) { + const k = stack.pop()!; + if (seen.has(k) || k === start) continue; + seen.add(k); + stack.push(...next(k)); + } + return seen; + }; + const keys = [...waits.keys()]; + return { + up: new Map(keys.map((k) => [k, closure(k, (x) => waits.get(x) ?? [])])), + down: new Map(keys.map((k) => [k, closure(k, (x) => after.get(x) ?? [])])) + }; +} + +/** + * The edges lit when a card is hovered or selected. For a member, its whole upstream and downstream: + * every drawn edge between members it waits for, directly or not, and between members waiting for + * it, so the waits the drawing leaves implied still light up; plus what touches its box. For a node + * inside a member, just the edges that touch it. + */ +export function litEdges(focus: string, edges: readonly Edge[], nodes: readonly Pick[], reach: ReturnType): Set { + const lit = edgesTouching(focus, edges, nodes); + const prefix = memberNodeId(''); + const member = focus.startsWith(prefix) ? focus.slice(prefix.length) : null; + if (member === null) return lit; + const up = reach.up.get(member) ?? new Set(); + const down = reach.down.get(member) ?? new Set(); + for (const e of edges) { + const d = e.data as Partial | undefined; + if (!d?.from || !d.to) continue; + for (const { from, to } of d.pairs ?? [{ from: d.from, to: d.to }]) { + if ((up.has(from) && (to === member || up.has(to))) || (down.has(to) && (from === member || down.has(from)))) lit.add(e.id); + } + } + return lit; +} + +/** + * Why each member waits for another, node by node, as the server publishes it (`reasons`), keyed + * by `pairKey` like `memberLinks`; a wait with none is direct with no lineage, such as an ordering. + * Null when no member has a reason: a snapshot published before reasons were recorded sends every + * member's list empty, which can't be told from a group whose waits are all orderings, so the page + * works the reasons out from lineage instead (`memberLinks`), which finds none for the latter too. + */ +export function publishedLinks(members: readonly JobGroupMember[]): Map | null { + if (!members.some((m) => m.reasons.length)) return null; + const links = new Map(); + for (const [jobKey, waits] of waitsWithin(members)) for (const upstream of waits) links.set(pairKey(upstream, jobKey), []); + for (const m of members) { + for (const r of m.reasons) { + const key = pairKey(r.upstreamJob, m.jobKey); + if (!links.has(key)) continue; + links.get(key)!.push({ from: r.fromNode, to: r.toNode }); + } + } + return links; +} + +/** + * Why each member waits for another, node by node, worked out from lineage, for a server that + * doesn't publish it (`publishedLinks`): a link from a node of the upstream member to a + * node of the waiting member it feeds, directly or through nodes no member selects (such as a + * source between a loader job and the models reading it). Lineage through another member's node + * is that member's link, not this pair's. Keyed by `pairKey`; a pair the server reports with no + * link here waits for a reason the lineage doesn't show, such as an ordering. + */ +export function memberLinks(members: readonly JobGroupMember[], info: ReadonlyMap, catalog: readonly CatalogNode[]): Map { + const byKey = new Map(catalog.map((n) => [n.key, n])); + const nodesOf = new Map(members.map((m) => [m.jobKey, new Set(info.get(m.jobKey)?.nodes ?? [])])); + const inAny = new Set([...nodesOf.values()].flatMap((s) => [...s])); + // The member nodes upstream of a node no member selects, through other such nodes. + const reach = new Map(); + const reachOf = (key: string): string[] => { + const known = reach.get(key); + if (known) return known; + reach.set(key, []); // A cycle ends here rather than looping. + const found = new Set(); + for (const up of byKey.get(key)?.upstream ?? []) for (const k of inAny.has(up) ? [up] : reachOf(up)) found.add(k); + const list = [...found]; + reach.set(key, list); + return list; + }; + const links = new Map(); + for (const [jobKey, waits] of waitsWithin(members)) { + const own = nodesOf.get(jobKey)!; + for (const upstream of waits) { + const theirs = nodesOf.get(upstream)!; + const found: NodeLink[] = []; + const seen = new Set(); + for (const to of own) { + for (const up of byKey.get(to)?.upstream ?? []) { + if (own.has(up)) continue; + for (const from of inAny.has(up) ? [up] : reachOf(up)) { + if (!theirs.has(from) || own.has(from) || seen.has(`${from}>${to}`)) continue; + seen.add(`${from}>${to}`); + found.push({ from, to }); + } + } + } + links.set(pairKey(upstream, jobKey), found); + } + } + return links; +} + +export type MemberGraphOptions = { + catalog?: readonly CatalogNode[]; + /** From `memberLinks`; without it every wait is drawn between member cards. */ + links?: ReadonlyMap; + expanded?: ReadonlySet; + /** Open folders inside each expanded member drawn by folder. */ + openFolders?: ReadonlyMap>; + ontoggle?: (jobKey: string) => void; + oncloseFolder?: (jobKey: string, folder: string) => void; + /** Members drawn by the folder of their declaration: closed folders as cards, open ones as boxes. */ + folders?: { tree: MemberFolderTree; open: ReadonlySet; onclose?: (folder: string) => void }; +}; + +/** A closed folder of members, and an open one's box. */ +export const memberFolderCardId = (folder: string) => `jobfolder:${folder}`; +export const memberFolderBoxId = (folder: string) => `jobfolderbox:${folder}`; + + +/** + * Cards for the members, with each expanded member a box around its nodes (by folder when it has + * many, folders opening as on Lineage), parents before children as Svelte Flow needs. Edges are drawn + * at the level you can see: one per pair of collapsed members however many node links join them, + * ending at the nodes inside an expanded member, and node to node between two expanded members. + * Positions are left for `layoutFolderGraph`. + */ +export function memberGraph(members: readonly JobGroupMember[], info: ReadonlyMap, opts: MemberGraphOptions = {}): { nodes: Node[]; edges: Edge[] } { + const { catalog = [], links = new Map(), expanded = new Set(), openFolders = new Map(), ontoggle, oncloseFolder, folders } = opts; + const byKey = new Map(catalog.map((n) => [n.key, n])); + const waits = directWaits(members).direct; + const drawn: Node[] = []; + const internal: Edge[] = []; + /** For each expanded member, what stands for one of its nodes. */ + const standIns = new Map string>(); + + const drawMember = (m: JobGroupMember, box: string | null) => { + const jobKey = m.jobKey; + const placed = box ? { parentId: box, extent: 'parent' as const } : {}; + const keys = (info.get(jobKey)?.nodes ?? []).filter((k) => byKey.has(k)); + const open = expanded.has(jobKey) && keys.length > 0; + const data: MemberCardData = { + jobKey, + nodeCount: info.get(jobKey)?.nodeCount ?? null, + lastRun: info.get(jobKey)?.lastRun ?? null, + waitsFor: waits.get(jobKey)!.length, + needs: needsBuilding(info.get(jobKey)?.nodes ?? [], byKey), + expandable: keys.length > 0, + expanded: open, + ontoggle + }; + const id = memberNodeId(jobKey); + if (!open) { + drawn.push({ id, type: 'member', position: { x: 0, y: 0 }, width: MEMBER_WIDTH, height: MEMBER_HEIGHT, ...placed, data }); + return; + } + drawn.push({ id, type: 'memberBox', position: { x: 0, y: 0 }, width: MEMBER_WIDTH, height: MEMBER_HEIGHT, ...placed, data }); + const nodes = keys.map((k) => byKey.get(k)!); + const inside = (inner: string) => innerId(jobKey, inner); + // The member is a scope of its own, drawn by the same rule as Lineage: by folder when large. + const tree: FolderTree = folderTree(nodes); + const sub = scopeGraph(nodes, { open: openFolders.get(jobKey) ?? new Set(), tree, onclose: oncloseFolder && ((folder) => oncloseFolder(jobKey, folder)) }); + for (const n of sub.nodes) drawn.push({ ...n, id: inside(n.id), parentId: n.parentId ? inside(n.parentId) : id, extent: 'parent', data: { ...n.data, member: jobKey } }); + for (const e of sub.edges) internal.push({ ...e, id: inside(e.id), source: inside(e.source), target: inside(e.target), style: e.style ?? edgeStyle(1) }); + standIns.set(jobKey, (key) => inside(sub.standIn(key))); + }; + + // By folder: each closed folder a card, each open one a box around its subfolders and members. + const byMember = new Map(members.map((m) => [m.jobKey, m])); + const folderStandIn = folders ? memberStandIn(folders.tree, folders.open) : () => null; + if (folders) { + const { tree, open, onclose } = folders; + const facts = (folder: string) => { + const inside = membersIn(tree, folder).filter((k) => byMember.has(k)); + const status = (k: string) => info.get(k)?.lastRun?.status; + const needs = (k: string) => needsBuilding(info.get(k)?.nodes ?? [], byKey); + return { + count: inside.length, + ok: inside.filter((k) => status(k) === 'success').length, + stale: inside.filter((k) => needs(k).changed + needs(k).behind > 0).length, + failed: inside.filter((k) => status(k) === 'failed' || status(k) === 'partial').length + }; + }; + const place = (folder: string, box: string | null) => { + const f = tree.folders.get(folder)!; + const placed = box ? { parentId: box, extent: 'parent' as const } : {}; + const fact = facts(folder); + if (!fact.count) return; + if (!open.has(folder)) { + const data: GroupCard & { memberFolder: string } = { group: folder, label: f.label, ...fact, noun: 'job', empty: 'No runs', memberFolder: folder }; + drawn.push({ id: memberFolderCardId(folder), type: 'folder', position: { x: 0, y: 0 }, selectable: false, ...placed, data }); + return; + } + const data: FolderBoxData = { folder, label: f.label, nodeCount: fact.count, stale: fact.stale, failed: fact.failed, noun: 'job', ontoggle: onclose }; + const id = memberFolderBoxId(folder); + drawn.push({ id, type: 'folderBox', position: { x: 0, y: 0 }, width: MEMBER_WIDTH, height: MEMBER_HEIGHT, selectable: false, ...placed, data }); + for (const sub of f.folders) place(sub, folder); + for (const k of f.members) if (byMember.has(k)) drawMember(byMember.get(k)!, id); + }; + for (const folder of tree.roots) place(folder, null); + const inTree = new Set(tree.chains.keys()); + for (const m of members) if (!inTree.has(m.jobKey) || !(tree.chains.get(m.jobKey) ?? []).length) drawMember(m, null); + } else { + for (const m of members) drawMember(m, null); + } + + /** What stands for a member's node: the member's closed folder, its card, or what is drawn of the node inside it. */ + const memberAt = (jobKey: string) => { + const folder = folderStandIn(jobKey); + return folder === null ? memberNodeId(jobKey) : memberFolderCardId(folder); + }; + const at = (jobKey: string, key: string) => (folderStandIn(jobKey) === null ? (standIns.get(jobKey)?.(key) ?? memberNodeId(jobKey)) : memberAt(jobKey)); + const crossing: Edge[] = []; + for (const m of members) { + for (const upstream of waits.get(m.jobKey)!) { + const found = links.get(pairKey(upstream, m.jobKey)) ?? []; + if (!found.length) { + const [source, target] = [memberAt(upstream), memberAt(m.jobKey)]; + const data: MemberEdgeData = { count: 0, from: upstream, to: m.jobKey, title: '' }; + crossing.push({ id: `wait:${source}-${target}`, source, target, type: 'count', style: edgeStyle(1, 'neutral', true), data: { ...data, title: edgeTitle(data) } }); + continue; + } + // The same edges as Lineage's folders: one per visible pair, ends standing in for hidden nodes. + // Each end is keyed with its member, as members may share nodes. + const pairLinks = found.map((l: NodeLink) => ({ from: `${upstream}\u0000${l.from}`, to: `${m.jobKey}\u0000${l.to}` })); + const split = (k: string) => k.split('\u0000') as [string, string]; + const title = (count: number) => edgeTitle({ count, from: upstream, to: m.jobKey, title: '' }); + for (const e of visibleEdges(pairLinks, (k) => at(...split(k)), { title })) crossing.push({ ...e, data: { ...e.data, from: upstream, to: m.jobKey } satisfies MemberEdgeData }); + } + } + // Waits whose ends a closed folder stands in for are one edge per pair of what is drawn, as Lineage's + // folders are; a wait inside one closed folder isn't drawn. + const merged = new Map(); + for (const e of crossing) { + if (e.source === e.target) continue; + const key = `${e.source}>${e.target}`; + const d = e.data as MemberEdgeData; + const had = merged.get(key); + if (!had) { + merged.set(key, { ...e, data: { ...d, pairs: [{ from: d.from, to: d.to }] } }); + continue; + } + const h = had.data as MemberEdgeData; + const pairs = h.pairs!.some((p) => p.from === d.from && p.to === d.to) ? h.pairs! : [...h.pairs!, { from: d.from, to: d.to }]; + const count = h.count + d.count; + const data: MemberEdgeData = { ...h, count, pairs, title: pairs.length > 1 ? `${plural(pairs.length, 'wait')}, ${plural(count, 'node link')}` : h.title }; + merged.set(key, { ...had, id: `${had.source}-${had.target}`, style: edgeStyle(Math.max(1, count), 'neutral', count === 0), data }); + } + const visible = [...merged.values()]; + // A wait implied by a path of other drawn edges isn't drawn either, now that hidden nodes stand + // in for their member or folder; lineage between two open members' nodes always is. + const collapsed = (id: string) => id.startsWith(memberNodeId('')) || id.startsWith(memberFolderCardId('')); + const between = new Set(visible); + const reduced = new Set(transitiveReduction([...visible, ...internal], (e) => between.has(e) && (collapsed(e.source) || collapsed(e.target)))); + return { nodes: drawn, edges: [...visible.filter((e) => reduced.has(e)), ...internal] }; +} + +/** Space between the cards of the block of members that wait for nothing and that nothing waits for. */ +const BLOCK_GAP = 24; + +/** + * `layoutFolderGraph`, with the collapsed members no drawn edge touches set apart in a compact block + * under the graph, rows as wide as the graph (or a square, if the graph is narrow), rather than a + * long column beside it. + */ +export function layoutMemberGraph(nodes: Node[], edges: Edge[]): Node[] { + const touched = new Set(edges.flatMap((e) => [e.source, e.target])); + const alone = new Set(nodes.filter((n) => !n.parentId && n.type === 'member' && !touched.has(n.id)).map((n) => n.id)); + if (alone.size < 2) return layoutFolderGraph(nodes, edges); + const rest = nodes.filter((n) => !alone.has(n.id)); + const placed = rest.length ? layoutFolderGraph(rest, edges) : []; + const top = placed.filter((n) => !n.parentId); + const left = top.length ? Math.min(...top.map((n) => n.position.x)) : BLOCK_GAP; + const right = top.length ? Math.max(...top.map((n) => n.position.x + (n.width ?? MEMBER_WIDTH))) : left; + const below = top.length ? Math.max(...top.map((n) => n.position.y + (n.height ?? MEMBER_HEIGHT))) + 2 * BLOCK_GAP : BLOCK_GAP; + const size = (n: Node) => ({ width: n.measured?.width ?? n.width ?? MEMBER_WIDTH, height: n.measured?.height ?? n.height ?? MEMBER_HEIGHT }); + const cards = nodes.filter((n) => alone.has(n.id)).map(size); + const cardWidth = Math.max(...cards.map((c) => c.width)); + const cardHeight = Math.max(...cards.map((c) => c.height)); + const columns = Math.min(alone.size, Math.max(Math.ceil(Math.sqrt(alone.size)), Math.floor((right - left + BLOCK_GAP) / (cardWidth + BLOCK_GAP)))); + const at = new Map([...alone].map((id, i) => [id, { x: left + (i % columns) * (cardWidth + BLOCK_GAP), y: below + Math.floor(i / columns) * (cardHeight + BLOCK_GAP) }])); + const byId = new Map(placed.map((n) => [n.id, n])); + return nodes.map((n) => (alone.has(n.id) ? { ...n, position: at.get(n.id)! } : byId.get(n.id)!)); +} + +/** "orders waits for customers: 12 node links", for an edge's tooltip. */ +export function edgeTitle(data: MemberEdgeData): string { + if (!data.count) return `${data.to} waits for ${data.from}, not through lineage between their nodes`; + return `${data.to} waits for ${data.from}: ${plural(data.count, 'node link')}`; +} + +/** + * A member's waits with their node-level reasons, capped at `max` reasons each; `more` counts the + * rest. Direct waits come first; an implied wait names `through`, the direct wait it comes after. + */ +export function waitReasons( + jobKey: string, + members: readonly JobGroupMember[], + links: ReadonlyMap, + max = 5 +): { upstream: string; reasons: NodeLink[]; more: number; through?: string }[] { + const { direct, implied } = directWaits(members); + const { up } = memberReach(members); + const reasons = (upstream: string) => { + const all = links.get(pairKey(upstream, jobKey)) ?? []; + return { upstream, reasons: all.slice(0, max), more: Math.max(0, all.length - max) }; + }; + const mine = direct.get(jobKey) ?? []; + return [...mine.map(reasons), ...(implied.get(jobKey) ?? []).map((upstream) => ({ ...reasons(upstream), through: mine.find((d) => up.get(d)?.has(upstream)) }))]; +} diff --git a/ui/src/lib/lineage/member-node.svelte b/ui/src/lib/lineage/member-node.svelte new file mode 100644 index 0000000000..c4968aee2d --- /dev/null +++ b/ui/src/lib/lineage/member-node.svelte @@ -0,0 +1,184 @@ + + +
+ +
+ + {d.jobKey} + Open job ↗ +
+
+ {#if d.expandable}{/if} + {#if d.nodeCount !== null}{plural(d.nodeCount, 'node')}{/if} + {#if d.waitsFor}waits for {d.waitsFor}{/if} + {#if d.needs.changed || d.needs.behind} + + {#if d.needs.changed}{d.needs.changed} changed{/if} + {#if d.needs.behind}{d.needs.changed ? '+' : ''}{d.needs.behind} behind{/if} + + {/if} +
+
+ + {#if d.lastRun} + {runStatusLabel[d.lastRun.status]} + {#if when}{when}{/if} + {:else} + No runs yet + {/if} +
+ + +
+ + diff --git a/ui/src/lib/lineage/scope-graph.svelte b/ui/src/lib/lineage/scope-graph.svelte index e2665aeea0..4241751124 100644 --- a/ui/src/lib/lineage/scope-graph.svelte +++ b/ui/src/lib/lineage/scope-graph.svelte @@ -1,15 +1,24 @@ -
0}> - + (grouped = !grouped) } : undefined} + expand={folderMode && !tooMany ? { all: expandEverything, collapse: () => ((open = new Set()), (expandNote = '')), canCollapse: open.size > 0, note: expandNote } : undefined} + focus={{ on: !!focusKeys, can: selectedKeys.length > 0, toggle: toggleFocus, title: 'Show only the selected nodes and what they read or feed' }} + onreset={reset} + /> + node.type === 'folder' && openFolder(String(node.data.group))} onselectionchange={({ nodes: picked }) => onselect(picked.filter((n) => n.type === 'asset').map((n) => n.id))} - > - - - - + /> + {#if tooMany} + + {#snippet actions()} + {#if canGroup && !grouped}{/if} + {#if folderMode && open.size}{/if} + {/snippet} + + {/if}
diff --git a/ui/src/lib/lineage/scoped-lineage.svelte b/ui/src/lib/lineage/scoped-lineage.svelte index 676b9adecc..be51c29adb 100644 --- a/ui/src/lib/lineage/scoped-lineage.svelte +++ b/ui/src/lib/lineage/scoped-lineage.svelte @@ -1,12 +1,17 @@
-
- - {#if cursorType} - - {/if} - -
+ {#if fullRefresh} -

Every model in this run is rebuilt from scratch, replacing all of its data. Recorded in the audit log.

+ +

Rebuilds every model from scratch, replacing all data. Audited.

{/if} {#if ranged} {#if cursorType === 'timestamp'} -
- {#each PRESETS as p (p.key)} - - {/each} - -
+ ({ value: p.key as string, label: p.label })), { value: 'custom', label: 'Custom' }]} + onselect={pickPreset} + /> {/if} {#if !preset}
@@ -89,7 +114,8 @@ {/if}
{/if} -

Incremental models reprocess this range, replacing what was built for it. Recorded in the audit log.

+ +

Replaces data built for this range. Audited.

{/if}
@@ -100,29 +126,6 @@ align-items: flex-start; gap: 8px; } - .pills { - display: flex; - flex-wrap: wrap; - gap: 4px; - } - .pills button { - padding: 2px 10px; - border: 1px solid var(--border); - border-radius: 999px; - color: var(--muted-foreground); - font-size: 11.5px; - } - .pills button[aria-pressed='true'] { - border-color: var(--border-strong); - background: var(--sidebar-accent); - color: var(--foreground); - } - .pills button:hover:not(:disabled) { - color: var(--foreground); - } - .pills button:disabled { - opacity: 0.5; - } .custom { display: flex; align-items: center; diff --git a/ui/src/lib/run/ignored-audits.test.ts b/ui/src/lib/run/ignored-audits.test.ts new file mode 100644 index 0000000000..597e7a4755 --- /dev/null +++ b/ui/src/lib/run/ignored-audits.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest'; +import type { Check, CheckEvaluation } from '$lib/api'; +import { ignoredFailuresOf } from './ignored-audits'; + +const check = (id: string, latest: Partial | null): Check => + ({ id, name: id, latest: latest && { id: `ar-${id}`, runId: null, fingerprintRunId: 'other', evaluatedAt: '2026-11-17T00:00:00Z', outcome: 'fail', violationCount: 1, measuredValue: null, evidenceCount: 1, evidenceTruncated: false, ignored: false, ...latest } }) as Check; + +describe('a run’s ignored audit failures', () => { + it('are the checks whose newest result is a failure this run, or a member run, let through', () => { + const checks = [ + check('by-run-id', { ignored: true, fingerprintRunId: 'run-1' }), + check('by-scheduler-run', { ignored: true, runId: 'run-2' }), + check('not-ignored', { fingerprintRunId: 'run-1' }), + check('another-run', { ignored: true, fingerprintRunId: 'run-9' }), + check('never-ran', null) + ]; + expect(ignoredFailuresOf(checks, new Set(['run-1', 'run-2'])).map((c) => c.id)).toEqual(['by-run-id', 'by-scheduler-run']); + }); +}); diff --git a/ui/src/lib/run/ignored-audits.ts b/ui/src/lib/run/ignored-audits.ts new file mode 100644 index 0000000000..a8a815f58c --- /dev/null +++ b/ui/src/lib/run/ignored-audits.ts @@ -0,0 +1,11 @@ +// The audit failures a run let through (`ignore_audit_failures`), for its run page. Pure. + +import type { Check } from '$lib/api'; + +/** + * The checks whose newest result is a failure one of `runIds` let through. The server lists only each + * check's newest result, so a failure a later run has since replaced isn't among them. + */ +export function ignoredFailuresOf(checks: Check[], runIds: ReadonlySet): Check[] { + return checks.filter((c) => !!c.latest?.ignored && (runIds.has(c.latest.runId ?? '') || runIds.has(c.latest.fingerprintRunId))); +} diff --git a/ui/src/lib/run/plan-browser.svelte b/ui/src/lib/run/plan-browser.svelte new file mode 100644 index 0000000000..31d386d188 --- /dev/null +++ b/ui/src/lib/run/plan-browser.svelte @@ -0,0 +1,479 @@ + + +
+ + + {#if group} +
+
+

{group.title} {group.count.toLocaleString()}

+ {#if group.subtitle}

{group.subtitle}

{/if} +
+ + {#if group.key === 'warnings'} + {#if warningsRead < group.count}

{loading ? 'Reading the warnings…' : `Grouping the first ${warningsRead.toLocaleString()} of ${group.count.toLocaleString()} warnings.`}

{/if} + {#each warningGroups as w (w.template)} +
+
+ {w.template} + {w.label} + +
+ {#if openWarnings[w.template]} + {@const all = allWarnings[w.template]} +
    + {#each all ? w.warnings : w.warnings.slice(0, WARNINGS_SHOWN) as one, i (i)}
  • {#if one.code}{one.code} {/if}{one.message}
  • {/each} +
+ {#if !all && w.warnings.length > WARNINGS_SHOWN} + + {/if} + {/if} +
+ {/each} + {:else} + + +
+ {#each visible as r (r.name + r.action)} + {@const link = href(r)} + {@const open = openDiff[diffKey(r)]} +
+
+ {#if link}{r.name}{:else}{r.name}{/if} + {r.action} + {r.detail}{#if r.diff || r.fetch}{r.detail ? ' · ' : ''}{/if} +
+ {#each r.notes as note, i (i)}
{note}
{/each} + {#each r.warnings as w, i (i)}

{warningText(w)}

{/each} + {#if open && r.diff} + {@const preview = open === 'full' ? { lines: r.diff, cut: false, more: 0 } : diffPreview(r.diff)} +
{#each preview.lines as l, i (i)}{l.text}{'\n'}{/each}
+ {#if preview.cut}{/if} + {:else if open && r.fetch === 'diff'} +

Reading the diff…

+ {:else if open && r.fetch === 'unavailable'} +

diff not available

+ {/if} +
+ {/each} + {#if filtering && !shownGroup}

Finding {group.title.toLowerCase()}…

+ {:else if filtering && !rowCount && !loading}

No {group.title.toLowerCase()} match “{filter.trim()}”.

{/if} +
+ {#if loadError}

Couldn't load more rows: {loadError}

{/if} + {#if remaining > 0} + + {:else if loading}

Loading…

{/if} + {/if} +
+ {/if} +
+ + diff --git a/ui/src/lib/run/plan-index.test.ts b/ui/src/lib/run/plan-index.test.ts new file mode 100644 index 0000000000..ebf6083489 --- /dev/null +++ b/ui/src/lib/run/plan-index.test.ts @@ -0,0 +1,214 @@ +import { describe, expect, it } from 'vitest'; +import type { PlanDocument, PlanModel } from '$lib/api'; +import { configChange, cursorWords, diffPreview, diffStats, sectionNote, groupWarnings, planGroups, planSummaryLine, selectedGroup, sharedNote, whitespaceOnly, type PlanRow } from './plan-index'; + +const empty: PlanDocument = { + selected_count: 0, + models: [], + seeds: [], + source_loads: [], + functions: [], + python_nodes: [], + warnings: [], + migrations: [], + column_migrations: [], + table_type_conversions: [], + retention: [], + synced_copy: null, + changed: null +}; +const model = (name: string, reason: string, extra: Partial = {}): PlanModel => ({ + name, + materialization_type: 'table', + action: 'create_table', + reason, + backfill: { action: 'forward', duration: null }, + ...extra +}); +const head = ['--- previous', '+++ current', '@@ -1,3 +1,3 @@']; + +describe('whitespace-only query changes', () => { + it('labels a diff that only moves whitespace', () => { + expect(whitespaceOnly([...head, '-select id, name', '-from orders', '+select id,', '+ name from orders'])).toBe(true); + }); + + it("doesn't label a change of keyword case: it's a change", () => { + expect(whitespaceOnly([...head, '-select id from orders', '+SELECT id FROM orders'])).toBe(false); + }); + + it("doesn't label a change of what the query reads", () => { + expect(whitespaceOnly([...head, '-select id from orders', '+select id from orders_v2'])).toBe(false); + }); + + it('counts the changed lines, not the diff headers', () => { + expect(diffStats([...head, ' select', '- a', '+ b', '+ c'])).toEqual({ added: 2, removed: 1 }); + }); +}); + +describe('config changes in one line', () => { + it('reads up to two changed keys of the config JSON', () => { + expect(configChange([...head, '- "full_refresh": true,', '+ "full_refresh": false,'])).toBe('full_refresh true → false'); + expect(configChange(['- "cursor": "observed_at",', '+ "cursor": "event_date",', '+ "grain": "day"'])).toBe('cursor observed_at → event_date; grain (unset) → day'); + }); + + it('leaves more keys, or a nested value, to the diff', () => { + expect(configChange(['- "a": 1,', '+ "a": 2,', '- "b": 1,', '+ "b": 2,', '- "c": 1,', '+ "c": 2,'])).toBeNull(); + expect(configChange(['+ "meta": {'])).toBeNull(); + }); +}); + +describe('cursor ranges in words', () => { + it("words the executor's bound placeholders", () => { + expect(cursorWords('range: __SQB_CURSOR_START__ → __SQB_CURSOR_END__')).toBe('range: from the last run → the latest data when the run starts'); + expect(cursorWords('range: 2026-11-01 → __SQB_CURSOR_END__')).toBe('range: 2026-11-01 → the latest data when the run starts'); + }); +}); + +describe('notes every row shares', () => { + const row = (name: string, detail: string, notes: string[] = []): PlanRow => ({ name, action: 'table', detail, notes, diff: null, diffOf: null, warnings: [], linked: true, fetch: null, pendingConfig: false }); + + it('finds the note every row carries, and none when one differs', () => { + expect(sharedNote([row('a', 'no completed record'), row('b', 'no completed record')])).toBe('no completed record'); + expect(sharedNote([row('a', 'no completed record'), row('b', 'reads orders')])).toBeNull(); + expect(sharedNote([row('a', 'no completed record')])).toBeNull(); + }); + + it('moves it to the group subtitle, off the rows', () => { + const plan: PlanDocument = { + ...empty, + models: ['orders', 'customers'].map((n) => model(n, 'no_change')), + changed: { counts: ['definition'], reach: 'all', runs: ['orders', 'customers'].map((n) => ({ node: `model:${n}`, reason: 'no completed record', downstream_of: null })), held_back: [], unchanged: [] } + }; + const [unchanged] = planGroups(plan); + expect(unchanged.key).toBe('unchanged'); + expect(unchanged.subtitle).toBe('no completed record'); + expect(unchanged.rows.map((r) => r.detail)).toEqual(['', '']); + }); +}); + +describe('warnings grouped by message', () => { + it('groups one message about different names into one row, counting the names it opens with', () => { + const warnings = [ + ...['a', 'b'].flatMap((s) => ['x', 'y', 'z'].map((m) => ({ severity: 'warning', message: `Scenario '${s}' provides fixture for unknown model '${m}'.` }))), + { severity: 'warning', message: 'orders: schema change detected but on_schema_change is set to ignore', model_name: 'orders' }, + ...[1, 2].map((k) => ({ severity: 'warning', message: `Scenario 't${k}' requires source 'raw.r${k}', but no fixture was provided. Add a CTE named __source__raw__r${k} AS (...).` })) + ]; + const groups = groupWarnings(warnings); + expect(groups.map((g) => [g.template, g.label, g.warnings.length])).toEqual([ + ['Scenario … provides fixture for unknown model …', '6 in 2 scenarios', 6], + ['Scenario … requires source …, but no fixture was provided. Add a CTE named … AS (...).', '2 warnings', 2], + ['schema change detected but on_schema_change is set to ignore', '1 warning', 1] + ]); + }); +}); + +describe('the plan index', () => { + const plan: PlanDocument = { + ...empty, + models: [ + model('orders', 'query_changed', { query_diff: [...head, '-select 1', '+select 2'] }), + model('payments', 'query_changed', { query_diff: [...head, '-select 1', '+select 1'] }), + model('refunds', 'config_changed', { config_diff: ['- "full_refresh": true,', '+ "full_refresh": false,'] }), + model('customers', 'first_run'), + model('regions', 'no_change') + ], + python_nodes: [{ name: 'export_orders', kind: 'asset', phase: 'after', identity_status: 'known' }], + warnings: [{ severity: 'warning', message: "Scenario 'a' provides fixture for unknown model 'x'." }] + }; + const groups = planGroups(plan); + + it('lists the non-empty groups in order, with the bulky ones after the warnings', () => { + expect(groups.map((g) => [g.key, g.count])).toEqual([ + ['changed', 3], + ['first_run', 1], + ['python', 1], + ['warnings', 1], + ['unchanged', 1] + ]); + expect(planSummaryLine(groups)).toBe('3 changed, 1 new, 1 Python · 1 unchanged'); + }); + + it('breaks the changes down by kind, with line counts, whitespace-only and inline config changes', () => { + const changed = groups[0]; + expect(changed.subtitle).toBe('2 query · 1 config'); + expect(Object.fromEntries(changed.rows.map((r) => [r.name, r.detail]))).toEqual({ orders: 'query +1 −1', payments: 'whitespace only', refunds: 'config · full_refresh true → false' }); + expect(changed.rows.find((r) => r.name === 'refunds')!.diffOf).toBeNull(); + expect(changed.rows.find((r) => r.name === 'orders')!.diffOf).toBe('query'); + }); + + it('opens on the group the address names, else the first', () => { + expect(selectedGroup(groups, 'python')?.key).toBe('python'); + expect(selectedGroup(groups, 'nonsense')?.key).toBe('changed'); + expect(selectedGroup(groups, null)?.key).toBe('changed'); + }); +}); + +describe('an opened diff', () => { + it('shows the first changed lines, then says the rest is cut off', () => { + const diff = [{ text: '@@', cls: 'hunk' as const }, ...Array.from({ length: 20 }, (_, i) => ({ text: `+${i}`, cls: 'add' as const }))]; + const preview = diffPreview(diff, 12); + expect(preview.lines).toHaveLength(13); + expect(preview.cut).toBe(true); + expect(preview.more).toBe(8); + expect(diffPreview(diff.slice(0, 5), 12)).toEqual({ lines: diff.slice(0, 5), cut: false, more: 0 }); + }); +}); + +describe('a plan from a server summary', () => { + it('says the note the counts show every row shares as "all N", and a majority as "N of M"', () => { + const notes = (count: number) => [{ kind: 'run_reason', value: 'no completed record', count }, { kind: 'materialization', value: 'table', count: 1507 }]; + expect(sectionNote({ section: 'routine_models', total: 1507, notes: notes(1507) })).toEqual({ row: 'no completed record', all: true, text: 'no completed record (all 1,507)' }); + expect(sectionNote({ section: 'routine_models', total: 1507, notes: notes(1200) })?.text).toBe('no completed record (1,200 of 1,507)'); + expect(sectionNote({ section: 'routine_models', total: 1507, notes: notes(700) })).toBeNull(); + expect(sectionNote({ section: 'routine_models', total: 3, notes: [{ kind: 'reads', value: 'orders', count: 3 }] })?.text).toBe('reads orders (all 3)'); + }); + + const summary: PlanDocument = { + ...empty, + models: [ + model('orders', 'query_changed', { diff_stats: { query: { added: 82, removed: 104, whitespace_only: false } } }), + model('payments', 'query_changed', { diff_stats: { query: { added: 2, removed: 2, whitespace_only: true } } }), + model('refunds', 'config_changed', { diff_stats: { config: { added: 1, removed: 1, whitespace_only: false } } }), + model('regions', 'no_change') + ], + sections: [ + { section: 'query_changed', total: 2, notes: [] }, + { section: 'config_changed', total: 1, notes: [] }, + { section: 'routine_models', total: 1507, notes: [{ kind: 'run_reason', value: 'no completed record', count: 1507 }] }, + { section: 'seeds', total: 10, notes: [{ kind: 'reason', value: 'first_run', count: 2 }] }, + { section: 'unchanged', total: 300, notes: [] } + ], + pages: { query_changed: 2, config_changed: 1, routine_models: 1507, seeds: 10, unchanged: 300 }, + loaded: { query_changed: 2, config_changed: 1, routine_models: 1 }, + changed: { counts: ['definition'], reach: 'all', runs: ['orders', 'payments', 'refunds', 'regions'].map((n) => ({ node: `model:${n}`, reason: n === 'regions' ? 'no completed record' : 'definition changed', downstream_of: null })), held_back: [], unchanged: [] } + }; + + it("counts groups from the server's sections, and pages their rows in plan order", () => { + const groups = planGroups(summary); + expect(groups.map((g) => [g.key, g.count])).toEqual([ + ['changed', 3], + ['unchanged', 1517], + ['skipped', 300] + ]); + const unchanged = groups[1]; + expect(unchanged.subtitle).toBe('1,507 models · 10 seeds · models: no completed record (all 1,507) · seeds: 2 first run'); + expect(unchanged.rows.map((r) => [r.name, r.action, r.detail])).toEqual([['regions', 'table', '']]); + expect(unchanged.pages).toEqual([ + { section: 'routine_models', loaded: 1, total: 1507 }, + { section: 'seeds', loaded: 0, total: 10 } + ]); + }); + + it("reads changed rows' counts from diff_stats, and reads a diff it doesn't carry on demand", () => { + const rows = (diffs?: Map) => Object.fromEntries(planGroups(summary, undefined, diffs)[0].rows.map((r) => [r.name, r])); + const before = rows(); + expect(before.orders).toMatchObject({ detail: 'query +82 −104', diff: null, fetch: 'diff', pendingConfig: false }); + expect(before.payments.detail).toBe('whitespace only'); + // A config change of up to two lines a side is read to show inline. + expect(before.refunds).toMatchObject({ detail: 'config', fetch: 'diff', pendingConfig: true }); + const read = rows(new Map([['refunds', { model: 'refunds', config_diff: ['- "full_refresh": true,', '+ "full_refresh": false,'] }], ['orders', null]])); + expect(read.refunds).toMatchObject({ detail: 'config · full_refresh true → false', fetch: null, diffOf: null }); + // The server has no diff to give, as without permission to read the model's SQL. + expect(read.orders).toMatchObject({ detail: 'query +82 −104', diff: null, fetch: 'unavailable' }); + }); +}); diff --git a/ui/src/lib/run/plan-index.ts b/ui/src/lib/run/plan-index.ts new file mode 100644 index 0000000000..099f2cc097 --- /dev/null +++ b/ui/src/lib/run/plan-index.ts @@ -0,0 +1,441 @@ +// The plan page's list as an index and a pane: the plan's sections (plan-text.ts) gathered into a few +// groups, one shown at a time, each row one line of name · action · detail. Pure. + +import type { PlanDiff, PlanDocument, PlanPageSection, PlanSectionCount, PlanWarning } from '$lib/api'; +import { plural } from '$lib/format/plural'; +import { changedSections, loadedRows, looseWarnings, planSections, sectionSize, type PlanEntry, type PlanSection } from './plan-text'; + +/** The groups in the index, in its order; the ones after the divider are the bulky, quiet ones. */ +export type PlanGroupKey = 'full_refresh' | 'changed' | 'renames' | 'first_run' | 'loads' | 'python' | 'held_back' | 'warnings' | 'unchanged' | 'skipped'; +const ORDER: PlanGroupKey[] = ['full_refresh', 'changed', 'renames', 'first_run', 'loads', 'python', 'held_back', 'warnings', 'unchanged', 'skipped']; +/** The first group below the index's divider. */ +export const AFTER_DIVIDER: PlanGroupKey = 'warnings'; +const TITLES: Record = { + full_refresh: 'Full refresh', + changed: 'Changed', + renames: 'Renames and migrations', + first_run: 'First run', + loads: 'Sources to load', + python: 'Python', + held_back: 'Held back', + warnings: 'Warnings', + unchanged: 'Unchanged', + skipped: 'Skipped by --changed' +}; +/** Each server section's group in the index. */ +const GROUP_OF: Record = { + full_refresh: 'full_refresh', + query_changed: 'changed', + function_changed: 'changed', + config_changed: 'changed', + schema_changed: 'changed', + changed_functions: 'changed', + renamed: 'renames', + migrations: 'renames', + column_migrations: 'renames', + table_type_conversions: 'renames', + retention: 'renames', + first_run: 'first_run', + loader_reloads: 'loads', + loader_loads: 'loads', + source_reloads: 'loads', + source_loads: 'loads', + python_nodes: 'python', + held_back: 'held_back', + warnings: 'warnings', + routine_models: 'unchanged', + functions: 'unchanged', + seeds: 'unchanged', + unchanged: 'skipped' +}; +/** The kind of change a Changed section counts as, for its breakdown. */ +const CHANGE_KIND: Partial> = { + query_changed: 'query', + config_changed: 'config', + function_changed: 'function', + changed_functions: 'function', + schema_changed: 'schema' +}; +/** A section's rows in words, for the breakdown of a group of several sections. */ +const SECTION_WORD: Partial> = { + routine_models: 'models', + functions: 'functions', + seeds: 'seeds', + renamed: 'renamed', + migrations: 'migrations', + column_migrations: 'column migrations', + table_type_conversions: 'table type changes', + retention: 'retention', + loader_reloads: 'loader reloads', + loader_loads: 'loader loads', + source_reloads: 'source reloads', + source_loads: 'source loads' +}; +const sectionWord = (section: PlanPageSection) => SECTION_WORD[section] ?? section.replace(/_/g, ' '); +/** Sections whose rows are catalog models, so they link to the catalog. */ +const MODEL_ROWS = new Set(['query_changed', 'function_changed', 'config_changed', 'schema_changed', 'first_run', 'renamed', 'routine_models', 'migrations', 'column_migrations', 'table_type_conversions', 'retention', 'held_back', 'unchanged']); +/** A whole plan's full refresh, which plan-text sums by materialization rather than listing as a section of models. */ +const TITLE_SECTION: Record = { 'Full refresh': 'full_refresh' }; +/** The notes a section's subtitle can say once for its rows: why `--changed` runs them. */ +const ROW_NOTES = new Set(['run_reason', 'reads']); +/** A seed's plan reason in words, for the Seeds breakdown; a current seed goes unsaid. */ +const SEED_REASON: Record = { first_run: 'first run', config_changed: 'changed' }; + +export type ChangeKind = 'query' | 'config' | 'function' | 'schema'; + +/** A line of a diff the row can open: `cls` colours it. */ +export type DiffLine = { text: string; cls: 'add' | 'del' | 'hunk' | '' }; + +export type PlanRow = { + name: string; + /** What happens to it: an action, a materialization, a decision. */ + action: string; + /** The one-line detail: for a change, its kind with line counts or the config change. */ + detail: string; + /** More detail lines, such as a cursor range. */ + notes: string[]; + /** The diff the row opens, when it carries one: config, schema and query parts. */ + diff: DiffLine[] | null; + /** Which part of the diff the "diff" link opens: the query, or the config when it isn't inline. */ + diffOf: 'query' | 'config' | 'schema' | null; + warnings: PlanWarning[]; + /** Rows of plan-text's catalog-linked sections link to the catalog. */ + linked: boolean; + /** A changed row's kinds of change. */ + kinds?: string[]; + /** The row's diff is read on demand ('diff'), or the server has none to give ('unavailable'). */ + fetch: 'diff' | 'unavailable' | null; + /** A config change of at most two keys, read on demand to show inline. */ + pendingConfig: boolean; +}; + +export type PlanGroup = { + key: PlanGroupKey; + title: string; + /** Rows in the group, loaded or not. */ + count: number; + /** The breakdown under the title, such as "87 query · 6 config · 1 function", and a note every row shares. */ + subtitle: string; + rows: PlanRow[]; + /** Its server sections' rows loaded so far, in the order their rows read. */ + pages: { section: PlanPageSection; loaded: number; total: number }[]; + /** The Warnings group's warnings, grouped by message. */ + warningGroups?: WarningGroup[]; + /** The server sections the group gathers, in plan order. */ + sections: PlanPageSection[]; +}; + +/** A warnings group: one message template and the warnings that fill it in. */ +export type WarningGroup = { template: string; warnings: PlanWarning[]; label: string }; + +const DIFF_HEAD = /^(---|\+\+\+|@@)/; +const lineClass = (line: string): DiffLine['cls'] => (DIFF_HEAD.test(line) ? 'hunk' : line.startsWith('+') ? 'add' : line.startsWith('-') ? 'del' : ''); +const changedLines = (diff: string[], sign: '+' | '-') => diff.filter((l) => l.startsWith(sign) && !DIFF_HEAD.test(l)); + +/** A unified diff's changed lines: `+N −M`. */ +export function diffStats(diff: string[]): { added: number; removed: number } { + return { added: changedLines(diff, '+').length, removed: changedLines(diff, '-').length }; +} + +/** + * Whether a query diff changes only whitespace: its removed and added lines read the same with all + * whitespace taken out. A display hint only; the planner decides what rebuilds. A change of keyword + * case is a change. + */ +export function whitespaceOnly(diff: string[]): boolean { + const side = (sign: '+' | '-') => + changedLines(diff, sign) + .map((l) => l.slice(1)) + .join('') + .replace(/\s+/g, ''); + const [removed, added] = [changedLines(diff, '-'), changedLines(diff, '+')]; + return (removed.length > 0 || added.length > 0) && side('-') === side('+'); +} + +/** + * A config diff's changes in one line, such as `full_refresh true → false`, when it changes at most + * `most` top-level keys with plain values; null otherwise, and the row links to the diff instead. + */ +export function configChange(diff: string[], most = 2): string | null { + const values = new Map(); + for (const line of diff) { + if (DIFF_HEAD.test(line) || !(line.startsWith('+') || line.startsWith('-'))) continue; + const m = /^[-+]\s*"([^"]+)":\s*(.*?),?\s*$/.exec(line); + // A nested value, or a line that isn't one key's value, can't be read in one line. + if (!m || /[{[]$/.test(m[2])) return null; + const entry = values.get(m[1]) ?? {}; + entry[line.startsWith('-') ? 'old' : 'new'] = m[2].replace(/^"(.*)"$/, '$1'); + values.set(m[1], entry); + } + if (!values.size || values.size > most) return null; + return [...values].map(([key, v]) => `${key} ${v.old ?? '(unset)'} → ${v.new ?? '(unset)'}`).join('; '); +} + +/** The executor's placeholders for a microbatch's bounds, which a preview reports as the range. */ +const CURSOR_START = '__SQB_CURSOR_START__'; +const CURSOR_END = '__SQB_CURSOR_END__'; +/** A cursor range's placeholders in words: "from the last run → the latest data when the run starts". */ +export function cursorWords(line: string): string { + return line.replace(new RegExp(`${CURSOR_START}`, 'g'), 'from the last run').replace(new RegExp(`${CURSOR_END}`, 'g'), 'the latest data when the run starts'); +} + +/** The order a row's kinds of change read in, as "config + query". */ +const KIND_ORDER: ChangeKind[] = ['config', 'query', 'function', 'schema']; + +type Stats = { added: number; removed: number; whitespace_only: boolean }; + +/** + * A changed model's row: the kinds of change, line counts (from the diff text when the row has it, + * else the server's counts) or the config change, and the diff it opens. `fetched` is the model's diff + * read on demand, null when the server has none to give. + */ +function changedRow(e: PlanEntry, kind: ChangeKind | undefined, linked: boolean, fetched?: PlanDiff | null): PlanRow { + const queryText = e.queryDiff ?? fetched?.query_diff; + const configText = e.configDiff ?? fetched?.config_diff; + const query: Stats | undefined = queryText?.length ? { ...diffStats(queryText), whitespace_only: whitespaceOnly(queryText) } : e.diffStats?.query; + const configStats: Stats | undefined = configText?.length ? { ...diffStats(configText), whitespace_only: false } : e.diffStats?.config; + const kinds = new Set(); + if (kind) kinds.add(kind); + if (query) kinds.add('query'); + if (configStats) kinds.add('config'); + if (e.schema?.length) kinds.add('schema'); + const cause = e.detail.find((d) => d.startsWith('cause: function ')); + if (cause) kinds.add('function'); + const config = configText?.length ? configChange(configText) : null; + // A config change of at most two keys reads inline once its text is read; until then the row waits for it. + const pendingConfig = !configText && !!configStats && configStats.added <= 2 && configStats.removed <= 2 && fetched === undefined; + const quiet = !!query?.whitespace_only; + const label = KIND_ORDER.filter((k) => kinds.has(k)).join(' + '); + const counts = query && !quiet ? `+${query.added} −${query.removed}` : null; + const parts: string[] = []; + if (quiet && kinds.size === 1) parts.push('whitespace only'); + else { + parts.push(counts && kinds.size === 1 ? `${label} ${counts}` : label); + if (config) parts.push(config); + if (counts && kinds.size > 1) parts.push(counts); + if (quiet) parts.push('query whitespace only'); + } + if (cause) parts.push(cause.slice('cause: function '.length)); + if (e.schema?.length) parts.push(plural(e.schema.length, 'column')); + // The diff the link opens: the query, else a config too long to read inline, else the schema. + const diffOf = query ? 'query' : configStats && !config && !pendingConfig ? 'config' : e.schema?.length ? 'schema' : null; + // Whether the row holds the text of the diff its link opens; if not, opening it reads it. + const held = diffOf === 'query' ? !!queryText : diffOf === 'config' ? !!configText : true; + const diff: DiffLine[] | null = + diffOf && held + ? [ + ...(configText?.length && !config ? configText.map((text) => ({ text, cls: lineClass(text) })) : []), + ...(e.schema ?? []).map((f) => ({ text: `${f.sign} ${f.text}`, cls: (f.sign === '+' ? 'add' : f.sign === '-' ? 'del' : 'hunk') as DiffLine['cls'] })), + ...(queryText ?? []).map((text) => ({ text, cls: lineClass(text) })) + ] + : null; + return { + name: e.name, + action: e.value, + detail: parts.filter(Boolean).join(' · '), + notes: e.detail.filter((d) => d !== cause).map(cursorWords), + diff, + diffOf, + warnings: e.warnings ?? [], + linked, + kinds: [...kinds], + fetch: (diffOf && !held && fetched === undefined) || pendingConfig ? 'diff' : diffOf && !held ? 'unavailable' : null, + pendingConfig + }; +} + +const plainRow = (e: PlanEntry, linked: boolean): PlanRow => { + const [first = '', ...rest] = e.detail.map(cursorWords); + return { name: e.name, action: e.value, detail: first, notes: rest, diff: null, diffOf: null, warnings: e.warnings ?? [], linked, fetch: null, pendingConfig: false }; +}; + +/** A note every loaded row shares, such as "no completed record": for a whole plan, which comes without counted notes. */ +export function sharedNote(rows: PlanRow[]): string | null { + if (rows.length < 2) return null; + const lines = (r: PlanRow) => [r.detail, ...r.notes].filter(Boolean); + const first = lines(rows[0]); + return first.find((line) => rows.every((r) => lines(r).includes(line))) ?? null; +} + +function withoutNote(row: PlanRow, note: string): PlanRow { + if (row.detail === note) { + const [next = '', ...rest] = row.notes; + return { ...row, detail: next, notes: rest }; + } + return { ...row, notes: row.notes.filter((n) => n !== note) }; +} + +/** + * The note most of a section's rows share, from the server's counts: its text on a row, and as the + * subtitle says it, "no completed record (all 1,507)" or "(1,200 of 1,507)". Said only when more than + * half the rows carry it; `all` when every row does, so the rows can leave it off. + */ +export function sectionNote(count: PlanSectionCount): { row: string; text: string; all: boolean } | null { + const top = count.notes.filter((n) => ROW_NOTES.has(n.kind) && n.value).sort((a, b) => b.count - a.count)[0]; + if (!top || top.count * 2 <= count.total) return null; + const row = top.kind === 'reads' ? `reads ${top.value}` : top.value!; + const all = top.count >= count.total; + return { row, all, text: `${row} (${all ? `all ${count.total.toLocaleString()}` : `${top.count.toLocaleString()} of ${count.total.toLocaleString()}`})` }; +} + +/** + * A warning's message as a template: the names it quotes, a leading model name, and names that look + * generated (with a double underscore or a digit) taken out, so one message about many names reads once. + */ +export function warningTemplate(w: PlanWarning): string { + let message = w.message; + if (w.model_name && message.startsWith(`${w.model_name}:`)) message = message.slice(w.model_name.length + 1).trim(); + // A sentence ending on an elided name ends on the ellipsis, not "…." as well. + return message + .replace(/'[^']*'|`[^`]*`|"[^"]*"/g, '…') + .replace(/\b\w*(?:__|\d)\w*\b/g, '…') + .replace(/…\.$/, '…'); +} + +/** + * Warnings grouped by template, largest first. The label counts them, and when the template opens with + * a named thing, such as a scenario, how many of those: "71 in 24 scenarios". + */ +export function groupWarnings(warnings: PlanWarning[]): WarningGroup[] { + const groups = new Map(); + for (const w of warnings) { + const t = warningTemplate(w); + groups.set(t, [...(groups.get(t) ?? []), w]); + } + return [...groups] + .map(([template, list]) => { + const noun = /^(\w+) …/.exec(template)?.[1]; + const firsts = new Set(list.map((w) => /'([^']*)'|`([^`]*)`|"([^"]*)"/.exec(w.message)?.slice(1).find(Boolean) ?? '')); + const label = noun && list.length > 1 && firsts.size < list.length ? `${list.length} in ${plural(firsts.size, noun.toLowerCase())}` : plural(list.length, 'warning'); + return { template, warnings: list, label }; + }) + .sort((a, b) => b.warnings.length - a.warnings.length); +} + +/** The plan's sections with their sizes and notes: the server summary's, else counted from a whole plan's rows. */ +export function sectionCounts(plan: PlanDocument, sections: PlanSection[]): PlanSectionCount[] { + if (plan.sections) return plan.sections; + const counts: PlanSectionCount[] = sections.flatMap((s) => { + const section = s.section ?? TITLE_SECTION[s.title]; + return section ? [{ section, total: sectionSize(s), notes: [] }] : []; + }); + const warnings = looseWarnings(plan, sections).length; + return warnings ? [...counts, { section: 'warnings', total: warnings, notes: [] }] : counts; +} + +/** + * The plan's groups in index order, empty ones left out: counts and notes from the server's sections, + * rows those loaded so far. `refresh` words what refreshes a held-back job's nodes; `diffs` holds the + * diffs read on demand, by model (null: the server has none to give). + */ +export function planGroups(plan: PlanDocument, refresh: (job: string) => string = (job) => `job ${job}`, diffs: ReadonlyMap = new Map()): PlanGroup[] { + const sections: PlanSection[] = [...planSections(plan), ...(plan.changed ? changedSections(plan.changed, refresh, plan.pages) : [])]; + const textOf = new Map(sections.map((s) => [s.section ?? TITLE_SECTION[s.title], s])); + const counts = sectionCounts(plan, sections); + const warnings = looseWarnings(plan, sections); + const grouped = warnings.length ? groupWarnings(warnings) : []; + // A whole plan (an older server's, or the mock's) has every row and no counted notes. + const whole = !plan.sections; + return ORDER.flatMap((key) => { + const mine = counts.filter((c) => GROUP_OF[c.section] === key && c.total > 0); + if (!mine.length) return []; + const count = mine.reduce((n, c) => n + c.total, 0); + const notes: string[] = []; + const rows: PlanRow[] = []; + const pages: PlanGroup['pages'] = []; + let complete = true; + for (const c of mine) { + const entries = textOf.get(c.section)?.entries ?? []; + const loaded = key === 'warnings' ? loadedRows(plan, 'warnings') : entries.length; + if (!whole) pages.push({ section: c.section, loaded, total: c.total }); + if (c.section === 'seeds') { + const reasons = c.notes.filter((n) => n.kind === 'reason' && n.value && SEED_REASON[n.value]).map((n) => `${n.count.toLocaleString()} ${SEED_REASON[n.value!]}`); + if (reasons.length) notes.push(`seeds: ${reasons.join(', ')}`); + } + // A section's rows follow the one before only once that one is all loaded, so rows read in order. + if (key === 'warnings' || !complete) continue; + const linked = MODEL_ROWS.has(c.section); + let sectionRows = entries.map((e) => (key === 'changed' ? changedRow(e, CHANGE_KIND[c.section], linked, diffs.get(e.name)) : plainRow(e, linked))); + // A changed row's run reason is its change, which the row already says. + const note = key === 'changed' ? null : whole ? sharedNote(sectionRows) : sectionNote(c); + const said = typeof note === 'string' ? note : note?.text; + if (said) notes.push(mine.length > 1 ? `${sectionWord(c.section)}: ${said}` : said); + // A note every row carries is said once, in the subtitle, and left off the rows. + const off = typeof note === 'string' ? note : note?.all ? note.row : null; + if (off) sectionRows = sectionRows.map((r) => withoutNote(r, off)); + rows.push(...sectionRows); + complete = whole || entries.length >= c.total; + } + const kinds = new Map(); + if (key === 'changed') for (const c of mine) kinds.set(CHANGE_KIND[c.section] ?? sectionWord(c.section), (kinds.get(CHANGE_KIND[c.section] ?? sectionWord(c.section)) ?? 0) + c.total); + const breakdown = + key === 'changed' + ? [...kinds].sort((a, b) => b[1] - a[1]).map(([k, n]) => `${n.toLocaleString()} ${k}`) + : key === 'warnings' + ? grouped.length && (whole || loadedRows(plan, 'warnings') >= count) + ? [plural(grouped.length, 'kind')] + : [] + : mine.length > 1 + ? mine.map((c) => `${c.total.toLocaleString()} ${sectionWord(c.section)}`) + : []; + return [{ key, title: TITLES[key], count, subtitle: [...breakdown, ...notes].join(' · '), rows, pages, sections: mine.map((c) => c.section), ...(key === 'warnings' ? { warningGroups: grouped } : {}) }]; + }); +} + +/** `plan` without its loaded rows: the start of a name-filtered view of it, whose pages hold only matching rows. */ +export function withoutRows(plan: PlanDocument): PlanDocument { + return { + ...plan, + models: [], + seeds: [], + source_loads: [], + functions: [], + python_nodes: [], + warnings: [], + migrations: [], + column_migrations: [], + table_type_conversions: [], + retention: [], + loaded: {}, + changed: plan.changed && { ...plan.changed, runs: [], held_back: [], unchanged: [] }, + synced_copy: plan.synced_copy && { ...plan.synced_copy, cursor_window_starts: {} } + }; +} + +/** The index's one-line summary, such as "94 changed, 672 new, 121 renames, 64 loads, 433 Python · 1,507 unchanged". */ +export function planSummaryLine(groups: PlanGroup[]): string { + const n = (key: PlanGroupKey) => groups.find((g) => g.key === key)?.count ?? 0; + const word: [PlanGroupKey, string][] = [ + ['full_refresh', 'full refresh'], + ['changed', 'changed'], + ['first_run', 'new'], + ['renames', 'renames'], + ['loads', 'loads'], + ['python', 'Python'], + ['held_back', 'held back'] + ]; + const main = word.filter(([key]) => n(key)).map(([key, w]) => `${n(key).toLocaleString()} ${w}`); + const quiet = (['unchanged', 'skipped'] as PlanGroupKey[]).filter((key) => n(key)).map((key) => `${n(key).toLocaleString()} ${key === 'unchanged' ? 'unchanged' : 'skipped'}`); + return [main.join(', '), quiet.join(', ')].filter(Boolean).join(' · '); +} + +/** The group a `?section=` names, else the first in the index. */ +export function selectedGroup(groups: PlanGroup[], wanted: string | null): PlanGroup | null { + return groups.find((g) => g.key === wanted) ?? groups[0] ?? null; +} + +/** How many changed lines an opened diff shows before "open the full diff". */ +export const DIFF_PREVIEW = 12; + +/** The start of a diff (its headers, context and first `most` changed lines), and how many lines it leaves out. */ +export function diffPreview(diff: DiffLine[], most = DIFF_PREVIEW): { lines: DiffLine[]; cut: boolean; more: number } { + let changed = 0; + for (let i = 0; i < diff.length; i++) { + if (diff[i].cls === 'add' || diff[i].cls === 'del') changed++; + if (changed === most) return { lines: diff.slice(0, i + 1), cut: i + 1 < diff.length, more: diff.length - i - 1 }; + } + return { lines: diff, cut: false, more: 0 }; +} + +/** How many warnings an opened warning group lists before "show all". */ +export const WARNINGS_SHOWN = 20; diff --git a/ui/src/lib/run/plan-report.svelte b/ui/src/lib/run/plan-report.svelte deleted file mode 100644 index 1ac0937a21..0000000000 --- a/ui/src/lib/run/plan-report.svelte +++ /dev/null @@ -1,218 +0,0 @@ - - -
- {#each sections as s (s.title)} - {#if s.collapsed && !expanded[s.title]} - - {:else} - {@const shown = expanded[s.title] ? s.entries : s.entries.slice(0, limit)} -
-

{s.title} ({s.entries.length})

- {#each shown as e (`${e.name}:${e.value}`)} - {@const link = href(e, s.title)} - {@const hasDiff = !!(e.queryDiff?.length || e.configDiff?.length || e.schema?.length)} -
-
- {#if link}{e.name}{:else}{e.name}{/if} - {e.value} -
- {#each e.detail as d, i (i)}
{d}
{/each} - {#each e.warnings ?? [] as w, i (i)}

{warningText(w)}

{/each} - {#if hasDiff} - {#if e.schema?.length} -
schema diff:
-
{#each e.schema as f (f.text)}{f.sign} {f.text}{'\n'}{/each}
- {/if} - {#if e.configDiff?.length} -
config diff:
-
{#each e.configDiff as l, i (i)}{l}{'\n'}{/each}
- {/if} - {#if e.queryDiff?.length} -
query diff:
-
{#each e.queryDiff as l, i (i)}{l}{'\n'}{/each}
- {/if} - {/if} -
- {/each} - {#if s.entries.length > shown.length} - - {/if} -
- {/if} - {/each} - {#if loose.length} -
-

Warnings ({loose.length})

- {#each loose as w, i (i)} -
- {#if w.model_name}{w.model_name}{/if} - {#if w.code}{w.code} {/if}{w.message} -
- {/each} -
- {/if} -
- - diff --git a/ui/src/lib/run/plan-text.test.ts b/ui/src/lib/run/plan-text.test.ts index 6c48fed5eb..86723bb729 100644 --- a/ui/src/lib/run/plan-text.test.ts +++ b/ui/src/lib/run/plan-text.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import type { PlanDocument, PlanModel } from '$lib/api'; -import { actionText, changedSections, cloneCommands, jobRefreshText, looseWarnings, planSections, warningText } from './plan-text'; +import { actionText, changedSections, cloneCommands, jobRefreshText, loadedRows, looseWarnings, planSections, sectionSize, warningText } from './plan-text'; const model = (name: string, over: Partial = {}): PlanModel => ({ name, @@ -173,11 +173,11 @@ describe('plan under --changed', () => { expect(jobRefreshText('nightly', [trigger('nightly', 'running', '2026-01-01T02:00:00Z'), trigger('nightly', 'running', '2026-01-01T05:00:00Z')], now)).toBe('job nightly · next run in 2h'); // While no server runs the scheduler, neither a schedule nor a sensor starts the job, so it shows no countdown. const stopped = true; - expect(jobRefreshText('nightly', [trigger('nightly', 'running', '2026-01-01T02:00:00Z')], now, stopped)).toBe('job nightly · can’t start until the scheduler is running again'); - expect(jobRefreshText('finance_daily', [trigger('finance_daily', 'running', null, 'sensor')], now, stopped)).toBe('job finance_daily · can’t start until the scheduler is running again'); + expect(jobRefreshText('nightly', [trigger('nightly', 'running', '2026-01-01T02:00:00Z')], now, stopped)).toBe('job nightly · scheduler stopped'); + expect(jobRefreshText('finance_daily', [trigger('finance_daily', 'running', null, 'sensor')], now, stopped)).toBe('job finance_daily · scheduler stopped'); // A paused job stays paused whether or not the scheduler runs. expect(jobRefreshText('finance_daily', [trigger('finance_daily', 'paused', '2026-01-01T05:00:00Z')], now, stopped)).toBe('job finance_daily is paused'); - expect(jobRefreshText('finance_daily', [trigger('finance_daily', 'paused', null), trigger('finance_daily', 'running', null, 'sensor')], now)).toBe('job finance_daily · when its sensor fires'); + expect(jobRefreshText('finance_daily', [trigger('finance_daily', 'paused', null), trigger('finance_daily', 'running', null, 'sensor')], now)).toBe('job finance_daily · on sensor'); // A running schedule with no next run, such as a finished calendar, is not a sensor. expect(jobRefreshText('finance_daily', [trigger('finance_daily', 'running', null)], now)).toBe('job finance_daily'); expect(jobRefreshText('crm', [trigger('finance_daily', 'running', '2026-01-01T05:00:00Z')], now)).toBe('job crm'); @@ -185,6 +185,22 @@ describe('plan under --changed', () => { }); +describe('plan from a server summary', () => { + const summary = plan([model('orders', { reason: 'query_changed' }), model('customers')], { + pages: { routine_models: 3, unchanged: 120 }, + loaded: { routine_models: 1, unchanged: 1 }, + changed: { counts: ['definition'], reach: 'all', runs: [{ node: 'model:orders', reason: 'definition changed', downstream_of: null }, { node: 'model:customers', reason: 'upstream changed', downstream_of: 'model:orders' }], held_back: [], unchanged: ['model:events'] } + }); + it.each([ + { description: 'routine models count the whole section and page from what is loaded', title: 'Models', expected: { size: 3, loaded: 1, page: 'routine_models' } }, + { description: 'unchanged nodes count the whole section', title: 'Unchanged', expected: { size: 120, loaded: 1, page: 'unchanged' } } + ])('$description', ({ title, expected }) => { + const section = [...planSections(summary), ...changedSections(summary.changed!, (job) => job, summary.pages)].find((s) => s.title === title)!; + expect({ size: sectionSize(section), loaded: section.entries.length, page: section.page }).toEqual(expected); + expect(loadedRows(summary, section.page!)).toBe(expected.loaded); + }); +}); + describe('cloneCommands', () => { it('returns each origin\'s clone command with every missing name, and nothing for other errors', () => { const error = [ diff --git a/ui/src/lib/run/plan-text.ts b/ui/src/lib/run/plan-text.ts index a34d36e3ae..938ca72d52 100644 --- a/ui/src/lib/run/plan-text.ts +++ b/ui/src/lib/run/plan-text.ts @@ -2,8 +2,9 @@ // terminal read the same: work that runs first, one section per change reason with each model's detail, // then the routine models, functions, seeds and warnings. -import type { PlanChanged, PlanDocument, PlanModel, PlanWarning, Trigger } from '$lib/api'; +import type { PlanChanged, PlanDocument, PlanModel, PlanPageSection, PlanWarning, Trigger } from '$lib/api'; import { formatIn } from '$lib/format/time'; +import { plural } from '$lib/format/plural'; export type PlanEntry = { name: string; @@ -13,16 +14,25 @@ export type PlanEntry = { detail: string[]; queryDiff?: string[]; configDiff?: string[]; + /** A server page's diff counts, in place of the diff text, which is read on demand. */ + diffStats?: PlanModel['diff_stats']; schema?: { sign: '+' | '-' | '~'; text: string }[]; /** The plan's warnings about this model, such as the CLI's forward-only replay warning, shown with it. */ warnings?: PlanWarning[]; }; -/** `collapsed` sections show one line, such as "4 unchanged, skipped by --changed", until opened. */ -export type PlanSection = { title: string; entries: PlanEntry[]; changed: boolean; collapsed?: boolean }; +/** + * `collapsed` sections show one line, such as "4 unchanged, skipped by --changed", until opened. A `page`d section + * holds the rows loaded so far of its `total`; the rest load from the server's plan a page at a time. + */ +/** `section` is the server's section the rows are; `page` and `total` are set when its rows come a page at a time. */ +export type PlanSection = { title: string; entries: PlanEntry[]; changed: boolean; collapsed?: boolean; section?: PlanPageSection; page?: PlanPageSection; total?: number }; + +/** The section's size: its paged total, or the rows it holds. */ +export const sectionSize = (s: PlanSection) => s.total ?? s.entries.length; /** The change reasons that get their own section, in the CLI's order. */ -const REASON_GROUPS: [string, string][] = [ +const REASON_GROUPS: [PlanPageSection, string][] = [ ['full_refresh', 'Full refresh'], ['query_changed', 'Query changed'], ['function_changed', 'Function changed'], @@ -41,9 +51,22 @@ export function builtModels(plan: PlanDocument): PlanModel[] { return plan.models.filter((m) => m.action !== 'skip' && runsUnderChanged(plan, `model:${m.name}`)); } +/** Rows of a paged section the plan holds so far, which is the offset of its next page. */ +export function loadedRows(plan: PlanDocument, section: PlanPageSection): number { + return plan.loaded?.[section] ?? 0; +} + +/** The names of every model the run builds, including routine ones a server summary has not loaded yet. */ + +/** Each plan's `--changed` run nodes as a set, worked out once per plan: a large plan has thousands. */ +const runNodes = new WeakMap>(); + /** Without `--changed` every planned node runs; with it, only those its decision runs. */ function runsUnderChanged(plan: PlanDocument, key: string): boolean { - return !plan.changed || plan.changed.runs.some((r) => r.node === key); + if (!plan.changed) return true; + let nodes = runNodes.get(plan.changed); + if (!nodes) runNodes.set(plan.changed, (nodes = new Set(plan.changed.runs.map((r) => r.node)))); + return nodes.has(key); } /** The seeds the run loads or reports on: under `--changed`, only those it runs. */ @@ -102,6 +125,7 @@ function changedEntry(m: PlanModel, reason: string, windowStart?: string): PlanE ], queryDiff: m.query_diff, configDiff: m.config_diff, + diffStats: m.diff_stats, schema: m.schema_findings?.map((f) => { const kind = SCHEMA_SIGNS[f.kind] ?? { sign: '~' as const, label: f.kind }; const types = f.kind === 'column_type_changed' ? ` ${f.expected_type ?? 'None'} → ${f.actual_type ?? 'None'}` : f.expected_type ? ` ${f.expected_type}` : ''; @@ -113,47 +137,48 @@ function changedEntry(m: PlanModel, reason: string, windowStart?: string): PlanE /** Whether the plan reads as one full refresh: every model being built is one, as in the CLI. */ export function isFullRefresh(plan: PlanDocument): boolean { const active = builtModels(plan); - return active.length > 0 && active.every((m) => m.reason === 'full_refresh'); + return active.length > 0 && !plan.pages?.routine_models && active.every((m) => m.reason === 'full_refresh'); } /** The plan's sections in the order `sqb plan` prints them. Skipped models are left out, as there. */ export function planSections(plan: PlanDocument): PlanSection[] { const active = builtModels(plan); const sections: PlanSection[] = []; - const add = (title: string, entries: PlanEntry[], changed = false) => { - if (entries.length) sections.push({ title, entries, changed }); + const add = (title: string, entries: PlanEntry[], changed = false, page?: PlanPageSection) => { + const total = page && plan.pages?.[page]; + if (entries.length || total) sections.push({ title, entries, changed, ...(page ? { section: page } : {}), ...(page && total !== undefined ? { page, total } : {}) }); }; // Work that runs before any model builds comes first, as in the CLI. - add('Migrations', plan.migrations.map((r) => ({ name: r.model, value: [r.origin_model && `from ${r.origin_model}`, r.decision, r.completed_at ? 'completed' : 'pending'].filter(Boolean).join(' · '), detail: [] })), true); - add('Column migrations', plan.column_migrations.map((r) => ({ name: r.model, value: `${r.origin_column} → ${r.destination_column} · ${r.decision}`, detail: [] })), true); - add('Table type changes', plan.table_type_conversions.map((r) => ({ name: r.model, value: [r.actual_type, r.desired_type].filter(Boolean).join(' → '), detail: [] })), true); - add('Retention', plan.retention.map((r) => ({ name: r.model, value: r.action ?? '', detail: [] })), true); + add('Migrations', plan.migrations.map((r) => ({ name: r.model, value: [r.origin_model && `from ${r.origin_model}`, r.decision, r.completed_at ? 'completed' : 'pending'].filter(Boolean).join(' · '), detail: [] })), true, 'migrations'); + add('Column migrations', plan.column_migrations.map((r) => ({ name: r.model, value: `${r.origin_column} → ${r.destination_column} · ${r.decision}`, detail: [] })), true, 'column_migrations'); + add('Table type changes', plan.table_type_conversions.map((r) => ({ name: r.model, value: [r.actual_type, r.desired_type].filter(Boolean).join(' → '), detail: [] })), true, 'table_type_conversions'); + add('Retention', plan.retention.map((r) => ({ name: r.model, value: r.action ?? '', detail: [] })), true, 'retention'); for (const kind of ['loader', 'source'] as const) { const loads = plan.source_loads.filter((s) => s.kind === kind); const label = kind === 'loader' ? 'Loaders' : 'Sources'; - add(`${label} to reload`, loads.filter((s) => s.is_reload).map((s) => ({ name: s.name, value: s.write_strategy ?? '', detail: [] }))); - add(`${label} to load`, loads.filter((s) => !s.is_reload).map((s) => ({ name: s.name, value: s.write_strategy ?? '', detail: s.cursor_column ? [`cursor: ${s.cursor_column}`] : [] }))); + add(`${label} to reload`, loads.filter((s) => s.is_reload).map((s) => ({ name: s.name, value: s.write_strategy ?? '', detail: [] })), false, `${kind}_reloads`); + add(`${label} to load`, loads.filter((s) => !s.is_reload).map((s) => ({ name: s.name, value: s.write_strategy ?? '', detail: s.cursor_column ? [`cursor: ${s.cursor_column}`] : [] })), false, `${kind}_loads`); } const functionEntry = (f: PlanDocument['functions'][number]) => ({ name: f.name, value: f.language, detail: [] }); - add('Changed functions', plan.functions.filter((f) => f.reason && f.reason !== 'no_change').map(functionEntry), true); + add('Changed functions', plan.functions.filter((f) => f.reason && f.reason !== 'no_change').map(functionEntry), true, 'changed_functions'); if (isFullRefresh(plan)) { const counts = new Map(); for (const m of active) counts.set(m.materialization_type, (counts.get(m.materialization_type) ?? 0) + 1); - add('Full refresh', [...counts].map(([kind, n]) => ({ name: kind, value: `${n} ${n === 1 ? 'model' : 'models'}`, detail: [] })), true); + add('Full refresh', [...counts].map(([kind, n]) => ({ name: kind, value: `${plural(n, 'model')}`, detail: [] })), true); } else { const starts = plan.synced_copy?.cursor_window_starts ?? {}; - for (const [reason, title] of REASON_GROUPS) add(title, active.filter((m) => m.reason === reason).map((m) => changedEntry(m, reason, starts[m.name])), true); + for (const [reason, title] of REASON_GROUPS) add(title, active.filter((m) => m.reason === reason).map((m) => changedEntry(m, reason, starts[m.name])), true, reason); // Under --changed a routine model runs because something it reads changed, or for its own reason, such as no completed record. const why = changedRuns(plan); add('Models', active.filter((m) => ROUTINE.has(m.reason)).map((m) => { const run = why.get(m.name); return { name: m.name, value: materializationLabel(m), detail: [...cursorLines(m, true, starts[m.name]), ...(run ? [run.downstream_of ? `reads ${keyName(run.downstream_of)}` : run.reason] : [])] }; - })); + }), false, 'routine_models'); } - add('Functions', plan.functions.filter((f) => !f.reason || f.reason === 'no_change').map(functionEntry)); + add('Functions', plan.functions.filter((f) => !f.reason || f.reason === 'no_change').map(functionEntry), false, 'functions'); const seedLabel: Record = { first_run: 'first_run', config_changed: 'seed_changed', no_change: 'current' }; - add('Seeds', builtSeeds(plan).map((s) => ({ name: s.name, value: seedLabel[s.reason] ?? '', detail: [] }))); - add('Python', plan.python_nodes.map((p) => ({ name: p.name, value: `${p.kind} · ${p.identity_status}`, detail: [] }))); + add('Seeds', builtSeeds(plan).map((s) => ({ name: s.name, value: seedLabel[s.reason] ?? '', detail: [] })), false, 'seeds'); + add('Python', plan.python_nodes.map((p) => ({ name: p.name, value: `${p.kind} · ${p.identity_status}`, detail: [] })), false, 'python_nodes'); // A planner warning names its model in `model_name`, or opens its message with it as the CLI prints it; it shows once, with the first entry for that model. const claimed = new Set(); for (const entry of sections.filter((s) => s.changed).flatMap((s) => s.entries)) { @@ -188,11 +213,12 @@ export function changedModelKeys(plan: PlanDocument): string[] { } /** The Held back and Unchanged sections of a `--changed` plan; `refresh` words what refreshes a job's nodes. */ -export function changedSections(changed: PlanChanged, refresh: (job: string) => string): PlanSection[] { +export function changedSections(changed: PlanChanged, refresh: (job: string) => string, pages: PlanDocument['pages'] = {}): PlanSection[] { const sections: PlanSection[] = []; + const paged = (page: 'held_back' | 'unchanged') => (pages?.[page] !== undefined ? { page, total: pages[page] } : {}); const held = changed.held_back.map((h) => ({ name: keyName(h.node), value: 'refreshed_by' in h ? h.refreshed_by.map(refresh).join(' · ') : `reads ${keyName(h.reads)}, which is held back`, detail: [] })); - if (held.length) sections.push({ title: 'Held back', entries: held, changed: false }); - if (changed.unchanged.length) sections.push({ title: 'Unchanged', entries: changed.unchanged.map((key) => ({ name: keyName(key), value: 'skipped by --changed', detail: [] })), changed: false, collapsed: true }); + if (held.length || pages?.held_back) sections.push({ title: 'Held back', entries: held, changed: false, section: 'held_back', ...paged('held_back') }); + if (changed.unchanged.length || pages?.unchanged) sections.push({ title: 'Unchanged', section: 'unchanged', entries: changed.unchanged.map((key) => ({ name: keyName(key), value: 'skipped by --changed', detail: [] })), changed: false, collapsed: true, ...paged('unchanged') }); return sections; } @@ -204,10 +230,10 @@ export function jobRefreshText(job: string, triggers: Pick t.jobKey === job); const live = mine.filter((t) => t.state === 'running'); if (mine.length && !live.length) return `job ${job} is paused`; - if (schedulerStopped && live.length) return `job ${job} · can’t start until the scheduler is running again`; + if (schedulerStopped && live.length) return `job ${job} · scheduler stopped`; const next = live.flatMap((t) => (t.nextDueAt ? [t.nextDueAt] : [])).sort()[0]; if (next) return `job ${job} · next run ${formatIn(next, now)}`; - return live.some((t) => t.type === 'sensor') ? `job ${job} · when its sensor fires` : `job ${job}`; + return live.some((t) => t.type === 'sensor') ? `job ${job} · on sensor` : `job ${job}`; } /** The `sqb clone` commands a plan error offers for missing upstreams (S301), each ready to copy as is. */ diff --git a/ui/src/lib/run/plan-timings.svelte b/ui/src/lib/run/plan-timings.svelte new file mode 100644 index 0000000000..96a9784494 --- /dev/null +++ b/ui/src/lib/run/plan-timings.svelte @@ -0,0 +1,101 @@ + + + + + Planned in {seconds(timings.total_seconds)} s + + +
+

Where the planning time went

+ + + + + {#each rows as r (r.name)} + + + + + + {/each} + + +
StepSecondsShare
{r.name}{r.seconds}{r.percent}
Total{seconds(timings.total_seconds)}
+

{timingFootnote(plan, timings)}

+
+
+
+ + diff --git a/ui/src/lib/run/plan-timings.test.ts b/ui/src/lib/run/plan-timings.test.ts new file mode 100644 index 0000000000..d7152cd026 --- /dev/null +++ b/ui/src/lib/run/plan-timings.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from 'vitest'; +import type { PlanDocument, PlanTimings } from '$lib/api'; +import { timingFootnote, timingRows } from './plan-timings'; + +const timings: PlanTimings = { + total_seconds: 10, + steps: [ + { name: 'Writing the plan', seconds: 0.8 }, + { name: 'Other', seconds: 0.4 }, + { name: 'Reading warehouse state', seconds: 3.1 }, + { name: 'Detecting changes', seconds: 1.75 } + ], + planned_from: 'synced_state' +}; + +describe('plan timings', () => { + it('lists the steps longest first, Other last, each with its share of the total', () => { + expect(timingRows(timings).map((r) => [r.name, r.seconds, r.percent])).toEqual([ + ['Reading warehouse state', '3.1', '31%'], + ['Detecting changes', '1.8', '18%'], + ['Writing the plan', '0.8', '8%'], + ['Other', '0.4', '4%'] + ]); + }); + + it('says where the plan ran and how many nodes it planned', () => { + const plan = { selected_count: 2418 } as PlanDocument; + expect(timingFootnote(plan, timings)).toBe('On the location, from the synced warehouse state · 2,418 nodes'); + expect(timingFootnote(plan, { ...timings, planned_from: 'live' })).toBe('On the location, live from the warehouse · 2,418 nodes'); + }); +}); diff --git a/ui/src/lib/run/plan-timings.ts b/ui/src/lib/run/plan-timings.ts new file mode 100644 index 0000000000..4215f659b1 --- /dev/null +++ b/ui/src/lib/run/plan-timings.ts @@ -0,0 +1,26 @@ +// Where a plan's time went, for the plan page's "Planned in 10.6 s" popover. Pure. + +import type { PlanDocument, PlanTimings } from '$lib/api'; +import { plural } from '$lib/format/plural'; + +export type TimingRow = { name: string; seconds: string; share: number; percent: string }; + +/** Seconds as the popover shows them: one decimal. */ +export const seconds = (s: number) => s.toFixed(1); + +/** The steps, longest first with `Other` last, each with its share of the total. */ +export function timingRows(timings: PlanTimings): TimingRow[] { + const total = timings.total_seconds || timings.steps.reduce((n, s) => n + s.seconds, 0); + const other = timings.steps.filter((s) => s.name === 'Other'); + const steps = [...timings.steps.filter((s) => s.name !== 'Other').sort((a, b) => b.seconds - a.seconds), ...other]; + return steps.map((s) => { + const share = total > 0 ? Math.min(1, s.seconds / total) : 0; + return { name: s.name, seconds: seconds(s.seconds), share, percent: `${Math.round(share * 100)}%` }; + }); +} + +/** Where the plan ran and from what, with the nodes it planned. */ +export function timingFootnote(plan: PlanDocument, timings: PlanTimings): string { + const from = timings.planned_from === 'live' ? 'On the location, live from the warehouse' : 'On the location, from the synced warehouse state'; + return `${from} · ${plural(plan.selected_count, 'node')}`; +} diff --git a/ui/src/lib/run/range-calendar.svelte b/ui/src/lib/run/range-calendar.svelte index 8c9b4c0321..2348c57ca6 100644 --- a/ui/src/lib/run/range-calendar.svelte +++ b/ui/src/lib/run/range-calendar.svelte @@ -1,4 +1,6 @@ - + diff --git a/ui/src/routes/catalog/[name]/checks-tab.svelte b/ui/src/routes/catalog/[name]/checks-tab.svelte index 98a29eef68..4423f5704d 100644 --- a/ui/src/routes/catalog/[name]/checks-tab.svelte +++ b/ui/src/routes/catalog/[name]/checks-tab.svelte @@ -1,4 +1,6 @@ - - - - {#each node.checkResults as c (c.id)} + c.id} empty="No checks."> + {#snippet row(c)} {@const o = c.latest?.outcome ?? 'skipped'} openRow(e, `/checks/${c.id}`)}> - - + + - {:else} - - {/each} - -
CheckOutcomeSeverityChecksResult typeResultLast checked
{c.name}{#if c.column}
column {c.column}
{/if}
{c.severity}{runScopePhaseLabel[c.runScopePhase]}{evaluationModeLabel[c.evaluationMode]}{runScopePhaseLabel[c.runScopePhase]}{evaluationModeLabel[c.evaluationMode]} {checkValue(c.latest, c.evaluationMode)}
No audits or checks attach to this node.
+ {/snippet} + diff --git a/ui/src/routes/catalog/[name]/runs-tab.svelte b/ui/src/routes/catalog/[name]/runs-tab.svelte index 9f270331f7..14429447c0 100644 --- a/ui/src/routes/catalog/[name]/runs-tab.svelte +++ b/ui/src/routes/catalog/[name]/runs-tab.svelte @@ -1,4 +1,6 @@ {#if costs.length} @@ -23,10 +34,8 @@ {#if costs.some((c) => c.usd === null)}

{CREDIT_PRICE_HINT}

{/if} {/if} - - {#if costs.length}{/if} - - {#each node.runs ?? [] as r, i (`${r.runId ?? r.fingerprintRunId}-${i}`)} + ({ ...r, i }))} key={(r) => `${r.runId ?? r.fingerprintRunId}-${r.i}`} empty="No builds yet."> + {#snippet row(r)} - + {#if costs.length}{@const c = costOf(r)}{/if} - {:else} - - {/each} - -
OutcomeRunJobRowsDurationEst. costStarted
@@ -34,19 +43,16 @@ {#if r.runId}{r.runId} - {:else}from the command line · {shortRunId(r.fingerprintRunId ?? '')}{/if} + {:else}CLI · {shortRunId(r.fingerprintRunId ?? '')}{/if} {#if r.jobKey}{r.jobKey}{:else}{r.runId ? 'started by hand' : '—'}{/if}{#if r.jobKey}{r.jobKey}{:else}{r.runId ? 'Manual' : '—'}{/if} {formatNumber(r.rows)} {formatMs(r.durationMs)}{costText(c)}{c?.status === 'partial' ? '*' : ''}
No builds recorded in this location.
+ {/snippet} + diff --git a/ui/src/routes/checks/[id]/check-view.svelte b/ui/src/routes/checks/[id]/check-view.svelte index 242b07a90e..ace681a254 100644 --- a/ui/src/routes/checks/[id]/check-view.svelte +++ b/ui/src/routes/checks/[id]/check-view.svelte @@ -1,5 +1,10 @@ {#if c && app.client.supports('launchAdhocRun')} - goto(to(planHref(c?.node ? [c.node] : [], 'audit')))}>▷ Run check + goto(to(planHref(c?.node ? [c.node] : [], 'audit')))}>Run check {/if} @@ -78,26 +90,22 @@ {c?.name ?? id} {#if c} - {c.severity} severity · {runScopePhaseLabel[c.runScopePhase].toLowerCase()} · {evaluationModeLabel[c.evaluationMode].toLowerCase()} + {c.severity} · {runScopePhaseLabel[c.runScopePhase].toLowerCase()} · {evaluationModeLabel[c.evaluationMode].toLowerCase()} {/if}
{#if res.error}
Couldn't load this check: {res.error}
{:else if res.data === null}
There's no check {id} in the version deployed to {app.location?.name}.
{/if} {#if c}
-
+ {#if c.description}

{c.description}

{/if} - + e.outcome)} tones={checkOutcomeTone} caption="Last {c.history.length} results" /> - - - - - - - {#each c.history as e (e.id)} + + e.id} empty="No results recorded yet."> + {#snippet row(e)} - {/each} - -
CheckedOutcome{c.evaluationMode === 'measurement' ? 'Measured' : 'Violations'}RunEvidence
{formatAgo(e.evaluatedAt, app.now)} @@ -110,34 +118,28 @@ {#if e.evidenceCount} - showEvidence(e.id)}>{e.evidenceCount}{e.evidenceTruncated ? '+' : ''} rows + showEvidence(e.id)}>{e.evidenceCount}{e.evidenceTruncated ? '+' : ''} rows {:else}—{/if}
+ {/snippet} +
{#if evidenceError}
{evidenceError}
{/if} {#if evidence} - - {#snippet actions()}{/snippet} - - {#each evidence.columns as col (col)}{/each} - - {#each evidence.rows as row, i (i)} - {#each row as v, j (j)}{/each} - {:else} - - {/each} - -
{col}
{cellText(v)}
No rows were stored.
- {#if evidence.truncated}

More rows matched than were stored.

{/if} + + {#snippet actions()}{/snippet} + ({ id: `${i}`, label: col, minWidth: 100 }))} rows={evidence.rows.map((cells, i) => ({ cells, i }))} key={(r) => r.i} empty="No rows were stored."> + {#snippet row(r)} + {#each r.cells as v, j (j)}{cellText(v)}{/each} + {/snippet} + + {#if evidence.truncated}

Truncated

{/if}
{/if} - + {#if sql} {#if sql.executionError}
{sql.executionError}
{/if} {#if sql.evidenceError}
Evidence couldn't be collected: {sql.evidenceError}
{/if} @@ -147,13 +149,13 @@

No SQL was stored with the latest result.

{/each} {:else if !app.can('view.sql')} -

You don't have permission to see SQL (view.sql)

+

No access (view.sql)

{:else} - Show SQL and errors + Show SQL {#if sqlError}
{sqlError}
{/if} {/if}
-
+