From 63e3bf574ed9c8330f667afefdddd2313dfd8b45 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:49:35 +0000 Subject: [PATCH 01/20] fix(compiler): read authored SQL with the project's lexical rules in every scanner Snowflake strings escape quotes with a backslash, but the scanners that find declarations, Python and SQL macro calls and @@ variables read SQL under generic rules, so an escaped quote hid later markers or reported an unclosed string. Local scenario replay also compiled with DuckDB's rules, so the reference scanner failed on Snowflake SQL. Every scanner on the compile path now reads quoted text and comments with the project's lexical rules, and local replay compiles with the project adapter's rules. --- Cargo.lock | 1 + Makefile | 1 + .../compiler/_helpers/sql_tests/assembly.rs | 2 +- .../_helpers/sql_tests/macro_call_names.rs | 5 +- .../sql_macros/_helpers/parsing/definition.rs | 2 +- .../src/sql_macros/_helpers/parsing/lexing.rs | 39 +++++- .../_helpers/rendering/expansion.rs | 17 ++- .../src/sql_macros/main/bare_macro_calls.rs | 5 +- .../src/sql_macros/main/sql_macro_calls.rs | 8 +- .../main/with_list_sql_macro_calls.rs | 2 +- .../src/sql_macros/tests/test_expansion.rs | 2 +- .../sqlbuild-analysis/src/sql_macros/types.rs | 6 + .../sql_expansion_context.rs | 4 + .../compile_driver/assembly_analysis.rs | 5 + .../compile_driver/assembly_models.rs | 1 + .../_helpers/compile_driver/inputs.rs | 4 + .../_helpers/compile_driver/pipeline.rs | 4 + .../_helpers/compile_driver/project.rs | 23 +++- .../_helpers/compile_records/constants.rs | 6 + .../_helpers/compiler_records/constants.rs | 1 + .../_helpers/input_loops/model_loop.rs | 18 ++- .../_helpers/input_loops/model_loop_hooks.rs | 4 +- .../_helpers/input_loops/source_loop.rs | 4 +- .../_helpers/input_loops/sql_test_loop.rs | 13 +- .../_helpers/record_behaviours/lexical.rs | 25 +--- .../bindings/_helpers/render/authored_sql.rs | 24 +++- .../bindings/_helpers/render/macro_calls.rs | 8 +- .../_helpers/render/macro_evaluation.rs | 6 +- .../_helpers/render/macro_expansion.rs | 14 +- .../_helpers/render/sql_macro_host.rs | 20 ++- .../_helpers/scenario_command/test_command.rs | 4 + .../sqltext/declaration_references.rs | 11 +- .../_helpers/sqltext/lexical_syntax.rs | 65 ++++++++++ .../bindings/_helpers/sqltext/static_sql.rs | 12 +- crates/sqlbuild-render/Cargo.toml | 1 + .../src/macro_calls/_helpers/scanner.rs | 119 +++++++++++------ .../src/macro_calls/main/macro_call_names.rs | 11 +- .../macro_calls/main/scan_macro_call_sites.rs | 11 +- .../tests/test_macro_call_names.rs | 4 +- .../tests/test_scan_macro_call_sites.rs | 63 ++++++++- .../src/relationship_names/_helpers/facts.rs | 8 +- .../declaration_references/scanner.rs | 51 +++++++- .../sql_interpolation/substitution.rs | 10 +- .../compiler/main/declaration_references.rs | 6 +- .../src/compiler/tests/helpers.rs | 14 +- .../tests/test_declaration_references.rs | 42 +++++- crates/sqlbuild-sqltext/src/compiler/types.rs | 6 + src/sqlbuild/_native.pyi | 4 + .../cli/commands/main/scenario/_test_types.py | 10 ++ .../scenario/test_local_escaped_quotes.py | 121 ++++++++++++++++++ .../main/test/test_dialect_string_escapes.py | 4 +- 51 files changed, 700 insertions(+), 151 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_escaped_quotes.py diff --git a/Cargo.lock b/Cargo.lock index de4e83322b..a35dbfd5f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1431,6 +1431,7 @@ dependencies = [ "serde_json", "sqlbuild-cache", "sqlbuild-core", + "sqlbuild-sqltext", "tempfile", ] diff --git a/Makefile b/Makefile index aff8652a6a..27f29d1384 100644 --- a/Makefile +++ b/Makefile @@ -156,6 +156,7 @@ E2E_DUCKDB_CLI_DATA_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_fixture_columns.py \ tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py \ tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_lambda.py \ + tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_escaped_quotes.py \ tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_snapshot_staleness.py \ tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_unmocked_inputs.py diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/assembly.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/assembly.rs index a768faf024..53a4e5c280 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/assembly.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/assembly.rs @@ -28,7 +28,7 @@ pub(crate) fn assemble_batch(batch: &SqlTestAssemblyBatch) -> Vec macro_call_names(sql), + Some(sql) if tests_macros => macro_call_names(sql, &batch.lexical_syntax), Some(_) => Ok(Vec::new()), None => Ok(model.macro_deps.clone()), }) diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/macro_call_names.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/macro_call_names.rs index 95169f6769..7094bfffdf 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/macro_call_names.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/macro_call_names.rs @@ -1,14 +1,15 @@ //! Python's `find_macro_call_names` over a model's pre-macro SQL. use sqlbuild_render::macro_calls::main::macro_call_names::macro_call_names as scanned_names; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; /// Declaration calls expansion substitutes rather than running as macros. const DECLARATION_CALLS: [&str; 2] = ["enum", "const"]; /// The names Python's scan returns, or the message of the `CompileInputError` it raises. -pub(crate) fn macro_call_names(sql: &str) -> Result, String> { +pub(crate) fn macro_call_names(sql: &str, syntax: &LexicalSyntax) -> Result, String> { let mut names: Vec = - scanned_names(sql, false).map_err(|error| error.message().to_owned())?; + scanned_names(sql, false, Some(syntax)).map_err(|error| error.message().to_owned())?; names.retain(|name| !DECLARATION_CALLS.contains(&name.as_str())); Ok(names) } diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/definition.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/definition.rs index f29f195c90..a969a868f0 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/definition.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/definition.rs @@ -135,7 +135,7 @@ pub(crate) fn body_form(body: &str, scanned: &[Token]) -> SqlMacroForm { /// The SQL macros the body calls, rejecting Python and bare macro calls. fn body_calls(name: &str, body: &str, in_body: Locate<'_>) -> Result, SqlMacroError> { let constructs: Vec = - at_constructs(body).map_err(|_| unclosed_body(name, in_body(0)))?; + at_constructs(body, None).map_err(|_| unclosed_body(name, in_body(0)))?; let mut calls: Vec = Vec::new(); for construct in constructs { match construct { diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/lexing.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/lexing.rs index 2fc6bdd706..b1e2db4df2 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/lexing.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/lexing.rs @@ -7,6 +7,7 @@ use sqlbuild_sqltext::header_strings::main::follows_description_key::follows_des use sqlbuild_sqltext::header_strings::main::opens_triple_quote::opens_triple_quote; use sqlbuild_sqltext::header_strings::main::prose_end::prose_end; use sqlbuild_sqltext::header_strings::main::triple_quoted_end::triple_quoted_end; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::sql_macros::constants::{HEADER_KEYWORD, SQL_KIND_WORD}; use crate::sql_macros::errors::LexError; @@ -57,15 +58,36 @@ pub(crate) const DECLARATION_WORDS: [&str; 3] = ["const", "enum", "param"]; /// Every token of `sql`; text unclosed under doubled-quote escapes is reread with backslash ones. pub(crate) fn tokens(sql: &str) -> Result, LexError> { - match scanned_tokens(sql, false) { + tokens_with(sql, None) +} + +/// Every token of `sql`, with `syntax`'s backslash escapes, or as [`tokens`] reads it. +pub(crate) fn tokens_with( + sql: &str, + syntax: Option<&LexicalSyntax>, +) -> Result, LexError> { + if let Some(syntax) = syntax { + let escapes = |quote: u8| { + syntax + .backslash_escape_quotes + .iter() + .any(|value| value.as_bytes() == [quote]) + }; + return scanned_tokens(sql, &escapes); + } + match scanned_tokens(sql, &|_| false) { Err(LexError::UnclosedQuote(start)) => { - scanned_tokens(sql, true).map_err(|_| LexError::UnclosedQuote(start)) + scanned_tokens(sql, &|_| true).map_err(|_| LexError::UnclosedQuote(start)) } scanned => scanned, } } -fn scanned_tokens(sql: &str, backslash_escapes: bool) -> Result, LexError> { +/// Every token of `sql`; a backslash escapes inside quotes for which `backslash_escapes` holds. +fn scanned_tokens( + sql: &str, + backslash_escapes: &dyn Fn(u8) -> bool, +) -> Result, LexError> { let bytes: &[u8] = sql.as_bytes(); let mut found: Vec = Vec::new(); let mut index: usize = 0; @@ -103,12 +125,12 @@ fn scanned_tokens(sql: &str, backslash_escapes: bool) -> Result, LexE TokenKind::Quoted } b'\'' => { - index = quote_end(bytes, index, backslash_escapes) + index = quote_end(bytes, index, backslash_escapes(byte)) .ok_or(LexError::UnclosedQuote(start))?; TokenKind::Text } b'"' | b'`' => { - index = quote_end(bytes, index, backslash_escapes) + index = quote_end(bytes, index, backslash_escapes(byte)) .ok_or(LexError::UnclosedQuote(start))?; TokenKind::Quoted } @@ -366,11 +388,14 @@ pub(crate) enum AtConstruct { } /// Every SQL macro call, Python macro call and bare call in `sql`, top-level and nested. -pub(crate) fn at_constructs(sql: &str) -> Result, LexError> { +pub(crate) fn at_constructs( + sql: &str, + syntax: Option<&LexicalSyntax>, +) -> Result, LexError> { if !sql.contains('@') { return Ok(Vec::new()); } - let scanned: Vec = tokens(sql)?; + let scanned: Vec = tokens_with(sql, syntax)?; let mut found: Vec = Vec::new(); for (index, token) in scanned.iter().enumerate() { if !token.is(sql, "@") { diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/expansion.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/expansion.rs index d221b84b62..f3e661c474 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/expansion.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/expansion.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use crate::sql_macros::_helpers::parsing::body::{expand_body, with_line_comment_closed}; use crate::sql_macros::_helpers::parsing::lexing::{AtConstruct, at_constructs, outer_sql_calls}; -use crate::sql_macros::_helpers::parsing::lexing::{Token, line_column, tokens}; +use crate::sql_macros::_helpers::parsing::lexing::{Token, line_column, tokens, tokens_with}; use crate::sql_macros::_helpers::parsing::literal_positions::{ LiteralRules, literal_error, non_literal_use, }; @@ -31,10 +31,14 @@ use crate::sql_macros::models::{ SqlMacroForm, SqlMacroLocation, TextContext, TypedCall, }; use crate::sql_macros::types::{ConstantShape, SqlMacroHost, SqlMacroScope}; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; /// The outer SQL macro calls of `text`, or why its `@` constructs could not be read. -pub(crate) fn sql_macro_calls(text: &str) -> Result, SqlMacroError> { - let constructs: Vec = at_constructs(text).map_err(|_| { +pub(crate) fn sql_macro_calls( + text: &str, + syntax: Option<&LexicalSyntax>, +) -> Result, SqlMacroError> { + let constructs: Vec = at_constructs(text, syntax).map_err(|_| { SqlMacroError::new( INVALID_DEFINITION_CODE, "SQL has an unclosed quote or comment", @@ -62,7 +66,7 @@ pub(crate) fn sql_macro_calls(text: &str) -> Result, SqlMacroE if calls.iter().all(|call| call.in_python) { return Ok(calls); } - let Ok(scanned) = tokens(text) else { + let Ok(scanned) = tokens_with(text, syntax) else { return Ok(calls); }; Ok(calls @@ -128,14 +132,15 @@ pub(crate) fn expand_text( host: &H, context: &TextContext<'_>, ) -> Result> { - let calls: Vec = sql_macro_calls(context.text)?; + let syntax: Option<&LexicalSyntax> = host.lexical_syntax(); + let calls: Vec = sql_macro_calls(context.text, syntax)?; if calls.is_empty() { return Ok(ExpandedText { sql: host.render_declarations(context.scope, context.text)?.sql, calls: Vec::new(), }); } - let scanned: Vec = tokens(context.text).map_err(|_| { + let scanned: Vec = tokens_with(context.text, syntax).map_err(|_| { SqlMacroError::new( INVALID_DEFINITION_CODE, "SQL has an unclosed quote or comment", diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/bare_macro_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/main/bare_macro_calls.rs index 5f36056974..d73cd82628 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/bare_macro_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/bare_macro_calls.rs @@ -1,14 +1,15 @@ use crate::sql_macros::_helpers::parsing::lexing::{AtConstruct, at_constructs}; use crate::sql_macros::constants::NON_ASCII_BYTE; use sqlbuild_render::macro_calls::main::opens_bare_call::opens_bare_call; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; /// The name and byte offset of each `@name(...)` call in `text` that names no macro kind. #[must_use] -pub fn bare_macro_calls(text: &str) -> Vec<(String, usize)> { +pub fn bare_macro_calls(text: &str, syntax: Option<&LexicalSyntax>) -> Vec<(String, usize)> { if !may_hold_bare_call(text) { return Vec::new(); } - at_constructs(text) + at_constructs(text, syntax) .unwrap_or_default() .into_iter() .filter_map(|construct| match construct { diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_calls.rs index aa5ad4a182..27a006b165 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_calls.rs @@ -1,10 +1,14 @@ use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::models::SqlMacroCall; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; /// The SQL macro calls of `text` not nested in another SQL macro call, in byte offsets. -pub fn sql_macro_calls(text: &str) -> Result, SqlMacroError> { +pub fn sql_macro_calls( + text: &str, + syntax: Option<&LexicalSyntax>, +) -> Result, SqlMacroError> { if !text.contains(crate::sql_macros::constants::SQL_CALL_PREFIX) { return Ok(Vec::new()); } - crate::sql_macros::_helpers::rendering::expansion::sql_macro_calls(text) + crate::sql_macros::_helpers::rendering::expansion::sql_macro_calls(text, syntax) } diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/with_list_sql_macro_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/main/with_list_sql_macro_calls.rs index 9669935131..26800dacd2 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/with_list_sql_macro_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/with_list_sql_macro_calls.rs @@ -6,7 +6,7 @@ use crate::sql_macros::models::{CallPosition, SqlMacroCall}; /// The SQL macro calls standing between the CTEs of the top-level `WITH` list of `text`. #[must_use] pub fn with_list_sql_macro_calls(text: &str) -> Vec { - let Ok(calls) = sql_macro_calls(text) else { + let Ok(calls) = sql_macro_calls(text, None) else { return Vec::new(); }; if calls.is_empty() { diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs index fe0701e975..3c6f1d606b 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs @@ -852,7 +852,7 @@ fn given_a_sql_macro_call_in_python_macro_arguments_when_expanding_then_python_r panic!("{} does not expand", test_case.description); }; - let scan = scan_macro_call_sites(&expanded); + let scan = scan_macro_call_sites(&expanded, None); let arguments: Option = scan.sites.first().map(|site| { expanded .chars() diff --git a/crates/sqlbuild-analysis/src/sql_macros/types.rs b/crates/sqlbuild-analysis/src/sql_macros/types.rs index 0c6fb2ae32..e86fe663dc 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/types.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/types.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use crate::sql_macros::errors::ExpansionFailure; use crate::sql_macros::models::{RenderedDeclarations, SqlMacroCheck, SqlMacroDefinition}; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; /// Whose scope a piece of SQL resolves its declarations and macros in. #[derive(Clone, Debug)] @@ -55,6 +56,11 @@ pub trait SqlMacroHost { "" } + /// The lexical rules the project's SQL is authored in; `None` reads it under generic rules. + fn lexical_syntax(&self) -> Option<&LexicalSyntax> { + None + } + /// Keep what the caller's analysed SQL must show for a call the caller wrote to fit. fn record_check(&self, _check: SqlMacroCheck, _call: (usize, usize)) {} } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs index db6644020c..3e0e8df43d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs @@ -165,6 +165,10 @@ pub(crate) fn sql_expansion_inputs(sources: ExpansionSources<'_>) -> PyResult { pub(crate) analyses: Option>, /// The native rows of `column_types`, `nullability` and `complete_schemas`. pub(crate) declared: DeclaredShapes, + /// The lexical rules the project's SQL is authored in. + pub(crate) syntax: Option, } /// Whether `model_input` is selected for analysis by `names` (`None` selects every model). @@ -122,6 +126,7 @@ pub(crate) fn analysis_context<'py>( dynamic_families, analyses: None, declared, + syntax: optional_record_syntax(&native_field(inputs, "sql_lexical_syntax")?)?, }; if context.sql_analysis_enabled.is_truthy()? { let declared: DeclaredShapes = std::mem::take(&mut context.declared); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index a5b9e0a289..af5593c480 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -165,6 +165,7 @@ fn compiled_model<'py>( top_level_macro_call_names( py, &native_field(model_input, "macro_source_sql")?.extract::()?, + context.syntax.as_ref(), )? .into_any() }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs index 40a7468d44..772a5d19c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs @@ -295,6 +295,10 @@ fn target_context<'py>( "_collection_rendering", native_field(adapter_context, "collection_rendering")?, ), + ( + "_lexical_syntax", + native_field(adapter_context, "sql_lexical_syntax")?, + ), ( "_enforce_explicit_references", native_field(&project_config, "references")?.getattr("enforce_explicit")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index 42cb2d73d6..c5a9f107fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -102,6 +102,10 @@ pub(crate) fn compiled_phase<'py>( ), ("resolved_connection", connection_config.clone()), ("analysis_selection", selection), + ( + "source_lexical_syntax", + native_field(&options, "source_lexical_syntax")?, + ), ], )?), ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index 2b63c48a50..4d298aaa6f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -19,6 +19,8 @@ use crate::bindings::_helpers::compile_driver::targets::{ }; use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, bridged, produced}; use crate::bindings::_helpers::records::construct::{enum_member, new_record_with}; +use crate::bindings::_helpers::sqltext::lexical_syntax::{optional_record_syntax, syntax_record}; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compile_driver::assembly::assemble_project; @@ -59,6 +61,8 @@ struct ProjectRequest { external_sql_reference_resolver: Py, resolved_connection: Py, analysis_selection: Py, + /// Rules the project's SQL is authored in where they differ from the adapter's, as in replay. + source_lexical_syntax: Option, } /// `column_lineage_mode`, defaulting to `ColumnLineageMode.FAST` when not passed. @@ -97,6 +101,7 @@ pub(crate) fn build_compiled_project<'py>( "external_sql_reference_resolver", "resolved_connection", "analysis_selection", + "source_lexical_syntax", ], ), )?; @@ -119,10 +124,23 @@ impl ProjectRequest { .unbind(), resolved_connection: arguments.value("resolved_connection")?.unbind(), analysis_selection: arguments.value("analysis_selection")?.unbind(), + source_lexical_syntax: optional_record_syntax( + &arguments.value("source_lexical_syntax")?, + )?, }) } } +impl ProjectRequest { + /// The lexical rules the project's SQL is read with: the source syntax, else the adapter's. + fn lexical_syntax<'py>(&self, py: Python<'py>) -> PyResult> { + match &self.source_lexical_syntax { + Some(syntax) => syntax_record(py, syntax), + None => native_field(self.adapter.bind(py), "sql_lexical_syntax"), + } + } +} + /// The compiled project produced through the `COMPILED_PROJECT` frontier. fn compiled_project(py: Python<'_>, request: ProjectRequest) -> PyResult> { produced( @@ -264,10 +282,7 @@ fn frontier_compile_inputs<'py>( "python_functions_inherit_default_namespace", adapter.call_method0("python_functions_inherit_default_namespace")?, ), - ( - "sql_lexical_syntax", - native_field(adapter, "sql_lexical_syntax")?, - ), + ("sql_lexical_syntax", request.lexical_syntax(py)?), ], )?; let inputs: InputsRequest = InputsRequest { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs index 2af9f2acbb..33cf4c8205 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs @@ -1303,6 +1303,12 @@ const R_MACRO_CONTEXT: RecordSpec = RecordSpec { compared: false, shown: false, }, + FieldSpec { + name: "_lexical_syntax", + default: FieldDefault::None, + compared: false, + shown: false, + }, ], }; const R_DECLARATION_RESOLUTION_CONTEXT: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs index 055a9891d2..e9a5c04769 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs @@ -404,6 +404,7 @@ const R_COMPILE_PIPELINE_OPTIONS: RecordSpec = RecordSpec { "sql_test_selection", FieldDefault::NativeCall(RecordId::SqlTestSelection), ), + FieldSpec::new("source_lexical_syntax", FieldDefault::None), ], }; const R_PYTHON_PLAN_ENTRY: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs index 447cc98dbb..78ef4e81d3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs @@ -209,6 +209,8 @@ pub(crate) struct LoopContext<'a, 'py> { pub(crate) host: &'a Bound<'py, ConstantRendering>, pub(crate) classes: &'a LoopClasses<'py>, pub(crate) contracts: &'a ContractClasses<'py>, + /// The lexical rules the project's SQL is authored in. + pub(crate) syntax: &'a LexicalSyntax, } /// The declarations a model sees and the identity its usages are recorded for. @@ -295,7 +297,7 @@ fn index_sql_hook_definitions(loop_classes: &Bound<'_, PyDict>) -> PyResult<()> /// Each model's `@@`-interpolated query and declaration scan, all interpolated before any runs. fn prepared_rows<'py>( py: Python<'py>, - classes: &LoopClasses<'py>, + (classes, syntax): (&LoopClasses<'py>, &LexicalSyntax), model_files: Vec>, ) -> PyResult> { let hooks = &classes.sql_hooks; @@ -313,6 +315,7 @@ fn prepared_rows<'py>( hooks.environment.clone(), None, hooks.render_project_var.clone(), + Some(syntax.clone()), ), &sqls, )?; @@ -330,7 +333,9 @@ fn prepared_rows<'py>( error, } = interpolated; let sql: String = rendered.unwrap_or(authored); - let scan: Option = error.is_none().then(|| declaration_scan(&sql)); + let scan: Option = error + .is_none() + .then(|| declaration_scan(&sql, Some(syntax))); let reads: Vec<(String, String)> = reads .into_iter() .map(|(kind, name)| (kind.to_owned(), name)) @@ -360,6 +365,7 @@ impl NativeModelLoop { host: &host, classes: &classes, contracts: &contracts, + syntax: &self.syntax, }; project .borrow_mut() @@ -427,7 +433,8 @@ impl NativeModelLoop { ) -> PyResult>> { let py: Python<'py> = slf.py(); let classes: LoopClasses<'py> = slf.get().loop_classes.bind(py).extract()?; - let mut models: Vec = prepared_rows(py, &classes, model_files)?; + let mut models: Vec = + prepared_rows(py, (&classes, &slf.get().syntax), model_files)?; let headers: Vec> = models.iter().map(native_header).collect(); let facts: Vec> = py .compiler_detach(|| Ok(all_header_facts(&headers))) @@ -456,6 +463,7 @@ impl NativeModelLoop { host, classes: &classes, contracts: &contracts, + syntax: &self.syntax, }; let attached: Vec = attached_sources(self, &context, rows, settings)?; Ok(retain_sources(project, attached)) @@ -476,6 +484,7 @@ impl NativeModelLoop { host, classes: &classes, contracts: &contracts, + syntax: &self.syntax, }; attached_audits(self, &context, (files, owners), settings) } @@ -495,6 +504,7 @@ impl NativeModelLoop { host, classes: &classes, contracts: &contracts, + syntax: &self.syntax, }; let validator: Bound<'py, NativeModelValidator> = self.validator.bind(py).downcast()?.clone(); @@ -562,6 +572,7 @@ impl NativeModelLoop { host, classes: &classes, contracts: &contracts, + syntax: &self.syntax, }; let attached: Vec = attached_functions(self, &context, rows, settings)?; checked_table_function_arities(&context, models, &attached)?; @@ -645,6 +656,7 @@ impl NativeModelLoop { query: authored_query, }), scopes: Some(&context.classes.visibility), + syntax: Some(context.syntax), }), })?; let macros: Option = if calls_macros(&declarations.spliced.sql) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs index 9db2bb03c0..11bdd8abdc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs @@ -386,6 +386,7 @@ impl<'py> HookRun<'_, '_, 'py> { hooks.environment.clone(), Some(self.context_values.clone()), hooks.render_project_var.clone(), + Some(self.context.syntax.clone()), ), sql, scope.file_path, @@ -487,7 +488,7 @@ impl<'py> HookRun<'_, '_, 'py> { declarations: &Bound<'py, PyAny>, scope: &Scope<'_, 'py>, ) -> PyResult> { - let (references, stop) = declaration_scan(sql); + let (references, stop) = declaration_scan(sql, Some(&self.model_loop.syntax)); let classes: ExpansionClasses<'py> = ExpansionClasses { usage_record: self.context.classes.usage_record.clone(), compile_error: self.context.classes.compile_error.clone(), @@ -515,6 +516,7 @@ impl<'py> HookRun<'_, '_, 'py> { ), origin: None, scopes: Some(&self.context.classes.visibility), + syntax: Some(self.context.syntax), }), }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs index a24e4bb3a1..4fb220bf8a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs @@ -213,6 +213,7 @@ pub(crate) fn expanded_authored_sql<'py>( hooks.environment.clone(), None, hooks.render_project_var.clone(), + Some(context.syntax.clone()), ), sql, file_path, @@ -227,7 +228,7 @@ pub(crate) fn expanded_authored_sql<'py>( } let interpolated_sql: String = interpolated_sql.unwrap_or_else(|| sql.to_owned()); let consumer: Bound<'py, PyAny> = native_field(declarations, "consumer")?; - let (references, stop) = declaration_scan(&interpolated_sql); + let (references, stop) = declaration_scan(&interpolated_sql, Some(context.syntax)); let mut usages: Vec> = Vec::new(); let mut argument_references: Vec = Vec::new(); let mut expanded_sql: String = interpolated_sql.clone(); @@ -252,6 +253,7 @@ pub(crate) fn expanded_authored_sql<'py>( loaded_macros: Some(context.classes.macros.loaded_macros.bind(py).as_any()), origin: None, scopes: Some(&context.classes.visibility), + syntax: Some(context.syntax), }), })?; expanded_sql = expanded.spliced.sql; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs index 1ecfc04e84..24d4275ef2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs @@ -418,7 +418,9 @@ impl<'py> TestScope<'_, 'py> { ))); } let mut names: Vec = with_list_macros.to_vec(); - for name in top_level_macro_names(&actual.1).map_err(|message| self.error(message))? { + for name in top_level_macro_names(&actual.1, &self.model_loop.syntax) + .map_err(|message| self.error(message))? + { if !names.contains(&name) { names.push(name); } @@ -519,6 +521,7 @@ impl<'py> TestScope<'_, 'py> { sqlbuild_sqltext::compiler::main::declaration_references::scan_declaration_references( &[sql.to_owned()], true, + Some(&self.model_loop.syntax), ); if let Some(stop) = scans.iter().find_map(|scan| scan.stop) { let problem: &str = match stop { @@ -560,7 +563,9 @@ impl<'py> TestScope<'_, 'py> { native_field(declarations, MACRO_RECORDS)?.downcast_into()?; let macro_visibility: Bound<'py, PyDict> = native_field(declarations, MACRO_VISIBILITY)?.downcast_into()?; - for name in top_level_macro_names(sql).map_err(|message| self.error(message))? { + for name in top_level_macro_names(sql, &self.model_loop.syntax) + .map_err(|message| self.error(message))? + { let Some(record) = macro_records.get_item(&name)? else { continue; }; @@ -1021,8 +1026,8 @@ fn test_visibility<'py>(records: &Bound<'py, PyAny>) -> PyResult Result, &'static str> { - Ok(macro_call_names(sql, false) +fn top_level_macro_names(sql: &str, syntax: &LexicalSyntax) -> Result, &'static str> { + Ok(macro_call_names(sql, false, Some(syntax)) .map_err(sqlbuild_render::macro_calls::errors::ScanError::message)? .into_iter() .filter(|name| !DECLARATION_CALLS.contains(&name.as_str())) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lexical.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lexical.rs index 51478550f0..e9e75ffd86 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lexical.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lexical.rs @@ -11,35 +11,12 @@ use sqlbuild_sqltext::sql_scan::main::unsupported_line_comment_prefixes::unsuppo use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::bindings::_helpers::records::construct::error_class; -use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::ErrorId; +use crate::bindings::_helpers::sqltext::lexical_syntax::{record_syntax, sorted_texts}; /// `SqlLexicalSyntax.line_comment_prefixes` position. const LINE_COMMENT_PREFIXES_FIELD: usize = 5; -/// The sorted texts of a set of strings. -fn sorted_texts(values: &Bound<'_, PyAny>) -> PyResult> { - let mut texts: Vec = values - .try_iter()? - .map(|value| value?.extract::()) - .collect::>()?; - texts.sort_unstable(); - Ok(texts) -} - -/// The lexical rules a `SqlLexicalSyntax` record holds. -fn record_syntax(instance: &Bound<'_, PyAny>) -> PyResult { - let flag = |name: &str| -> PyResult { native_field(instance, name)?.is_truthy() }; - Ok(LexicalSyntax { - backslash_escape_quotes: sorted_texts(&native_field(instance, "backslash_escape_quotes")?)?, - escape_string_prefix: flag("escape_string_prefix")?, - raw_string_prefix: flag("raw_string_prefix")?, - triple_quoted_strings: flag("triple_quoted_strings")?, - nested_block_comments: flag("nested_block_comments")?, - line_comment_prefixes: sorted_texts(&native_field(instance, "line_comment_prefixes")?)?, - }) -} - /// `SqlLexicalSyntax`: reject line comment prefixes the scanners cannot read. pub(crate) fn finish_lexical_syntax( py: Python<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs index 1c561f4e82..5b295f94ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs @@ -6,6 +6,7 @@ use pyo3::prelude::{ }; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::model_config::template_data::record_reads; @@ -28,10 +29,13 @@ use crate::bindings::_helpers::render::macro_call_rows::{span_row, span_tuple}; use crate::bindings::_helpers::render::macro_expansion_result::{ExpansionRequest, macro_output}; use crate::bindings::_helpers::render::sql_macro_host::SqlMacroSource; use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; +use crate::bindings::_helpers::sqltext::lexical_syntax::optional_record_syntax; use crate::bindings::_helpers::sqltext::static_sql::{ InterpolationRow, InterpolationSources, interpolation_batch, }; +/// The macro context field holding the project's `SqlLexicalSyntax`. +const LEXICAL_SYNTAX_FIELD: &str = "_lexical_syntax"; const DECLARATION_FIELDS: [&str; 4] = [ "enums", "constants", @@ -39,11 +43,20 @@ const DECLARATION_FIELDS: [&str; 4] = [ "inaccessible_constants", ]; +/// The project's lexical rules the macro context holds, if any. +fn context_syntax(macro_context: &Bound<'_, PyAny>) -> PyResult> { + if macro_context.is_none() { + return Ok(None); + } + optional_record_syntax(&native_field(macro_context, LEXICAL_SYNTAX_FIELD)?) +} + /// `sql` with `@@` tokens interpolated and each substitution's span, raising its error. fn substituted<'py>( sql: &Bound<'py, PyAny>, file_path: &Bound<'py, PyAny>, (effective_vars, context_values): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + syntax: Option<&LexicalSyntax>, ) -> PyResult<(String, Vec)> { let py: Python<'py> = sql.py(); let sources = InterpolationSources( @@ -55,6 +68,7 @@ fn substituted<'py>( Some(context_values.downcast()?.clone()) }, registry::native(py, NativeAttr::RenderProjectVarText)?, + syntax.cloned(), ); let text: String = sql.extract()?; let path: String = file_path.str()?.extract()?; @@ -123,9 +137,10 @@ fn declarations_expanded<'py>( &Bound<'py, PyAny>, &Bound<'py, PyAny>, ), + syntax: Option<&LexicalSyntax>, ) -> PyResult<(String, Vec)> { let py: Python<'py> = file_path.py(); - let (references, stop) = declaration_scan(sql); + let (references, stop) = declaration_scan(sql, syntax); if references.is_empty() && stop.is_none() { return Ok((sql.to_owned(), Vec::new())); } @@ -155,6 +170,7 @@ fn declarations_expanded<'py>( loaded_macros: Some(loaded_macros), origin: None, scopes: None, + syntax, }), })?; let spans: Vec = span_rows(&expanded.spliced); @@ -184,10 +200,12 @@ struct AuthoredSources<'a, 'py> { fn authored_expansion(sources: AuthoredSources<'_, '_>) -> PyResult { let py: Python<'_> = sources.sql.py(); + let syntax: Option = context_syntax(sources.macro_context)?; let (interpolated, interpolation_spans) = substituted( sources.sql, sources.file_path, (sources.effective_vars, sources.context_values), + syntax.as_ref(), )?; let (declared, declaration_spans) = declarations_expanded( &interpolated, @@ -197,6 +215,7 @@ fn authored_expansion(sources: AuthoredSources<'_, '_>) -> PyResult = ExpansionRequest { sql: PyString::new(py, &declared), @@ -368,7 +387,8 @@ fn substitute_sql_vars_with_spans<'py>( let py: Python<'py> = sql.py(); let context_values: Bound<'py, PyAny> = context_values.unwrap_or_else(|| py.None().into_bound(py)); - let (substituted_sql, spans) = substituted(sql, file_path, (effective_vars, &context_values))?; + let (substituted_sql, spans) = + substituted(sql, file_path, (effective_vars, &context_values), None)?; Ok((substituted_sql, span_objects(py, &spans)?)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs index 4b051769a3..652146bb70 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs @@ -18,6 +18,7 @@ use crate::bindings::_helpers::render::macro_call_rows::{ }; use crate::bindings::_helpers::render::macro_expansion::raised; use crate::bindings::types::CompilerDetach; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; const DECLARATION_CALLS: [&str; 2] = ["enum", "const"]; @@ -31,7 +32,7 @@ pub(crate) fn scan_macro_call_sites( sql: &str, ) -> PyResult<(Vec, Option)> { let MacroCallScan { sites, failure } = py - .compiler_detach(|| Ok(scan_sites(sql))) + .compiler_detach(|| Ok(scan_sites(sql, None))) .map_err(compiler_error)?; Ok(( sites.into_iter().map(site_row).collect(), @@ -49,9 +50,10 @@ pub(crate) fn scan_macro_call_sites( pub(crate) fn top_level_macro_call_names<'py>( py: Python<'py>, sql: &str, + syntax: Option<&LexicalSyntax>, ) -> PyResult> { let names = py - .compiler_detach(|| Ok(scan_names(sql, false))) + .compiler_detach(|| Ok(scan_names(sql, false, syntax))) .map_err(compiler_error)?; match names { Ok(names) => PyTuple::new( @@ -71,7 +73,7 @@ pub(crate) fn top_level_macro_call_names<'py>( /// Unique authored macro call names in encounter order, declaration calls excluded. #[pyfunction] fn find_macro_call_names<'py>(py: Python<'py>, sql: &str) -> PyResult> { - top_level_macro_call_names(py, sql) + top_level_macro_call_names(py, sql, None) } /// Splice rendered macro outputs over their call sites, returning the SQL and substitution spans. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs index cdae298365..7ae04b53df 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs @@ -226,8 +226,8 @@ fn final_sql(run: &Evaluation<'_, '_>, name: &str, value: &Bound<'_, PyAny>) -> return Err(PyErr::from_value(raised_error)); } let sql: String = value.extract()?; - let names: Vec = - scan_names(&sql, false).map_err(|error| raised(&classes.compile_error, error.message()))?; + let names: Vec = scan_names(&sql, false, expansion.syntax.as_ref()) + .map_err(|error| raised(&classes.compile_error, error.message()))?; if names.is_empty() { return Ok(sql); } @@ -256,7 +256,7 @@ fn call_arguments<'py>( return Ok((PyTuple::empty(py), PyDict::new(py))); } let text: CallText<'_> = CallText::new(args_source); - let scan: MacroCallScan = scan_sites(args_source); + let scan: MacroCallScan = scan_sites(args_source, run.expansion.syntax.as_ref()); let mut values: Vec> = Vec::with_capacity(scan.sites.len()); for site in &scan.sites { values.push(evaluate_call(run, (&text, site), false)?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs index 5065fec0ad..3a295db1eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs @@ -16,6 +16,7 @@ use sqlbuild_render::macro_calls::main::splice_macro_calls::splice_macro_calls a use sqlbuild_render::macro_calls::models::{ MacroCallEntry, MacroCallEvent, MacroCallScan, MacroCallSite, MacroScanFailure, MacroSpliceSpan, }; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::bindings::_helpers::records::construct::new_helped_error; use crate::bindings::_helpers::records::fields::{field as native_field, field_text}; @@ -31,9 +32,11 @@ use crate::bindings::_helpers::render::macro_expansion_facts::{ExpansionFacts, R use crate::bindings::_helpers::render::macro_generated_references::{ GeneratedOrigin, GeneratedReferenceClasses, report_generated_references, }; +use crate::bindings::_helpers::sqltext::lexical_syntax::optional_record_syntax; const RELATION_PREFIX: &str = "__sqlbuild_relation_"; const MEMO_ATTRIBUTE: &str = "memo"; +const LEXICAL_SYNTAX_ATTRIBUTE: &str = "_lexical_syntax"; const OBSERVE_MODULES_METHOD: &str = "observe_modules"; const RELATION_SUFFIX: &str = "__"; const CONSTANT_KIND: &str = "constant"; @@ -121,6 +124,8 @@ pub(crate) struct NativeMacroExpansion { enforce_explicit: bool, classes: Py, pub(crate) facts: RefCell, + /// The lexical rules the authored SQL is written in, from the macro context, if it has them. + pub(crate) syntax: Option, } positional_record! { @@ -151,7 +156,14 @@ impl NativeMacroExpansion { declarations, consumer, } = state; + let context: &Bound<'_, PyAny> = macro_context.bind(file_path.py()); + let syntax: Option = if context.is_none() { + None + } else { + optional_record_syntax(&native_field(context, LEXICAL_SYNTAX_ATTRIBUTE)?)? + }; Ok(Self { + syntax, file_path: file_path.clone().unbind(), file_label: file_path.str()?.to_string(), loaded_macros, @@ -728,7 +740,7 @@ pub(crate) fn expanded<'py>( classes: &classes, }; let text: CallText<'_> = CallText::new(sql); - let scan: MacroCallScan = scan_sites(text.text); + let scan: MacroCallScan = scan_sites(text.text, this.syntax.as_ref()); let memo: Option> = if bridge.is_none() { None } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs index bbeb5bc735..8b379d006b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs @@ -24,6 +24,7 @@ use sqlbuild_analysis::sql_macros::models::{ SqlMacroDefinition, SqlMacroLocation, TextContext, TextSplice, }; use sqlbuild_analysis::sql_macros::types::{ConstantShape, SqlMacroHost, SqlMacroScope}; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::bindings::_helpers::records::construct::{ enum_member, new_helped_error, new_record, new_record_with, @@ -66,6 +67,8 @@ pub(crate) struct SqlMacroSource<'a, 'py> { pub(crate) origin: Option>, /// Where macro bodies resolve their declarations; `None` outside the input loops. pub(crate) scopes: Option<&'a dyn MacroScopes<'py>>, + /// The lexical rules the authored SQL is written in; `None` reads it under the generic rules. + pub(crate) syntax: Option<&'a LexicalSyntax>, } /// The declaration contexts of macro files in a scoped compile. @@ -102,8 +105,9 @@ pub(crate) fn expanded_sql_macro_calls<'py>( starts: &[usize], ) -> PyResult<(Vec, SqlMacroFacts<'py>)> { let sql: &str = request.sql; + let syntax: Option<&LexicalSyntax> = request.sql_macros.and_then(|source| source.syntax); let calls: Vec = - sql_macro_calls(sql).map_err(|error| macro_error(request, &error))?; + sql_macro_calls(sql, syntax).map_err(|error| macro_error(request, &error))?; let by_char: HashMap = calls .iter() .map(|call| (sql[..call.start].chars().count(), call)) @@ -112,6 +116,7 @@ pub(crate) fn expanded_sql_macro_calls<'py>( loaded_macros: None, origin: None, scopes: None, + syntax: None, }); let host: PyHost<'_, '_, 'py> = PyHost { request, @@ -219,7 +224,12 @@ pub(crate) fn reject_bare_macro_calls(request: &DeclarationExpansion<'_, '_>) -> let Some(origin) = request.sql_macros.and_then(|source| source.origin) else { return Ok(()); }; - let Some((name, start)) = bare_macro_calls(request.sql).into_iter().next() else { + let Some((name, start)) = bare_macro_calls( + request.sql, + request.sql_macros.and_then(|source| source.syntax), + ) + .into_iter() + .next() else { return Ok(()); }; let error: SqlMacroError = SqlMacroError::new_public( @@ -418,6 +428,10 @@ impl<'py> PyHost<'_, '_, 'py> { impl<'py> SqlMacroHost for PyHost<'_, '_, 'py> { type Error = PyErr; + fn lexical_syntax(&self) -> Option<&LexicalSyntax> { + self.source.syntax + } + fn definition( &self, scope: &SqlMacroScope, @@ -475,7 +489,7 @@ impl<'py> SqlMacroHost for PyHost<'_, '_, 'py> { scope: &SqlMacroScope, text: &str, ) -> Result> { - let (references, stop) = declaration_scan(text); + let (references, stop) = declaration_scan(text, self.source.syntax); if references.is_empty() && stop.is_none() { return Ok(RenderedDeclarations { sql: text.to_owned(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs index 0f4c8df59c..0d0caf736a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs @@ -427,6 +427,10 @@ pub(crate) fn run_scenario_test_command( ("no_sql_validation", boolean(py, no_sql_validation)), ("source_deferral_enabled", boolean(py, false)), ("connection_config", run.connection_config.clone()), + ( + "source_lexical_syntax", + native_field(&run.project_adapter, "sql_lexical_syntax")?, + ), ], planning: &planning, }, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs index 29ad58f545..e990f03a98 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs @@ -7,6 +7,7 @@ use sqlbuild_sqltext::compiler::models::{ DeclarationReference, DeclarationReferenceKind, DeclarationReferenceScan, DeclarationReferenceStop, }; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; /// The reference kind of an outer SQL macro call. const SQL_MACRO_REFERENCE_KIND: u8 = 2; @@ -24,20 +25,20 @@ pub(crate) struct ScannedReference { type ScanRow = (Vec, Option); /// One SQL string's references and stop code, scanned without walking past malformed ones. -pub(crate) fn declaration_scan(sql: &str) -> ScanRow { +pub(crate) fn declaration_scan(sql: &str, syntax: Option<&LexicalSyntax>) -> ScanRow { let scanned: Vec = - scan_declaration_references(&[sql.to_owned()], false); + scan_declaration_references(&[sql.to_owned()], false, syntax); let row: ScanRow = scanned.into_iter().next().map(scan_row).unwrap_or_default(); if !sql.contains(SQL_CALL_PREFIX) { return row; } - with_sql_macro_calls(sql, row) + with_sql_macro_calls(sql, row, syntax) } /// The scan with each outer SQL macro call as one kind-2 reference, replacing those inside it. -fn with_sql_macro_calls(sql: &str, row: ScanRow) -> ScanRow { +fn with_sql_macro_calls(sql: &str, row: ScanRow, syntax: Option<&LexicalSyntax>) -> ScanRow { let (references, stop) = row; - let Ok(calls) = sql_macro_calls(sql) else { + let Ok(calls) = sql_macro_calls(sql, syntax) else { return (references, stop); }; if calls.is_empty() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/lexical_syntax.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/lexical_syntax.rs index d760f5cd3a..600dd863ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/lexical_syntax.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/lexical_syntax.rs @@ -1,8 +1,14 @@ //! One adapter's `SqlLexicalSyntax`, read from a Python mapping. use pyo3::FromPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyFrozenSet; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; +use crate::bindings::_helpers::records::construct::new_record_with; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::RecordId; + /// One adapter's `SqlLexicalSyntax`, read from a Python mapping. #[derive(Clone, FromPyObject)] #[pyo3(from_item_all)] @@ -27,3 +33,62 @@ impl From for LexicalSyntax { } } } + +/// The sorted texts of a set of strings. +pub(crate) fn sorted_texts(values: &Bound<'_, PyAny>) -> PyResult> { + let mut texts: Vec = values + .try_iter()? + .map(|value| value?.extract::()) + .collect::>()?; + texts.sort_unstable(); + Ok(texts) +} + +/// The lexical rules a `SqlLexicalSyntax` record holds. +pub(crate) fn record_syntax(instance: &Bound<'_, PyAny>) -> PyResult { + let flag = |name: &str| -> PyResult { native_field(instance, name)?.is_truthy() }; + Ok(LexicalSyntax { + backslash_escape_quotes: sorted_texts(&native_field(instance, "backslash_escape_quotes")?)?, + escape_string_prefix: flag("escape_string_prefix")?, + raw_string_prefix: flag("raw_string_prefix")?, + triple_quoted_strings: flag("triple_quoted_strings")?, + nested_block_comments: flag("nested_block_comments")?, + line_comment_prefixes: sorted_texts(&native_field(instance, "line_comment_prefixes")?)?, + }) +} + +/// The lexical rules a `SqlLexicalSyntax` record holds, or `None` for `None`. +pub(crate) fn optional_record_syntax(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + record_syntax(value).map(Some) +} + +/// A `SqlLexicalSyntax` record holding `syntax`. +pub(crate) fn syntax_record<'py>( + py: Python<'py>, + syntax: &LexicalSyntax, +) -> PyResult> { + let flag = |value: bool| -> PyResult> { + Ok(pyo3::types::PyBool::new(py, value).to_owned().into_any()) + }; + new_record_with( + py, + RecordId::SqlLexicalSyntax, + &[ + ( + "backslash_escape_quotes", + PyFrozenSet::new(py, &syntax.backslash_escape_quotes)?.into_any(), + ), + ("escape_string_prefix", flag(syntax.escape_string_prefix)?), + ("raw_string_prefix", flag(syntax.raw_string_prefix)?), + ("triple_quoted_strings", flag(syntax.triple_quoted_strings)?), + ("nested_block_comments", flag(syntax.nested_block_comments)?), + ( + "line_comment_prefixes", + PyFrozenSet::new(py, &syntax.line_comment_prefixes)?.into_any(), + ), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs index a60ec2f168..a85311b919 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs @@ -1,11 +1,12 @@ //! Interpolate `@@` tokens in authored SQL and extract static SQL references. +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use std::cell::RefCell; +use pyo3::PyErr; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyString, PyStringMethods}; -use pyo3::{FromPyObject, PyErr}; use sqlbuild_sqltext::compiler::main::sql_interpolation::interpolate_sql; use sqlbuild_sqltext::compiler::models::InterpolationRead; use sqlbuild_sqltext::compiler::types::InterpolationHost; @@ -24,13 +25,14 @@ pub(crate) struct InterpolationRow { pub(crate) error: Option, } -/// Project variables, the environment, context values (or `None`) and the variable renderer. -#[derive(FromPyObject)] +/// Variables, environment, context values (or `None`), variable renderer and lexical rules. pub(crate) struct InterpolationSources<'py>( pub(crate) Bound<'py, PyDict>, pub(crate) Bound<'py, PyAny>, pub(crate) Option>, pub(crate) Bound<'py, PyAny>, + /// The lexical rules the SQL is authored in; `None` reads it under the generic rules. + pub(crate) Option, ); struct PythonInterpolationHost<'py> { @@ -130,6 +132,10 @@ impl InterpolationHost for PythonInterpolationHost<'_> { } } + fn lexical_syntax(&self) -> Option<&LexicalSyntax> { + self.sources.4.as_ref() + } + fn context_names(&self) -> Vec { self.sources .2 diff --git a/crates/sqlbuild-render/Cargo.toml b/crates/sqlbuild-render/Cargo.toml index 74d7b19ee8..4ee1c4318c 100644 --- a/crates/sqlbuild-render/Cargo.toml +++ b/crates/sqlbuild-render/Cargo.toml @@ -10,6 +10,7 @@ rust-version.workspace = true [dependencies] sqlbuild-cache.workspace = true sqlbuild-core.workspace = true +sqlbuild-sqltext.workspace = true serde_json.workspace = true [dev-dependencies] diff --git a/crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs b/crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs index 9cbc71cc21..237f008f17 100644 --- a/crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs +++ b/crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs @@ -1,6 +1,8 @@ //! Byte offset port of the Python macro call scanner, with the errors Python's scan raises. use sqlbuild_core::text::main::is_letter::is_letter; +use sqlbuild_sqltext::sql_scan::main::dialect_non_code_end::dialect_non_code_end; +use sqlbuild_sqltext::sql_scan::models::{LexicalSyntax, Unclosed}; use crate::macro_calls::errors::ScanError; use crate::macro_calls::main::opens_bare_call::opens_bare_call; @@ -29,20 +31,30 @@ pub(crate) struct ScannedFailure { } /// Every top-level Python macro call of `text` in order, up to the first error the scan raises. -pub(crate) fn top_level_calls(text: &str) -> (Vec, Option) { - top_level_calls_of(text, false) +pub(crate) fn top_level_calls( + text: &str, + syntax: Option<&LexicalSyntax>, +) -> (Vec, Option) { + top_level_calls_of(text, false, syntax) } /// Every top-level Python or SQL macro call of `text`, as static name scans read them. -pub(crate) fn top_level_named_calls(text: &str) -> (Vec, Option) { - top_level_calls_of(text, true) +pub(crate) fn top_level_named_calls( + text: &str, + syntax: Option<&LexicalSyntax>, +) -> (Vec, Option) { + top_level_calls_of(text, true, syntax) } -fn top_level_calls_of(text: &str, with_sql: bool) -> (Vec, Option) { +fn top_level_calls_of( + text: &str, + with_sql: bool, + syntax: Option<&LexicalSyntax>, +) -> (Vec, Option) { let mut calls: Vec = Vec::new(); let mut cursor = 0; while cursor < text.len() { - let start = match next_macro_start(text, cursor) { + let start = match next_macro_start(text, cursor, syntax) { Ok(Some(start)) => start, Ok(None) => break, Err(error) => { @@ -53,7 +65,7 @@ fn top_level_calls_of(text: &str, with_sql: bool) -> (Vec, Option { cursor = call.close + 1; calls.push(call); @@ -126,11 +138,14 @@ pub(crate) fn call_name(text: &str, start: usize) -> String { } /// Every Python or SQL macro call start's name in order, nested calls included. -pub(crate) fn every_call_name(text: &str) -> Result, ScanError> { +pub(crate) fn every_call_name( + text: &str, + syntax: Option<&LexicalSyntax>, +) -> Result, ScanError> { let mut names: Vec = Vec::new(); let mut cursor = 0; while cursor < text.len() { - let Some(start) = next_macro_start(text, cursor)? else { + let Some(start) = next_macro_start(text, cursor, syntax)? else { break; }; if let Some(error) = kind_error(call_start_at(text, start), true) { @@ -146,18 +161,21 @@ pub(crate) fn every_call_name(text: &str) -> Result, ScanError> { } /// Names of every call nested in `args` in source order, in one pass without recursion. -pub(crate) fn nested_names(args: &str) -> Result, ScanError> { +pub(crate) fn nested_names( + args: &str, + syntax: Option<&LexicalSyntax>, +) -> Result, ScanError> { let bytes: &[u8] = args.as_bytes(); let mut names: Vec = Vec::new(); let mut open_call_depths: Vec = Vec::new(); let mut depth: usize = 0; let mut index: usize = 0; while index < bytes.len() { + if let Some(end) = non_code_end(bytes, index, syntax)? { + index = end; + continue; + } match bytes[index] { - b'\'' | b'"' | b'`' => index = quote_end(bytes, index)?, - b'$' => index = dollar_quote_end(bytes, index)?.unwrap_or(index + 1), - b'-' if bytes.get(index + 1) == Some(&b'-') => index = line_comment_end(bytes, index), - b'/' if bytes.get(index + 1) == Some(&b'*') => index = block_comment_end(bytes, index)?, b'@' if macro_call_starts_at(args, index) => { if let Some(error) = kind_error(call_start_at(args, index), false) { return Err(error); @@ -194,12 +212,17 @@ pub(crate) fn mentions_typed_reference(args: &str) -> bool { TYPED_REFERENCE_NAMES.iter().any(|name| args.contains(name)) } -fn call_at(text: &str, start: usize, with_sql: bool) -> Result { +fn call_at( + text: &str, + start: usize, + with_sql: bool, + syntax: Option<&LexicalSyntax>, +) -> Result { if let Some(error) = kind_error(call_start_at(text, start), with_sql) { return Err(error); } let (name, open) = call_header(text, start)?; - let close = matching_paren(text.as_bytes(), open)?; + let close = matching_paren(text.as_bytes(), open, syntax)?; Ok(ScannedCall { start, open, @@ -218,15 +241,19 @@ fn call_header(text: &str, start: usize) -> Result<(String, usize), ScanError> { Ok((text[from..name_end].to_owned(), open)) } -fn next_macro_start(text: &str, from: usize) -> Result, ScanError> { +fn next_macro_start( + text: &str, + from: usize, + syntax: Option<&LexicalSyntax>, +) -> Result, ScanError> { let bytes = text.as_bytes(); let mut index = from; while index < bytes.len() { + if let Some(end) = non_code_end(bytes, index, syntax)? { + index = end; + continue; + } match bytes[index] { - b'\'' | b'"' | b'`' => index = quote_end(bytes, index)?, - b'$' => index = dollar_quote_end(bytes, index)?.unwrap_or(index + 1), - b'-' if bytes.get(index + 1) == Some(&b'-') => index = line_comment_end(bytes, index), - b'/' if bytes.get(index + 1) == Some(&b'*') => index = block_comment_end(bytes, index)?, b'@' => { if macro_call_starts_at(text, index) { return Ok(Some(index)); @@ -292,29 +319,19 @@ fn char_at(text: &str, index: usize) -> Option { text.get(index..).and_then(|rest| rest.chars().next()) } -fn matching_paren(bytes: &[u8], open: usize) -> Result { +fn matching_paren( + bytes: &[u8], + open: usize, + syntax: Option<&LexicalSyntax>, +) -> Result { let mut depth = 1usize; let mut index = open + 1; while index < bytes.len() { + if let Some(end) = non_code_end(bytes, index, syntax)? { + index = end; + continue; + } match bytes[index] { - b'\'' | b'"' | b'`' => { - index = quote_end(bytes, index)?; - continue; - } - b'$' => { - if let Some(end) = dollar_quote_end(bytes, index)? { - index = end; - continue; - } - } - b'-' if bytes.get(index + 1) == Some(&b'-') => { - index = line_comment_end(bytes, index); - continue; - } - b'/' if bytes.get(index + 1) == Some(&b'*') => { - index = block_comment_end(bytes, index)?; - continue; - } b'(' => depth += 1, b')' => { depth -= 1; @@ -329,6 +346,28 @@ fn matching_paren(bytes: &[u8], open: usize) -> Result { Err(ScanError::UnclosedParenthesis) } +/// The end of the comment or quoted text at `index`, under `syntax` or the generic rules. +fn non_code_end( + bytes: &[u8], + index: usize, + syntax: Option<&LexicalSyntax>, +) -> Result, ScanError> { + if let Some(syntax) = syntax { + return dialect_non_code_end(bytes, index, syntax).map_err(|unclosed| match unclosed { + Unclosed::BlockComment => ScanError::UnclosedComment, + Unclosed::Quote => ScanError::UnclosedQuote, + Unclosed::Parenthesis => ScanError::UnclosedParenthesis, + }); + } + match bytes[index] { + b'\'' | b'"' | b'`' => quote_end(bytes, index).map(Some), + b'$' => dollar_quote_end(bytes, index), + b'-' if bytes.get(index + 1) == Some(&b'-') => Ok(Some(line_comment_end(bytes, index))), + b'/' if bytes.get(index + 1) == Some(&b'*') => block_comment_end(bytes, index).map(Some), + _ => Ok(None), + } +} + fn quote_end(bytes: &[u8], start: usize) -> Result { let quote = bytes[start]; let doubled_escapes = quote == b'\'' || quote == b'"'; diff --git a/crates/sqlbuild-render/src/macro_calls/main/macro_call_names.rs b/crates/sqlbuild-render/src/macro_calls/main/macro_call_names.rs index 50c65a9c27..d093294685 100644 --- a/crates/sqlbuild-render/src/macro_calls/main/macro_call_names.rs +++ b/crates/sqlbuild-render/src/macro_calls/main/macro_call_names.rs @@ -2,16 +2,21 @@ use crate::macro_calls::_helpers::scanner::{every_call_name, top_level_named_calls}; use crate::macro_calls::errors::ScanError; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; /// Python's `_scan_sqlbuild_call_names`: unique Python and SQL macro call names in encounter order. -pub fn macro_call_names(sql: &str, nested: bool) -> Result, ScanError> { +pub fn macro_call_names( + sql: &str, + nested: bool, + syntax: Option<&LexicalSyntax>, +) -> Result, ScanError> { if !sql.contains('@') { return Ok(Vec::new()); } let scanned: Vec = if nested { - every_call_name(sql)? + every_call_name(sql, syntax)? } else { - let (calls, failure) = top_level_named_calls(sql); + let (calls, failure) = top_level_named_calls(sql, syntax); if let Some(failure) = failure { return Err(failure.error); } diff --git a/crates/sqlbuild-render/src/macro_calls/main/scan_macro_call_sites.rs b/crates/sqlbuild-render/src/macro_calls/main/scan_macro_call_sites.rs index 1d5ae84f51..79292f1315 100644 --- a/crates/sqlbuild-render/src/macro_calls/main/scan_macro_call_sites.rs +++ b/crates/sqlbuild-render/src/macro_calls/main/scan_macro_call_sites.rs @@ -5,10 +5,11 @@ use crate::macro_calls::_helpers::scanner::{ call_name, mentions_typed_reference, nested_names, top_level_calls, }; use crate::macro_calls::models::{MacroCallScan, MacroCallSite, MacroScanFailure}; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; /// Every complete top-level call site in order, and where Python's scan raises, if it does. -pub fn scan_macro_call_sites(sql: &str) -> MacroCallScan { - let (calls, failure) = top_level_calls(sql); +pub fn scan_macro_call_sites(sql: &str, syntax: Option<&LexicalSyntax>) -> MacroCallScan { + let (calls, failure) = top_level_calls(sql, syntax); let mut boundaries: Vec = calls .iter() .flat_map(|call| [call.start, call.open + 1, call.close + 1]) @@ -21,7 +22,7 @@ pub fn scan_macro_call_sites(sql: &str) -> MacroCallScan { .zip(offsets.as_chunks::<3>().0) .map(|(call, offsets)| { let args = &sql[call.open + 1..call.close]; - let tree_names: Option> = call_tree_names(&call.name, args); + let tree_names: Option> = call_tree_names(&call.name, args, syntax); MacroCallSite { start: offsets[0], args_start: offsets[1], @@ -46,8 +47,8 @@ fn failed_call_name(sql: &str, start: Option) -> Option { } /// The call's name, then each nested call's once; None where a nested scan raises. -fn call_tree_names(name: &str, args: &str) -> Option> { - let nested: Vec = match nested_names(args) { +fn call_tree_names(name: &str, args: &str, syntax: Option<&LexicalSyntax>) -> Option> { + let nested: Vec = match nested_names(args, syntax) { Ok(nested) => nested, Err(_scan_error) => return None, }; diff --git a/crates/sqlbuild-render/src/macro_calls/tests/test_macro_call_names.rs b/crates/sqlbuild-render/src/macro_calls/tests/test_macro_call_names.rs index 522c08f0af..4910585c87 100644 --- a/crates/sqlbuild-render/src/macro_calls/tests/test_macro_call_names.rs +++ b/crates/sqlbuild-render/src/macro_calls/tests/test_macro_call_names.rs @@ -64,8 +64,8 @@ fn given_sql_when_listing_macro_call_names_then_python_scan_agrees() { for test_case in test_cases { assert_eq!( ( - macro_call_names(test_case.sql, false), - macro_call_names(test_case.sql, true) + macro_call_names(test_case.sql, false, None), + macro_call_names(test_case.sql, true, None) ), ( owned_names(test_case.expected_top_level), diff --git a/crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs b/crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs index 35f1d718bc..431e0127db 100644 --- a/crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs +++ b/crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs @@ -1,5 +1,6 @@ use crate::macro_calls::errors::ScanError; use crate::macro_calls::main::scan_macro_call_sites::scan_macro_call_sites; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use std::time::Instant; use crate::macro_calls::tests::test_types::{ @@ -170,7 +171,7 @@ fn given_authored_sql_when_scanning_macro_calls_then_sites_match_python_expansio ]; for test_case in test_cases { - let scan = scan_macro_call_sites(test_case.sql); + let scan = scan_macro_call_sites(test_case.sql, None); let actual: Vec = scan .sites .into_iter() @@ -238,7 +239,7 @@ fn given_deeply_nested_calls_when_scanning_then_time_is_linear_and_nothing_overf ); let started = Instant::now(); - let scan = scan_macro_call_sites(&sql); + let scan = scan_macro_call_sites(&sql, None); assert!( started.elapsed().as_secs_f64() < test_case.expected_max_seconds, @@ -298,7 +299,7 @@ fn given_calls_when_scanning_then_argument_starts_and_failing_call_names_match_p }, ]; for test_case in test_cases { - let scan = scan_macro_call_sites(test_case.sql); + let scan = scan_macro_call_sites(test_case.sql, None); assert_eq!( ( @@ -322,3 +323,59 @@ fn given_calls_when_scanning_then_argument_starts_and_failing_call_names_match_p ); } } + +#[test] +fn given_backslash_escapes_when_scanning_macro_calls_then_dialect_quotes_are_text() { + let syntax = LexicalSyntax { + backslash_escape_quotes: vec!["'".to_owned()], + line_comment_prefixes: vec!["--".to_owned(), "//".to_owned()], + ..LexicalSyntax::default() + }; + let test_cases = [ + ScanMacroCallSitesTestCase { + description: "an escaped quote before a call stays inside its string", + sql: "'O\\'Brien @py.hidden()' @py.m('a\\')')", + expected_sites: &[(24, 37, "m", Some(&["m"]), false)], + expected_failure: None, + }, + ScanMacroCallSitesTestCase { + description: "a backslash at the end of a literal escapes itself", + sql: "'C:\\\\' @py.m(')')", + expected_sites: &[(7, 17, "m", Some(&["m"]), false)], + expected_failure: None, + }, + ScanMacroCallSitesTestCase { + description: "a dialect line comment hides a call", + sql: "// it's @py.hidden()\n@py.m()", + expected_sites: &[(21, 28, "m", Some(&["m"]), false)], + expected_failure: None, + }, + ]; + + for test_case in test_cases { + let scan = scan_macro_call_sites(test_case.sql, Some(&syntax)); + let actual: Vec<(usize, usize, String)> = scan + .sites + .into_iter() + .map(|site| (site.start, site.end, site.name)) + .collect(); + + assert_eq!( + ( + actual, + scan.failure + .map(|failure| (failure.call_start, failure.error)) + ), + ( + test_case + .expected_sites + .iter() + .map(|(start, end, name, _, _)| (*start, *end, (*name).to_owned())) + .collect::>(), + test_case.expected_failure + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-scopes/src/relationship_names/_helpers/facts.rs b/crates/sqlbuild-scopes/src/relationship_names/_helpers/facts.rs index c19845b8c6..ab217b7a3f 100644 --- a/crates/sqlbuild-scopes/src/relationship_names/_helpers/facts.rs +++ b/crates/sqlbuild-scopes/src/relationship_names/_helpers/facts.rs @@ -42,7 +42,7 @@ pub(crate) fn block_fact( syntax: &LexicalSyntax, ) -> Result { let expected_models: Vec = names?; - let called_macros: Vec = call_names(block.sql, true)?; + let called_macros: Vec = call_names(block.sql, true, syntax)?; let tested_macros: Vec = if block.mode == MACRO_MODE { let ctes: Vec<(String, String)> = match scan_top_level_ctes(block.sql, relative_path, RelationshipSource::Test, syntax) { @@ -50,7 +50,7 @@ pub(crate) fn block_fact( TopLevelCtes::Failed(message) => return Err(message), }; match ctes.iter().find(|(name, _)| name == MACRO_ACTUAL_CTE) { - Some((_, body)) => call_names(body, false)?, + Some((_, body)) => call_names(body, false, syntax)?, None => Vec::new(), } } else { @@ -68,9 +68,9 @@ pub(crate) fn block_fact( } /// `find_macro_call_names` or `find_nested_macro_call_names`, or the scan's message. -fn call_names(sql: &str, nested: bool) -> Result, String> { +fn call_names(sql: &str, nested: bool, syntax: &LexicalSyntax) -> Result, String> { let names: Vec = - macro_call_names(sql, nested).map_err(|error| error.message().to_owned())?; + macro_call_names(sql, nested, Some(syntax)).map_err(|error| error.message().to_owned())?; Ok(names .into_iter() .filter(|name| !DECLARATION_CALLS.contains(&name.as_str())) diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/scanner.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/scanner.rs index 39ea4b3718..49c341c48f 100644 --- a/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/scanner.rs +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/scanner.rs @@ -7,6 +7,8 @@ use crate::compiler::models::{ DeclarationReference, DeclarationReferenceKind, DeclarationReferenceScan, DeclarationReferenceStop, }; +use crate::sql_scan::main::dialect_non_code_end::dialect_non_code_end; +use crate::sql_scan::models::{LexicalSyntax, Unclosed}; const SPECIAL_BYTES: &[u8] = b"'\"`$@/-"; const ENUM_KEYWORD: &str = "@enum"; @@ -22,13 +24,21 @@ enum NextStart<'sql> { } /// Python's references in `sql` (code points) and stop; `skip_malformed` resumes past malformed. -pub(crate) fn scan_references(sql: &str, skip_malformed: bool) -> DeclarationReferenceScan { +pub(crate) fn scan_references( + sql: &str, + skip_malformed: bool, + syntax: Option<&LexicalSyntax>, +) -> DeclarationReferenceScan { let mut references: Vec = Vec::new(); let mut offsets: CharOffsets = CharOffsets::default(); let mut cursor: usize = 0; let mut stop: Option = None; while cursor < sql.len() { - let (start, matched) = match next_reference_start(sql, cursor) { + let next: NextStart<'_> = match syntax { + Some(syntax) => next_dialect_reference_start(sql, cursor, syntax), + None => next_reference_start(sql, cursor), + }; + let (start, matched) = match next { NextStart::Reference(start, matched) => (start, matched), NextStart::End => break, NextStart::Malformed(_, keyword_end) if skip_malformed => { @@ -102,6 +112,43 @@ fn next_reference_start(sql: &str, start: usize) -> NextStart<'_> { NextStart::End } +/// The next reference start at or after `start`, skipping text the dialect reads as non-code. +fn next_dialect_reference_start<'sql>( + sql: &'sql str, + start: usize, + syntax: &LexicalSyntax, +) -> NextStart<'sql> { + let rest: &str = &sql[start..]; + if !rest.contains("@enum") && !rest.contains("@const") { + return NextStart::End; + } + let bytes: &[u8] = sql.as_bytes(); + let mut index: usize = start; + while index < bytes.len() { + if bytes[index] == b'@' { + match match_reference(sql, index) { + ReferenceSyntax::NotReference => index += 1, + ReferenceSyntax::Matched(matched) => return NextStart::Reference(index, matched), + ReferenceSyntax::Malformed(kind) => { + return NextStart::Malformed(kind, index + keyword_length(kind)); + } + } + continue; + } + index = match dialect_non_code_end(bytes, index, syntax) { + Ok(Some(end)) => end, + Ok(None) => index + 1, + Err(Unclosed::BlockComment) => { + return NextStart::Stop(DeclarationReferenceStop::UnclosedBlockComment); + } + Err(Unclosed::Quote | Unclosed::Parenthesis) => { + return NextStart::Stop(DeclarationReferenceStop::UnclosedQuote); + } + }; + } + NextStart::End +} + fn keyword_length(kind: DeclarationReferenceKind) -> usize { match kind { DeclarationReferenceKind::Enum => ENUM_KEYWORD.len(), diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs index 6d583b0502..827216b2b5 100644 --- a/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs @@ -3,6 +3,7 @@ use crate::compiler::models::CharSpan; use crate::compiler::models::{InterpolatedSql, InterpolationFailure, InterpolationRead}; use crate::compiler::types::InterpolationHost; +use crate::sql_scan::main::dialect_non_code_end::dialect_non_code_end; use crate::sql_scan::main::non_code_end::non_code_end; use crate::sql_scan::models::{QuotePolicy, Unclosed}; use sqlbuild_core::text::main::is_letter::is_letter; @@ -70,7 +71,11 @@ impl Interpolation<'_, H> { let bytes = sql.as_bytes(); let mut index = 0; while index < bytes.len() { - let text_end = match python_non_code_end(bytes, index) { + let non_code = match self.host.lexical_syntax() { + Some(syntax) => dialect_non_code_end(bytes, index, syntax), + None => python_non_code_end(bytes, index), + }; + let text_end = match non_code { Ok(Some(end)) => end, Ok(None) => index, Err(Unclosed::BlockComment) => { @@ -81,8 +86,7 @@ impl Interpolation<'_, H> { } }; if text_end > index { - let comment = - bytes[index..].starts_with(b"--") || bytes[index..].starts_with(b"/*"); + let comment = !matches!(bytes[index], b'\'' | b'"' | b'`' | b'$'); if comment { self.output.push_str(&sql[index..text_end]); } else { diff --git a/crates/sqlbuild-sqltext/src/compiler/main/declaration_references.rs b/crates/sqlbuild-sqltext/src/compiler/main/declaration_references.rs index 635eddf148..52da0c1a83 100644 --- a/crates/sqlbuild-sqltext/src/compiler/main/declaration_references.rs +++ b/crates/sqlbuild-sqltext/src/compiler/main/declaration_references.rs @@ -4,14 +4,16 @@ use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; use crate::compiler::_helpers::declaration_references::scanner::scan_references; use crate::compiler::models::DeclarationReferenceScan; +use crate::sql_scan::models::LexicalSyntax; -/// Each SQL string's references and stopping error; `skip_malformed` walks past malformed starts. +/// Each SQL string's references and stop; `syntax` decides where quoted text and comments end. #[must_use] pub fn scan_declaration_references( sqls: &[String], skip_malformed: bool, + syntax: Option<&LexicalSyntax>, ) -> Vec { sqls.par_iter() - .map(|sql| scan_references(sql, skip_malformed)) + .map(|sql| scan_references(sql, skip_malformed, syntax)) .collect() } diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs index 660f9d5fb6..36d9296d69 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs @@ -10,6 +10,7 @@ use crate::compiler::models::{ InterpolationRead, }; use crate::compiler::tests::test_types::{HeaderNestingTestCase, MapHost}; +use crate::sql_scan::models::LexicalSyntax; use std::collections::BTreeMap; use std::thread; @@ -139,17 +140,22 @@ pub(crate) fn batch_sizes_bound_workers_by_contract() -> bool { /// Spell a scan as `kind:name[.member]@start..end, ...` plus ` | stop:` when it stops. pub(crate) fn scanned_references(sql: &str) -> Option { - spelled_scan(sql, false) + spelled_scan(sql, false, None) +} + +/// The same spelling of a scan under an adapter's lexical rules. +pub(crate) fn scanned_dialect_references(sql: &str, syntax: &LexicalSyntax) -> Option { + spelled_scan(sql, false, Some(syntax)) } /// The same spelling of the usage walk, which steps past malformed references. pub(crate) fn scanned_usages(sql: &str) -> Option { - spelled_scan(sql, true) + spelled_scan(sql, true, None) } -fn spelled_scan(sql: &str, skip_malformed: bool) -> Option { +fn spelled_scan(sql: &str, skip_malformed: bool, syntax: Option<&LexicalSyntax>) -> Option { let scan: DeclarationReferenceScan = - scan_declaration_references(&[sql.to_owned()], skip_malformed).pop()?; + scan_declaration_references(&[sql.to_owned()], skip_malformed, syntax).pop()?; let references: String = scan .references .iter() diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/test_declaration_references.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_declaration_references.rs index c550aef3bd..326185e1a8 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/test_declaration_references.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/test_declaration_references.rs @@ -1,5 +1,8 @@ -use crate::compiler::tests::helpers::{scanned_references, scanned_usages}; +use crate::compiler::tests::helpers::{ + scanned_dialect_references, scanned_references, scanned_usages, +}; use crate::compiler::tests::test_types::DeclarationReferenceScanTestCase; +use crate::sql_scan::models::LexicalSyntax; #[test] fn given_sql_when_scanning_declaration_references_then_python_offsets_are_returned() { @@ -208,3 +211,40 @@ fn given_sql_when_walking_declaration_usages_then_malformed_references_are_skipp ); } } + +#[test] +fn given_backslash_escapes_when_scanning_declaration_references_then_dialect_quotes_are_text() { + let syntax = LexicalSyntax { + backslash_escape_quotes: vec!["'".to_owned()], + line_comment_prefixes: vec!["--".to_owned(), "//".to_owned()], + ..LexicalSyntax::default() + }; + let test_cases = [ + DeclarationReferenceScanTestCase { + description: "an escaped quote stays inside the string", + sql: "SELECT 'O\\'Brien @const(\"a\")', @const(\"b\")", + expected_references: Some("const:b@31..42"), + }, + DeclarationReferenceScanTestCase { + description: "a backslash at the end of a literal escapes itself", + sql: "SELECT 'C:\\\\', @const(\"b\")", + expected_references: Some("const:b@15..26"), + }, + DeclarationReferenceScanTestCase { + description: "a dialect line comment is text", + sql: "SELECT 1 // it's @const(\"a\")\n, @const(\"b\")", + expected_references: Some("const:b@31..42"), + }, + ]; + + for test_case in test_cases { + let scanned = scanned_dialect_references(test_case.sql, &syntax); + + assert_eq!( + scanned.as_deref(), + test_case.expected_references, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-sqltext/src/compiler/types.rs b/crates/sqlbuild-sqltext/src/compiler/types.rs index 95b3a0bf9d..09f1a3ce5a 100644 --- a/crates/sqlbuild-sqltext/src/compiler/types.rs +++ b/crates/sqlbuild-sqltext/src/compiler/types.rs @@ -1,3 +1,5 @@ +use crate::sql_scan::models::LexicalSyntax; + /// The project variables, process environment and context values one interpolation reads. pub trait InterpolationHost { /// The text of project variable `name`, `Err(message)` when it cannot be text, or `None`. @@ -12,4 +14,8 @@ pub trait InterpolationHost { fn context(&self, name: &str) -> Option>; /// Every known context key. fn context_names(&self) -> Vec; + /// The lexical rules the SQL is authored in; `None` reads it under the generic compiler rules. + fn lexical_syntax(&self) -> Option<&LexicalSyntax> { + None + } } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 2bd38fd736..2b8218d67a 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -10019,6 +10019,7 @@ class CompilePipelineOptions(NativeRecord): record_migration_fingerprints: bool accepts_unit_test_selectors: bool sql_test_selection: SqlTestSelection + source_lexical_syntax: SqlLexicalSyntax | None def __init__( self, selected_target: str | None = ..., @@ -10044,6 +10045,7 @@ class CompilePipelineOptions(NativeRecord): record_migration_fingerprints: bool = ..., accepts_unit_test_selectors: bool = ..., sql_test_selection: SqlTestSelection = ..., + source_lexical_syntax: SqlLexicalSyntax | None = ..., ) -> None: ... class PythonPlanEntry(NativeRecord): @@ -15162,6 +15164,7 @@ class MacroContext(NativeRecord): _collection_rendering: CollectionRendering _constant_declarations: Mapping[str, ConstantDeclaration] _enforce_explicit_references: bool + _lexical_syntax: SqlLexicalSyntax | None def __init__( self, adapter_name: str, @@ -15174,6 +15177,7 @@ class MacroContext(NativeRecord): _collection_rendering: CollectionRendering = ..., _constant_declarations: Mapping[str, ConstantDeclaration] = ..., _enforce_explicit_references: bool = ..., + _lexical_syntax: SqlLexicalSyntax | None = ..., ) -> None: ... def render_constant(self, name: str) -> str: ... def render_enum_member(self, *, enum_name: str, member_name: str) -> str: ... diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py index fb52762696..2972c895dc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py @@ -354,6 +354,16 @@ class CapturedVariantTextScenarioTestCase: expected_stdout_fragment: str +@dataclass(frozen=True) +class EscapedQuoteScenarioTestCase: + """A Snowflake project whose SQL holds backslash escapes before compile markers.""" + + description: str + scenario_name: str + expected_compiled_fragments: tuple[str, ...] + expected_stdout_fragment: str + + @dataclass(frozen=True) class CapturedLambdaScenarioTestCase: """A Snowflake-captured scenario whose model uses an arrow lambda, replayed in DuckDB.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_escaped_quotes.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_escaped_quotes.py new file mode 100644 index 0000000000..cd1c42e1c4 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_escaped_quotes.py @@ -0,0 +1,121 @@ +"""E2E coverage that Snowflake backslash escapes hide nothing from compile-time scanners.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.scenario._test_types import ( + EscapedQuoteScenarioTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.scenario.helpers import ( + SNOWFLAKE_ORDER_IDS_FIXTURE_SQL, + build_offline_snowflake_project_toml, + write_snowflake_order_ids_snapshot, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_NOTE_SQL: str = '\'it\\\'s __ref("ghost") @py.ghost() @sql.ghost(1) @const("ghost")\'' +_EXPECTED_ROW_SQL: str = ( + "SELECT 1 AS order_id, 'O\\'Brien' AS customer_name, 'EU' AS region, " + "'2025-01-01' AS start_date, 'web' AS channel, 'A\\'B' AS shouted, 'tag' AS tag, " + f"'C:\\\\ducks\\\\' AS folder, {_NOTE_SQL} AS note" +) +_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": build_offline_snowflake_project_toml() + '\n[vars]\nregion = "EU"\n', + "sources/raw.yml": ( + "sources:\n - name: raw_orders\n description: Raw orders.\n" + " schema: main\n table: raw_orders\n columns:\n" + ' - {name: order_id, type: "NUMBER(38,0)"}\n' + ), + "models/_constants/start_date.sql": 'CONSTANT (name start_date, value "2025-01-01");\n', + "models/_enums/channel.sql": 'ENUM (name channel, members (WEB "web", APP "app"));\n', + "models/_macros/sql/shout.sql": ( + 'MACRO (description "Upper case.", arguments (v VARCHAR), returns VARCHAR);\n\nUPPER(@v)\n' + ), + "models/_macros/python/tag.py": ( + '"""Order tag macros."""\n\n\n' + "def tag() -> str:\n" + ' """Return the order tag literal."""\n' + " return \"'tag'\"\n" + ), + "models/order_notes.sql": ( + 'MODEL (description "Order notes holding backslash escapes.");\n' + "SELECT\n" + " o.order_id,\n" + " 'O\\'Brien' AS customer_name,\n" + " '@@region' AS region,\n" + ' @const("start_date") AS start_date,\n' + ' @enum("channel").WEB AS channel,\n' + " @sql.shout('a\\'b') AS shouted,\n" + " @py.tag() AS tag,\n" + " 'C:\\\\ducks\\\\' AS folder, // it's a path\n" + f" {_NOTE_SQL} AS note\n" + 'FROM __source("raw_orders") AS o\n' + ), + "tests/unit/test_order_notes.sql": ( + 'TEST (name "order_notes__keeps_escaped_quotes");\n\n' + "WITH __source__raw_orders AS (SELECT 1 AS order_id, 'it\\'s' AS unused),\n\n" + f"__expected__order_notes AS ({_EXPECTED_ROW_SQL})\n" + ), + "tests/scenarios/order_notes__escapes.sql": ( + "SCENARIO (description 'Escaped quotes replay locally.');\n" + "WITH\n" + f"__source__raw_orders AS ({SNOWFLAKE_ORDER_IDS_FIXTURE_SQL}),\n" + f"__expected__order_notes AS ({_EXPECTED_ROW_SQL})\n" + "SELECT 1\n" + ), +} + + +@pytest.mark.parametrize( + "test_case", + [ + EscapedQuoteScenarioTestCase( + description="escaped quotes and trailing backslashes before every compile marker", + scenario_name="order_notes__escapes", + expected_compiled_fragments=( + "'2025-01-01' AS start_date", + "'web' AS channel", + "UPPER('a\\'b') AS shouted", + "'tag' AS tag", + "'EU' AS region", + _NOTE_SQL, + ), + expected_stdout_fragment="PASS=1 FAIL=0 ERROR=0 SKIP=0 TOTAL=1", + ), + ], + ids=lambda case: case.description, +) +def test_given_snowflake_backslash_escapes_when_compiling_and_replaying_locally_then_passes( + test_case: EscapedQuoteScenarioTestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="escaped_quotes", repo_files=_PROJECT_FILES + ) + write_snowflake_order_ids_snapshot( + project_dir=project_dir, scenario_name=test_case.scenario_name + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), project_dir=project_dir + ) + replayed: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "scenario", "test", "--local", "--strict", test_case.scenario_name), + project_dir=project_dir, + ) + + assert compiled.returncode == 0, compiled.stdout + compiled.stderr + compiled_sql: str = ( + project_dir / "target" / "compiled" / "models" / "order_notes.sql" + ).read_text() + for fragment in test_case.expected_compiled_fragments: + assert fragment in compiled_sql, compiled_sql + assert replayed.returncode == 0, replayed.stdout + replayed.stderr + assert test_case.expected_stdout_fragment in replayed.stdout, replayed.stdout + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_dialect_string_escapes.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_dialect_string_escapes.py index 875899670b..db31ac7f17 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_dialect_string_escapes.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_dialect_string_escapes.py @@ -18,7 +18,7 @@ "models/stg_orders.sql": "MODEL (description 'Test model stg_orders.', materialized table);\n\nSELECT 1 AS order_id\n", "models/order_names.sql": ( "MODEL (description 'Test model order_names.', materialized table);\n\n" - "SELECT E'O\\'Brien' AS customer_name, order_id\n" + "SELECT E'O\\'Brien' AS customer_name, 'C:\\' AS folder, order_id\n" 'FROM __ref("stg_orders")\n' ), "tests/unit/test_order_names.sql": ( @@ -39,7 +39,7 @@ "test_case", ( SqlTestE2ETestCase( - description="escape strings before model references and test markers", + description="escape strings and a plain trailing backslash before references and markers", expected_exit_code=0, expected_stdout_fragment="PASS=1 FAIL=0", ), From 2b9da1eab7b4d24ca102ee4a60c132f7d41922b3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:57:37 +0000 Subject: [PATCH 02/20] docs(benchmarks): replay DuckShop scenarios locally in the project itself Local replay now compiles the edge-case area, so the runbook replays the core scenarios in place and the README drops the escaped-quote limitation. --- benchmarks/duckshop/README.md | 5 +---- benchmarks/duckshop/RUNBOOK.md | 7 ++----- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/benchmarks/duckshop/README.md b/benchmarks/duckshop/README.md index f3166cfac9..51073c3fcf 100644 --- a/benchmarks/duckshop/README.md +++ b/benchmarks/duckshop/README.md @@ -156,7 +156,7 @@ Last verified on Snowflake with SQLBuild 0.175.0 at `data_scale = "1"`: | `sqb test` | 266 of 266 pass | | `sqb scenario test` | 8 of 8 pass | | `sqb scenario capture` | Captures all 8; snapshots are in `project/tests/_scenario_snapshots/` | -| `sqb scenario test --local --strict` | In the core copy, 0 of 8 pass: 5 fail in translation, 3 report a stale snapshot | +| `sqb scenario test --local --strict --exclude edge_cases` | 0 of 8 pass: 5 fail in translation, 3 report a stale snapshot | Local replay translates each scenario's Snowflake SQL to DuckDB on a best-effort basis. These five failures are the baseline that better translation should remove: @@ -219,9 +219,6 @@ noted here: header cannot declare a quoted column name. `stg_edge_partner_listings` has a reviewed `SQBRTEST202` ignore, `rpt_edge_partner_listing_export` a reviewed `SQBRTEST201` ignore, and the edge build excludes `stg_edge_partner_listings+`. -- Local scenario replay stops at compile (`P001`, unclosed quoted string) on a model with a - backslash-escaped quote in a string literal, `'O\'Brien'` in `stg_edge_ticket_notes`, so local - replay runs in the core copy. - `SQBRSQL018` can only take proof from single-column keys. Append-only feeds redeliver rows, so their id is unique only together with `_loaded_at` (the `unique_per_delivery` audit), and the staging models that keep the latest delivery carry reviewed exceptions. diff --git a/benchmarks/duckshop/RUNBOOK.md b/benchmarks/duckshop/RUNBOOK.md index 6cd3759b96..45c53584ce 100644 --- a/benchmarks/duckshop/RUNBOOK.md +++ b/benchmarks/duckshop/RUNBOOK.md @@ -133,16 +133,13 @@ on Snowflake. Report any failure rather than editing the expectations to match. ```bash sqb scenario capture --exclude edge_cases -python ../scripts/core_project.py /tmp/duckshop-core -(cd /tmp/duckshop-core && sqb scenario test --local --strict) +sqb scenario test --local --strict --exclude edge_cases ``` Capture materialises each scenario's fixtures in Snowflake, downloads them as JSONL into `tests/_scenario_snapshots/`, and drops what it created. The local run then replays all 8 scenarios on your machine from those files, with no warehouse connection; `--strict` fails on a -missing or stale snapshot instead of skipping it. It runs in a copy of the project without the -edge-case area, because local replay of the full project stops at compile on an edge-case model -(see Known limitations in the README). Delete the copy afterwards. +missing or stale snapshot instead of skipping it. With SQLBuild 0.175.0 the local run is expected to end with `ERROR=8`: five scenarios fail while translating Snowflake SQL to DuckDB, and three report `X602` (stale snapshot) because of a From 05205c3d0eb1a34da60ce0e0b67f78598c12b29a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:58:11 +0000 Subject: [PATCH 03/20] test(benchmarks): call a DuckShop edge macro after an escaped quote stg_edge_ticket_notes now normalises its SKU with a SQL macro after the backslash-escaped quote, so the edge area covers a compile marker that a scanner reading generic quote rules would hide. --- .../models/staging/edge_cases/stg_edge_ticket_notes.sql | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_ticket_notes.sql b/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_ticket_notes.sql index d187342865..6fac313799 100644 --- a/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_ticket_notes.sql +++ b/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_ticket_notes.sql @@ -1,8 +1,9 @@ MODEL ( description """ One row per support ticket note with order references and reasons found by regular expressions, - written with escaped and dollar-quoted patterns, campaign and discount codes matched by pattern - lists and escapes, and SKUs split at edge positions + written with escaped and dollar-quoted patterns, a SKU normalised by a macro after an escaped + quote, campaign and discount codes matched by pattern lists and escapes, and SKUs split at edge + positions """, tags [edge_case], columns ( @@ -16,6 +17,7 @@ MODEL ( refund_word (type VARCHAR), body_collapsed (type VARCHAR), quotes_customer (type BOOLEAN), + sku_canonical (type VARCHAR), second_hash_tag (type VARCHAR), is_newsletter_visit (type BOOLEAN), is_customer_code (type BOOLEAN), @@ -48,8 +50,9 @@ final AS ( REGEXP_COUNT(s.body, 'DS-O-\\d+')::INT AS order_ref_count, REGEXP_SUBSTR(s.body, 'refund(ed)?', 1, 1, 'i')::VARCHAR AS refund_word, REGEXP_REPLACE(s.body, '\\s+', ' ')::VARCHAR AS body_collapsed, - // A backslash-escaped quote inside a single-quoted literal + // A backslash-escaped quote inside a single-quoted literal, then a macro call to expand CONTAINS(s.body, 'O\'Brien')::BOOLEAN AS quotes_customer, + @sql.normalise_sku(s.sku) AS sku_canonical, SPLIT_PART(s.tags_text, '#', 2)::VARCHAR AS second_hash_tag, ( s.utm_source ILIKE ANY ('%mail%', 'news%') From ca7f919360de955725aaf0151f8ab95d10d52dcb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:01:25 +0000 Subject: [PATCH 04/20] fix(benchmarks): make three DuckShop core models deterministic int_cart_funnel sorts its distinct added SKUs, int_delivery_slot_utilisation takes the weekday average as an exact decimal sum over a count and the earliest of equally busy slots, and rpt_delivery_exceptions_monthly breaks ties for the top exception code alphabetically instead of using MODE. The core-copy compile guard records the new checksum and the README notes the change for benchmark comparisons. --- benchmarks/duckshop/README.md | 8 +++ .../clickstream/int_cart_funnel.sql | 14 ++-- .../int_delivery_slot_utilisation.sql | 29 ++++++-- .../rpt_delivery_exceptions_monthly.sql | 71 +++++++++++++++++-- .../test_duckshop_benchmark_compile.py | 2 +- 5 files changed, 109 insertions(+), 15 deletions(-) diff --git a/benchmarks/duckshop/README.md b/benchmarks/duckshop/README.md index 51073c3fcf..956a858bda 100644 --- a/benchmarks/duckshop/README.md +++ b/benchmarks/duckshop/README.md @@ -145,6 +145,14 @@ Benchmarks and core measurements leave the area out: with `--exclude edge_cases`; - the construct census counts the core project unless given `--area edge` or `--area all`. +Changes to the core project's compiled SQL, after which benchmark results are not comparable with +earlier ones: + +- `int_cart_funnel`, `int_delivery_slot_utilisation` and `rpt_delivery_exceptions_monthly` give the + same results on every run: added SKUs are sorted, the weekday average utilisation is an exact + decimal sum over a count, the busiest slot of equally busy ones is the earliest, and the top + exception code of equally frequent ones is the first in alphabetical order. + ## Status Last verified on Snowflake with SQLBuild 0.175.0 at `data_scale = "1"`: diff --git a/benchmarks/duckshop/project/models/intermediate/clickstream/int_cart_funnel.sql b/benchmarks/duckshop/project/models/intermediate/clickstream/int_cart_funnel.sql index df1ff3cef5..c021646d33 100644 --- a/benchmarks/duckshop/project/models/intermediate/clickstream/int_cart_funnel.sql +++ b/benchmarks/duckshop/project/models/intermediate/clickstream/int_cart_funnel.sql @@ -1,7 +1,8 @@ MODEL ( description """ One row per web session with its cart activity worked out from event item arrays using FILTER - and TRANSFORM lambdas: items added, removed, net units, and the checkout steps reached. + and TRANSFORM lambdas: items added (distinct SKUs in sorted order), removed, net units, and the + checkout steps reached. """, columns ( session_id (type VARCHAR, nullable false, audits [not_null, unique]), @@ -91,10 +92,13 @@ final AS ( s.session_id, s.anonymous_id, s.session_start_utc, - ARRAY_DISTINCT(TRANSFORM( - ARRAY_FLATTEN(COALESCE(sc.added_item_lists, ARRAY_CONSTRUCT())), - i -> UPPER(i:sku::VARCHAR) - ))::ARRAY AS added_skus, + -- ARRAY_AGG and ARRAY_DISTINCT leave element order unspecified; sort for a stable result. + ARRAY_SORT( + ARRAY_DISTINCT(TRANSFORM( + ARRAY_FLATTEN(COALESCE(sc.added_item_lists, ARRAY_CONSTRUCT())), + i -> UPPER(i:sku::VARCHAR) + )) + )::ARRAY AS added_skus, COALESCE(sc.removed_sku_count, 0)::INT AS removed_sku_count, COALESCE(sc.net_units, 0)::INT AS net_units, sc.max_added_unit_minor::INT AS max_added_unit_minor, diff --git a/benchmarks/duckshop/project/models/intermediate/fulfilment/int_delivery_slot_utilisation.sql b/benchmarks/duckshop/project/models/intermediate/fulfilment/int_delivery_slot_utilisation.sql index a78bb34b5e..53130a6923 100644 --- a/benchmarks/duckshop/project/models/intermediate/fulfilment/int_delivery_slot_utilisation.sql +++ b/benchmarks/duckshop/project/models/intermediate/fulfilment/int_delivery_slot_utilisation.sql @@ -49,6 +49,21 @@ warehouse_regions AS ( ON w.country_code = c.country_code ), +busiest_slots AS ( + SELECT + s.warehouse_code, + s.slot_start_local::DATE AS slot_date_local, + s.slot_start_local AS busiest_slot_start_local + FROM slots AS s + WHERE s.capacity <> 0 + -- The earliest of equally busy slots, so ties give one answer. + QUALIFY + ROW_NUMBER() OVER ( + PARTITION BY s.warehouse_code, s.slot_start_local::DATE + ORDER BY s.booked / s.capacity DESC, s.slot_start_local, s.slot_id + ) = 1 +), + by_day AS ( SELECT s.warehouse_code, @@ -56,8 +71,7 @@ by_day AS ( COUNT(*) AS slot_count, SUM(s.capacity) AS total_capacity, SUM(s.booked) AS total_booked, - COUNT_IF(s.remaining_capacity = 0) AS full_slot_count, - MAX_BY(s.slot_start_local, s.booked / NULLIF(s.capacity, 0)) AS busiest_slot_start_local + COUNT_IF(s.remaining_capacity = 0) AS full_slot_count FROM slots AS s GROUP BY s.warehouse_code, @@ -72,13 +86,16 @@ with_calendar AS ( d.total_capacity, d.total_booked, d.full_slot_count, - d.busiest_slot_start_local, + b.busiest_slot_start_local, r.region_code, ( h.holiday_date IS NOT NULL ) AS is_holiday, d.total_booked / NULLIF(d.total_capacity, 0) AS utilisation_raw FROM by_day AS d + LEFT JOIN busiest_slots AS b + ON d.warehouse_code = b.warehouse_code + AND d.slot_date_local = b.slot_date_local LEFT JOIN warehouse_regions AS r ON d.warehouse_code = r.warehouse_code LEFT JOIN holidays AS h @@ -108,9 +125,13 @@ final AS ( @const("unknown_region_code") )::VARCHAR AS region_code, d.is_holiday::BOOLEAN AS is_holiday, + -- The average as an exact decimal sum over a count, so rounding gives one answer. ROUND( d.utilisation_raw - - AVG(d.utilisation_raw) OVER ( + - SUM(d.utilisation_raw) OVER ( + PARTITION BY d.warehouse_code, DAYOFWEEKISO(d.slot_date_local) + ) + / COUNT(d.utilisation_raw) OVER ( PARTITION BY d.warehouse_code, DAYOFWEEKISO(d.slot_date_local) ), 4 diff --git a/benchmarks/duckshop/project/models/marts/fulfilment/rpt_delivery_exceptions_monthly.sql b/benchmarks/duckshop/project/models/marts/fulfilment/rpt_delivery_exceptions_monthly.sql index d9708b05dc..64b2660099 100644 --- a/benchmarks/duckshop/project/models/marts/fulfilment/rpt_delivery_exceptions_monthly.sql +++ b/benchmarks/duckshop/project/models/marts/fulfilment/rpt_delivery_exceptions_monthly.sql @@ -1,6 +1,7 @@ MODEL ( description """ - One row per month, carrier and responsibility with delivery exceptions and compensation paid + One row per month, carrier and responsibility with delivery exceptions, compensation paid and + the most frequent exception code (the first in alphabetical order on a tie) """, columns ( exception_month_key (type VARCHAR, nullable false, audits [not_null, unique]), @@ -20,7 +21,47 @@ WITH exceptions AS ( FROM __ref("int_delivery_exceptions_classified") ), -final AS ( +dated_codes AS ( + SELECT + DATE_TRUNC('month', CONVERT_TIMEZONE('UTC', e.occurred_at))::DATE AS exception_month, + e.carrier_code, + e.responsibility, + e.exception_code + FROM exceptions AS e + WHERE e.exception_code IS NOT NULL +), + +code_counts AS ( + SELECT + d.exception_month, + d.carrier_code, + d.responsibility, + d.exception_code, + COUNT(*) AS code_exceptions + FROM dated_codes AS d + GROUP BY + d.exception_month, + d.carrier_code, + d.responsibility, + d.exception_code +), + +top_codes AS ( + SELECT + cc.exception_month, + cc.carrier_code, + cc.responsibility, + cc.exception_code + FROM code_counts AS cc + -- MODE breaks ties arbitrarily; take the first code in alphabetical order instead. + QUALIFY + ROW_NUMBER() OVER ( + PARTITION BY cc.exception_month, cc.carrier_code, cc.responsibility + ORDER BY cc.code_exceptions DESC, cc.exception_code + ) = 1 +), + +counts AS ( SELECT ( TO_VARCHAR( @@ -29,14 +70,16 @@ final AS ( ) || ':' || e.carrier_code || ':' || e.responsibility )::VARCHAR AS exception_month_key, - DATE_TRUNC('month', CONVERT_TIMEZONE('UTC', e.occurred_at))::DATE AS exception_month, + DATE_TRUNC( + 'month', + CONVERT_TIMEZONE('UTC', e.occurred_at) + )::DATE AS exception_month, e.carrier_code, e.responsibility, COUNT(*)::INT AS exceptions, COUNT(DISTINCT e.shipment_id)::INT AS shipments_affected, ROUND(MEDIAN(e.hours_to_resolve), 2)::NUMBER(10, 2) AS median_hours_to_resolve, - SUM(e.compensation_usd)::NUMBER(18, 2) AS compensation_usd, - MODE(e.exception_code)::VARCHAR AS top_exception_code + SUM(e.compensation_usd)::NUMBER(18, 2) AS compensation_usd FROM exceptions AS e GROUP BY DATE_TRUNC( @@ -45,6 +88,24 @@ final AS ( ), e.carrier_code, e.responsibility +), + +final AS ( + SELECT + c.exception_month_key, + c.exception_month, + c.carrier_code, + c.responsibility, + c.exceptions, + c.shipments_affected, + c.median_hours_to_resolve, + c.compensation_usd, + t.exception_code::VARCHAR AS top_exception_code + FROM counts AS c + LEFT JOIN top_codes AS t + ON c.exception_month = t.exception_month + AND c.carrier_code = t.carrier_code + AND c.responsibility = t.responsibility ) SELECT * diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_benchmark_compile.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_benchmark_compile.py index f93c80252f..184b99c169 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_benchmark_compile.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_benchmark_compile.py @@ -73,7 +73,7 @@ def test_given_benchmark_project_when_compiling_then_reports_no_errors_or_warnin expected_errors=0, expected_warnings=0, expected_compiled_checksum=( - "4568ab62e8033a5698793db6993f19467ff68bd919c3ae2e2f72055b8ac2cde0" + "a36dfda0afbe642c86441adc078d026cf6eefc95921173c32df74f2783a04f87" ), ) ], From eb93903718fccda357a8870ef6422a5767cf422c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:06:50 +0000 Subject: [PATCH 05/20] fix(duckdb): follow SET search_path in relation existence lookups SET search_path and SET schema change how unqualified names resolve, but were classified as data statements, so the run's relation metadata cache kept answers from before them. They now invalidate every cached lookup. DuckDB's unqualified existence check also looks only in the schemas on the session's search path, as name resolution does. --- .../src/ddl_effects/_helpers/relations.rs | 21 +++++-- .../src/ddl_effects/constants.rs | 6 ++ .../src/ddl_effects/tests/test_ddl_effects.rs | 42 ++++++++++++++ .../contract/classes/duckdb_backed_adapter.py | 2 + .../sqlbuild/adapters/duckdb/_test_types.py | 10 ++++ .../sqlbuild/adapters/duckdb/test_client.py | 57 +++++++++++++++++++ 6 files changed, 134 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/relations.rs b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/relations.rs index 7937e89afe..e7c2183d64 100644 --- a/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/relations.rs +++ b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/relations.rs @@ -13,10 +13,11 @@ use crate::ddl_effects::_helpers::names::{UnquotedPart, part_name_key, qualified use crate::ddl_effects::constants::{ ALTER_FOLLOWING_WORDS, ALTER_SESSION_SET, ALTER_SESSION_UNSET, COMPUTE_ONLY_STATEMENTS, COPY_INTO, CREATE_FOLLOWING_WORDS, CREATE_SCHEMA_IF_NOT_EXISTS, DROP_FOLLOWING_WORDS, - FROM_WORD, IF_EXISTS, IF_NOT_EXISTS, INTO, NEUTRAL_OBJECT_KINDS, NEUTRAL_OBJECT_LEADS, - NEUTRAL_OBJECT_MODIFIERS, NEUTRAL_SESSION_PARAMETERS, OR_REPLACE, QUERY_STATEMENTS, - READ_OR_DATA_STATEMENTS, RELATION_DDL_STATEMENTS, RELATION_KIND_MODIFIERS, RELATION_KINDS, - RENAMING_ALTERS, STATEMENT_SEPARATOR, TRANSACTION_END_STATEMENTS, + FROM_WORD, IF_EXISTS, IF_NOT_EXISTS, INTO, NAME_RESOLUTION_SETTINGS, NEUTRAL_OBJECT_KINDS, + NEUTRAL_OBJECT_LEADS, NEUTRAL_OBJECT_MODIFIERS, NEUTRAL_SESSION_PARAMETERS, OR_REPLACE, + QUERY_STATEMENTS, READ_OR_DATA_STATEMENTS, RELATION_DDL_STATEMENTS, RELATION_KIND_MODIFIERS, + RELATION_KINDS, RENAMING_ALTERS, SET_SCOPE_MODIFIERS, SET_WORD, STATEMENT_SEPARATOR, + TRANSACTION_END_STATEMENTS, }; use crate::ddl_effects::models::StatementMetadataEffect; @@ -240,6 +241,15 @@ fn sets_neutral_session_parameter(statement: &str) -> bool { .is_some_and(|end| only_whitespace(statement, end)) } +/// Whether the statement is `SET [SESSION|LOCAL|GLOBAL] search_path|schema ...`. +fn sets_name_resolution(statement: &str) -> bool { + let Some(after_set) = phrase_then_whitespace(statement, 0, &[SET_WORD]) else { + return false; + }; + let start: usize = repeated_modifiers(statement, after_set, SET_SCOPE_MODIFIERS); + bounded_phrase(statement, start, NAME_RESOLUTION_SETTINGS).is_some() +} + /// Whether the statement changes only session state that relations and name resolution ignore. fn is_session_only(statement: &str) -> bool { bounded_phrase(statement, 0, COMPUTE_ONLY_STATEMENTS) @@ -265,6 +275,9 @@ pub(crate) fn code_effect(code: Option<&str>) -> StatementMetadataEffect { if QUERY_STATEMENTS.contains(&keyword.as_str()) { return select_into_effect(statement); } + if sets_name_resolution(statement) { + return all_relations(); + } if READ_OR_DATA_STATEMENTS.contains(&keyword.as_str()) { return StatementMetadataEffect::default(); } diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/constants.rs b/crates/sqlbuild-sqltext/src/ddl_effects/constants.rs index 9576778176..4856b5f3e7 100644 --- a/crates/sqlbuild-sqltext/src/ddl_effects/constants.rs +++ b/crates/sqlbuild-sqltext/src/ddl_effects/constants.rs @@ -44,6 +44,12 @@ pub(crate) const NEUTRAL_SESSION_PARAMETERS: &[&[&str]] = &[ &["STATEMENT_TIMEOUT_IN_SECONDS"], &["USE_CACHED_RESULT"], ]; +/// `SET` before the setting it changes. +pub(crate) const SET_WORD: &[&str] = &["SET"]; +/// Scope modifiers allowed between `SET` and the setting. +pub(crate) const SET_SCOPE_MODIFIERS: &[&[&str]] = &[&["SESSION"], &["LOCAL"], &["GLOBAL"]]; +/// Settings, such as DuckDB's and PostgreSQL's, that change how unqualified names resolve. +pub(crate) const NAME_RESOLUTION_SETTINGS: &[&[&str]] = &[&["search_path"], &["schema"]]; /// Statements that end the current transaction. pub(crate) const TRANSACTION_END_STATEMENTS: &[&str] = &["COMMIT", "ROLLBACK", "END", "ABORT"]; /// Statements whose relation name is read from the statement. diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_ddl_effects.rs b/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_ddl_effects.rs index 7784b468b2..620a917e47 100644 --- a/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_ddl_effects.rs +++ b/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_ddl_effects.rs @@ -198,6 +198,48 @@ fn given_executed_statement_when_classifying_then_reports_possible_metadata_chan expected_invalidates_all: false, expected_ends_transaction: false, }, + StatementEffectTestCase { + description: "duckdb search path changes how unqualified names resolve", + sql: "SET search_path = 'staging,main'", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "scoped search path in another case changes unqualified names", + sql: "set session SEARCH_PATH to staging", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "duckdb default schema changes unqualified names", + sql: "SET schema = 'staging'", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "reset search path changes unqualified names", + sql: "RESET search_path", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "other settings and session variables leave names alone", + sql: "SET threads = 4", + expected_relation_names: &[], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "a session variable named like a schema setting prefix is neutral", + sql: "SET schema_owner = 'orders'", + expected_relation_names: &[], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, StatementEffectTestCase { description: "schema for the session changes unqualified names", sql: "USE SCHEMA analytics.marts", diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index d79c83879c..d9da48eca2 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -1233,6 +1233,8 @@ def relation_exists( ) if schema is not None: query += f" AND table_schema = {_duckdb_string_literal(schema)}" + else: + query += " AND table_schema IN (SELECT UNNEST(current_schemas(true)))" if database is not None: query += f" AND table_catalog = {_duckdb_string_literal(database)}" result: Any = self.execute(connection=connection, sql=query).fetchone() diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py b/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py index 0e9e638a17..492b5c02ef 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py @@ -292,3 +292,13 @@ class QualifiedViewDefinitionTestCase: name: str sql: str expected_match: bool + + +@dataclass(frozen=True) +class SearchPathCacheTestCase: + """An unqualified existence lookup cached before and after a statement changes the path.""" + + description: str + hook_sql: str + expected_before: bool + expected_after: bool diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py index 42f5af9db4..6d5f2cf105 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py @@ -23,6 +23,10 @@ SchemaDiffResult, ) from sqlbuild.adapter.contract.types import FunctionNullabilityRule +from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists +from sqlbuild.adapter.relations.main.relation_metadata_cache_scope import ( + relation_metadata_cache_scope, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from tests.integration.src.sqlbuild.adapters.duckdb._test_types import ( CompositeRowDiffSamplingTestCase, @@ -47,6 +51,7 @@ RecorderWriteTestCase, RelationExistsTestCase, RenameTestCase, + SearchPathCacheTestCase, StorageVersionConnectTestCase, SwapTestCase, TransactionalAtomicityTestCase, @@ -304,6 +309,26 @@ def test_given_sql_when_querying_then_returns_normalized_query_result( name="missing_table", expected_exists=False, ), + RelationExistsTestCase( + description="an unqualified name off the search path does not resolve", + setup_sql=("CREATE SCHEMA staging", "CREATE TABLE staging.raw_orders (id INTEGER)"), + database=None, + schema=None, + name="raw_orders", + expected_exists=False, + ), + RelationExistsTestCase( + description="an unqualified name on the search path resolves", + setup_sql=( + "CREATE SCHEMA staging", + "CREATE TABLE staging.raw_orders (id INTEGER)", + "SET search_path = 'staging'", + ), + database=None, + schema=None, + name="raw_orders", + expected_exists=True, + ), ], ids=lambda case: case.description, ) @@ -1776,3 +1801,35 @@ def test_given_database_qualified_view_when_comparing_definition_then_catalog_pr ) assert matches is test_case.expected_match + + +@pytest.mark.parametrize( + "test_case", + [ + SearchPathCacheTestCase( + description="a hook's SET search_path between two models changes the cached answer", + hook_sql="SET search_path = 'staging'", + expected_before=False, + expected_after=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_cached_unqualified_lookup_when_hook_sets_search_path_then_lookup_reads_again( + test_case: SearchPathCacheTestCase, + adapter: DuckDbAdapter, + connection: Any, +) -> None: + connection.execute("CREATE SCHEMA staging") + connection.execute("CREATE TABLE staging.raw_orders (id INTEGER)") + + with relation_metadata_cache_scope(adapter=adapter): + before: bool = cached_relation_exists( + adapter=adapter, connection=connection, database=None, schema=None, name="raw_orders" + ) + adapter.execute(connection=connection, sql=test_case.hook_sql) + after: bool = cached_relation_exists( + adapter=adapter, connection=connection, database=None, schema=None, name="raw_orders" + ) + + assert (before, after) == (test_case.expected_before, test_case.expected_after) From 9792129e4414f79e0f74e3965f2dda20f7ce5a5c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:25:17 +0000 Subject: [PATCH 06/20] perf(snowflake): reuse the planner's schema listings in the build's run A first build listed each schema during planning and again when the run's relation cache first needed it. The build now keeps one relation metadata cache from planning to the end of the run, and the planner's Snowflake schema listings go into it. Statements the build issues after a listing still invalidate it, and a planner listing older than a minute is read again. --- .../src/entries/_helpers/adapter.rs | 5 + crates/sqlbuild-host/src/entries/models.rs | 1 + .../_helpers/build_command/command.rs | 11 ++- fensu.toml | 5 + src/sqlbuild/_native_host.py | 4 + .../relations/classes/inspection_catalog.py | 39 +++++++- .../classes/relation_metadata_cache.py | 52 ++++++++-- src/sqlbuild/adapter/relations/constants.py | 1 + src/sqlbuild/adapter/relations/models.py | 1 + src/sqlbuild/adapter/relations/types.py | 11 +++ .../snowflake/classes/snowflake_adapter.py | 7 ++ .../build/snowflake_metadata/_test_types.py | 18 ++++ .../build/snowflake_metadata/helpers.py | 35 +++++++ .../test_planner_listing_reuse.py | 96 +++++++++++++++++++ 14 files changed, 273 insertions(+), 13 deletions(-) create mode 100644 tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_planner_listing_reuse.py diff --git a/crates/sqlbuild-host/src/entries/_helpers/adapter.rs b/crates/sqlbuild-host/src/entries/_helpers/adapter.rs index 9f98191093..d3cf753392 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/adapter.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/adapter.rs @@ -224,4 +224,9 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Function, edge: HostEdge::Adapter, }, + HostEntry { + name: "relation_metadata_cache_scope", + kind: HostKind::Function, + edge: HostEdge::Adapter, + }, ]; diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index 86621a8546..5c8482411a 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -106,6 +106,7 @@ pub enum AdapterHost { RunRecallCommand, InsertRecallRows, CloseRecallStore, + RelationMetadataCacheScope, } impl AdapterHost { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 1802b57764..8dbd45566b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -4,7 +4,7 @@ use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; +use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost, UserCodeHost}; use crate::bindings::_helpers::boundary::python_calls::{call_with, within}; use crate::bindings::_helpers::build_command::completion::{ @@ -39,6 +39,15 @@ fn run_with_providers(context: &BuildContext<'_>) -> PyResult { fn run_planned_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) -> PyResult { run_defer_clone_prephase(context)?; + let cache_scope: Bound<'_, PyAny> = call_with( + &host::entry(context.py(), AdapterHost::RelationMetadataCacheScope)?, + &[("adapter", context.get("adapter")?)], + )?; + within(&cache_scope, |_| run_cached_build(context, providers)) +} + +/// Plan and run the build, sharing one relation metadata cache from planning to the run's end. +fn run_cached_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) -> PyResult { let pipeline: Bound<'_, PyAny> = compile_build_plan(context)?; write_plan_text(context, &pipeline)?; enforce_build_plan_policies(context, &pipeline)?; diff --git a/fensu.toml b/fensu.toml index d64899dc9b..af975cc307 100644 --- a/fensu.toml +++ b/fensu.toml @@ -122,6 +122,11 @@ modules = ["sqlbuild.adapter.relations.main.relation_lookup"] symbols = ["build_relation_lookup"] reason = "Native code resolves these through the sqlbuild._native_host name table." +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.relation_metadata_cache_scope"] +symbols = ["relation_metadata_cache_scope"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + [[targets.python.dead_code.roots]] modules = ["sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name"] symbols = ["resolve_relation_location_qualified_name"] diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index 94b3745544..9dca6ce419 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -115,6 +115,10 @@ "sqlbuild.adapters.clickhouse._helpers.recall_store", "close_recall_store", ), + "relation_metadata_cache_scope": ( + "sqlbuild.adapter.relations.main.relation_metadata_cache_scope", + "relation_metadata_cache_scope", + ), "python_resource_rows": ( "sqlbuild.compiler.compile._helpers.diagnostics.python_resources", "python_resource_rows", diff --git a/src/sqlbuild/adapter/relations/classes/inspection_catalog.py b/src/sqlbuild/adapter/relations/classes/inspection_catalog.py index 078f055e3b..f4dc93bf8e 100644 --- a/src/sqlbuild/adapter/relations/classes/inspection_catalog.py +++ b/src/sqlbuild/adapter/relations/classes/inspection_catalog.py @@ -9,9 +9,14 @@ from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapter.relations._helpers.inspection_context import emit_inspection_record +from sqlbuild.adapter.relations._helpers.relation_metadata_context import ( + current_relation_metadata_cache, +) +from sqlbuild.adapter.relations.classes.relation_metadata_cache import RelationMetadataCache from sqlbuild.adapter.relations.constants import ( INSPECTION_DESCRIBED_NAME_LIMIT, INSPECTION_IN_LIST_LIMIT, + PLANNER_LISTING_MAX_AGE_SECONDS, ) from sqlbuild.adapter.relations.main.run_bounded_inspections import run_bounded_inspections from sqlbuild.adapter.relations.models import ( @@ -19,7 +24,11 @@ SchemaColumnListing, SchemaRelationListing, ) -from sqlbuild.adapter.relations.types import SchemaScopedMetadataReader, ViewDefinitionBatchReader +from sqlbuild.adapter.relations.types import ( + RelationListingScoper, + SchemaScopedMetadataReader, + ViewDefinitionBatchReader, +) type _RelationIdentity = tuple[str | None, str | None, str] type _SchemaKey = tuple[str | None, str] @@ -301,9 +310,7 @@ def _bind_relation_listing( ) -> Callable[[], SchemaRelationListing | None]: def read_listing() -> SchemaRelationListing | None: try: - return reader.read_schema_relation_listing( - connection=self.connection, database=database, schema=schema - ) + return self._read_relation_listing(reader=reader, database=database, schema=schema) except Exception: if best_effort: return None @@ -311,6 +318,30 @@ def read_listing() -> SchemaRelationListing | None: return read_listing + def _read_relation_listing( + self, *, reader: SchemaScopedMetadataReader, database: str | None, schema: str + ) -> SchemaRelationListing: + """Read one schema's relations, and keep them for the build's run when one is open.""" + + cache: RelationMetadataCache | None = current_relation_metadata_cache(adapter=self.adapter) + if cache is None or not isinstance(self.adapter, RelationListingScoper): + return reader.read_schema_relation_listing( + connection=self.connection, database=database, schema=schema + ) + versions: tuple[int, dict[str, int]] = cache.listing_versions() + listing: SchemaRelationListing = reader.read_schema_relation_listing( + connection=self.connection, database=database, schema=schema + ) + _ = cache.keep_schema_listing( + scope=self.adapter.relation_listing_scope( + connection=self.connection, database=database, schema=schema + ), + relations={entry.stored_name: entry.relation for entry in listing.entries}, + versions=versions, + max_age_seconds=PLANNER_LISTING_MAX_AGE_SECONDS, + ) + return listing + def _read_columns_directly(self, *, relations: tuple[RelationInfo, ...]) -> None: started: float = time.monotonic() columns: dict[_RelationIdentity, tuple[ColumnInfo, ...]] = ( diff --git a/src/sqlbuild/adapter/relations/classes/relation_metadata_cache.py b/src/sqlbuild/adapter/relations/classes/relation_metadata_cache.py index 4e93832592..dd5c89846e 100644 --- a/src/sqlbuild/adapter/relations/classes/relation_metadata_cache.py +++ b/src/sqlbuild/adapter/relations/classes/relation_metadata_cache.py @@ -3,6 +3,7 @@ from __future__ import annotations import threading +import time from collections.abc import Callable from typing import cast @@ -24,8 +25,11 @@ class RelationMetadataCache: """Serve repeated identical metadata lookups until SQL may have changed the relation.""" - def __init__(self, *, transactional_ddl: bool) -> None: + def __init__( + self, *, transactional_ddl: bool, clock: Callable[[], float] = time.monotonic + ) -> None: self._transactional_ddl: bool = transactional_ddl + self._clock: Callable[[], float] = clock self._lock: threading.Lock = threading.Lock() self._entries: dict[RelationCacheKey, object] = {} self._name_versions: dict[str, int] = {} @@ -70,7 +74,11 @@ def schema_relation( listing: RunSchemaListing | None with self._lock: listing = self._listings.get(scope) - if listing is None: + expired: bool = listing is not None and self._expired(listing=listing) + if expired: + del self._listings[scope] + listing = None + if listing is None and not expired: lookups: int = self._schema_lookups.get(scope, 0) + 1 self._schema_lookups[scope] = lookups if lookups < SCHEMA_LISTING_LOOKUP_THRESHOLD: @@ -92,11 +100,34 @@ def refresh_schema_listing( ) -> RunSchemaListing: """List the schema again; every relation in it is current as of this listing.""" + versions: tuple[int, dict[str, int]] = self.listing_versions() + return self.keep_schema_listing(scope=scope, relations=read_listing(), versions=versions) + + def listing_versions(self) -> tuple[int, dict[str, int]]: + """The cache versions a listing read from now on is current as of.""" + with self._lock: - global_version: int = self._global_version - name_versions: dict[str, int] = dict(self._name_versions) + return self._global_version, dict(self._name_versions) + + def keep_schema_listing( + self, + *, + scope: SchemaListingScope, + relations: dict[str, RelationInfo], + versions: tuple[int, dict[str, int]], + max_age_seconds: float | None = None, + ) -> RunSchemaListing: + """Keep a listing read after ``listing_versions`` returned ``versions``, for a time.""" + + global_version: int = versions[0] + expires_at: float | None = ( + None if max_age_seconds is None else self._clock() + max_age_seconds + ) listing: RunSchemaListing = RunSchemaListing( - relations=read_listing(), global_version=global_version, name_versions=name_versions + relations=relations, + global_version=global_version, + name_versions=versions[1], + expires_at=expires_at, ) with self._lock: if self._global_version == global_version: @@ -116,9 +147,14 @@ def listed_relation( def _listed_fresh(self, *, listing: RunSchemaListing, name: str) -> bool: name_key: str = relation_name_key(name) - return listing.global_version == self._global_version and listing.name_versions.get( - name_key, 0 - ) == self._name_versions.get(name_key, 0) + return ( + not self._expired(listing=listing) + and listing.global_version == self._global_version + and listing.name_versions.get(name_key, 0) == self._name_versions.get(name_key, 0) + ) + + def _expired(self, *, listing: RunSchemaListing) -> bool: + return listing.expires_at is not None and self._clock() >= listing.expires_at def invalidate_relations(self, *, qualified_names: tuple[str, ...]) -> None: """Evict relations changed outside SQL statements, such as by a warehouse copy job.""" diff --git a/src/sqlbuild/adapter/relations/constants.py b/src/sqlbuild/adapter/relations/constants.py index bc8d6175ff..431f3acc5f 100644 --- a/src/sqlbuild/adapter/relations/constants.py +++ b/src/sqlbuild/adapter/relations/constants.py @@ -7,4 +7,5 @@ RELATION_EXISTS_LOOKUP: str = "exists" RELATION_TYPE_LOOKUP: str = "type" SCHEMA_LISTING_LOOKUP_THRESHOLD: int = 2 +PLANNER_LISTING_MAX_AGE_SECONDS: float = 60.0 UNKNOWN_RELATION_READER: str = "unknown user" diff --git a/src/sqlbuild/adapter/relations/models.py b/src/sqlbuild/adapter/relations/models.py index 2cf2f8af24..d9762f4f60 100644 --- a/src/sqlbuild/adapter/relations/models.py +++ b/src/sqlbuild/adapter/relations/models.py @@ -60,3 +60,4 @@ class RunSchemaListing: relations: dict[str, RelationInfo] global_version: int name_versions: dict[str, int] + expires_at: float | None = None diff --git a/src/sqlbuild/adapter/relations/types.py b/src/sqlbuild/adapter/relations/types.py index cc3f0cf0d4..c67c1359a2 100644 --- a/src/sqlbuild/adapter/relations/types.py +++ b/src/sqlbuild/adapter/relations/types.py @@ -23,6 +23,17 @@ class RelationLocation(Protocol): qualified_name: str | None +@runtime_checkable +class RelationListingScoper(Protocol): + """Adapter whose builds keep schema listings under a stored database and schema name.""" + + def relation_listing_scope( + self, *, connection: Any, database: str | None, schema: str + ) -> SchemaListingScope: + """The stored names that key a build's listing of one schema.""" + ... + + @runtime_checkable class SchemaScopedMetadataReader(Protocol): """Adapter able to list a whole schema's relations and columns in one read each.""" diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 74157a4ec1..487064de9b 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -327,6 +327,13 @@ def _schema_relations( ) return {entry.stored_name: entry.relation for entry in listing.entries} + def relation_listing_scope( + self, *, connection: Any, database: str | None, schema: str + ) -> SchemaListingScope: + """The stored database and schema names that key a build's listing of one schema.""" + + return self._listing_scope(connection=connection, database=database, schema=schema) + def _listing_scope( self, *, connection: Any, database: str | None, schema: str ) -> SchemaListingScope: diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py index 3df1107d3b..ac787310bf 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from collections.abc import Callable from dataclasses import dataclass @@ -82,3 +83,20 @@ class ExternalChangeTestCase: change: Callable[[SimulatedSnowflakeWarehouse], None] before_statement: str expected_table_types: tuple[str, ...] + + +@dataclass(frozen=True) +class PlannerListingReuseTestCase: + """Two real ``sqb build`` runs and the schema listings each issues. + + Each external change runs once, when a build reaches its statement pattern, as another + session changing the schema between planning and the run. + """ + + description: str + expected_listings: tuple[dict[str, int], ...] + expected_table_types: tuple[str, ...] + listing_max_age_seconds: float = 60.0 + external_changes: tuple[ + tuple[re.Pattern[str], Callable[[SimulatedSnowflakeWarehouse], None]], ... + ] = () diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py index 71fce24b0a..f65ad62662 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py @@ -13,14 +13,19 @@ from pathlib import Path from typing import Any +import pytest + from sqlbuild._native import ( CompilePipelineOptions, CompilePipelineResult, DiscoveredProjectInputs, discover_project_inputs, + run_cli, run_compile_pipeline, ) from sqlbuild.adapter.contract.types import TablePromotionMode +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_connection import _SnowflakeConnection from sqlbuild.executor.build.models import BuildExecutionResult, BuildRuntimeParams from sqlbuild.executor.pipeline.main.run import run_build_pipeline from sqlbuild.executor.scheduling.types import ExecutionStatus @@ -766,3 +771,33 @@ def write_model_copies( 'SELECT order_id, ordered_at, amount FROM __ref("orders_table")\n', encoding="utf-8", ) + + +def build_with_cli( + *, project_dir: Path, warehouse: SimulatedSnowflakeWarehouse, monkeypatch: pytest.MonkeyPatch +) -> tuple[int, tuple[str, ...]]: + """Run the real ``sqb build`` against the simulated warehouse; its exit code and statements.""" + + monkeypatch.setattr( + SnowflakeAdapter, + "connect", + lambda _adapter, _config: _SnowflakeConnection(warehouse.raw_connection()), + ) + warehouse.reset() + exit_code: int = run_cli(["--project-dir", str(project_dir), "--no-color", "build"]) + return exit_code, tuple(warehouse.attempted_sql) + + +_SCHEMA_LISTING_STATEMENT: re.Pattern[str] = re.compile( + r"^SHOW (?PTABLES|VIEWS) IN SCHEMA (?P\S+)" +) + + +def schema_listing_counts(statements: tuple[str, ...]) -> dict[str, int]: + """How often each schema was listed, keyed by ``TABLES``/``VIEWS`` and the schema.""" + + searches: list[re.Match[str] | None] = [ + _SCHEMA_LISTING_STATEMENT.search(statement) for statement in statements + ] + matches: list[re.Match[str]] = list(filter(None, searches)) + return dict(Counter(f"{match['kind']} {match['scope']}" for match in matches)) diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_planner_listing_reuse.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_planner_listing_reuse.py new file mode 100644 index 0000000000..43535cb195 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_planner_listing_reuse.py @@ -0,0 +1,96 @@ +"""A Snowflake build's run reuses the schema listings its planning read, through the real CLI.""" + +from __future__ import annotations + +import re +from functools import partial +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.executor.build.snowflake_metadata._test_types import ( + PlannerListingReuseTestCase, +) +from tests.integration.src.sqlbuild.executor.build.snowflake_metadata.helpers import ( + SimulatedSnowflakeWarehouse, + build_with_cli, + create_external_table, + orders_warehouse, + relation_types, + schema_listing_counts, + write_orders_project, +) + +_SCHEMA_LISTING: re.Pattern[str] = re.compile( + r"^SHOW (?PTABLES|VIEWS) IN SCHEMA (?P\S+)" +) +_PLANNER_LISTING_MAX_AGE: str = ( + "sqlbuild.adapter.relations.classes.inspection_catalog.PLANNER_LISTING_MAX_AGE_SECONDS" +) +_ONE_LISTING_EACH: dict[str, int] = { + 'TABLES "ANALYTICS"."MARTS"': 1, + 'TABLES "ANALYTICS"."RAW"': 1, + 'VIEWS "ANALYTICS"."MARTS"': 1, + 'VIEWS "ANALYTICS"."RAW"': 1, +} + +_RELISTED_MARTS: dict[str, int] = { + **_ONE_LISTING_EACH, + 'TABLES "ANALYTICS"."MARTS"': 2, + 'VIEWS "ANALYTICS"."MARTS"': 2, +} + + +@pytest.mark.parametrize( + "test_case", + [ + PlannerListingReuseTestCase( + description="a first build and a rebuild list each schema once", + expected_listings=(_ONE_LISTING_EACH, _ONE_LISTING_EACH), + expected_table_types=("BASE TABLE",), + ), + PlannerListingReuseTestCase( + description="a planning listing past its age is read again by the run", + listing_max_age_seconds=0.0, + expected_listings=(_RELISTED_MARTS, _RELISTED_MARTS), + expected_table_types=("BASE TABLE",), + ), + PlannerListingReuseTestCase( + description="another session creates a model's table between planning and the run", + external_changes=( + ( + re.compile(r"^CREATE SCHEMA IF NOT EXISTS "), + partial(create_external_table, name="ORDERS_TABLE"), + ), + ), + expected_listings=(_ONE_LISTING_EACH, _ONE_LISTING_EACH), + expected_table_types=("BASE TABLE",), + ), + ], + ids=lambda case: case.description, +) +def test_given_planned_build_when_running_then_run_reuses_planner_schema_listings( + test_case: PlannerListingReuseTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + write_orders_project(project_dir=tmp_path) + warehouse: SimulatedSnowflakeWarehouse = orders_warehouse() + monkeypatch.setattr(_PLANNER_LISTING_MAX_AGE, test_case.listing_max_age_seconds) + warehouse.external_changes.extend(test_case.external_changes) + + builds: tuple[tuple[int, tuple[str, ...]], ...] = tuple( + build_with_cli(project_dir=tmp_path, warehouse=warehouse, monkeypatch=monkeypatch) + for _ in test_case.expected_listings + ) + + output: str = capsys.readouterr().out + assert tuple(build[0] for build in builds) == (0,) * len(builds), output + assert warehouse.external_changes == [] + assert tuple(schema_listing_counts(build[1]) for build in builds) == test_case.expected_listings + assert relation_types(warehouse, "ORDERS_TABLE") == test_case.expected_table_types + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From f2341489f21d0b310f246281a7359e61ccc5607a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:30:58 +0000 Subject: [PATCH 07/20] fix: read quoted mixed-case column names end to end A source or model column created with a quoted mixed-case name, such as "listingId", could not be used: a MODEL header could not declare it, unit-test and scenario fixtures rejected it, and once the table existed Snowflake planning lower-cased it and failed to bind. Header column keys may now be double-quoted identifiers, fixtures and contract checks match a quoted name to its unquoted spelling, Snowflake keeps stored mixed-case column names quoted, adapters render quoted names as they are, and quoted aliases a final SELECT * reads keep their exact case. DuckShop's partner-listings edge models now build, test and replay without the exclusion or the rule ignores. --- Makefile | 3 +- benchmarks/duckshop/README.md | 17 ++- benchmarks/duckshop/RUNBOOK.md | 6 +- .../rpt_edge_partner_listing_export.sql | 3 + .../duckshop/project/sqlbuild_project.toml | 17 --- .../test_stg_edge_partner_listings.sql | 30 +++++ .../src/contracts/_helpers/columns.rs | 23 ++-- .../_helpers/metadata_checks/checks.rs | 11 +- .../semantic_validation/_helpers/catalog.rs | 34 ++++++ .../_helpers/planner_resolution/sources.rs | 11 +- .../planner_strategy/fixture_context.rs | 5 +- .../planner_strategy/fixture_metadata.rs | 5 +- .../planner_strategy/fixture_values.rs | 9 +- .../semantic_shapes/source_bindings.rs | 9 +- .../_helpers/model_headers/tokenization.rs | 19 ++- .../src/resolution/main/column_name_key.rs | 9 ++ .../src/resolution/main/mod.rs | 2 + .../src/resolution/main/source_cast_plan.rs | 8 +- .../resolution/main/unquoted_column_name.rs | 10 ++ fensu.toml | 1 - .../contract/classes/duckdb_backed_adapter.py | 3 + src/sqlbuild/adapter/contract/constants.py | 2 + .../contract/main/is_quoted_identifier.py | 19 +++ .../snowflake/_helpers/column_names.py | 11 ++ .../snowflake/classes/snowflake_adapter.py | 22 ++-- .../cli/commands/main/build/_test_types.py | 10 ++ .../build/test_quoted_mixed_case_columns.py | 113 ++++++++++++++++++ .../build/snowflake_metadata/_test_types.py | 9 ++ .../test_quoted_mixed_case_columns.py | 92 ++++++++++++++ 29 files changed, 441 insertions(+), 72 deletions(-) create mode 100644 benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_partner_listings.sql create mode 100644 crates/sqlbuild-sqltext/src/resolution/main/column_name_key.rs create mode 100644 crates/sqlbuild-sqltext/src/resolution/main/unquoted_column_name.rs create mode 100644 src/sqlbuild/adapter/contract/main/is_quoted_identifier.py create mode 100644 src/sqlbuild/adapters/snowflake/_helpers/column_names.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/build/test_quoted_mixed_case_columns.py create mode 100644 tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_quoted_mixed_case_columns.py diff --git a/Makefile b/Makefile index 27f29d1384..1753e39f05 100644 --- a/Makefile +++ b/Makefile @@ -116,7 +116,8 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_check_outcomes.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_attach.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_unbuilt_dependencies.py \ - tests/e2e/src/sqlbuild/cli/commands/main/build/test_validation_failures.py + tests/e2e/src/sqlbuild/cli/commands/main/build/test_validation_failures.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_quoted_mixed_case_columns.py E2E_DUCKDB_BUILD_INCREMENTAL_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay \ diff --git a/benchmarks/duckshop/README.md b/benchmarks/duckshop/README.md index 956a858bda..2f87e3e9d2 100644 --- a/benchmarks/duckshop/README.md +++ b/benchmarks/duckshop/README.md @@ -119,7 +119,7 @@ project and does not change it: - raw `edge__*` sources in `sources/edge_cases.yml`, loaded by `python/loaders/edge_cases.py` from fixed rows that `data_scale` does not change, so core data and core expectations stay the same. -It has 38 models (7 staging, 22 intermediate, 9 marts), 61 unit tests and 4 scenarios, in four +It has 38 models (7 staging, 22 intermediate, 9 marts), 62 unit tests and 4 scenarios, in four groups: | Group | Covers | @@ -189,12 +189,12 @@ the same rows, apart from the snapshots, which record build times, and three mod vary from one run to the next (`int_cart_funnel`, `int_delivery_slot_utilisation` and `rpt_delivery_exceptions_monthly`). -The edge-case area, verified on Snowflake with SQLBuild 0.184.0 on the `edge` target: +The edge-case area, verified on Snowflake with SQLBuild built after 0.185.0 on the `edge` target: | Step | Result | |------|--------| -| `sqb build --select tag:edge_case --exclude stg_edge_partner_listings+` | Passes; the disabled model is skipped | -| `sqb test --select tag:edge_case` | 61 of 61 pass | +| `sqb build --select tag:edge_case` | Passes; the disabled model is skipped | +| `sqb test --select tag:edge_case` | 62 of 62 pass | | `sqb scenario test --select edge_cases` | 4 of 4 pass | | Two-load replay of the delivery change log | The merged status equals the full rebuild after each load | @@ -221,12 +221,9 @@ noted here: - Compilation accepts SQL that Snowflake rejects: `CONNECT_BY_ISLEAF`, which Snowflake does not have, and a CTE whose body is a bare `VALUES` list. `int_category_paths` finds leaf categories with a join instead, and test case tables use `SELECT ... FROM VALUES ... AS v(...)`. -- A source column whose name is quoted mixed case, such as `"listingId"`, cannot be read once the - table exists: planning folds the quoted reference to lower case and fails with `B002`. Fixtures - cannot name such columns either (`B302` in unit tests, `S510` in scenarios), and a `MODEL` - header cannot declare a quoted column name. `stg_edge_partner_listings` has a reviewed - `SQBRTEST202` ignore, `rpt_edge_partner_listing_export` a reviewed `SQBRTEST201` ignore, and the - edge build excludes `stg_edge_partner_listings+`. +- Static type inference cannot follow a quoted mixed-case column, such as `"unitPrice"`, from a CTE + through `SELECT *` on Snowflake, so `rpt_edge_partner_listing_export` declares only the name and + audits of `"listingId"`, without types. - `SQBRSQL018` can only take proof from single-column keys. Append-only feeds redeliver rows, so their id is unique only together with `_loaded_at` (the `unique_per_delivery` audit), and the staging models that keep the latest delivery carry reviewed exceptions. diff --git a/benchmarks/duckshop/RUNBOOK.md b/benchmarks/duckshop/RUNBOOK.md index 45c53584ce..93a836b905 100644 --- a/benchmarks/duckshop/RUNBOOK.md +++ b/benchmarks/duckshop/RUNBOOK.md @@ -172,7 +172,7 @@ collation). Check with `SHOW PARAMETERS LIKE 'DEFAULT_DDL_COLLATION' IN SCHEMA < ### Load, build and test ```bash -sqb build --target edge --select tag:edge_case --exclude stg_edge_partner_listings+ --concurrency 8 +sqb build --target edge --select tag:edge_case --concurrency 8 sqb test --target edge --select tag:edge_case --concurrency 8 sqb scenario test --target edge --select edge_cases ``` @@ -180,10 +180,6 @@ sqb scenario test --target edge --select edge_cases The build loads the `edge__*` sources from fixed rows that `data_scale` does not change, then builds the edge models and runs their tests and audits. `fct_edge_delivery_events` is disabled and reports `skip: disabled` (see [Pending acceptance cases](#pending-acceptance-cases)). -`stg_edge_partner_listings` and its downstream models are excluded because planning fails once the -`edge__partner_listings` table exists (see Known limitations in the README); the -`edge_partner_listings__star_contract_and_quoted_export` scenario covers the models downstream of -it. ### Replay the delivery change log as two loads diff --git a/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_partner_listing_export.sql b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_partner_listing_export.sql index 0712d418b8..ad91f766fa 100644 --- a/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_partner_listing_export.sql +++ b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_partner_listing_export.sql @@ -4,6 +4,9 @@ MODEL ( and prices in major units """, tags [edge_case], + columns ( + "listingId" (nullable false, audits [not_null, unique]), + ), ); WITH listings AS ( diff --git a/benchmarks/duckshop/project/sqlbuild_project.toml b/benchmarks/duckshop/project/sqlbuild_project.toml index d93428c719..569b4fa2a0 100644 --- a/benchmarks/duckshop/project/sqlbuild_project.toml +++ b/benchmarks/duckshop/project/sqlbuild_project.toml @@ -286,21 +286,4 @@ The edge case selects the key columns that FULL OUTER JOIN ... USING merges. The side, so qualifying them with either alias would drop the keys of rows found only on the other side. """ -[[rules.rule_ignores]] -rules = ["SQBRTEST201"] -paths = ["models/marts/edge_cases/rpt_edge_partner_listing_export.sql"] -reason = """ -The partner feed export names its columns in quoted camelCase. A MODEL header cannot declare a -quoted column name, so the model has no columns block to attach audits to, and a generic audit that -names "listingId" fails to bind because the quoted name is folded to lower case. -""" - -[[rules.rule_ignores]] -rules = ["SQBRTEST202"] -paths = ["models/staging/edge_cases/stg_edge_partner_listings.sql"] -reason = """ -Unit-test and scenario fixtures for a source whose declared columns are quoted camelCase names are -rejected (B302 and S510), so this model is covered by building the edge-case area on Snowflake, -where it reads the loaded quoted columns, instead of by a unit test. -""" # <<< edge-case area diff --git a/benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_partner_listings.sql b/benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_partner_listings.sql new file mode 100644 index 0000000000..80e3c2560e --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_partner_listings.sql @@ -0,0 +1,30 @@ +TEST (name "stg_edge_partner_listings__reads_quoted_camel_case_columns_by_exact_name"); + +WITH __source__edge__partner_listings AS ( + SELECT + v."listingId", + v."unitPriceMinor"::NUMBER(38, 0) AS "unitPriceMinor", + v."listingStatus", + '2026-03-01 10:00:00'::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('LS-MALLARD-01', 1299, 'LIVE'), + ('LS-TEAL-02', 899, 'Out_Of_Stock'), + ('LS-EIDER-03', 5, NULL) AS v("listingId", "unitPriceMinor", "listingStatus") +), + +__expected__stg_edge_partner_listings AS ( + SELECT + 'LS-MALLARD-01' AS listing_id, + 1299::NUMBER(38, 0) AS unit_price_minor, + 'live' AS listing_status + UNION ALL + SELECT + 'LS-TEAL-02' AS listing_id, + 899::NUMBER(38, 0) AS unit_price_minor, + 'out_of_stock' AS listing_status + UNION ALL + SELECT + 'LS-EIDER-03' AS listing_id, + 5::NUMBER(38, 0) AS unit_price_minor, + CAST(NULL AS VARCHAR) AS listing_status +) diff --git a/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs b/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs index 64a04fda35..da6625e7a1 100644 --- a/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs +++ b/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs @@ -17,6 +17,7 @@ use crate::contracts::models::{ DeclaredColumn, InferredOutputColumn, RelatedOutput, }; use crate::type_system::errors::TypeNormalizationError; +use sqlbuild_sqltext::resolution::main::unquoted_column_name::unquoted_column_name as output_column_name; /// Python's `_requires_contract_evaluation`. pub(crate) fn requires_contract_evaluation(model: &ContractModel, implicit: bool) -> bool { @@ -85,12 +86,17 @@ pub(crate) fn model_contract_diagnostics( .map(String::as_str) .collect(); for (index, declared) in schema.columns.iter().enumerate() { - let Some(inferred) = inferred_by_name.get(declared.name.as_str()).or_else(|| { - types - .folds_upper() - .then(|| inferred_by_name.get(declared.name.to_uppercase().as_str())) - .flatten() - }) else { + let output: &str = output_column_name(&declared.name); + let Some(inferred) = inferred_by_name + .get(declared.name.as_str()) + .or_else(|| inferred_by_name.get(output)) + .or_else(|| { + types + .folds_upper() + .then(|| inferred_by_name.get(declared.name.to_uppercase().as_str())) + .flatten() + }) + else { if validate_declared_shape && !model.fast_lineage_has_star { diagnostics.push(missing_column(model, index, declared, contract_enforced)); } @@ -168,8 +174,11 @@ fn extra_columns( .columns .iter() .flat_map(|column| { + let output: &str = output_column_name(&column.name); let folded: Option = folds_upper.then(|| column.name.to_uppercase()); - std::iter::once(column.name.clone()).chain(folded) + std::iter::once(column.name.clone()) + .chain(std::iter::once(output.to_owned())) + .chain(folded) }) .collect(); let help: &str = if schema.named_schema { diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/checks.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/checks.rs index 5a272e337d..74e70129b8 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/checks.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/checks.rs @@ -1,5 +1,6 @@ //! Python's `get_semantic_metadata_diagnostics`, apart from the resource SQL checks. +use sqlbuild_sqltext::resolution::main::column_name_key::column_name_key; use std::collections::{HashMap, HashSet}; use std::sync::LazyLock; @@ -128,11 +129,11 @@ fn model_error( }) } -/// Python's `{column.casefold() for column in shape}`. +/// Each shape column's case-folded name, without the quotes of a quoted identifier. fn folded_names(shape: &[(String, String)]) -> Result, SemanticFailure> { let mut names: HashSet = HashSet::with_capacity(shape.len()); for (name, _) in shape { - names.insert(casefold(name)); + names.insert(column_name_key(name)); } Ok(names) } @@ -162,7 +163,7 @@ fn reference_errors( } let mut errors: Vec = Vec::new(); for (key, name, reference_shape) in references { - if !folded_names(reference_shape)?.contains(&casefold(name)) { + if !folded_names(reference_shape)?.contains(&column_name_key(name)) { let message = format!("{key} references unknown column '{name}'"); errors.push(model_error( model, @@ -232,7 +233,7 @@ fn source_errors( else { continue; }; - if folded_names(shape)?.contains(&casefold(cursor)) { + if folded_names(shape)?.contains(&column_name_key(cursor)) { continue; } let (line, column) = text_position(&source.contents, cursor, 0)?; @@ -267,7 +268,7 @@ fn sql_test_errors( }; let available: HashSet = folded_names(shape)?; for column_name in columns { - if available.contains(&casefold(column_name)) { + if available.contains(&column_name_key(column_name)) { continue; } let offset: usize = find_code_point(&test.contents, cte_name).unwrap_or(0); diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs index 7f892c622e..32fd8e8198 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs @@ -23,9 +23,41 @@ use polyglot_sql::{ use rayon::iter::{IntoParallelIterator, ParallelIterator}; const ANALYSIS_WORKERS: usize = 4; + const ANALYSIS_WORKER_STACK_BYTES: usize = 16 * 1024 * 1024; const DEFAULT_BINDING_MESSAGE: &str = "SQL binding failed"; +/// Names some projection quotes, as a CTE's `AS "listingId"` read by `SELECT *`, and none leaves bare. +fn nested_quoted_aliases(parsed: &[polyglot_sql::Expression]) -> HashMap { + use polyglot_sql::{Expression, ExpressionWalk}; + let mut quoted: HashMap = HashMap::new(); + let mut unquoted: std::collections::HashSet = std::collections::HashSet::new(); + for expression in parsed { + for node in expression.dfs() { + let Expression::Select(select) = node else { + continue; + }; + for projection in &select.expressions { + let identifier = match projection { + Expression::Alias(alias) => &alias.alias, + Expression::Column(column) => &column.name, + _ => continue, + }; + if identifier.quoted { + quoted.insert( + identifier.name.clone(), + format!("\"{}\"", identifier.name.replace('"', "\"\"")), + ); + } else { + unquoted.insert(identifier.name.clone()); + } + } + } + } + quoted.retain(|name, _| !unquoted.contains(name)); + quoted +} + impl ProjectCatalog { /// Map inferred column types onto the binding names the query projects, in column order. pub fn inferred_schema( @@ -62,6 +94,7 @@ impl ProjectCatalog { break; } } + let nested_quoted: HashMap = nested_quoted_aliases(&parsed); let mut exact_inputs: HashMap = HashMap::new(); for shape in inputs.values() { for (name, _) in &shape.0 { @@ -76,6 +109,7 @@ impl ProjectCatalog { for (name, column_type) in columns.0 { let binding_name = identifiers .get(&name) + .or_else(|| nested_quoted.get(&name)) .or_else(|| exact_inputs.get(&name)) .cloned() .unwrap_or(name); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs index 44e0752476..f3475da690 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs @@ -17,6 +17,7 @@ use pyo3::prelude::{ }; use pyo3::types::{PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_sqltext::resolution::main::column_name_key::column_name_key; use sqlbuild_sqltext::resolution::main::{ reference_spans::reference_spans, source_alias::source_alias, }; @@ -49,11 +50,11 @@ fn check_columns( let available = rows(columns)?; let available: std::collections::HashMap> = available .into_iter() - .map(|(name, kind)| (name.to_lowercase(), kind)) + .map(|(name, kind)| (column_name_key(&name), kind)) .collect(); let missing = declared .iter() - .filter(|(name, _)| !available.contains_key(&name.to_lowercase())) + .filter(|(name, _)| !available.contains_key(&column_name_key(name))) .map(|(name, _)| name.as_str()) .collect::>(); if !missing.is_empty() { @@ -77,7 +78,7 @@ fn check_columns( let Some(kind) = kind else { continue; }; - let Some(Some(actual)) = available.get(&name.to_lowercase()) else { + let Some(Some(actual)) = available.get(&column_name_key(&name)) else { continue; }; if actual.is_empty() { @@ -158,11 +159,11 @@ fn cast_source( if expression { let name_map: std::collections::HashMap = names .iter() - .map(|name| (name.to_lowercase(), *name)) + .map(|name| (column_name_key(name), *name)) .collect(); let missing = declared .iter() - .filter(|(name, _)| !name_map.contains_key(&name.to_lowercase())) + .filter(|(name, _)| !name_map.contains_key(&column_name_key(name))) .map(|(name, _)| name.as_str()) .collect::>(); if !missing.is_empty() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs index e641f17398..9d5f12e701 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs @@ -9,7 +9,6 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; use pyo3::{pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::fixture_schema::fixture_schema; use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; -use sqlbuild_core::text::main::python_casefold::python_casefold; use crate::bindings::_helpers::analysis_session::fixture_inferences::FixtureColumnInferences; use crate::bindings::_helpers::compiled_views::model_reads; @@ -23,7 +22,7 @@ use crate::bindings::_helpers::planner_strategy::constants::{ use crate::bindings::_helpers::planner_strategy::fixture_metadata::model_fixture_relation; use crate::bindings::_helpers::planner_strategy::fixture_reads::lineage_reads; use crate::bindings::_helpers::planner_strategy::fixture_types::projections; -use crate::bindings::_helpers::planner_strategy::fixture_values::columns; +use crate::bindings::_helpers::planner_strategy::fixture_values::{columns, folded_name}; use crate::bindings::_helpers::planner_strategy::models::{ FixtureReadCacheHost, FixtureRelation, ModelFixtureFacts, RelationFixturePlanningContext, }; @@ -180,7 +179,7 @@ impl RelationFixturePlanningContext { let mut fixture_columns: Vec = Vec::with_capacity(columns.len()); for (name, sql_type, nullable) in columns { fixture_columns.push(FixtureColumn { - folded_name: python_casefold(&name), + folded_name: folded_name(&name), name, sql_type, nullable, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs index 3d0bf2cf0c..edff03de7e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs @@ -3,13 +3,12 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::planner_fixtures::main::merge_metadata::merge_metadata; use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; -use sqlbuild_core::text::main::python_casefold::python_casefold; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::model_reads::{inferred_rows, star_flags}; use crate::bindings::_helpers::model_config::typed_config::{config_value, model_values}; use crate::bindings::_helpers::planner_strategy::constants::ENFORCED_CONTRACT; -use crate::bindings::_helpers::planner_strategy::fixture_values::columns; +use crate::bindings::_helpers::planner_strategy::fixture_values::{columns, folded_name}; use crate::bindings::_helpers::planner_strategy::models::FixtureRelation; use crate::bindings::_helpers::records::fields::field as native_field; use sqlbuild_project::types::InferredRow; @@ -29,7 +28,7 @@ pub(crate) fn model_fixture_relation(model: &Bound<'_, PyAny>) -> PyResult) -> PyResult { - value.call_method0("casefold")?.extract() + Ok(folded_name(&value.extract::()?)) +} + +/// The case-folded column name, without the quotes of a quoted identifier such as `"listingId"`. +pub(crate) fn folded_name(name: &str) -> String { + python_casefold(unquoted_column_name(name)) } pub(crate) fn columns(values: &Bound<'_, PyAny>, inferred: bool) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs index 327c4bb470..96c23d32cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs @@ -28,6 +28,7 @@ use crate::bindings::_helpers::semantic_shapes::shapes::{ model_inputs, project_shapes, published_shape, }; use crate::bindings::models::BindingPositions; +use sqlbuild_sqltext::resolution::main::unquoted_column_name::unquoted_column_name; const BINDING_UNKNOWN_TABLE_INTERNAL_CODE: &str = "B000"; const GRAPHLIB_MODULE: &str = "graphlib"; @@ -37,7 +38,7 @@ const UNKNOWN: &str = "UNKNOWN"; const GENERIC_DIALECT: &str = "generic"; /// Dialects whose unquoted identifiers already bind with their exact case. const EXACT_UNQUOTED_DIALECTS: [&str; 1] = ["clickhouse"]; -/// The dialect whose unquoted identifiers resolve as upper case; the adapter lists them lowercased. +/// The upper-case folding dialect; the adapter lists folded names lowercased and others quoted. const UPPERCASE_FOLDING_DIALECT: &str = "snowflake"; type Object<'py> = Bound<'py, PyAny>; @@ -109,7 +110,11 @@ pub(crate) fn source_binding_diagnostics<'py>( } let quoted: Bound<'py, PyDict> = PyDict::new(py); for (column, column_type) in physical { - let name: String = if exact_unquoted || (folds_upper && is_folded_identifier(&column)) { + let listed_quoted: bool = folds_upper && unquoted_column_name(&column) != column; + let name: String = if exact_unquoted + || listed_quoted + || (folds_upper && is_folded_identifier(&column)) + { column } else { format!("\"{}\"", column.replace('"', "\"\"")) diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/tokenization.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/tokenization.rs index 7b6d9d100f..5169ad48f4 100644 --- a/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/tokenization.rs +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/tokenization.rs @@ -133,7 +133,11 @@ impl<'a> HeaderParser<'a> { continue; } let key_token = self.peek().clone(); - let key = self.consume_key()?; + let key = if options.column_entries { + self.consume_column_key()? + } else { + self.consume_key()? + }; if values.iter().any(|(existing, _)| existing == &key) { return Err(format!("duplicate key '{key}'")); } @@ -458,6 +462,19 @@ impl<'a> HeaderParser<'a> { Ok(token.1) } + /// A column name: a bare word, or a double-quoted identifier kept with its quotes. + fn consume_column_key(&mut self) -> Result { + let token = self.peek().clone(); + let quoted: bool = token.0 == STRING_TOKEN + && self.characters.get(token.2) == Some(&'"') + && !opens_triple_quote(self.characters, token.2); + if !quoted { + return self.consume_key(); + } + self.advance(); + Ok(format!("\"{}\"", token.1)) + } + fn at_end_symbol(&self, symbol: Option<&str>) -> bool { let token = self.peek(); symbol.map_or(token.0 == END_TOKEN, |expected| { diff --git a/crates/sqlbuild-sqltext/src/resolution/main/column_name_key.rs b/crates/sqlbuild-sqltext/src/resolution/main/column_name_key.rs new file mode 100644 index 0000000000..0ef40fc505 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/resolution/main/column_name_key.rs @@ -0,0 +1,9 @@ +//! The key that matches a declared column name to a warehouse or inferred column name. + +use crate::resolution::main::unquoted_column_name::unquoted_column_name; + +/// The case-folded unquoted name, so `"listingId"`, `listingId` and `LISTINGID` share one key. +#[must_use] +pub fn column_name_key(name: &str) -> String { + unquoted_column_name(name).to_lowercase() +} diff --git a/crates/sqlbuild-sqltext/src/resolution/main/mod.rs b/crates/sqlbuild-sqltext/src/resolution/main/mod.rs index 4b60837f85..037d057ac1 100644 --- a/crates/sqlbuild-sqltext/src/resolution/main/mod.rs +++ b/crates/sqlbuild-sqltext/src/resolution/main/mod.rs @@ -1,8 +1,10 @@ //! Public SQL resolution decisions. +pub mod column_name_key; pub mod cursor_subquery; pub mod quoted_name; pub mod ref_alias; pub mod reference_spans; pub mod source_alias; pub mod source_cast_plan; +pub mod unquoted_column_name; diff --git a/crates/sqlbuild-sqltext/src/resolution/main/source_cast_plan.rs b/crates/sqlbuild-sqltext/src/resolution/main/source_cast_plan.rs index b9ad75fcd2..3a530ac84e 100644 --- a/crates/sqlbuild-sqltext/src/resolution/main/source_cast_plan.rs +++ b/crates/sqlbuild-sqltext/src/resolution/main/source_cast_plan.rs @@ -1,5 +1,6 @@ //! Plan source casts without making adapter rendering decisions. +use crate::resolution::main::column_name_key::column_name_key; use crate::resolution::models::{SourceCastPlan, SourceProjection}; use crate::resolution::types::SourceColumns; use std::collections::{HashMap, HashSet}; @@ -24,11 +25,11 @@ pub fn source_cast_plan( if expression { let names: HashMap = available .iter() - .map(|(name, _)| (name.to_lowercase(), name.as_str())) + .map(|(name, _)| (column_name_key(name), name.as_str())) .collect(); let mut casts: HashMap<&str, (&str, &str)> = HashMap::new(); for (declared, kind) in enforced { - if let Some(actual) = names.get(&declared.to_lowercase()) { + if let Some(actual) = names.get(&column_name_key(declared)) { casts.insert(*actual, (declared, kind)); } } @@ -42,9 +43,10 @@ pub fn source_cast_plan( }; } let names: HashSet<&str> = available.iter().map(|(name, _)| name.as_str()).collect(); + let keys: HashSet = names.iter().map(|name| column_name_key(name)).collect(); let projections: Vec = enforced .into_iter() - .filter(|(name, _)| names.contains(name)) + .filter(|(name, _)| names.contains(name) || keys.contains(&column_name_key(name))) .map(|(name, kind)| SourceProjection { name: name.to_owned(), cast: Some((name.to_owned(), kind.to_owned())), diff --git a/crates/sqlbuild-sqltext/src/resolution/main/unquoted_column_name.rs b/crates/sqlbuild-sqltext/src/resolution/main/unquoted_column_name.rs new file mode 100644 index 0000000000..022326b062 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/resolution/main/unquoted_column_name.rs @@ -0,0 +1,10 @@ +//! A column name without the double quotes of a quoted identifier. + +/// The name without the double quotes of a quoted identifier, such as `listingId` for `"listingId"`. +#[must_use] +pub fn unquoted_column_name(name: &str) -> &str { + name.strip_prefix('"') + .and_then(|rest| rest.strip_suffix('"')) + .filter(|inner| !inner.is_empty() && !inner.contains('"')) + .unwrap_or(name) +} diff --git a/fensu.toml b/fensu.toml index af975cc307..4d1674dafa 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2076,7 +2076,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs=1", diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index d9da48eca2..aa8559d41a 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -49,6 +49,7 @@ QUALIFIED_NAME_SEPARATOR, ) from sqlbuild.adapter.contract.exceptions import AdapterUserError, ViewRenameRefusedError +from sqlbuild.adapter.contract.main.is_quoted_identifier import is_quoted_identifier from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition from sqlbuild.adapter.contract.main.unsupported_relation_usage import unsupported_relation_usage @@ -399,6 +400,8 @@ def default_promotion_strategy(self) -> PromotionStrategy: def render_identifier(self, name: str) -> str: """Render one DuckDB identifier using double-quote escaping.""" + if is_quoted_identifier(name): + return name return '"' + name.replace('"', '""') + '"' def render_exact_identifier(self, name: str) -> str: diff --git a/src/sqlbuild/adapter/contract/constants.py b/src/sqlbuild/adapter/contract/constants.py index e37c2b2c11..6e00b10343 100644 --- a/src/sqlbuild/adapter/contract/constants.py +++ b/src/sqlbuild/adapter/contract/constants.py @@ -23,3 +23,5 @@ DUCKDB_STORAGE_VERSION_KEY: str = "storage_compatibility_version" LATEST_DUCKDB_STORAGE_VERSION: str = "latest" +QUOTED_IDENTIFIER_DELIMITER: str = '"' +MIN_QUOTED_IDENTIFIER_LENGTH: int = 3 diff --git a/src/sqlbuild/adapter/contract/main/is_quoted_identifier.py b/src/sqlbuild/adapter/contract/main/is_quoted_identifier.py new file mode 100644 index 0000000000..cbd7a5a740 --- /dev/null +++ b/src/sqlbuild/adapter/contract/main/is_quoted_identifier.py @@ -0,0 +1,19 @@ +"""Column names declared as quoted identifiers, such as `"listingId"`.""" + +from __future__ import annotations + +from sqlbuild.adapter.contract.constants import ( + MIN_QUOTED_IDENTIFIER_LENGTH, + QUOTED_IDENTIFIER_DELIMITER, +) + + +def is_quoted_identifier(name: str) -> bool: + """Whether ``name`` is one double-quoted identifier that keeps its exact spelling.""" + + return ( + len(name) >= MIN_QUOTED_IDENTIFIER_LENGTH + and name.startswith(QUOTED_IDENTIFIER_DELIMITER) + and name.endswith(QUOTED_IDENTIFIER_DELIMITER) + and QUOTED_IDENTIFIER_DELIMITER not in name[1:-1] + ) diff --git a/src/sqlbuild/adapters/snowflake/_helpers/column_names.py b/src/sqlbuild/adapters/snowflake/_helpers/column_names.py new file mode 100644 index 0000000000..ba33c2cbc6 --- /dev/null +++ b/src/sqlbuild/adapters/snowflake/_helpers/column_names.py @@ -0,0 +1,11 @@ +"""SQLBuild names for the column names Snowflake stores.""" + +from __future__ import annotations + + +def logical_column_name(stored: str) -> str: + """Lower case for an unquoted name Snowflake stores in upper case, else the quoted name.""" + + if stored == stored.upper(): + return stored.lower() + return '"' + stored.replace('"', '""') + '"' diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 487064de9b..215694451f 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -49,6 +49,7 @@ from sqlbuild.adapter.contract.main.complete_table_freshness_results import ( complete_table_freshness_results, ) +from sqlbuild.adapter.contract.main.is_quoted_identifier import is_quoted_identifier from sqlbuild.adapter.contract.main.normalize_seed_csv_value import normalize_seed_csv_value from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition @@ -123,6 +124,7 @@ from sqlbuild.adapter.type_system.main.first_arg_nullability import first_arg_nullability from sqlbuild.adapter.type_system.main.normalize_numeric_family import normalize_numeric_family from sqlbuild.adapter.type_system.main.types_equal import types_equal +from sqlbuild.adapters.snowflake._helpers.column_names import logical_column_name from sqlbuild.adapters.snowflake._helpers.cost import cost_failure_cause from sqlbuild.adapters.snowflake._helpers.grants import ( render_snowflake_view_grants, @@ -1566,6 +1568,8 @@ def default_promotion_strategy(self) -> PromotionStrategy: def render_identifier(self, name: str) -> str: """Render a logical identifier using Snowflake's unquoted uppercase semantics.""" + if is_quoted_identifier(name): + return name return '"' + name.upper().replace('"', '""') + '"' def render_exact_identifier(self, name: str) -> str: @@ -2241,7 +2245,7 @@ def read_schema_column_listing( continue columns.setdefault(str(row["table_name"]), []).append( ColumnInfo( - name=str(row["column_name"]).lower(), + name=logical_column_name(str(row["column_name"])), type=show_columns_type(row["data_type"]), ) ) @@ -2312,7 +2316,7 @@ def _read_information_schema_column_listing( for row in rows: columns.setdefault(str(row[0]), []).append( ColumnInfo( - name=str(row[1]).lower(), + name=logical_column_name(str(row[1])), type=information_schema_type( data_type=str(row[2]), numeric_precision=row[3], @@ -2572,7 +2576,7 @@ def get_columns( continue return tuple( ColumnInfo( - name=str(row["column_name"]).lower(), + name=logical_column_name(str(row["column_name"])), type=show_columns_type(row["data_type"]), ) for row in rows @@ -2612,7 +2616,7 @@ def _information_schema_columns( cursor.close() return tuple( ColumnInfo( - name=str(row[0]).lower(), + name=logical_column_name(str(row[0])), type=self._build_information_schema_type( data_type=str(row[1]), numeric_precision=row[2], @@ -2662,7 +2666,7 @@ def get_all_columns( table_name: str = str(row[0]).lower() result.setdefault(table_name, []).append( ColumnInfo( - name=str(row[1]).lower(), + name=logical_column_name(str(row[1])), type=self._build_information_schema_type( data_type=str(row[2]), numeric_precision=row[3], @@ -2729,7 +2733,7 @@ def fetch() -> list[tuple[Any, ...]]: rows: list[tuple[Any, ...]] = run_recorded_inspection_query(sql=query, run=fetch) return tuple( ColumnInfo( - name=str(row[2]).lower(), + name=logical_column_name(str(row[2])), type=show_columns_type(row[3]), ) for row in rows @@ -2762,7 +2766,7 @@ def _get_columns_for_relations_bulk( ) result.setdefault(identity, []).append( ColumnInfo( - name=str(row[3]).lower(), + name=logical_column_name(str(row[3])), type=self._build_information_schema_type( data_type=str(row[4]), numeric_precision=row[5], @@ -3933,7 +3937,9 @@ def describe_relation(self, *, connection: Any, relation: str) -> tuple[ColumnIn rows: list[tuple[Any, ...]] = cursor.fetchall() finally: cursor.close() - return tuple(ColumnInfo(name=str(row[0]).lower(), type=str(row[1])) for row in rows) + return tuple( + ColumnInfo(name=logical_column_name(str(row[0])), type=str(row[1])) for row in rows + ) def build_cursor_filter( self, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 9b1c1f78a4..9c40100751 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1494,3 +1494,13 @@ class DuckDbAttachE2ETestCase: expected_archived_rows: list[tuple[object, ...]] attach_path: str = "archive.duckdb" expected_archive_root: str = "project" + + +@dataclass(frozen=True) +class QuotedMixedCaseColumnsBuildE2ETestCase: + """A source and a model output with quoted mixed-case columns, through every command.""" + + description: str + expected_export_rows: list[tuple[object, ...]] + expected_test_fragment: str + expected_scenario_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_quoted_mixed_case_columns.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_quoted_mixed_case_columns.py new file mode 100644 index 0000000000..93e7c6b482 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_quoted_mixed_case_columns.py @@ -0,0 +1,113 @@ +"""E2E coverage that quoted mixed-case column names work in headers, fixtures, plans and builds.""" + +from __future__ import annotations + +import subprocess +from decimal import Decimal +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + QuotedMixedCaseColumnsBuildE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + prepare_inline_project, + query_duckdb, + run_sqb, +) + +_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "partner_feed"\nadapter = "duckdb"\n\n[connection]\ndatabase = "partner_feed.duckdb"\n' + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_listings\n description: Partner listings in quoted camelCase.\n" + " schema: main\n table: raw_listings\n columns:\n" + " - name: '\"listingId\"'\n type: VARCHAR\n" + " - name: '\"unitPriceMinor\"'\n type: BIGINT\n" + ), + "models/stg_listings.sql": ( + "MODEL (description 'Listings renamed to snake case.', materialized table);\n\n" + 'SELECT s."listingId" AS listing_id, s."unitPriceMinor" AS unit_price_minor\n' + 'FROM __source("raw_listings") AS s\n' + ), + "models/listing_export.sql": ( + "MODEL (\n description 'Listings in the partner feed format.',\n materialized table,\n" + " columns (\n" + ' "listingId" (type VARCHAR, nullable false, audits [not_null, unique]),\n' + ' "unitPrice" (type DECIMAL(18, 2)),\n ),\n);\n\n' + "WITH final AS (\n" + ' SELECT l.listing_id::VARCHAR AS "listingId", (l.unit_price_minor / 100)::DECIMAL(18, 2) AS "unitPrice"\n' + ' FROM __ref("stg_listings") AS l\n)\n\nSELECT *\nFROM final\n' + ), + "tests/unit/test_stg_listings.sql": ( + 'TEST (name "stg_listings__renames_quoted_columns");\n\n' + 'WITH __source__raw_listings AS (SELECT \'L1\' AS "listingId", 1299 AS "unitPriceMinor"),\n\n' + "__expected__stg_listings AS (SELECT 'L1' AS listing_id, 1299 AS unit_price_minor)\n" + ), + "tests/scenarios/listings__export.sql": ( + "SCENARIO (description 'Listings reach the export in quoted camelCase.');\n" + "WITH\n" + '__source__raw_listings AS (SELECT \'L1\' AS "listingId", 1299 AS "unitPriceMinor"),\n' + '__expected__listing_export AS (SELECT \'L1\' AS "listingId", 12.99::DECIMAL(18, 2) AS "unitPrice")\n' + "SELECT 1\n" + ), +} + + +@pytest.mark.parametrize( + "test_case", + [ + QuotedMixedCaseColumnsBuildE2ETestCase( + description="quoted camelCase columns compile, build twice, test and replay", + expected_export_rows=[("L1", Decimal("12.99"))], + expected_test_fragment="PASS=1 FAIL=0", + expected_scenario_fragment="PASS=1 FAIL=0", + ), + ], + ids=lambda case: case.description, +) +def test_given_quoted_mixed_case_columns_when_running_commands_then_each_succeeds( + test_case: QuotedMixedCaseColumnsBuildE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="partner_feed", repo_files=_PROJECT_FILES + ) + db_path: Path = project_dir / "partner_feed.duckdb" + execute_duckdb( + db_path=db_path, + sql='CREATE TABLE main.raw_listings AS SELECT \'L1\' AS "listingId", 1299::BIGINT AS "unitPriceMinor"', + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), project_dir=project_dir + ) + builds: tuple[subprocess.CompletedProcess[str], ...] = tuple( + run_sqb(command=("--no-color", "build"), project_dir=project_dir) for _ in range(2) + ) + tested: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "test"), project_dir=project_dir + ) + replayed: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "scenario", "test"), project_dir=project_dir + ) + + assert compiled.returncode == 0, compiled.stdout + compiled.stderr + assert "0 errors, 0 warnings" in compiled.stdout, compiled.stdout + for build in builds: + assert build.returncode == 0, build.stdout + build.stderr + rows: list[tuple[object, ...]] = query_duckdb( + db_path=db_path, sql='SELECT "listingId", "unitPrice" FROM main.listing_export' + ) + assert rows == test_case.expected_export_rows + assert tested.returncode == 0, tested.stdout + tested.stderr + assert test_case.expected_test_fragment in tested.stdout, tested.stdout + assert replayed.returncode == 0, replayed.stdout + replayed.stderr + assert test_case.expected_scenario_fragment in replayed.stdout, replayed.stdout + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py index ac787310bf..4175f1733f 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py @@ -100,3 +100,12 @@ class PlannerListingReuseTestCase: external_changes: tuple[ tuple[re.Pattern[str], Callable[[SimulatedSnowflakeWarehouse], None]], ... ] = () + + +@dataclass(frozen=True) +class QuotedColumnBuildTestCase: + """A source stored with quoted mixed-case column names, built twice through the real CLI.""" + + description: str + stored_columns: tuple[str, ...] + expected_fragment: str diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_quoted_mixed_case_columns.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_quoted_mixed_case_columns.py new file mode 100644 index 0000000000..9cc4b4d12b --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_quoted_mixed_case_columns.py @@ -0,0 +1,92 @@ +"""A Snowflake source whose columns are stored in quoted mixed case plans and builds offline.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.executor.build.snowflake_metadata._test_types import ( + QuotedColumnBuildTestCase, +) +from tests.integration.src.sqlbuild.executor.build.snowflake_metadata.helpers import ( + SimulatedSnowflakeWarehouse, + build_with_cli, +) +from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import ( + FakeColumn, + FakeRelation, +) + +_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "partner_feed"\nadapter = "snowflake"\n\n' + '[connection]\naccount = "example"\nuser = "example"\n' + 'database = "analytics"\nschema = "marts"\n\n' + '[defaults]\ndatabase = "analytics"\nschema = "marts"\nmaterialized = "table"\n' + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_listings\n description: Partner listings in quoted camelCase.\n" + " database: analytics\n schema: raw\n table: listings\n" + " columns:\n" + " - name: '\"listingId\"'\n type: VARCHAR\n" + " - name: '\"unitPriceMinor\"'\n type: NUMBER(38,0)\n" + ), + "models/stg_listings.sql": ( + "MODEL (description 'Listings renamed to snake case.');\n\n" + 'SELECT s."listingId" AS listing_id, s."unitPriceMinor" AS unit_price_minor\n' + 'FROM __source("raw_listings") AS s\n' + ), + "models/listing_export.sql": ( + "MODEL (\n description 'Listings in the partner feed format.',\n" + ' columns ("listingId" (nullable false, audits [not_null]),),\n);\n\n' + 'WITH final AS (\n SELECT l.listing_id AS "listingId"\n' + ' FROM __ref("stg_listings") AS l\n)\n\nSELECT *\nFROM final\n' + ), +} + + +@pytest.mark.parametrize( + "test_case", + [ + QuotedColumnBuildTestCase( + description="quoted camelCase source columns bind in planning on a first build and rebuild", + stored_columns=("listingId", "unitPriceMinor"), + expected_fragment='"listingId" IS NULL', + ), + ], + ids=lambda case: case.description, +) +def test_given_quoted_mixed_case_source_columns_when_building_then_planning_binds_them( + test_case: QuotedColumnBuildTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + for relative, content in _PROJECT_FILES.items(): + path: Path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(content, encoding="utf-8") + columns: tuple[FakeColumn, ...] = tuple( + FakeColumn(name=name, data_type="TEXT") for name in test_case.stored_columns + ) + warehouse: SimulatedSnowflakeWarehouse = SimulatedSnowflakeWarehouse( + relations=( + FakeRelation(database="ANALYTICS", schema="RAW", name="LISTINGS", columns=columns), + ), + ) + + builds: tuple[tuple[int, tuple[str, ...]], ...] = tuple( + build_with_cli(project_dir=tmp_path, warehouse=warehouse, monkeypatch=monkeypatch) + for _ in range(2) + ) + + output: str = capsys.readouterr().out + assert tuple(build[0] for build in builds) == (0, 0), output + assert "B002" not in output, output + assert any(test_case.expected_fragment in statement for statement in builds[1][1]), builds[1][1] + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 303a432e749b6d285be83150991df746b3d33f8c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:48:45 +0000 Subject: [PATCH 08/20] build: pin polyglot-sql-chio 0.13.10 The new wheel writes DuckDB lambdas as LAMBDA x : body, so the local scenario replay of a lambda now passes and its strict xfail goes. --- pyproject.toml | 2 +- .../commands/main/scenario/test_local_lambda.py | 4 ---- uv.lock | 16 ++++++++-------- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 29ca817a64..78dd656058 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ "duckdb>=2.0.0", "filelock>=3.20.3", "orjson>=3.11.7", - "polyglot-sql-chio==0.13.9", + "polyglot-sql-chio==0.13.10", "pydantic-settings>=2,<3", "PyYAML>=6.0.2", "sqlbuild-cli==0.185.0", # x-release-please-version diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_lambda.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_lambda.py index 642a7575f9..707c1e5e6f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_lambda.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_lambda.py @@ -17,10 +17,6 @@ from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb -@pytest.mark.xfail( - strict=True, - reason="needs the polyglot-sql-chio wheel that writes DuckDB lambdas as LAMBDA x : body", -) @pytest.mark.parametrize( "test_case", [ diff --git a/uv.lock b/uv.lock index 701bac993b..ecce22e058 100644 --- a/uv.lock +++ b/uv.lock @@ -2964,15 +2964,15 @@ wheels = [ [[package]] name = "polyglot-sql-chio" -version = "0.13.9" +version = "0.13.10" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/80/60/57a3b6fe1cea85c5229f32dc6becb3ad445f8318dd53623fcfaafdbaaa0c/polyglot_sql_chio-0.13.9.tar.gz", hash = "sha256:99655df81e3fb2d1b4017bdb382ed39eac104e7afdb502f9fb32d2ef4a63d166", size = 2112062, upload-time = "2026-10-08T01:26:30.773Z" } +sdist = { url = "https://files.pythonhosted.org/packages/18/8c/b48d0fcca36c2e6df0a2ab8aa24e029373264b6dfa9db683eb533f6b6249/polyglot_sql_chio-0.13.10.tar.gz", hash = "sha256:4365617889c3779fd1a3ebdd15ab03c92c517cb1be36dedf82e5853b31d39781", size = 2279296, upload-time = "2026-10-28T23:07:03.083Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/52/de/b8b877381aa2cf530dcc74e83d47a636f68a777d3682fc9fb080e03d4f3c/polyglot_sql_chio-0.13.9-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:ee8246059495e7808a0ac3a3306375124d268088912b0e3020d5b2a64587c4ad", size = 12727017, upload-time = "2026-10-08T01:26:18.545Z" }, - { url = "https://files.pythonhosted.org/packages/87/9e/8ed75a9f9cd324d0305080e1f6d4b54c17d88d21a2935200f38b422dfd0c/polyglot_sql_chio-0.13.9-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:63d38d1b20731c4cd6f3a370e34a8cd0023d99071b4b665fd645943b36d03617", size = 11972559, upload-time = "2026-10-08T01:26:20.877Z" }, - { url = "https://files.pythonhosted.org/packages/43/32/ed55ff16069f60d8d6f3fe744a28026be0cbaf1b4965220940bf9883847d/polyglot_sql_chio-0.13.9-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9c881b7e4c7c098ab4ec605918c45bbc25f2b953a36e1d9902305cd2c18157f", size = 12831250, upload-time = "2026-10-08T01:26:23.269Z" }, - { url = "https://files.pythonhosted.org/packages/8b/34/f016ab19dc804321c49564ab67a15b2bd232a6e6facbd6c7d98252fbae1d/polyglot_sql_chio-0.13.9-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32bd401b19431e96b148a38f08f610d75f06fbf7f4f72d5bd790dae5fb4397f6", size = 11997845, upload-time = "2026-10-08T01:26:26.139Z" }, - { url = "https://files.pythonhosted.org/packages/de/ea/79b0e54a12b0dd9a0fb7df57184df3c8fb22edfe8438702994659f2eb813/polyglot_sql_chio-0.13.9-cp312-abi3-win_amd64.whl", hash = "sha256:afacd08235f149348893f4a574d357bb54d8cd42875527c1934de586941323df", size = 11695291, upload-time = "2026-10-08T01:26:28.451Z" }, + { url = "https://files.pythonhosted.org/packages/bb/7a/c66b36a40c0931d9647215f315f90e2e0d83328165dce7275d624f694381/polyglot_sql_chio-0.13.10-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:a1abe78cde4b54d145f42356ca4c3a7585775aa4efee954209084ea993633f90", size = 13701198, upload-time = "2026-10-28T23:06:50.006Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e2/b2ba58e445163b9e29a715d9e3eab5fd9e276e8bbe58f5710bd77aae70d7/polyglot_sql_chio-0.13.10-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:a7d01559916bff555fd0c258f4a52a98e3f2fa25b1f4c482ef35cb868704c3b6", size = 12941723, upload-time = "2026-10-28T23:06:52.738Z" }, + { url = "https://files.pythonhosted.org/packages/c6/08/a9e5cb57c9938551edeb0884cc1b57ef54bbb38eb5f371bb146b663881ab/polyglot_sql_chio-0.13.10-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e5ee843dccffe87501113238ba333c17dce55b31867fcdc1b81e245b9c3eb4", size = 12895624, upload-time = "2026-10-28T23:06:55.52Z" }, + { url = "https://files.pythonhosted.org/packages/a0/9e/0879b3a267899906c231270d0167d8c32c14dcc2ae346b86834c4f7bb8b2/polyglot_sql_chio-0.13.10-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd2ca982a19c7fbeb460914eceae60d9c5f7e6e631473f0141585b778772c1ce", size = 12661001, upload-time = "2026-10-28T23:06:58.287Z" }, + { url = "https://files.pythonhosted.org/packages/e7/13/dbdbce022153042edb59abe07e64c51a0e0a9a8f4c86cf2043f1c42a1da7/polyglot_sql_chio-0.13.10-cp312-abi3-win_amd64.whl", hash = "sha256:f366391d63fbf455f71c07fb03e4ac1cf860609de7fd875759647ffe98659320", size = 12301030, upload-time = "2026-10-28T23:07:00.945Z" }, ] [[package]] @@ -4323,7 +4323,7 @@ requires-dist = [ { name = "ingestr", marker = "extra == 'ingestr'", specifier = ">=1.1.73" }, { name = "jinja2", marker = "extra == 'dbt'", specifier = ">=3.1" }, { name = "orjson", specifier = ">=3.11.7" }, - { name = "polyglot-sql-chio", specifier = "==0.13.9" }, + { name = "polyglot-sql-chio", specifier = "==0.13.10" }, { name = "psycopg", extras = ["binary"], marker = "extra == 'postgres'", specifier = ">=3.2.0" }, { name = "pydantic-settings", specifier = ">=2,<3" }, { name = "pymssql", marker = "extra == 'sqlserver'", specifier = ">=2.3.0" }, From f1c7ba8504b0f155f2f7d9e24460ea62e61bb5af Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:48:47 +0000 Subject: [PATCH 09/20] perf(query): show progress only for slow steps and lighten imports sqb query no longer prints a Project discovery lifecycle for reading two configuration files. Connecting and running the query show a transient stderr line on a terminal only once the step has taken 300 ms, named for the step; piped and JSON output hold only the result. The user-code import hook first searches only the search path entries under a project root, so installed and standard-library imports skip the full path search they used to repeat. The Snowflake adapter sets SNOWFLAKE_DISABLE_PLATFORM_DETECTION=true unless the user set it, which removes the connector's cloud metadata probes, its AWS identity call and the botocore warnings it printed. The adapter page documents how to turn detection back on. --- .../build_progress/transient_status.rs | 45 +++++++++-- .../_helpers/discovery/configuration.rs | 51 ++++++------ .../bindings/_helpers/query_command/query.rs | 41 +++++++--- .../docs/concepts/adapters/snowflake.md | 6 ++ .../snowflake/classes/snowflake_adapter.py | 6 +- src/sqlbuild/adapters/snowflake/constants.py | 2 + .../user_code/classes/user_code_finder.py | 28 ++++++- .../cli/commands/main/query/_test_types.py | 9 +++ .../main/query/test_query_progress.py | 79 +++++++++++++++++++ .../adapters/snowflake/_test_types.py | 7 ++ .../adapters/snowflake/test_client.py | 31 ++++++++ .../runtime/user_code/classes/__init__.py | 0 .../runtime/user_code/classes/_test_types.py | 13 +++ .../classes/test_user_code_finder.py | 59 ++++++++++++++ .../docs/docs/concepts/adapters/snowflake.mdx | 6 ++ 15 files changed, 342 insertions(+), 41 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_progress.py create mode 100644 tests/unit/src/sqlbuild/runtime/user_code/classes/__init__.py create mode 100644 tests/unit/src/sqlbuild/runtime/user_code/classes/_test_types.py create mode 100644 tests/unit/src/sqlbuild/runtime/user_code/classes/test_user_code_finder.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs index a64ef9dc5e..1ceaa66c6b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs @@ -1,6 +1,8 @@ //! One transient status line with a spinner on a terminal and plain lines elsewhere. +use std::sync::mpsc::{self, RecvTimeoutError}; use std::sync::{Mutex, MutexGuard, PoisonError}; +use std::time::Duration; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; @@ -315,24 +317,57 @@ pub(crate) fn progress_spinners_disabled() -> bool { .is_ok_and(|value| PROGRESS_DISABLED_VALUES.contains(&value.trim().to_lowercase().as_str())) } +/// A spinner reporter on stderr, or `None` when stderr is not a terminal or progress is off. +fn stderr_reporter(py: Python<'_>) -> PyResult>> { + let stderr: Bound<'_, PyAny> = py.import("sys")?.getattr("stderr")?; + if progress_spinners_disabled() || !tty_flag(&stderr)? { + return Ok(None); + } + Py::new(py, TransientStatusReporter::new(py, &stderr, false, true)?).map(Some) +} + /// Run `body` under a stderr spinner when `enabled` and stderr is an interactive terminal. pub(crate) fn maybe_status<'py, T>( py: Python<'py>, (message, enabled): (&str, bool), body: impl FnOnce() -> PyResult, ) -> PyResult { - let stderr: Bound<'py, PyAny> = py.import("sys")?.getattr("stderr")?; - if !enabled || progress_spinners_disabled() || !tty_flag(&stderr)? { + let Some(reporter) = enabled.then(|| stderr_reporter(py)).transpose()?.flatten() else { return body(); - } - let reporter: Bound<'py, TransientStatusReporter> = - Bound::new(py, TransientStatusReporter::new(py, &stderr, false, true)?)?; + }; + let reporter: Bound<'py, TransientStatusReporter> = reporter.into_bound(py); TransientStatusReporter::start(&reporter, message.to_owned())?; let result: PyResult = body(); TransientStatusReporter::close(&reporter)?; result } +/// Run `body`, drawing a stderr spinner named `message` only once it has run for `delay`. +pub(crate) fn delayed_status<'py, T>( + py: Python<'py>, + (message, delay): (&str, Duration), + body: impl FnOnce() -> PyResult, +) -> PyResult { + let Some(reporter) = stderr_reporter(py)? else { + return body(); + }; + let (finished, waiting) = mpsc::channel::<()>(); + let pending: Py = reporter.clone_ref(py); + let message: String = message.to_owned(); + let timer = std::thread::spawn(move || -> PyResult<()> { + if waiting.recv_timeout(delay) != Err(RecvTimeoutError::Timeout) { + return Ok(()); + } + Python::attach(|py| TransientStatusReporter::start(pending.bind(py), message)) + }); + let result: PyResult = body(); + let _ = finished.send(()); + let started: PyResult<()> = py.detach(|| timer.join()).unwrap_or_else(|_| Ok(())); + TransientStatusReporter::close(reporter.bind(py))?; + started?; + result +} + /// A native status reporter writing to `stream`. pub(crate) fn status_reporter<'py>( stream: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs index 96c80164de..032e560736 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs @@ -23,12 +23,12 @@ pub(crate) fn discover_project_configuration<'py>( configuration_inputs(py, project_dir, false) } -/// `discover_project_configuration` with the target warehouses and connections validated. +/// The validated configuration without a discovery lifecycle, which reading two files never needs. pub(crate) fn discover_connection_configuration<'py>( py: Python<'py>, project_dir: &Bound<'py, PyAny>, ) -> PyResult> { - configuration_inputs(py, project_dir, true) + loaded_configuration(py, project_dir, true) } fn configuration_inputs<'py>( @@ -47,25 +47,32 @@ fn configuration_inputs<'py>( ], )?; within(&lifecycle, |_| { - let project_config: Bound<'py, PyAny> = - load_project_config_file(py, project_dir)?.into_bound(py); - let local_config: Bound<'py, PyAny> = - load_local_config_file(py, project_dir)?.into_bound(py); - if validate_connections { - reject_invalid_connection_configuration(&ProjectConfiguration { - project_dir, - project_config: &project_config, - local_config: &local_config, - })?; - } - new_record_with( - py, - RecordId::DiscoveredProjectInputs, - &[ - ("project_dir", project_dir.clone()), - ("project_config", project_config), - ("local_config", local_config), - ], - ) + loaded_configuration(py, project_dir, validate_connections) }) } + +fn loaded_configuration<'py>( + py: Python<'py>, + project_dir: &Bound<'py, PyAny>, + validate_connections: bool, +) -> PyResult> { + let project_config: Bound<'py, PyAny> = + load_project_config_file(py, project_dir)?.into_bound(py); + let local_config: Bound<'py, PyAny> = load_local_config_file(py, project_dir)?.into_bound(py); + if validate_connections { + reject_invalid_connection_configuration(&ProjectConfiguration { + project_dir, + project_config: &project_config, + local_config: &local_config, + })?; + } + new_record_with( + py, + RecordId::DiscoveredProjectInputs, + &[ + ("project_dir", project_dir.clone()), + ("project_config", project_config), + ("local_config", local_config), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs index e659c3c734..08a58f2c93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs @@ -1,5 +1,7 @@ //! `sqb query`: run ad hoc SQL on the project connection and print the rows. +use std::time::Duration; + use pyo3::exceptions::PyException; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -7,12 +9,14 @@ use pyo3::prelude::{ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyString, PyStringMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::json::models::JsonValue; +use sqlbuild_core::progress::main::connection_start_message::connection_start_message; use sqlbuild_core::query_command::main::render_query_json::render_query_json; use sqlbuild_core::query_command::models::QueryDisplay; use sqlbuild_host::entries::models::AdapterHost; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::build_progress::transient_status::delayed_status; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::native_host::host; @@ -30,6 +34,9 @@ use crate::bindings::_helpers::records::ids::ErrorId; const JSON_FORMAT: &str = "json"; const STATEMENT_OK: &str = "OK\n"; const DEFAULT_LIMIT: i64 = 20; +/// How long a step runs before its transient progress line appears. +const SLOW_STEP: Duration = Duration::from_millis(300); +const RUNNING_QUERY_MESSAGE: &str = "Running query..."; /// A `CliUserError` with its code and optional help. fn user_error(py: Python<'_>, message: &str, code: Option<&str>, help: Option<&str>) -> PyErr { @@ -160,22 +167,32 @@ fn run_sql<'py>( ], )?), )?; - let connection: Bound<'py, PyAny> = project - .adapter - .call_method1("connect", (connection_config,)) - .map_err(|error| connection_failure(&project.adapter, &error))?; + let adapter_name: String = native_field(&project.adapter, "adapter_name")?.extract()?; + let connection: Bound<'py, PyAny> = delayed_status( + py, + (&connection_start_message(&adapter_name, 1), SLOW_STEP), + || { + project + .adapter + .call_method1("connect", (connection_config,)) + .map_err(|error| connection_failure(&project.adapter, &error)) + }, + )?; let limit_value: Bound<'py, PyAny> = match limit { Some(limit) => PyInt::new(py, limit).into_any(), None => py.None().into_bound(py), }; - let queried: PyResult> = call_with( - &native_field(&project.adapter, "query")?, - &[ - ("connection", connection.clone()), - ("sql", sql), - ("limit", limit_value), - ], - ); + let queried: PyResult> = + delayed_status(py, (RUNNING_QUERY_MESSAGE, SLOW_STEP), || { + call_with( + &native_field(&project.adapter, "query")?, + &[ + ("connection", connection.clone()), + ("sql", sql), + ("limit", limit_value), + ], + ) + }); let queried: PyResult> = queried.map_err(|error| { let adapter_error: bool = host::entry(py, AdapterHost::AdapterUserError) .is_ok_and(|class| error.value(py).is_instance(&class).unwrap_or(false)); diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md index d82303fc86..ca81724ba2 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md @@ -46,6 +46,12 @@ schema = "RAW" Connection fields are passed directly to `snowflake.connector.connect()`. See the [Snowflake Connector documentation](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect) for all available options, including key-pair authentication, OAuth, and SSO. Put the authoritative database and schema on the target. +SQLBuild turns off the connector's platform detection by setting +`SNOWFLAKE_DISABLE_PLATFORM_DETECTION=true` when the variable is unset. Detection runs on every +connect: it probes cloud metadata endpoints, calls AWS to identify the caller for client +telemetry, and can print `botocore` warnings when an AWS SSO session has expired. To keep it on, +set the variable yourself before running `sqb`, for example `SNOWFLAKE_DISABLE_PLATFORM_DETECTION=false`. + ## Session initialization On connect, SQLBuild applies the connection's role and warehouse, then applies the active diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 215694451f..9dbd19841c 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -5,6 +5,7 @@ import csv import json import logging +import os from collections.abc import Callable from dataclasses import replace from datetime import date, datetime @@ -158,6 +159,8 @@ MAX_STATEMENT_TIMEOUT_SECONDS, MFA_AUTHENTICATOR, OAUTH_AUTHORIZATION_CODE_AUTHENTICATOR, + PLATFORM_DETECTION_DISABLED, + PLATFORM_DETECTION_VARIABLE, SECONDARY_ROLES_ALL, SECONDARY_ROLES_NONE, SHOW_LIKE_ESCAPE_CHARACTER, @@ -1915,8 +1918,9 @@ def supports_python_functions(self) -> bool: return True def connect(self, config: dict[str, Any]) -> _SnowflakeConnection: - """Open a Snowflake connection from the resolved connection config.""" + """Open a Snowflake connection, with platform detection off unless the user set it.""" + _ = os.environ.setdefault(PLATFORM_DETECTION_VARIABLE, PLATFORM_DETECTION_DISABLED) try: import snowflake.connector except ImportError as error: diff --git a/src/sqlbuild/adapters/snowflake/constants.py b/src/sqlbuild/adapters/snowflake/constants.py index e5f457a610..9d8e1fe456 100644 --- a/src/sqlbuild/adapters/snowflake/constants.py +++ b/src/sqlbuild/adapters/snowflake/constants.py @@ -47,3 +47,5 @@ QUOTED_IDENTIFIER_MARK: str = '"' NAME_SEPARATOR: str = "." FIXED_POINT_RESULT_TYPE: str = "FIXED" +PLATFORM_DETECTION_VARIABLE: str = "SNOWFLAKE_DISABLE_PLATFORM_DETECTION" +PLATFORM_DETECTION_DISABLED: str = "true" diff --git a/src/sqlbuild/runtime/user_code/classes/user_code_finder.py b/src/sqlbuild/runtime/user_code/classes/user_code_finder.py index cecc1f56ed..5981453b19 100644 --- a/src/sqlbuild/runtime/user_code/classes/user_code_finder.py +++ b/src/sqlbuild/runtime/user_code/classes/user_code_finder.py @@ -13,7 +13,7 @@ class UserCodeFinder: - """Find modules normally, switching source files under a project root to source-only loading.""" + """Load modules found under a project root from source; search only project entries first.""" def __init__(self) -> None: self.roots: set[Path] = set() @@ -26,6 +26,8 @@ def __init__(self) -> None: *sysconfig.get_paths().values(), } ) + self._project_entries: dict[str, bool] = {} + self._entry_roots: frozenset[Path] = frozenset() def find_spec( self, @@ -33,6 +35,12 @@ def find_spec( path: Sequence[str] | None = None, target: ModuleType | None = None, ) -> ModuleSpec | None: + entries: Sequence[str] = sys.path if path is None else path + project_entries: list[str] = [entry for entry in entries if self._is_project_entry(entry)] + if not project_entries: + return None + if PathFinder.find_spec(fullname, project_entries, target) is None: + return None spec: ModuleSpec | None = PathFinder.find_spec(fullname, path, target) if spec is None or spec.origin is None or not isinstance(spec.loader, SourceFileLoader): return None @@ -45,3 +53,21 @@ def find_spec( return None spec.loader = SourceOnlyLoader(fullname, spec.origin) return spec + + def _is_project_entry(self, entry: str) -> bool: + roots: frozenset[Path] = frozenset(self.roots) + if roots != self._entry_roots: + self._project_entries = {} + self._entry_roots = roots + known: bool | None = self._project_entries.get(entry) + if known is not None: + return known + try: + resolved: Path = Path(entry or ".").resolve() + except OSError: + resolved = Path(entry) + under_root: bool = any( + resolved.is_relative_to(root) or root.is_relative_to(resolved) for root in roots + ) + self._project_entries[entry] = under_root + return under_root diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/query/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/query/_test_types.py index 6bfcfc0b91..0cdc0c3210 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/query/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/query/_test_types.py @@ -62,3 +62,12 @@ class QueryConnectionConfigErrorTestCase: project_toml: str expected_stderr_fragment: str expected_returncode: int = 1 + + +@dataclass(frozen=True) +class QueryProgressTestCase: + description: str + sql: str + use_pty: bool + expected_fragments: tuple[str, ...] + unexpected_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_progress.py b/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_progress.py new file mode 100644 index 0000000000..ffdab51701 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_progress.py @@ -0,0 +1,79 @@ +"""`sqb query` prints progress only for slow steps, on a terminal, and never on piped stdout.""" + +from __future__ import annotations + +import subprocess +from collections.abc import Callable +from functools import partial +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.query._test_types import QueryProgressTestCase +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + run_sqb, + run_sqb_with_pty, +) + +_PROJECT_TOML: str = ( + 'name = "orders_project"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' +) +_SLOW_SQL: str = "SELECT COUNT(*) AS orders FROM range(3000000000) AS t(i) WHERE i % 7 = 3" + + +_RUNNERS: dict[bool, Callable[..., subprocess.CompletedProcess[str]]] = { + False: run_sqb, + True: partial(run_sqb_with_pty, timeout_seconds=300.0), +} + + +@pytest.mark.parametrize( + "test_case", + [ + QueryProgressTestCase( + description="piped output holds only the result", + sql="SELECT 7 AS order_id", + use_pty=False, + expected_fragments=("order_id", "1 row"), + unexpected_fragments=("Project discovery", "Connecting to", "Running query"), + ), + QueryProgressTestCase( + description="a fast query on a terminal prints only its result", + sql="SELECT 7 AS order_id", + use_pty=True, + expected_fragments=("order_id", "1 row"), + unexpected_fragments=("Project discovery", "Connecting to", "Running query"), + ), + QueryProgressTestCase( + description="a slow query on a terminal shows a transient line naming it", + sql=_SLOW_SQL, + use_pty=True, + expected_fragments=("Running query...", "1 row"), + unexpected_fragments=("Project discovery",), + ), + ], + ids=lambda case: case.description, +) +def test_given_query_when_running_then_progress_names_only_slow_steps( + test_case: QueryProgressTestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_project", + repo_files={"sqlbuild_project.toml": _PROJECT_TOML}, + ) + + completed: subprocess.CompletedProcess[str] = _RUNNERS[test_case.use_pty]( + command=("--no-color", "query", test_case.sql), project_dir=project_dir + ) + + assert completed.returncode == 0, completed.stdout + completed.stderr + for fragment in test_case.expected_fragments: + assert fragment in completed.stdout, completed.stdout + for fragment in test_case.unexpected_fragments: + assert fragment not in completed.stdout, completed.stdout + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index 8d8e943b24..30ed59be03 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -410,3 +410,10 @@ class SnowflakeRunListingTestCase: relations: tuple[FakeRelation, ...] lookups: tuple[str, ...] expected_listings: tuple[int, int] + + +@dataclass(frozen=True) +class SnowflakePlatformDetectionTestCase: + description: str + environment: dict[str, str] + expected_value: str diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py index 57f8ea7ea7..6a9dcf2cdd 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from datetime import UTC, datetime, timedelta, timezone from pathlib import Path from typing import Any, cast @@ -34,6 +35,7 @@ SnowflakeLoadSeedTestCase, SnowflakeMergeExclusionTestCase, SnowflakeMoveOrCopyRelationTestCase, + SnowflakePlatformDetectionTestCase, SnowflakePruneSqlTestCase, SnowflakeQualifiedColumnInspectionTestCase, SnowflakeQueryColumnNamesTestCase, @@ -335,6 +337,35 @@ def test_given_connection_config_when_connecting_then_uses_expected_connector_an assert tuple(raw_connection.executed_sql) == test_case.expected_session_statements +@pytest.mark.parametrize( + "test_case", + [ + SnowflakePlatformDetectionTestCase( + description="connector platform detection is turned off when the user has not chosen", + environment={}, + expected_value="true", + ), + SnowflakePlatformDetectionTestCase( + description="the user's own platform detection setting is kept", + environment={"SNOWFLAKE_DISABLE_PLATFORM_DETECTION": "false"}, + expected_value="false", + ), + ], + ids=lambda case: case.description, +) +def test_given_platform_detection_setting_when_connecting_then_user_choice_wins( + test_case: SnowflakePlatformDetectionTestCase, + monkeypatch: pytest.MonkeyPatch, +) -> None: + environment: dict[str, str] = dict(test_case.environment) + monkeypatch.setattr(os, "environ", environment) + _ = install_fake_snowflake_connector(monkeypatch) + + SnowflakeAdapter().connect({"account": "acct"}) + + assert environment["SNOWFLAKE_DISABLE_PLATFORM_DETECTION"] == test_case.expected_value + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/runtime/user_code/classes/__init__.py b/tests/unit/src/sqlbuild/runtime/user_code/classes/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/runtime/user_code/classes/_test_types.py b/tests/unit/src/sqlbuild/runtime/user_code/classes/_test_types.py new file mode 100644 index 0000000000..2a84a8d568 --- /dev/null +++ b/tests/unit/src/sqlbuild/runtime/user_code/classes/_test_types.py @@ -0,0 +1,13 @@ +"""Parameterized user-code finder test cases.""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class FinderSpecTestCase: + """One module to import, whether an earlier path entry shadows it, and the expected loader.""" + + description: str + module_name: str + earlier_entries: tuple[str, ...] + expected_source_only: bool diff --git a/tests/unit/src/sqlbuild/runtime/user_code/classes/test_user_code_finder.py b/tests/unit/src/sqlbuild/runtime/user_code/classes/test_user_code_finder.py new file mode 100644 index 0000000000..c832178ef0 --- /dev/null +++ b/tests/unit/src/sqlbuild/runtime/user_code/classes/test_user_code_finder.py @@ -0,0 +1,59 @@ +"""The user-code finder claims only project modules that normal import would load.""" + +import sys +from importlib.machinery import ModuleSpec +from pathlib import Path + +import pytest + +from sqlbuild.runtime.user_code.classes.source_only_loader import SourceOnlyLoader +from sqlbuild.runtime.user_code.classes.user_code_finder import UserCodeFinder +from tests.unit.src.sqlbuild.runtime.user_code.classes._test_types import FinderSpecTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + FinderSpecTestCase( + description="a project module loads from source", + module_name="order_rules", + earlier_entries=(), + expected_source_only=True, + ), + FinderSpecTestCase( + description="an installed module is left to normal import", + module_name="pytest", + earlier_entries=(), + expected_source_only=False, + ), + FinderSpecTestCase( + description="a project module shadowed by an earlier path entry is left alone", + module_name="order_rules", + earlier_entries=("shadow",), + expected_source_only=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_module_name_when_finding_spec_then_only_project_modules_load_from_source( + test_case: FinderSpecTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + project.mkdir() + _ = (project / "order_rules.py").write_text("LIMIT: int = 1\n", encoding="utf-8") + shadow: Path = tmp_path / "shadow" + shadow.mkdir() + _ = (shadow / "order_rules.py").write_text("LIMIT: int = 2\n", encoding="utf-8") + earlier: list[str] = [str(tmp_path / name) for name in test_case.earlier_entries] + monkeypatch.setattr(sys, "path", [*earlier, str(project), *sys.path]) + finder: UserCodeFinder = UserCodeFinder() + finder.roots.add(project.resolve()) + + spec: ModuleSpec | None = finder.find_spec(test_case.module_name) + + source_only: bool = spec is not None and isinstance(spec.loader, SourceOnlyLoader) + assert source_only == test_case.expected_source_only + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx index 2d07f75c78..477f4ea2bc 100644 --- a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx +++ b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx @@ -32,6 +32,12 @@ schema = "RAW" Connection fields are passed directly to `snowflake.connector.connect()`. See the [Snowflake Connector documentation](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect) for all available options, including key-pair authentication, OAuth, and SSO. Put the authoritative database and schema on the target. +SQLBuild turns off the connector's platform detection by setting +`SNOWFLAKE_DISABLE_PLATFORM_DETECTION=true` when the variable is unset. Detection runs on every +connect: it probes cloud metadata endpoints, calls AWS to identify the caller for client +telemetry, and can print `botocore` warnings when an AWS SSO session has expired. To keep it on, +set the variable yourself before running `sqb`, for example `SNOWFLAKE_DISABLE_PLATFORM_DETECTION=false`. + ## Session initialization On connect, SQLBuild applies the connection's role and warehouse, then applies the active From ef842e5c1cfabdb16a92775bc895d692432c56ea Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:26:27 +0000 Subject: [PATCH 10/20] fix(rules): reject a rule ignore whose literal path no longer exists A rule exception naming a missing file already fails compilation, but a rule ignore did not, so an ignore left behind by a renamed or deleted model stayed in the project silently. A rule ignore path without glob characters must now exist; globs and folders are unaffected. --- .../src/engine/_helpers/evaluation.rs | 11 +++++++++++ .../docs/concepts/rules/findings-and-exceptions.md | 4 +++- .../src/sqlbuild/cli/commands/main/test_rules.py | 14 ++++++++++++++ .../concepts/rules/findings-and-exceptions.mdx | 4 +++- 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs index 494165ea40..d38f6cb21d 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs @@ -533,6 +533,17 @@ fn validate_exception_paths(request: &EvaluateRequest) -> Result<(), String> { )); } } + let literal_ignore_paths = request + .config + .rule_ignores + .iter() + .flat_map(|entry| &entry.paths) + .filter(|path| !path.contains(['*', '?', '[', '{'])); + for path in literal_ignore_paths { + if !root.join(path).exists() { + return Err(format!("rule ignore path does not exist: {path}")); + } + } Ok(()) } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md index 32d07b90d3..41281df46d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md @@ -85,7 +85,9 @@ reason = "Examples retain intentionally minimal SQL." ``` Path ignores accept exact codes and family prefixes. Keep their scope narrow and explain why the -project differs from the selected requirement. +project differs from the selected requirement. A path without glob characters must exist, so an +ignore left behind by a renamed or deleted file fails compilation with +`rule ignore path does not exist: `; move the ignore to the new path or delete it. Use `selectors` when the exception follows graph-resource identities or lineage rather than files: diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 2a89641171..a8f1a66c9c 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -3029,6 +3029,20 @@ def required_prefix(*, model: Model, ctx: RuleContext) -> list[Finding]: 1, ("rule-ignore selector matches no rules: SQBRZZZ",), ), + EmptyRuleSelectionTestCase( + "missing literal ignore path is rejected", + '[[rules.rule_ignores]]\nrules = ["SQBRSQL"]\npaths = ["models/orders.sql", "models/gone.sql"]\n' + 'reason = "Legacy."\n', + 1, + ("rule ignore path does not exist: models/gone.sql",), + ), + EmptyRuleSelectionTestCase( + "ignore paths naming a folder or a glob that matches nothing compile", + '[[rules.rule_ignores]]\nrules = ["SQBRSQL"]\npaths = ["models", "models/legacy/**"]\n' + 'reason = "Legacy."\n', + 0, + (), + ), EmptyRuleSelectionTestCase( "unknown built-in rule option is rejected", "[rules.rule_options.SQBRSQL004]\nnope = 3\n", diff --git a/website/src/content/docs/docs/concepts/rules/findings-and-exceptions.mdx b/website/src/content/docs/docs/concepts/rules/findings-and-exceptions.mdx index 6088e8936f..3b063f1b7c 100644 --- a/website/src/content/docs/docs/concepts/rules/findings-and-exceptions.mdx +++ b/website/src/content/docs/docs/concepts/rules/findings-and-exceptions.mdx @@ -72,7 +72,9 @@ reason = "Examples retain intentionally minimal SQL." ``` Path ignores accept exact codes and family prefixes. Keep their scope narrow and explain why the -project differs from the selected requirement. +project differs from the selected requirement. A path without glob characters must exist, so an +ignore left behind by a renamed or deleted file fails compilation with +`rule ignore path does not exist: `; move the ignore to the new path or delete it. Use `selectors` when the exception follows graph-resource identities or lineage rather than files: From 8e2eb723bb405320bff003e9b9d95e138955bc07 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:26:28 +0000 Subject: [PATCH 11/20] docs(benchmarks): describe DuckShop's Snowflake type-inference limitation precisely --- benchmarks/duckshop/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/benchmarks/duckshop/README.md b/benchmarks/duckshop/README.md index 2f87e3e9d2..505f262150 100644 --- a/benchmarks/duckshop/README.md +++ b/benchmarks/duckshop/README.md @@ -221,9 +221,10 @@ noted here: - Compilation accepts SQL that Snowflake rejects: `CONNECT_BY_ISLEAF`, which Snowflake does not have, and a CTE whose body is a bare `VALUES` list. `int_category_paths` finds leaf categories with a join instead, and test case tables use `SELECT ... FROM VALUES ... AS v(...)`. -- Static type inference cannot follow a quoted mixed-case column, such as `"unitPrice"`, from a CTE - through `SELECT *` on Snowflake, so `rpt_edge_partner_listing_export` declares only the name and - audits of `"listingId"`, without types. +- On Snowflake, a model that names a column in quoted mixed case, such as `"unitPrice"`, loses the + static types of the columns it reads from a CTE through a table alias (`FROM listings AS l`), so + `rpt_edge_partner_listing_export` declares only the name and audits of `"listingId"`, without + types. - `SQBRSQL018` can only take proof from single-column keys. Append-only feeds redeliver rows, so their id is unique only together with `_loaded_at` (the `unique_per_delivery` audit), and the staging models that keep the latest delivery carry reviewed exceptions. From a899e7c1924183c0ea6f3b9f4a6ac75544a8c5ae Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:51:57 +0000 Subject: [PATCH 12/20] fix(compiler): end SQL macro strings and comments with the adapter's lexical rules The SQL macro lexer took only the adapter's backslash-escape quotes, so an E'...' string with an escaped quote on DuckDB or Postgres read as unclosed and the model failed to compile. With an adapter's rules it now ends quoted text and comments exactly as the shared dialect scanner does, including E-string and raw-string prefixes and dialect line comments. --- .../src/sql_macros/_helpers/parsing/lexing.rs | 93 +++++++++++++------ .../sqlbuild-analysis/src/sql_macros/tests.rs | 2 + .../src/sql_macros/tests/helpers.rs | 25 +++++ .../sql_macros/tests/test_lexical_rules.rs | 47 ++++++++++ .../src/sql_macros/tests/test_types.rs | 8 ++ .../cli/commands/main/compile/_test_types.py | 10 ++ .../compile/test_sql_macro_string_escapes.py | 87 +++++++++++++++++ 7 files changed, 242 insertions(+), 30 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/sql_macros/tests/test_lexical_rules.rs create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_sql_macro_string_escapes.py diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/lexing.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/lexing.rs index b1e2db4df2..f54b1fc42e 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/lexing.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/lexing.rs @@ -7,6 +7,7 @@ use sqlbuild_sqltext::header_strings::main::follows_description_key::follows_des use sqlbuild_sqltext::header_strings::main::opens_triple_quote::opens_triple_quote; use sqlbuild_sqltext::header_strings::main::prose_end::prose_end; use sqlbuild_sqltext::header_strings::main::triple_quoted_end::triple_quoted_end; +use sqlbuild_sqltext::sql_scan::main::dialect_non_code_end::dialect_non_code_end; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::sql_macros::constants::{HEADER_KEYWORD, SQL_KIND_WORD}; @@ -53,6 +54,11 @@ impl Token { } const PUNCTUATION: &[u8] = b"()[]{},.;"; + +/// Where the quoted text opened at an offset ends. +type TextEnd<'a> = dyn Fn(&[u8], usize) -> Result + 'a; +/// Where a comment opened at an offset ends, `None` when none opens there. +type CommentEnd<'a> = dyn Fn(&[u8], usize) -> Result, LexError> + 'a; /// The `@word(...)` declaration forms that are not macro calls. pub(crate) const DECLARATION_WORDS: [&str; 3] = ["const", "enum", "param"]; @@ -61,32 +67,72 @@ pub(crate) fn tokens(sql: &str) -> Result, LexError> { tokens_with(sql, None) } -/// Every token of `sql`, with `syntax`'s backslash escapes, or as [`tokens`] reads it. +/// Every token of `sql`, with quoted text ending where `syntax` ends it, or as [`tokens`] reads it. pub(crate) fn tokens_with( sql: &str, syntax: Option<&LexicalSyntax>, ) -> Result, LexError> { if let Some(syntax) = syntax { - let escapes = |quote: u8| { - syntax - .backslash_escape_quotes - .iter() - .any(|value| value.as_bytes() == [quote]) + let text_end = |bytes: &[u8], start: usize| match dialect_non_code_end(bytes, start, syntax) + { + Ok(Some(end)) => Ok(end), + _ => Err(LexError::UnclosedQuote(start)), }; - return scanned_tokens(sql, &escapes); + let comment_end = |bytes: &[u8], start: usize| dialect_comment_end(bytes, start, syntax); + return scanned_tokens(sql, &text_end, &comment_end); } - match scanned_tokens(sql, &|_| false) { - Err(LexError::UnclosedQuote(start)) => { - scanned_tokens(sql, &|_| true).map_err(|_| LexError::UnclosedQuote(start)) - } + match scanned_tokens( + sql, + &|bytes, start| quote_end(bytes, start, false).ok_or(LexError::UnclosedQuote(start)), + &generic_comment_end, + ) { + Err(LexError::UnclosedQuote(start)) => scanned_tokens( + sql, + &|bytes, start| quote_end(bytes, start, true).ok_or(LexError::UnclosedQuote(start)), + &generic_comment_end, + ) + .map_err(|_| LexError::UnclosedQuote(start)), scanned => scanned, } } -/// Every token of `sql`; a backslash escapes inside quotes for which `backslash_escapes` holds. +/// The end of a `--` or `/* */` comment opening at `start`, if one opens there. +fn generic_comment_end(bytes: &[u8], start: usize) -> Result, LexError> { + if bytes[start..].starts_with(b"--") { + return Ok(Some( + bytes[start..] + .iter() + .position(|byte| *byte == b'\n') + .map_or(bytes.len(), |offset| start + offset + 1), + )); + } + if bytes[start..].starts_with(b"/*") { + return bytes[start + 2..] + .windows(2) + .position(|pair| pair == b"*/") + .map(|offset| Some(start + 2 + offset + 2)) + .ok_or(LexError::UnclosedComment(start)); + } + Ok(None) +} + +/// The end of a comment opening at `start` under `syntax`'s comment rules, if one opens there. +fn dialect_comment_end( + bytes: &[u8], + start: usize, + syntax: &LexicalSyntax, +) -> Result, LexError> { + if matches!(bytes[start], b'\'' | b'"' | b'`' | b'$') { + return Ok(None); + } + dialect_non_code_end(bytes, start, syntax).map_err(|_| LexError::UnclosedComment(start)) +} + +/// Every token of `sql`; `text_end` and `comment_end` find where quoted text and comments end. fn scanned_tokens( sql: &str, - backslash_escapes: &dyn Fn(u8) -> bool, + text_end: &TextEnd<'_>, + comment_end: &CommentEnd<'_>, ) -> Result, LexError> { let bytes: &[u8] = sql.as_bytes(); let mut found: Vec = Vec::new(); @@ -98,19 +144,8 @@ fn scanned_tokens( index += 1; continue; } - if byte == b'-' && bytes.get(index + 1) == Some(&b'-') { - index = bytes[index..] - .iter() - .position(|byte| *byte == b'\n') - .map_or(bytes.len(), |offset| index + offset + 1); - continue; - } - if byte == b'/' && bytes.get(index + 1) == Some(&b'*') { - index = bytes[index + 2..] - .windows(2) - .position(|pair| pair == b"*/") - .map(|offset| index + 2 + offset + 2) - .ok_or(LexError::UnclosedComment(index))?; + if let Some(end) = comment_end(bytes, index)? { + index = end; continue; } let start: usize = index; @@ -125,13 +160,11 @@ fn scanned_tokens( TokenKind::Quoted } b'\'' => { - index = quote_end(bytes, index, backslash_escapes(byte)) - .ok_or(LexError::UnclosedQuote(start))?; + index = text_end(bytes, index)?; TokenKind::Text } b'"' | b'`' => { - index = quote_end(bytes, index, backslash_escapes(byte)) - .ok_or(LexError::UnclosedQuote(start))?; + index = text_end(bytes, index)?; TokenKind::Quoted } b'$' if dollar_end(bytes, index).is_some() => { diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests.rs b/crates/sqlbuild-analysis/src/sql_macros/tests.rs index 1f2cf23da5..1dc7f7981e 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests.rs @@ -4,6 +4,8 @@ mod helpers; mod test_definitions; #[path = "tests/test_expansion.rs"] mod test_expansion; +#[path = "tests/test_lexical_rules.rs"] +mod test_lexical_rules; #[path = "tests/test_relation_slots.rs"] mod test_relation_slots; #[path = "tests/test_typed_calls.rs"] diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs index fafd5775b0..229742aba3 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs @@ -186,3 +186,28 @@ pub(super) fn untyped_calls(sql: &str, texts: &[&str]) -> Vec { }) .collect() } + +pub(super) fn escape_prefix_rules() -> sqlbuild_sqltext::sql_scan::models::LexicalSyntax { + sqlbuild_sqltext::sql_scan::models::LexicalSyntax { + escape_string_prefix: true, + line_comment_prefixes: vec!["--".to_owned()], + ..sqlbuild_sqltext::sql_scan::models::LexicalSyntax::default() + } +} + +pub(super) fn backslash_rules() -> sqlbuild_sqltext::sql_scan::models::LexicalSyntax { + sqlbuild_sqltext::sql_scan::models::LexicalSyntax { + backslash_escape_quotes: vec!["'".to_owned()], + line_comment_prefixes: vec!["--".to_owned(), "//".to_owned()], + ..sqlbuild_sqltext::sql_scan::models::LexicalSyntax::default() + } +} + +pub(super) fn raw_prefix_rules() -> sqlbuild_sqltext::sql_scan::models::LexicalSyntax { + sqlbuild_sqltext::sql_scan::models::LexicalSyntax { + backslash_escape_quotes: vec!["'".to_owned(), "\"".to_owned()], + raw_string_prefix: true, + line_comment_prefixes: vec!["--".to_owned(), "#".to_owned()], + ..sqlbuild_sqltext::sql_scan::models::LexicalSyntax::default() + } +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_lexical_rules.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_lexical_rules.rs new file mode 100644 index 0000000000..5c4882a601 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_lexical_rules.rs @@ -0,0 +1,47 @@ +use crate::sql_macros::main::sql_macro_calls::sql_macro_calls; +use crate::sql_macros::tests::helpers::{backslash_rules, escape_prefix_rules, raw_prefix_rules}; +use crate::sql_macros::tests::test_types::LexicalRulesCallsTestCase; + +#[test] +fn given_adapter_lexical_rules_when_finding_sql_macro_calls_then_strings_end_where_the_adapter_ends_them() + { + let test_cases = [ + LexicalRulesCallsTestCase { + description: "an E-string's escaped quote on DuckDB and Postgres", + syntax: escape_prefix_rules(), + sql: "SELECT E'it\\'s' AS a, @sql.order_total(1) AS b", + expected_calls: &["order_total"], + }, + LexicalRulesCallsTestCase { + description: "a plain string ending in a backslash on DuckDB and Postgres", + syntax: escape_prefix_rules(), + sql: "SELECT 'C:\\' AS path, @sql.order_total(1) AS b", + expected_calls: &["order_total"], + }, + LexicalRulesCallsTestCase { + description: "a backslash-escaped quote on Snowflake", + syntax: backslash_rules(), + sql: "SELECT 'it\\'s' AS a, @sql.order_total(1) AS b", + expected_calls: &["order_total"], + }, + LexicalRulesCallsTestCase { + description: "a quote inside a Snowflake // comment", + syntax: backslash_rules(), + sql: "SELECT @sql.order_total(1) AS b // the customer's total\nFROM orders", + expected_calls: &["order_total"], + }, + LexicalRulesCallsTestCase { + description: "a raw string ending in a backslash on BigQuery", + syntax: raw_prefix_rules(), + sql: "SELECT r'C:\\' AS path, @sql.order_total(1) AS b", + expected_calls: &["order_total"], + }, + ]; + + for test_case in test_cases { + let calls = sql_macro_calls(test_case.sql, Some(&test_case.syntax)) + .unwrap_or_else(|error| panic!("{}: {error:?}", test_case.description)); + let names: Vec<&str> = calls.iter().map(|call| call.name.as_str()).collect(); + assert_eq!(names, test_case.expected_calls, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs index a3164977ff..b7400f466c 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs @@ -114,3 +114,11 @@ pub(super) struct ExpandedAnalysisTestCase { pub(super) sql: &'static str, pub(super) expected_sql: &'static str, } + +/// A query read under one adapter's lexical rules, with the SQL macro calls it holds. +pub(super) struct LexicalRulesCallsTestCase { + pub(super) description: &'static str, + pub(super) syntax: sqlbuild_sqltext::sql_scan::models::LexicalSyntax, + pub(super) sql: &'static str, + pub(super) expected_calls: &'static [&'static str], +} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index c774b30615..d6b0c48055 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1369,3 +1369,13 @@ class SqlTestCacheMissE2ETestCase: description: str change: Callable[[Path], None] expected_debug_line: str + + +@dataclass(frozen=True) +class SqlMacroStringEscapeTestCase: + """A model whose SQL macro call follows a string the adapter ends with its own rules.""" + + description: str + adapter_config: str + string_sql: str + expected_compiled_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_sql_macro_string_escapes.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_sql_macro_string_escapes.py new file mode 100644 index 0000000000..b4506dbda6 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_sql_macro_string_escapes.py @@ -0,0 +1,87 @@ +"""E2E coverage that SQL macro calls after E-strings and backslash-ended strings compile.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + SqlMacroStringEscapeTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_DUCKDB_CONFIG: str = ( + 'name = "order_notes"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' +) +_POSTGRES_CONFIG: str = ( + 'name = "order_notes"\nadapter = "postgres"\n\n[connection]\n' + 'database = "analytics"\nschema = "public"\n' +) +_SHOUT_MACRO: str = ( + 'MACRO (description "Upper case.", arguments (v VARCHAR), returns VARCHAR);\n\nUPPER(@v)\n' +) + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroStringEscapeTestCase( + description="an E-string escaped quote on DuckDB", + adapter_config=_DUCKDB_CONFIG, + string_sql="E'it\\'s'", + expected_compiled_fragment="UPPER('web')", + ), + SqlMacroStringEscapeTestCase( + description="an E-string escaped quote on Postgres", + adapter_config=_POSTGRES_CONFIG, + string_sql="E'it\\'s'", + expected_compiled_fragment="UPPER('web')", + ), + SqlMacroStringEscapeTestCase( + description="a plain string ending in a backslash on DuckDB", + adapter_config=_DUCKDB_CONFIG, + string_sql="'C:\\'", + expected_compiled_fragment="UPPER('web')", + ), + SqlMacroStringEscapeTestCase( + description="a plain string ending in a backslash on Postgres", + adapter_config=_POSTGRES_CONFIG, + string_sql="'C:\\'", + expected_compiled_fragment="UPPER('web')", + ), + ], + ids=lambda case: case.description, +) +def test_given_string_with_backslash_when_compiling_sql_macro_call_then_call_expands( + test_case: SqlMacroStringEscapeTestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="order_notes", + repo_files={ + "sqlbuild_project.toml": test_case.adapter_config, + "models/_macros/sql/shout.sql": _SHOUT_MACRO, + "models/order_notes.sql": ( + 'MODEL (description "Order notes after an escaped string.");\n' + f"SELECT {test_case.string_sql} AS note, @sql.shout('web') AS channel\n" + ), + }, + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), project_dir=project_dir + ) + + assert compiled.returncode == 0, compiled.stdout + compiled.stderr + assert "0 errors" in compiled.stdout, compiled.stdout + compiled_sql: str = ( + project_dir / "target" / "compiled" / "models" / "order_notes.sql" + ).read_text(encoding="utf-8") + assert test_case.expected_compiled_fragment in compiled_sql, compiled_sql + assert test_case.string_sql in compiled_sql, compiled_sql + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 7ea3248d323ec1fe56122b3ec549fe58962ed5da Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 29 Oct 2026 06:03:41 +0000 Subject: [PATCH 13/20] fix(compiler): read @py call arguments with Python's string rules The macro call scanner read @py arguments with the adapter's SQL rules, so on Snowflake an invalid argument such as 10 // 3 read as a line comment and reported an unclosed parenthesis instead of the clear not-a-literal error. Inside @py arguments it now uses Python's string rules with only the generic -- line comment, so dialect comment markers such as // and # read as code, as they did before the lexical-rules change. --- .../src/macro_calls/_helpers/scanner.rs | 25 ++++++- .../macro_calls/main/scan_macro_call_sites.rs | 11 ++- .../tests/test_scan_macro_call_sites.rs | 6 ++ .../cli/commands/main/compile/_test_types.py | 9 +++ .../test_python_macro_argument_errors.py | 68 +++++++++++++++++++ 5 files changed, 115 insertions(+), 4 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_macro_argument_errors.py diff --git a/crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs b/crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs index 237f008f17..6d27bb272f 100644 --- a/crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs +++ b/crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs @@ -9,6 +9,8 @@ use crate::macro_calls::main::opens_bare_call::opens_bare_call; /// The prefix of a Python macro call after its `@`. const PYTHON_PREFIX: &str = "py."; +/// The line comment every scan honours, the only one inside `@py.` call arguments. +const GENERIC_LINE_COMMENT_PREFIX: &str = "--"; /// The prefix of a SQL macro call after its `@`. const SQL_PREFIX: &str = "sql."; /// Declaration and SQL test parameter references, which scans read as calls that need no kind. @@ -22,6 +24,8 @@ pub(crate) struct ScannedCall { pub(crate) open: usize, pub(crate) close: usize, pub(crate) name: String, + /// A `@py.` call, whose arguments are Python literals rather than SQL. + pub(crate) python: bool, } /// Where Python's scan of the text raises: in the call starting at a byte offset, or between calls. @@ -218,19 +222,36 @@ fn call_at( with_sql: bool, syntax: Option<&LexicalSyntax>, ) -> Result { - if let Some(error) = kind_error(call_start_at(text, start), with_sql) { + let kind: CallStart = call_start_at(text, start); + if let Some(error) = kind_error(kind, with_sql) { return Err(error); } let (name, open) = call_header(text, start)?; - let close = matching_paren(text.as_bytes(), open, syntax)?; + let python: bool = kind == CallStart::Python; + let python_syntax: LexicalSyntax = python_argument_syntax(); + let argument_syntax: Option<&LexicalSyntax> = + if python { Some(&python_syntax) } else { syntax }; + let close = matching_paren(text.as_bytes(), open, argument_syntax)?; Ok(ScannedCall { start, open, close, name, + python, }) } +/// Python's string rules for `@py.` call arguments, with only the generic `--` line comment. +pub(crate) fn python_argument_syntax() -> LexicalSyntax { + LexicalSyntax { + backslash_escape_quotes: vec!["'".to_owned(), "\"".to_owned()], + raw_string_prefix: true, + triple_quoted_strings: true, + line_comment_prefixes: vec![GENERIC_LINE_COMMENT_PREFIX.to_owned()], + ..LexicalSyntax::default() + } +} + fn call_header(text: &str, start: usize) -> Result<(String, usize), ScanError> { let from: usize = name_start(text, start); let name_end = identifier_end(text, from); diff --git a/crates/sqlbuild-render/src/macro_calls/main/scan_macro_call_sites.rs b/crates/sqlbuild-render/src/macro_calls/main/scan_macro_call_sites.rs index 79292f1315..e93a0e976f 100644 --- a/crates/sqlbuild-render/src/macro_calls/main/scan_macro_call_sites.rs +++ b/crates/sqlbuild-render/src/macro_calls/main/scan_macro_call_sites.rs @@ -2,7 +2,7 @@ use crate::macro_calls::_helpers::offsets::char_offsets; use crate::macro_calls::_helpers::scanner::{ - call_name, mentions_typed_reference, nested_names, top_level_calls, + call_name, mentions_typed_reference, nested_names, python_argument_syntax, top_level_calls, }; use crate::macro_calls::models::{MacroCallScan, MacroCallSite, MacroScanFailure}; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; @@ -22,7 +22,14 @@ pub fn scan_macro_call_sites(sql: &str, syntax: Option<&LexicalSyntax>) -> Macro .zip(offsets.as_chunks::<3>().0) .map(|(call, offsets)| { let args = &sql[call.open + 1..call.close]; - let tree_names: Option> = call_tree_names(&call.name, args, syntax); + let python_syntax: LexicalSyntax = python_argument_syntax(); + let argument_syntax: Option<&LexicalSyntax> = if call.python { + Some(&python_syntax) + } else { + syntax + }; + let tree_names: Option> = + call_tree_names(&call.name, args, argument_syntax); MacroCallSite { start: offsets[0], args_start: offsets[1], diff --git a/crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs b/crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs index 431e0127db..77bcebda9a 100644 --- a/crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs +++ b/crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs @@ -350,6 +350,12 @@ fn given_backslash_escapes_when_scanning_macro_calls_then_dialect_quotes_are_tex expected_sites: &[(21, 28, "m", Some(&["m"]), false)], expected_failure: None, }, + ScanMacroCallSitesTestCase { + description: "a dialect line comment marker inside Python arguments is an operator", + sql: "@py.m(10 // 3) AS a", + expected_sites: &[(0, 14, "m", Some(&["m"]), false)], + expected_failure: None, + }, ]; for test_case in test_cases { diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index d6b0c48055..c1ac4c26c4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1379,3 +1379,12 @@ class SqlMacroStringEscapeTestCase: adapter_config: str string_sql: str expected_compiled_fragment: str + + +@dataclass(frozen=True) +class PythonMacroArgumentErrorTestCase: + """An invalid `@py` argument span on Snowflake and the error compile reports for it.""" + + description: str + call_sql: str + expected_message: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_macro_argument_errors.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_macro_argument_errors.py new file mode 100644 index 0000000000..53e0b833b0 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_macro_argument_errors.py @@ -0,0 +1,68 @@ +"""E2E coverage that `@py` arguments are read as Python, not with Snowflake's line comments.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + PythonMacroArgumentErrorTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.scenario.helpers import ( + build_offline_snowflake_project_toml, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_ORDER_MACRO: str = ( + '"""Order macros."""\n\n\n' + "def order_total(value: int) -> str:\n" + ' """Return the order total literal."""\n' + " return str(value)\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonMacroArgumentErrorTestCase( + description="floor division is not a literal, not a comment", + call_sql="@py.order_total(10 // 3)", + expected_message="must use only Python literals", + ), + PythonMacroArgumentErrorTestCase( + description="a Python comment in a multi-line call is not a comment", + call_sql="@py.order_total(\n 3 # it's three\n)", + expected_message="a quoted string in @py.order_total is not closed", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_python_macro_arguments_when_compiling_then_error_names_the_argument_problem( + test_case: PythonMacroArgumentErrorTestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="order_totals", + repo_files={ + "sqlbuild_project.toml": build_offline_snowflake_project_toml(), + "models/_macros/python/order_total.py": _ORDER_MACRO, + "models/order_totals.sql": ( + f'MODEL (description "Order totals.");\nSELECT {test_case.call_sql} AS total\n' + ), + }, + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), project_dir=project_dir + ) + + assert compiled.returncode != 0, compiled.stdout + compiled.stderr + assert test_case.expected_message in compiled.stdout + compiled.stderr, ( + compiled.stdout + compiled.stderr + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 5777f17b482b93ab88c281b084bbfd21de2d5a7b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 29 Oct 2026 06:07:54 +0000 Subject: [PATCH 14/20] fix(executor): match runtime column names with the compiler's quoted-name key Snowflake listings now keep a mixed-case column quoted, as in "listingId", but the runtime contract, type enforcement, incremental schema, load, migration and raw-query diff checks compared names with lower(). A quoted alias declared unquoted in the header then failed the runtime contract and skipped type enforcement, although compile accepted it. These checks now key names as the compiler does: case-folded and without identifier quotes. --- .../adapter/contract/main/column_name_key.py | 11 ++++ .../executor/diff/_helpers/execution.py | 47 +++++++++----- src/sqlbuild/executor/load/_helpers/schema.py | 5 +- .../executor/migrations/_helpers/columns.py | 6 +- .../_helpers/materializations/incremental.py | 21 +++--- .../run/_helpers/validation/contracts.py | 13 ++-- .../_helpers/validation/type_enforcement.py | 7 +- .../cli/commands/main/build/_test_types.py | 9 +++ .../build/test_quoted_mixed_case_columns.py | 52 +++++++++++++++ .../build/snowflake_metadata/_test_types.py | 10 +++ .../test_quoted_mixed_case_columns.py | 65 +++++++++++++++++++ .../adapter/contract/main/_test_types.py | 9 +++ .../contract/main/test_column_name_key.py | 37 +++++++++++ 13 files changed, 253 insertions(+), 39 deletions(-) create mode 100644 src/sqlbuild/adapter/contract/main/column_name_key.py create mode 100644 tests/unit/src/sqlbuild/adapter/contract/main/test_column_name_key.py diff --git a/src/sqlbuild/adapter/contract/main/column_name_key.py b/src/sqlbuild/adapter/contract/main/column_name_key.py new file mode 100644 index 0000000000..11890376a1 --- /dev/null +++ b/src/sqlbuild/adapter/contract/main/column_name_key.py @@ -0,0 +1,11 @@ +"""The key that matches a declared column name to a warehouse-listed one.""" + +from __future__ import annotations + +from sqlbuild.adapter.contract.main.is_quoted_identifier import is_quoted_identifier + + +def column_name_key(name: str) -> str: + """The lower-cased name without the quotes of a quoted identifier, as the compiler keys it.""" + + return (name[1:-1] if is_quoted_identifier(name) else name).lower() diff --git a/src/sqlbuild/executor/diff/_helpers/execution.py b/src/sqlbuild/executor/diff/_helpers/execution.py index 96de9b6fd2..f2601b4027 100644 --- a/src/sqlbuild/executor/diff/_helpers/execution.py +++ b/src/sqlbuild/executor/diff/_helpers/execution.py @@ -13,6 +13,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.unkeyed_diff import UnkeyedDiffMixin from sqlbuild.adapter.contract.constants import DIFF_LEFT_SIDE, DIFF_RIGHT_SIDE +from sqlbuild.adapter.contract.main.column_name_key import column_name_key from sqlbuild.adapter.contract.models import ( ColumnInfo, RowDiffCoverage, @@ -176,16 +177,18 @@ def execute_query_diff( ), unkeyed=options.unkeyed, ) - canonical_names: dict[str, str] = {column.name.lower(): column.name for column in left_columns} - unique_key = tuple(canonical_names[key.lower()] for key in unique_key) + canonical_names: dict[str, str] = { + column_name_key(column.name): column.name for column in left_columns + } + unique_key = tuple(canonical_names[column_name_key(key)] for key in unique_key) excluded_columns: tuple[str, ...] = tuple( - canonical_names[column.lower()] for column in options.excluded_columns_override + canonical_names[column_name_key(column)] for column in options.excluded_columns_override ) tolerance_overrides: RowDiffTolerances = options.tolerance_overrides or RowDiffTolerances() tolerances: RowDiffTolerances = replace( tolerance_overrides, by_column={ - canonical_names[column.lower()]: tolerance + canonical_names[column_name_key(column)]: tolerance for column, tolerance in tolerance_overrides.by_column.items() }, ) @@ -306,10 +309,14 @@ def _validate_query_diff_columns( tolerance_columns: tuple[str, ...], unkeyed: bool, ) -> None: - left_names: dict[str, str] = {column.name.lower(): column.name for column in left_columns} - right_names: dict[str, str] = {column.name.lower(): column.name for column in right_columns} + left_names: dict[str, str] = { + column_name_key(column.name): column.name for column in left_columns + } + right_names: dict[str, str] = { + column_name_key(column.name): column.name for column in right_columns + } for key in unique_key: - if key.lower() not in left_names or key.lower() not in right_names: + if column_name_key(key) not in left_names or column_name_key(key) not in right_names: raise ExecutorInputError( f"raw-query diff key column '{key}' is missing from one or both results", code="X307", @@ -317,7 +324,7 @@ def _validate_query_diff_columns( missing_exclusions: tuple[str, ...] = tuple( column for column in excluded_columns - if column.lower() not in left_names or column.lower() not in right_names + if column_name_key(column) not in left_names or column_name_key(column) not in right_names ) if missing_exclusions: raise ExecutorInputError( @@ -328,7 +335,7 @@ def _validate_query_diff_columns( missing_tolerances: tuple[str, ...] = tuple( column for column in tolerance_columns - if column.lower() not in left_names or column.lower() not in right_names + if column_name_key(column) not in left_names or column_name_key(column) not in right_names ) if missing_tolerances: raise ExecutorInputError( @@ -336,18 +343,22 @@ def _validate_query_diff_columns( + ", ".join(missing_tolerances), code="X311", ) - excluded_column_names: frozenset[str] = frozenset(item.lower() for item in excluded_columns) + excluded_column_names: frozenset[str] = frozenset( + column_name_key(item) for item in excluded_columns + ) intersection: tuple[str, ...] = tuple( - key for key in unique_key if key.lower() in excluded_column_names + key for key in unique_key if column_name_key(key) in excluded_column_names ) if intersection: raise ExecutorInputError( "raw-query diff excluded columns intersect --key: " + ", ".join(intersection), code="X309", ) - tolerance_column_names: frozenset[str] = frozenset(item.lower() for item in tolerance_columns) + tolerance_column_names: frozenset[str] = frozenset( + column_name_key(item) for item in tolerance_columns + ) tolerance_key_intersection: tuple[str, ...] = tuple( - key for key in unique_key if key.lower() in tolerance_column_names + key for key in unique_key if column_name_key(key) in tolerance_column_names ) if tolerance_key_intersection: raise ExecutorInputError( @@ -355,8 +366,10 @@ def _validate_query_diff_columns( + ", ".join(tolerance_key_intersection), code="X312", ) - excluded_names: frozenset[str] = frozenset(column.lower() for column in excluded_columns) - if unkeyed and all(column.name.lower() in excluded_names for column in left_columns): + excluded_names: frozenset[str] = frozenset( + column_name_key(column) for column in excluded_columns + ) + if unkeyed and all(column_name_key(column.name) in excluded_names for column in left_columns): raise ExecutorInputError( "unkeyed raw-query diff requires at least one compared column", code="X310", @@ -660,10 +673,10 @@ def _require_relation_keys( *, adapter: BaseAdapter, connection: Any, relation: str, keys: tuple[str, ...] ) -> None: columns: set[str] = { - column.name.lower() + column_name_key(column.name) for column in adapter.describe_relation(connection=connection, relation=relation) } - missing: tuple[str, ...] = tuple(key for key in keys if key.lower() not in columns) + missing: tuple[str, ...] = tuple(key for key in keys if column_name_key(key) not in columns) if not missing: return noun: str = "column" if len(missing) == 1 else "columns" diff --git a/src/sqlbuild/executor/load/_helpers/schema.py b/src/sqlbuild/executor/load/_helpers/schema.py index 9b7c93205b..a84107ff24 100644 --- a/src/sqlbuild/executor/load/_helpers/schema.py +++ b/src/sqlbuild/executor/load/_helpers/schema.py @@ -7,6 +7,7 @@ from sqlbuild._native import ContractPolicy, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.main.column_name_key import column_name_key from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.type_system.main.types_equal import types_equal from sqlbuild.errors.contracts.exceptions import ExecutorInputError @@ -25,12 +26,12 @@ def validate_and_evolve_existing_target( connection=connection, relation=target ) target_by_name: dict[str, ColumnInfo] = { - column.name.lower(): column for column in target_columns + column_name_key(column.name): column for column in target_columns } new_columns: list[ColumnInfo] = [] staging_column: ColumnInfo for staging_column in staging_columns: - target_column: ColumnInfo | None = target_by_name.get(staging_column.name.lower()) + target_column: ColumnInfo | None = target_by_name.get(column_name_key(staging_column.name)) if target_column is None: if source_entry.contract == ContractPolicy.ENFORCED: raise ExecutorInputError( diff --git a/src/sqlbuild/executor/migrations/_helpers/columns.py b/src/sqlbuild/executor/migrations/_helpers/columns.py index 612f484223..0d83861cc3 100644 --- a/src/sqlbuild/executor/migrations/_helpers/columns.py +++ b/src/sqlbuild/executor/migrations/_helpers/columns.py @@ -16,6 +16,7 @@ ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.main.column_name_key import column_name_key from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.executor.migrations._helpers.relation_for_location import ( migration_relation_for_location, @@ -75,11 +76,12 @@ def _rename_and_record( attempts: int = MIGRATION_WRITE_ATTEMPTS, ) -> None: live: frozenset[str] = frozenset( - column.name.lower() + column_name_key(column.name) for column in _live_columns(adapter=adapter, connection=connection, entry=entry) ) already_renamed: bool = ( - entry.origin_column.lower() not in live and entry.destination_column.lower() in live + column_name_key(entry.origin_column) not in live + and column_name_key(entry.destination_column) in live ) decision: ColumnMigrationDecision = entry.decision if entry.decision.renames and not already_renamed: diff --git a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py index 52e6654911..75c12b2cc9 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py @@ -18,6 +18,7 @@ from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.main.column_name_key import column_name_key from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, @@ -654,10 +655,10 @@ def _prepare_incremental_delta_types( inputs.declared_columns if entry.type_enforcement else () ) authoritative_names: frozenset[str] = frozenset( - column.name.lower() for column in authoritative_columns + column_name_key(column.name) for column in authoritative_columns ) delta_map: dict[str, ColumnInfo] = { - column.name.lower(): column for column in inputs.delta_columns + column_name_key(column.name): column for column in inputs.delta_columns } effective_policy: OnSchemaChange = entry.on_schema_change or _DEFAULT_ON_SCHEMA_CHANGE preserved_columns: list[tuple[ColumnInfo, ColumnInfo]] = [] @@ -665,10 +666,10 @@ def _prepare_incremental_delta_types( if effective_policy in (OnSchemaChange.APPEND_NEW_COLUMNS, OnSchemaChange.IGNORE): target_column: ColumnInfo for target_column in inputs.target_columns: - incoming_column: ColumnInfo | None = delta_map.get(target_column.name.lower()) + incoming_column: ColumnInfo | None = delta_map.get(column_name_key(target_column.name)) if ( incoming_column is not None - and target_column.name.lower() not in authoritative_names + and column_name_key(target_column.name) not in authoritative_names and not types_equal( left=target_column.type, right=incoming_column.type, @@ -732,8 +733,8 @@ def _apply_schema_change( guard: BoundViewGuard = bound_views or BoundViewGuard() - target_map: dict[str, str] = {col.name.lower(): col.type for col in target_columns} - delta_map: dict[str, str] = {col.name.lower(): col.type for col in delta_columns} + target_map: dict[str, str] = {column_name_key(col.name): col.type for col in target_columns} + delta_map: dict[str, str] = {column_name_key(col.name): col.type for col in delta_columns} added: list[ColumnInfo] = [] removed: list[str] = [] @@ -741,7 +742,7 @@ def _apply_schema_change( col: ColumnInfo for col in delta_columns: - col_lower: str = col.name.lower() + col_lower: str = column_name_key(col.name) if col_lower not in target_map: added.append(col) elif not types_equal( @@ -750,7 +751,7 @@ def _apply_schema_change( type_changed.append(col) for col in target_columns: - col_lower = col.name.lower() + col_lower = column_name_key(col.name) if col_lower not in delta_map: removed.append(col.name) @@ -885,9 +886,9 @@ def _execute_dml( strategy: str | None = entry.incremental_strategy unique_key: tuple[str, ...] = entry.unique_key - target_col_set: frozenset[str] = frozenset(col.name.lower() for col in target_columns) + target_col_set: frozenset[str] = frozenset(column_name_key(col.name) for col in target_columns) intersection_names: tuple[str, ...] = tuple( - col.name for col in delta_columns if col.name.lower() in target_col_set + col.name for col in delta_columns if column_name_key(col.name) in target_col_set ) dml_columns: tuple[str, ...] = intersection_names projection: str = ", ".join(intersection_names) diff --git a/src/sqlbuild/executor/run/_helpers/validation/contracts.py b/src/sqlbuild/executor/run/_helpers/validation/contracts.py index ae8fefe93b..8848a9e6bc 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/contracts.py +++ b/src/sqlbuild/executor/run/_helpers/validation/contracts.py @@ -3,6 +3,7 @@ from __future__ import annotations from sqlbuild._native import ModelPlanEntry, SchemaDynamicColumnFamily, matching_dynamic_families +from sqlbuild.adapter.contract.main.column_name_key import column_name_key from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.adapter.type_system.main.types_equal import types_equal @@ -32,16 +33,16 @@ def validate_runtime_contract( ) declared_by_name: dict[str, ColumnInfo] = { - column.name.lower(): column for column in entry.contract_columns + column_name_key(column.name): column for column in entry.contract_columns } actual_by_name: dict[str, ColumnInfo] = { - column.name.lower(): column for column in actual_columns + column_name_key(column.name): column for column in actual_columns } missing_names: tuple[str, ...] = tuple( column.name for column in entry.contract_columns - if column.name.lower() not in actual_by_name + if column_name_key(column.name) not in actual_by_name ) if missing_names: raise ExecutorInputError( @@ -50,7 +51,7 @@ def validate_runtime_contract( ) extra_columns: tuple[ColumnInfo, ...] = tuple( - column for column in actual_columns if column.name.lower() not in declared_by_name + column for column in actual_columns if column_name_key(column.name) not in declared_by_name ) extra_names: tuple[str, ...] = tuple( column.name @@ -86,7 +87,9 @@ def validate_runtime_contract( actual_column: ColumnInfo for actual_column in actual_columns: - declared_column: ColumnInfo | None = declared_by_name.get(actual_column.name.lower()) + declared_column: ColumnInfo | None = declared_by_name.get( + column_name_key(actual_column.name) + ) if declared_column is None: continue if not declared_column.type or not actual_column.type: diff --git a/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py b/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py index bec5e60b1b..09649f37a5 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py +++ b/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py @@ -7,6 +7,7 @@ from sqlbuild._native import fit_auxiliary_relation_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.main.column_name_key import column_name_key from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, @@ -38,7 +39,7 @@ def enforce_types_staged( schema=staging_schema, name=staging_table, ) - declared_map: dict[str, str] = {col.name.lower(): col.type for col in declared_columns} + declared_map: dict[str, str] = {column_name_key(col.name): col.type for col in declared_columns} needs_enforcement: bool = False col: ColumnInfo @@ -46,7 +47,7 @@ def enforce_types_staged( produced_type: str | None = None produced_col: ColumnInfo for produced_col in produced_columns: - if produced_col.name.lower() == col.name.lower(): + if column_name_key(produced_col.name) == column_name_key(col.name): produced_type = produced_col.type break if produced_type is not None and not types_equal( @@ -104,7 +105,7 @@ def _build_type_enforcement_projection( projection_parts: list[str] = [] produced_column: ColumnInfo for produced_column in produced_columns: - declared_type: str | None = declared_map.get(produced_column.name.lower()) + declared_type: str | None = declared_map.get(column_name_key(produced_column.name)) quoted_column: str = adapter.render_identifier(produced_column.name) if declared_type is None: projection_parts.append(quoted_column) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 9c40100751..c114f1cb3b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1504,3 +1504,12 @@ class QuotedMixedCaseColumnsBuildE2ETestCase: expected_export_rows: list[tuple[object, ...]] expected_test_fragment: str expected_scenario_fragment: str + + +@dataclass(frozen=True) +class QuotedContractE2ETestCase: + """An enforced contract whose declared name the model produces as a quoted camelCase alias.""" + + description: str + declared_name: str + expected_rows: list[tuple[object, ...]] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_quoted_mixed_case_columns.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_quoted_mixed_case_columns.py index 93e7c6b482..bfa49dd09c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_quoted_mixed_case_columns.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_quoted_mixed_case_columns.py @@ -9,6 +9,7 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + QuotedContractE2ETestCase, QuotedMixedCaseColumnsBuildE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( @@ -109,5 +110,56 @@ def test_given_quoted_mixed_case_columns_when_running_commands_then_each_succeed assert test_case.expected_scenario_fragment in replayed.stdout, replayed.stdout +@pytest.mark.parametrize( + "test_case", + [ + QuotedContractE2ETestCase( + description="an unquoted declaration enforces the quoted camelCase alias", + declared_name="listingId", + expected_rows=[("L1",)], + ), + QuotedContractE2ETestCase( + description="a quoted declaration enforces the quoted camelCase alias", + declared_name='"listingId"', + expected_rows=[("L1",)], + ), + ], + ids=lambda case: case.description, +) +def test_given_enforced_contract_on_quoted_alias_when_building_then_contract_and_types_hold( + test_case: QuotedContractE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="partner_feed", + repo_files={ + **_PROJECT_FILES, + "models/listing_export.sql": ( + "MODEL (\n description 'Listings in the partner feed format.',\n" + " materialized table,\n contract enforced,\n" + f" columns ({test_case.declared_name} (type VARCHAR),),\n);\n\n" + 'SELECT l.listing_id::VARCHAR AS "listingId"\nFROM __ref("stg_listings") AS l\n' + ), + }, + ) + db_path: Path = project_dir / "partner_feed.duckdb" + execute_duckdb( + db_path=db_path, + sql='CREATE TABLE main.raw_listings AS SELECT \'L1\' AS "listingId", 1299::BIGINT AS "unitPriceMinor"', + ) + + builds: tuple[subprocess.CompletedProcess[str], ...] = tuple( + run_sqb(command=("--no-color", "build"), project_dir=project_dir) for _ in range(2) + ) + + for build in builds: + assert build.returncode == 0, build.stdout + build.stderr + assert "runtime contract" not in build.stdout, build.stdout + rows: list[tuple[object, ...]] = query_duckdb( + db_path=db_path, sql='SELECT "listingId" FROM main.listing_export' + ) + assert rows == test_case.expected_rows + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py index 4175f1733f..e787f39fb2 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py @@ -109,3 +109,13 @@ class QuotedColumnBuildTestCase: description: str stored_columns: tuple[str, ...] expected_fragment: str + + +@dataclass(frozen=True) +class QuotedContractBuildTestCase: + """An enforced contract declaring a name the model's quoted mixed-case alias produces.""" + + description: str + declared_name: str + produced_columns: tuple[str, ...] + expected_exit_code: int = 0 diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_quoted_mixed_case_columns.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_quoted_mixed_case_columns.py index 9cc4b4d12b..1037660e61 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_quoted_mixed_case_columns.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_quoted_mixed_case_columns.py @@ -8,6 +8,7 @@ from tests.integration.src.sqlbuild.executor.build.snowflake_metadata._test_types import ( QuotedColumnBuildTestCase, + QuotedContractBuildTestCase, ) from tests.integration.src.sqlbuild.executor.build.snowflake_metadata.helpers import ( SimulatedSnowflakeWarehouse, @@ -88,5 +89,69 @@ def test_given_quoted_mixed_case_source_columns_when_building_then_planning_bind assert any(test_case.expected_fragment in statement for statement in builds[1][1]), builds[1][1] +@pytest.mark.parametrize( + "test_case", + [ + QuotedContractBuildTestCase( + description="an unquoted declaration matches the quoted camelCase alias", + declared_name="listingId", + produced_columns=("listingId",), + ), + QuotedContractBuildTestCase( + description="a quoted declaration matches the quoted camelCase alias", + declared_name='"listingId"', + produced_columns=("listingId",), + ), + ], + ids=lambda case: case.description, +) +def test_given_enforced_contract_on_quoted_alias_when_building_then_runtime_contract_passes( + test_case: QuotedContractBuildTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + files: dict[str, str] = { + **_PROJECT_FILES, + "models/listing_export.sql": ( + "MODEL (\n description 'Listings in the partner feed format.',\n" + " contract enforced,\n" + f" columns ({test_case.declared_name} (type VARCHAR),),\n);\n\n" + 'SELECT l.listing_id::VARCHAR AS "listingId"\nFROM __ref("stg_listings") AS l\n' + ), + } + for relative, content in files.items(): + path: Path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(content, encoding="utf-8") + warehouse: SimulatedSnowflakeWarehouse = SimulatedSnowflakeWarehouse( + relations=( + FakeRelation( + database="ANALYTICS", + schema="RAW", + name="LISTINGS", + columns=( + FakeColumn(name="listingId", data_type="TEXT"), + FakeColumn(name="unitPriceMinor", data_type="NUMBER"), + ), + ), + ), + ) + warehouse.set_model_columns( + model="listing_export", + columns=tuple( + FakeColumn(name=name, data_type="TEXT") for name in test_case.produced_columns + ), + ) + + exit_code, _statements = build_with_cli( + project_dir=tmp_path, warehouse=warehouse, monkeypatch=monkeypatch + ) + + output: str = capsys.readouterr().out + assert exit_code == test_case.expected_exit_code, output + assert "runtime contract" not in output, output + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapter/contract/main/_test_types.py b/tests/unit/src/sqlbuild/adapter/contract/main/_test_types.py index 23f19534a1..fa89409bf8 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/main/_test_types.py +++ b/tests/unit/src/sqlbuild/adapter/contract/main/_test_types.py @@ -23,3 +23,12 @@ class RelationProbeClassificationTestCase: expected_status: RelationReadStatus | None expected_role: str | None = None role: str | None = None + + +@dataclass(frozen=True) +class ColumnNameKeyTestCase: + """One declared or listed column name and the key it matches by.""" + + description: str + name: str + expected_key: str diff --git a/tests/unit/src/sqlbuild/adapter/contract/main/test_column_name_key.py b/tests/unit/src/sqlbuild/adapter/contract/main/test_column_name_key.py new file mode 100644 index 0000000000..247c42ae4a --- /dev/null +++ b/tests/unit/src/sqlbuild/adapter/contract/main/test_column_name_key.py @@ -0,0 +1,37 @@ +"""Column names match by the compiler's key: case-folded and without identifier quotes.""" + +import pytest + +from sqlbuild.adapter.contract.main.column_name_key import column_name_key +from tests.unit.src.sqlbuild.adapter.contract.main._test_types import ColumnNameKeyTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + ColumnNameKeyTestCase( + description="a quoted camelCase listing", name='"listingId"', expected_key="listingid" + ), + ColumnNameKeyTestCase( + description="an unquoted declaration", name="listingId", expected_key="listingid" + ), + ColumnNameKeyTestCase( + description="an upper-case listing", name="ORDER_ID", expected_key="order_id" + ), + ColumnNameKeyTestCase( + description="a name holding a quote is not unwrapped", + name='"a""b"', + expected_key='"a""b"', + ), + ColumnNameKeyTestCase(description="empty quotes are kept", name='""', expected_key='""'), + ], + ids=lambda case: case.description, +) +def test_given_column_name_when_keying_then_matches_compiler_key( + test_case: ColumnNameKeyTestCase, +) -> None: + assert column_name_key(test_case.name) == test_case.expected_key + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) From 679ace9216a205124f41da6fdd2f90d1f27f5005 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 29 Oct 2026 06:09:25 +0000 Subject: [PATCH 15/20] fix(adapter): expire every schema listing a build carries, not only the planner's A listing read through the cache's refresh path, including one made while planning, had no age limit, so a build's run could trust it for as long as no statement invalidated it. Every kept schema listing now expires after a minute, like the planner listings the run reuses. --- .../relations/classes/inspection_catalog.py | 4 +- .../classes/relation_metadata_cache.py | 14 ++++-- src/sqlbuild/adapter/relations/constants.py | 2 +- .../test_planner_listing_reuse.py | 2 +- .../relation_metadata_cache/_test_types.py | 9 ++++ .../test_relation_metadata_cache.py | 47 ++++++++++++++++++- 6 files changed, 70 insertions(+), 8 deletions(-) diff --git a/src/sqlbuild/adapter/relations/classes/inspection_catalog.py b/src/sqlbuild/adapter/relations/classes/inspection_catalog.py index f4dc93bf8e..870d551b88 100644 --- a/src/sqlbuild/adapter/relations/classes/inspection_catalog.py +++ b/src/sqlbuild/adapter/relations/classes/inspection_catalog.py @@ -16,7 +16,7 @@ from sqlbuild.adapter.relations.constants import ( INSPECTION_DESCRIBED_NAME_LIMIT, INSPECTION_IN_LIST_LIMIT, - PLANNER_LISTING_MAX_AGE_SECONDS, + SCHEMA_LISTING_MAX_AGE_SECONDS, ) from sqlbuild.adapter.relations.main.run_bounded_inspections import run_bounded_inspections from sqlbuild.adapter.relations.models import ( @@ -338,7 +338,7 @@ def _read_relation_listing( ), relations={entry.stored_name: entry.relation for entry in listing.entries}, versions=versions, - max_age_seconds=PLANNER_LISTING_MAX_AGE_SECONDS, + max_age_seconds=SCHEMA_LISTING_MAX_AGE_SECONDS, ) return listing diff --git a/src/sqlbuild/adapter/relations/classes/relation_metadata_cache.py b/src/sqlbuild/adapter/relations/classes/relation_metadata_cache.py index dd5c89846e..9f09095255 100644 --- a/src/sqlbuild/adapter/relations/classes/relation_metadata_cache.py +++ b/src/sqlbuild/adapter/relations/classes/relation_metadata_cache.py @@ -13,7 +13,10 @@ statement_metadata_effect, ) from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.adapter.relations.constants import SCHEMA_LISTING_LOOKUP_THRESHOLD +from sqlbuild.adapter.relations.constants import ( + SCHEMA_LISTING_LOOKUP_THRESHOLD, + SCHEMA_LISTING_MAX_AGE_SECONDS, +) from sqlbuild.adapter.relations.models import RunSchemaListing, StatementMetadataEffect from sqlbuild.adapter.relations.types import ( RelationCacheKey, @@ -98,10 +101,15 @@ def schema_relation( def refresh_schema_listing( self, *, scope: SchemaListingScope, read_listing: Callable[[], dict[str, RelationInfo]] ) -> RunSchemaListing: - """List the schema again; every relation in it is current as of this listing.""" + """List the schema again; every relation in it is current as of this listing, for a time.""" versions: tuple[int, dict[str, int]] = self.listing_versions() - return self.keep_schema_listing(scope=scope, relations=read_listing(), versions=versions) + return self.keep_schema_listing( + scope=scope, + relations=read_listing(), + versions=versions, + max_age_seconds=SCHEMA_LISTING_MAX_AGE_SECONDS, + ) def listing_versions(self) -> tuple[int, dict[str, int]]: """The cache versions a listing read from now on is current as of.""" diff --git a/src/sqlbuild/adapter/relations/constants.py b/src/sqlbuild/adapter/relations/constants.py index 431f3acc5f..89549a0934 100644 --- a/src/sqlbuild/adapter/relations/constants.py +++ b/src/sqlbuild/adapter/relations/constants.py @@ -7,5 +7,5 @@ RELATION_EXISTS_LOOKUP: str = "exists" RELATION_TYPE_LOOKUP: str = "type" SCHEMA_LISTING_LOOKUP_THRESHOLD: int = 2 -PLANNER_LISTING_MAX_AGE_SECONDS: float = 60.0 +SCHEMA_LISTING_MAX_AGE_SECONDS: float = 60.0 UNKNOWN_RELATION_READER: str = "unknown user" diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_planner_listing_reuse.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_planner_listing_reuse.py index 43535cb195..f15e2284ab 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_planner_listing_reuse.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_planner_listing_reuse.py @@ -25,7 +25,7 @@ r"^SHOW (?PTABLES|VIEWS) IN SCHEMA (?P\S+)" ) _PLANNER_LISTING_MAX_AGE: str = ( - "sqlbuild.adapter.relations.classes.inspection_catalog.PLANNER_LISTING_MAX_AGE_SECONDS" + "sqlbuild.adapter.relations.classes.inspection_catalog.SCHEMA_LISTING_MAX_AGE_SECONDS" ) _ONE_LISTING_EACH: dict[str, int] = { 'TABLES "ANALYTICS"."MARTS"': 1, diff --git a/tests/unit/src/sqlbuild/adapter/relations/classes/relation_metadata_cache/_test_types.py b/tests/unit/src/sqlbuild/adapter/relations/classes/relation_metadata_cache/_test_types.py index eb198fc12f..407e760646 100644 --- a/tests/unit/src/sqlbuild/adapter/relations/classes/relation_metadata_cache/_test_types.py +++ b/tests/unit/src/sqlbuild/adapter/relations/classes/relation_metadata_cache/_test_types.py @@ -31,3 +31,12 @@ class LiveInvalidationTestCase: expected_first_columns: tuple[str, ...] expected_second_columns: tuple[str, ...] expected_second_exists: bool + + +@dataclass(frozen=True) +class ListingExpiryTestCase: + """A schema listing refreshed at time zero and looked up again some seconds later.""" + + description: str + seconds_later: float + expected_listings: int diff --git a/tests/unit/src/sqlbuild/adapter/relations/classes/relation_metadata_cache/test_relation_metadata_cache.py b/tests/unit/src/sqlbuild/adapter/relations/classes/relation_metadata_cache/test_relation_metadata_cache.py index 99aa2b0f47..eaa5673eae 100644 --- a/tests/unit/src/sqlbuild/adapter/relations/classes/relation_metadata_cache/test_relation_metadata_cache.py +++ b/tests/unit/src/sqlbuild/adapter/relations/classes/relation_metadata_cache/test_relation_metadata_cache.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapter.relations.classes.relation_metadata_cache import RelationMetadataCache from sqlbuild.adapter.relations.main.cached_relation_columns import cached_relation_columns from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists @@ -17,6 +17,7 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from tests.unit.src.sqlbuild.adapter.relations.classes.relation_metadata_cache._test_types import ( CacheInvalidationTestCase, + ListingExpiryTestCase, LiveInvalidationTestCase, OverlappingReadTestCase, ) @@ -229,3 +230,47 @@ def test_given_adapter_ddl_when_looking_up_again_then_returns_fresh_metadata( if __name__ == "__main__": pytest.main([__file__, "-vv"]) + + +@pytest.mark.parametrize( + "test_case", + [ + ListingExpiryTestCase( + description="a recent listing answers the lookup", + seconds_later=30.0, + expected_listings=1, + ), + ListingExpiryTestCase( + description="a listing older than a minute is read again", + seconds_later=61.0, + expected_listings=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_refreshed_listing_when_looking_up_later_then_expired_listing_is_read_again( + test_case: ListingExpiryTestCase, +) -> None: + now: list[float] = [0.0] + cache: RelationMetadataCache = RelationMetadataCache( + transactional_ddl=False, clock=lambda: now[0] + ) + listings: list[int] = [] + orders: RelationInfo = RelationInfo( + database="ANALYTICS", schema="MARTS", name="ORDERS", relation_type="table" + ) + + def read_listing() -> dict[str, RelationInfo]: + listings.append(1) + return {"ORDERS": orders} + + _ = cache.refresh_schema_listing(scope=("ANALYTICS", "MARTS"), read_listing=read_listing) + now[0] = test_case.seconds_later + _ = cache.schema_relation( + scope=("ANALYTICS", "MARTS"), + name="analytics.marts.orders", + stored_name="ORDERS", + read_listing=read_listing, + ) + + assert len(listings) == test_case.expected_listings From 0f306926bd76725078db5d8c25887186d741348c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 29 Oct 2026 06:09:46 +0000 Subject: [PATCH 16/20] fix(duckdb): match search path schemas without regard to case DuckDB resolves an unqualified name through a search path schema written in any case, but relation existence compared the search path's spelling exactly, so after SET search_path = 'STAGING' a table DuckDB could read was reported missing. --- .../contract/classes/duckdb_backed_adapter.py | 6 +++++- .../src/sqlbuild/adapters/duckdb/test_client.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index aa8559d41a..a7b499bea5 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -1237,7 +1237,11 @@ def relation_exists( if schema is not None: query += f" AND table_schema = {_duckdb_string_literal(schema)}" else: - query += " AND table_schema IN (SELECT UNNEST(current_schemas(true)))" + query += ( + " AND lower(table_schema) IN " + "(SELECT lower(search_schema) FROM " + "(SELECT UNNEST(current_schemas(true)) AS search_schema))" + ) if database is not None: query += f" AND table_catalog = {_duckdb_string_literal(database)}" result: Any = self.execute(connection=connection, sql=query).fetchone() diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py index 6d5f2cf105..d7c27e5bf5 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py @@ -329,6 +329,18 @@ def test_given_sql_when_querying_then_returns_normalized_query_result( name="raw_orders", expected_exists=True, ), + RelationExistsTestCase( + description="a search path naming the schema in another case resolves as DuckDB does", + setup_sql=( + "CREATE SCHEMA staging", + "CREATE TABLE staging.raw_orders (id INTEGER)", + "SET search_path = 'STAGING'", + ), + database=None, + schema=None, + name="raw_orders", + expected_exists=True, + ), ], ids=lambda case: case.description, ) From 4d42294848b31829021c97459564d87c5dad1a90 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 29 Oct 2026 06:10:52 +0000 Subject: [PATCH 17/20] fix(runtime): resolve the working-directory import path entry per directory The import hook cached whether a relative search path entry such as "" lay under a project root, resolved against the working directory of its first lookup, so after a chdir it kept answering for the old directory. Relative entries are now keyed by the current working directory. --- .../user_code/classes/user_code_finder.py | 8 ++-- .../runtime/user_code/classes/_test_types.py | 10 +++++ .../classes/test_user_code_finder.py | 45 ++++++++++++++++++- 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/src/sqlbuild/runtime/user_code/classes/user_code_finder.py b/src/sqlbuild/runtime/user_code/classes/user_code_finder.py index 5981453b19..2d36a4916c 100644 --- a/src/sqlbuild/runtime/user_code/classes/user_code_finder.py +++ b/src/sqlbuild/runtime/user_code/classes/user_code_finder.py @@ -1,5 +1,6 @@ """A meta-path finder that loads modules under registered project roots from source.""" +import os import sys import sysconfig from collections.abc import Sequence @@ -26,7 +27,7 @@ def __init__(self) -> None: *sysconfig.get_paths().values(), } ) - self._project_entries: dict[str, bool] = {} + self._project_entries: dict[tuple[str, str], bool] = {} self._entry_roots: frozenset[Path] = frozenset() def find_spec( @@ -59,7 +60,8 @@ def _is_project_entry(self, entry: str) -> bool: if roots != self._entry_roots: self._project_entries = {} self._entry_roots = roots - known: bool | None = self._project_entries.get(entry) + directory: str = "" if os.path.isabs(entry) else os.getcwd() + known: bool | None = self._project_entries.get((entry, directory)) if known is not None: return known try: @@ -69,5 +71,5 @@ def _is_project_entry(self, entry: str) -> bool: under_root: bool = any( resolved.is_relative_to(root) or root.is_relative_to(resolved) for root in roots ) - self._project_entries[entry] = under_root + self._project_entries[(entry, directory)] = under_root return under_root diff --git a/tests/unit/src/sqlbuild/runtime/user_code/classes/_test_types.py b/tests/unit/src/sqlbuild/runtime/user_code/classes/_test_types.py index 2a84a8d568..68d6f86922 100644 --- a/tests/unit/src/sqlbuild/runtime/user_code/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/runtime/user_code/classes/_test_types.py @@ -11,3 +11,13 @@ class FinderSpecTestCase: module_name: str earlier_entries: tuple[str, ...] expected_source_only: bool + + +@dataclass(frozen=True) +class FinderWorkingDirectoryTestCase: + """A lookup through the working-directory path entry before and after changing directory.""" + + description: str + first_directory: str + second_directory: str + expected_source_only: tuple[bool, bool] diff --git a/tests/unit/src/sqlbuild/runtime/user_code/classes/test_user_code_finder.py b/tests/unit/src/sqlbuild/runtime/user_code/classes/test_user_code_finder.py index c832178ef0..0fd7708dc8 100644 --- a/tests/unit/src/sqlbuild/runtime/user_code/classes/test_user_code_finder.py +++ b/tests/unit/src/sqlbuild/runtime/user_code/classes/test_user_code_finder.py @@ -8,7 +8,10 @@ from sqlbuild.runtime.user_code.classes.source_only_loader import SourceOnlyLoader from sqlbuild.runtime.user_code.classes.user_code_finder import UserCodeFinder -from tests.unit.src.sqlbuild.runtime.user_code.classes._test_types import FinderSpecTestCase +from tests.unit.src.sqlbuild.runtime.user_code.classes._test_types import ( + FinderSpecTestCase, + FinderWorkingDirectoryTestCase, +) @pytest.mark.parametrize( @@ -55,5 +58,45 @@ def test_given_module_name_when_finding_spec_then_only_project_modules_load_from assert source_only == test_case.expected_source_only +@pytest.mark.parametrize( + "test_case", + [ + FinderWorkingDirectoryTestCase( + description="moving into the project makes its modules load from source", + first_directory="elsewhere", + second_directory="project", + expected_source_only=(False, True), + ), + FinderWorkingDirectoryTestCase( + description="moving out of the project leaves its modules to normal import", + first_directory="project", + second_directory="elsewhere", + expected_source_only=(True, False), + ), + ], + ids=lambda case: case.description, +) +def test_given_working_directory_entry_when_directory_changes_then_finder_follows_it( + test_case: FinderWorkingDirectoryTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + project.mkdir() + _ = (project / "order_rules.py").write_text("LIMIT: int = 1\n", encoding="utf-8") + (tmp_path / "elsewhere").mkdir() + monkeypatch.setattr(sys, "path", ["", *sys.path]) + finder: UserCodeFinder = UserCodeFinder() + finder.roots.add(project.resolve()) + + monkeypatch.chdir(tmp_path / test_case.first_directory) + first: ModuleSpec | None = finder.find_spec("order_rules") + monkeypatch.chdir(tmp_path / test_case.second_directory) + second: ModuleSpec | None = finder.find_spec("order_rules") + + assert ( + first is not None and isinstance(first.loader, SourceOnlyLoader), + second is not None and isinstance(second.loader, SourceOnlyLoader), + ) == test_case.expected_source_only + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) From 7a9ab9b4b6958c4b4e7df8e7ef8aeda389c4b170 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 29 Oct 2026 06:42:06 +0000 Subject: [PATCH 18/20] fix(compiler): keep backtick text undoubled when interpolating @@ variables With the adapter's lexical rules, @@ interpolation read a doubled backtick as an escape, so audit SQL holding `@@name`` reported an unclosed string instead of the unknown variable it names. Backtick text is read as before, ending at the next backtick; quotes and comments still follow the adapter's rules. --- .../src/compiler/_helpers/sql_interpolation/substitution.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs index 827216b2b5..6f4b7b1df6 100644 --- a/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs @@ -72,8 +72,8 @@ impl Interpolation<'_, H> { let mut index = 0; while index < bytes.len() { let non_code = match self.host.lexical_syntax() { - Some(syntax) => dialect_non_code_end(bytes, index, syntax), - None => python_non_code_end(bytes, index), + Some(syntax) if bytes[index] != b'`' => dialect_non_code_end(bytes, index, syntax), + _ => python_non_code_end(bytes, index), }; let text_end = match non_code { Ok(Some(end)) => end, From 9633260ff8bd0b4d4581b6eaa7d295d5acc96826 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 29 Oct 2026 07:56:31 +0000 Subject: [PATCH 19/20] chore: move repository links to the sqlstack-io organization --- Cargo.lock | 6 +++--- Cargo.toml | 4 ++-- README.md | 10 +++++----- .../skills/sqlbuild/references/docs/benchmarks.md | 4 ++-- .../sqlbuild/references/docs/concepts/adapters.md | 2 +- .../references/docs/concepts/adapters/bigquery.md | 2 +- .../references/docs/concepts/adapters/clickhouse.md | 2 +- .../references/docs/concepts/adapters/databricks.md | 2 +- .../references/docs/concepts/adapters/sqlserver.md | 2 +- website/astro.config.mjs | 2 +- website/public/index.html | 6 +++--- website/src/content/docs/docs/benchmarks.mdx | 4 ++-- website/src/content/docs/docs/concepts/adapters.mdx | 2 +- .../content/docs/docs/concepts/adapters/bigquery.mdx | 2 +- .../content/docs/docs/concepts/adapters/clickhouse.mdx | 2 +- .../content/docs/docs/concepts/adapters/databricks.mdx | 2 +- .../content/docs/docs/concepts/adapters/sqlserver.mdx | 2 +- 17 files changed, 28 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a35dbfd5f0..834f1217eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -711,7 +711,7 @@ checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "polyglot-sql" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=8767d688c5b504ad609645e27ab77a260d91c5a1#8767d688c5b504ad609645e27ab77a260d91c5a1" +source = "git+https://github.com/sqlstack-io/polyglot.git?rev=8767d688c5b504ad609645e27ab77a260d91c5a1#8767d688c5b504ad609645e27ab77a260d91c5a1" dependencies = [ "polyglot-sql-ast-derive", "polyglot-sql-function-catalogs", @@ -726,7 +726,7 @@ dependencies = [ [[package]] name = "polyglot-sql-ast-derive" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=8767d688c5b504ad609645e27ab77a260d91c5a1#8767d688c5b504ad609645e27ab77a260d91c5a1" +source = "git+https://github.com/sqlstack-io/polyglot.git?rev=8767d688c5b504ad609645e27ab77a260d91c5a1#8767d688c5b504ad609645e27ab77a260d91c5a1" dependencies = [ "proc-macro2", "quote", @@ -736,7 +736,7 @@ dependencies = [ [[package]] name = "polyglot-sql-function-catalogs" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=8767d688c5b504ad609645e27ab77a260d91c5a1#8767d688c5b504ad609645e27ab77a260d91c5a1" +source = "git+https://github.com/sqlstack-io/polyglot.git?rev=8767d688c5b504ad609645e27ab77a260d91c5a1#8767d688c5b504ad609645e27ab77a260d91c5a1" [[package]] name = "portable-atomic" diff --git a/Cargo.toml b/Cargo.toml index 3d2b9d2b97..2b19fb47d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,14 +72,14 @@ junction = { version = "2.1.0", default-features = false } mimalloc = { version = "0.1", default-features = false, features = ["local_dynamic_tls"] } proc-macro2 = { version = "1", features = ["span-locations"] } pyo3 = { version = "0.26", features = ["abi3-py312"] } -polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "8767d688c5b504ad609645e27ab77a260d91c5a1", default-features = false, features = [ +polyglot-sql = { git = "https://github.com/sqlstack-io/polyglot.git", rev = "8767d688c5b504ad609645e27ab77a260d91c5a1", default-features = false, features = [ "ast-tools", "builder", "all-dialects", "semantic", "stacker", ] } -polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "8767d688c5b504ad609645e27ab77a260d91c5a1", default-features = false, features = [ +polyglot-sql-function-catalogs = { git = "https://github.com/sqlstack-io/polyglot.git", rev = "8767d688c5b504ad609645e27ab77a260d91c5a1", default-features = false, features = [ "dialect-duckdb", ] } blake3 = { version = "1", features = ["rayon"] } diff --git a/README.md b/README.md index ed0a07cd5f..fa0130eeff 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- SQLBuild + SQLBuild

@@ -21,7 +21,7 @@ no external state database, no manifest files and no paid tier. Its compiler and planner run in Rust. Warehouse execution and extensions use Python.

- Moving and renaming an incremental model with sqb mv: sqb plan migrates the existing table instead of rebuilding it + Moving and renaming an incremental model with sqb mv: sqb plan migrates the existing table instead of rebuilding it

Move an incremental model to a new folder and name with `sqb mv`, and `sqb plan` migrates its table @@ -100,7 +100,7 @@ running on local DuckDB. `SELECT` fails at compile time, before anything reaches the warehouse.

- sqb compile fails because units_sold is not in the enforced contract and the declared waffles_sold column is missing + sqb compile fails because units_sold is not in the enforced contract and the declared waffles_sold column is missing

Both sides are reported: the new column isn't in the contract, and the declared one is gone. @@ -133,7 +133,7 @@ A mart that reads `__source("raw__payments")` directly now fails, and so does th has no mock for it:

- sqb compile reports the custom rule XSQBRARCH001 on the line that reads a raw source, and a unit test with no mock for that source + sqb compile reports the custom rule XSQBRARCH001 on the line that reads a raw source, and a unit test with no mock for that source

See [rules](https://sqlbuild.com/docs/concepts/rules/). @@ -143,7 +143,7 @@ See [rules](https://sqlbuild.com/docs/concepts/rules/). `sqb scope --as-path` previews moving a model before you move it.

- sqb scope previews moving daily_revenue: the enum and macro it uses would be lost, so both usages are invalidated + sqb scope previews moving daily_revenue: the enum and macro it uses would be lost, so both usages are invalidated

Look at `Lost` and `Invalidated usages`: the model uses an enum and a macro that are private to diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/benchmarks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/benchmarks.md index 4856edf6ec..fb7940ae49 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/benchmarks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/benchmarks.md @@ -18,7 +18,7 @@ Online: https://sqlbuild.com/docs/benchmarks/ SQLBuild 0.164.3 and dbt 2.0.9 compiling the same generated [10,000-model DuckDB project](#the-project), on a GitHub-hosted runner with 4 CPUs and 15 GiB RAM. Each cell is the median of three rounds from -[run 39543189999](https://github.com/chio-labs/sqlbuild/actions/runs/39543189999) (October 21, 2026). +[run 39543189999](https://github.com/sqlstack-io/sqlbuild/actions/runs/39543189999) (October 21, 2026). ## SQLBuild vs dbt v2 @@ -121,7 +121,7 @@ compiles in full. ## Reproduce The harness and the exact command for every variant are in -[`benchmarks/dbt-compare/`](https://github.com/chio-labs/sqlbuild/tree/main/benchmarks/dbt-compare). +[`benchmarks/dbt-compare/`](https://github.com/sqlstack-io/sqlbuild/tree/main/benchmarks/dbt-compare). To rerun it on GitHub Actions: ```bash diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md index beced17e68..6c562417b0 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md @@ -13,7 +13,7 @@ Online: https://sqlbuild.com/docs/concepts/adapters/ SQLBuild uses adapters to connect to different database engines. Snowflake is the main target. - **Supported:** ready for production use. -- **Beta:** builds, tests and plans work, but with less production use so far. Expect rough edges, and please [report issues](https://github.com/chio-labs/sqlbuild/issues). +- **Beta:** builds, tests and plans work, but with less production use so far. Expect rough edges, and please [report issues](https://github.com/sqlstack-io/sqlbuild/issues). | Adapter | Status | Install | |---------|--------|---------| diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/bigquery.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/bigquery.md index b450317391..6d7d3aa55f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/bigquery.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/bigquery.md @@ -6,7 +6,7 @@ Online: https://sqlbuild.com/docs/concepts/adapters/bigquery/ -The BigQuery adapter is in beta: builds, tests and plans work, but it has had less production use so far. Please [report issues](https://github.com/chio-labs/sqlbuild/issues). +The BigQuery adapter is in beta: builds, tests and plans work, but it has had less production use so far. Please [report issues](https://github.com/sqlstack-io/sqlbuild/issues). BigQuery requires the optional `google-cloud-bigquery` dependency: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md index 7c86acc95c..e042fd3154 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md @@ -21,7 +21,7 @@ Online: https://sqlbuild.com/docs/concepts/adapters/clickhouse/ - Limits - Recall store -The ClickHouse adapter is in beta and supports a single server. Clusters (`ON CLUSTER`, replicated and distributed tables) are not supported yet. Please [report issues](https://github.com/chio-labs/sqlbuild/issues). +The ClickHouse adapter is in beta and supports a single server. Clusters (`ON CLUSTER`, replicated and distributed tables) are not supported yet. Please [report issues](https://github.com/sqlstack-io/sqlbuild/issues). ClickHouse requires the optional `clickhouse-connect` dependency: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/databricks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/databricks.md index 0053f2e4fc..094ba0a9f7 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/databricks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/databricks.md @@ -6,7 +6,7 @@ Online: https://sqlbuild.com/docs/concepts/adapters/databricks/ -The Databricks adapter is in beta: builds, tests and plans work, but it has had less production use so far. Please [report issues](https://github.com/chio-labs/sqlbuild/issues). +The Databricks adapter is in beta: builds, tests and plans work, but it has had less production use so far. Please [report issues](https://github.com/sqlstack-io/sqlbuild/issues). Databricks requires the optional `databricks-sql-connector` dependency: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlserver.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlserver.md index 3939785771..3ba3f8f00c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlserver.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlserver.md @@ -6,7 +6,7 @@ Online: https://sqlbuild.com/docs/concepts/adapters/sqlserver/ -The SQL Server adapter is in beta: builds, tests and plans work, but it has had less production use so far. Please [report issues](https://github.com/chio-labs/sqlbuild/issues). +The SQL Server adapter is in beta: builds, tests and plans work, but it has had less production use so far. Please [report issues](https://github.com/sqlstack-io/sqlbuild/issues). SQL Server requires the optional `pymssql` dependency: diff --git a/website/astro.config.mjs b/website/astro.config.mjs index 17e6ee3681..3b47d41c91 100644 --- a/website/astro.config.mjs +++ b/website/astro.config.mjs @@ -24,7 +24,7 @@ export default defineConfig({ { tag: 'meta', attrs: { name: 'twitter:image', content: 'https://sqlbuild.com/assets/og-sqlbuild-docs.png' } }, ], social: [ - { icon: 'github', label: 'GitHub', href: 'https://github.com/chio-labs/sqlbuild' }, + { icon: 'github', label: 'GitHub', href: 'https://github.com/sqlstack-io/sqlbuild' }, { icon: 'discord', label: 'Discord', href: 'https://discord.gg/tYU4nXdsT' }, ], components: { diff --git a/website/public/index.html b/website/public/index.html index a3372ef917..8d7ac87117 100644 --- a/website/public/index.html +++ b/website/public/index.html @@ -43,7 +43,7 @@ How it works Docs Quickstart - + Read the docs - View on GitHub + View on GitHub
Change your warehouse like you change code SQLBuild · Apache-2.0 Docs · - GitHub · + GitHub · PyPI
diff --git a/website/src/content/docs/docs/benchmarks.mdx b/website/src/content/docs/docs/benchmarks.mdx index d09b5c9c66..4d046dd9c6 100644 --- a/website/src/content/docs/docs/benchmarks.mdx +++ b/website/src/content/docs/docs/benchmarks.mdx @@ -5,7 +5,7 @@ description: "SQLBuild and dbt v2 compiling the same 10,000-model project: cold, SQLBuild 0.164.3 and dbt 2.0.9 compiling the same generated [10,000-model DuckDB project](#the-project), on a GitHub-hosted runner with 4 CPUs and 15 GiB RAM. Each cell is the median of three rounds from -[run 39543189999](https://github.com/chio-labs/sqlbuild/actions/runs/39543189999) (October 21, 2026). +[run 39543189999](https://github.com/sqlstack-io/sqlbuild/actions/runs/39543189999) (October 21, 2026). ## SQLBuild vs dbt v2 @@ -108,7 +108,7 @@ compiles in full. ## Reproduce The harness and the exact command for every variant are in -[`benchmarks/dbt-compare/`](https://github.com/chio-labs/sqlbuild/tree/main/benchmarks/dbt-compare). +[`benchmarks/dbt-compare/`](https://github.com/sqlstack-io/sqlbuild/tree/main/benchmarks/dbt-compare). To rerun it on GitHub Actions: ```bash diff --git a/website/src/content/docs/docs/concepts/adapters.mdx b/website/src/content/docs/docs/concepts/adapters.mdx index b1d4a094cb..0bdbeff4a5 100644 --- a/website/src/content/docs/docs/concepts/adapters.mdx +++ b/website/src/content/docs/docs/concepts/adapters.mdx @@ -6,7 +6,7 @@ description: "Supported database engines and their connection configuration." SQLBuild uses adapters to connect to different database engines. Snowflake is the main target. - **Supported:** ready for production use. -- **Beta:** builds, tests and plans work, but with less production use so far. Expect rough edges, and please [report issues](https://github.com/chio-labs/sqlbuild/issues). +- **Beta:** builds, tests and plans work, but with less production use so far. Expect rough edges, and please [report issues](https://github.com/sqlstack-io/sqlbuild/issues). | Adapter | Status | Install | |---------|--------|---------| diff --git a/website/src/content/docs/docs/concepts/adapters/bigquery.mdx b/website/src/content/docs/docs/concepts/adapters/bigquery.mdx index 9bcc286861..0613aa982a 100644 --- a/website/src/content/docs/docs/concepts/adapters/bigquery.mdx +++ b/website/src/content/docs/docs/concepts/adapters/bigquery.mdx @@ -6,7 +6,7 @@ description: "BigQuery adapter configuration for SQLBuild." import Note from '../../../../../components/mintlify/Note.astro'; -The BigQuery adapter is in beta: builds, tests and plans work, but it has had less production use so far. Please [report issues](https://github.com/chio-labs/sqlbuild/issues). +The BigQuery adapter is in beta: builds, tests and plans work, but it has had less production use so far. Please [report issues](https://github.com/sqlstack-io/sqlbuild/issues). BigQuery requires the optional `google-cloud-bigquery` dependency: diff --git a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx index 27fb8d29cb..2444770b00 100644 --- a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx +++ b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx @@ -6,7 +6,7 @@ description: "ClickHouse adapter configuration for SQLBuild." import Note from '../../../../../components/mintlify/Note.astro'; -The ClickHouse adapter is in beta and supports a single server. Clusters (`ON CLUSTER`, replicated and distributed tables) are not supported yet. Please [report issues](https://github.com/chio-labs/sqlbuild/issues). +The ClickHouse adapter is in beta and supports a single server. Clusters (`ON CLUSTER`, replicated and distributed tables) are not supported yet. Please [report issues](https://github.com/sqlstack-io/sqlbuild/issues). ClickHouse requires the optional `clickhouse-connect` dependency: diff --git a/website/src/content/docs/docs/concepts/adapters/databricks.mdx b/website/src/content/docs/docs/concepts/adapters/databricks.mdx index c017c54238..8dcc5e3e2c 100644 --- a/website/src/content/docs/docs/concepts/adapters/databricks.mdx +++ b/website/src/content/docs/docs/concepts/adapters/databricks.mdx @@ -6,7 +6,7 @@ description: "Databricks adapter configuration for SQLBuild." import Note from '../../../../../components/mintlify/Note.astro'; -The Databricks adapter is in beta: builds, tests and plans work, but it has had less production use so far. Please [report issues](https://github.com/chio-labs/sqlbuild/issues). +The Databricks adapter is in beta: builds, tests and plans work, but it has had less production use so far. Please [report issues](https://github.com/sqlstack-io/sqlbuild/issues). Databricks requires the optional `databricks-sql-connector` dependency: diff --git a/website/src/content/docs/docs/concepts/adapters/sqlserver.mdx b/website/src/content/docs/docs/concepts/adapters/sqlserver.mdx index 7d9ef18ff7..991b99c390 100644 --- a/website/src/content/docs/docs/concepts/adapters/sqlserver.mdx +++ b/website/src/content/docs/docs/concepts/adapters/sqlserver.mdx @@ -6,7 +6,7 @@ description: "Microsoft SQL Server adapter configuration for SQLBuild." import Note from '../../../../../components/mintlify/Note.astro'; -The SQL Server adapter is in beta: builds, tests and plans work, but it has had less production use so far. Please [report issues](https://github.com/chio-labs/sqlbuild/issues). +The SQL Server adapter is in beta: builds, tests and plans work, but it has had less production use so far. Please [report issues](https://github.com/sqlstack-io/sqlbuild/issues). SQL Server requires the optional `pymssql` dependency: From a1c656d55f257803a57dd86b949a49b3d45741f4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 29 Oct 2026 08:09:06 +0000 Subject: [PATCH 20/20] fix(runtime): skip a relative import path entry when the working directory is gone The import hook read the working directory outside its error handling, so after the directory was deleted every import through a relative search path entry raised FileNotFoundError. The entry is skipped again, as before the hook keyed entries by directory. A SQL macro lexer test now names the raw-string prefix rules it uses. --- .../sql_macros/tests/test_lexical_rules.rs | 2 +- .../user_code/classes/user_code_finder.py | 5 ++- .../runtime/user_code/classes/_test_types.py | 9 ++++++ .../classes/test_user_code_finder.py | 32 +++++++++++++++++++ 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_lexical_rules.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_lexical_rules.rs index 5c4882a601..44c244c404 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_lexical_rules.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_lexical_rules.rs @@ -31,7 +31,7 @@ fn given_adapter_lexical_rules_when_finding_sql_macro_calls_then_strings_end_whe expected_calls: &["order_total"], }, LexicalRulesCallsTestCase { - description: "a raw string ending in a backslash on BigQuery", + description: "a raw string ending in a backslash under raw-string prefix rules", syntax: raw_prefix_rules(), sql: "SELECT r'C:\\' AS path, @sql.order_total(1) AS b", expected_calls: &["order_total"], diff --git a/src/sqlbuild/runtime/user_code/classes/user_code_finder.py b/src/sqlbuild/runtime/user_code/classes/user_code_finder.py index 2d36a4916c..b753137b9d 100644 --- a/src/sqlbuild/runtime/user_code/classes/user_code_finder.py +++ b/src/sqlbuild/runtime/user_code/classes/user_code_finder.py @@ -60,7 +60,10 @@ def _is_project_entry(self, entry: str) -> bool: if roots != self._entry_roots: self._project_entries = {} self._entry_roots = roots - directory: str = "" if os.path.isabs(entry) else os.getcwd() + try: + directory: str = "" if os.path.isabs(entry) else os.getcwd() + except OSError: + return False known: bool | None = self._project_entries.get((entry, directory)) if known is not None: return known diff --git a/tests/unit/src/sqlbuild/runtime/user_code/classes/_test_types.py b/tests/unit/src/sqlbuild/runtime/user_code/classes/_test_types.py index 68d6f86922..f148381de4 100644 --- a/tests/unit/src/sqlbuild/runtime/user_code/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/runtime/user_code/classes/_test_types.py @@ -21,3 +21,12 @@ class FinderWorkingDirectoryTestCase: first_directory: str second_directory: str expected_source_only: tuple[bool, bool] + + +@dataclass(frozen=True) +class FinderDeletedDirectoryTestCase: + """A lookup while the working directory, on the search path as "", has been deleted.""" + + description: str + module_name: str + expected_spec: None = None diff --git a/tests/unit/src/sqlbuild/runtime/user_code/classes/test_user_code_finder.py b/tests/unit/src/sqlbuild/runtime/user_code/classes/test_user_code_finder.py index 0fd7708dc8..7e8f72ecde 100644 --- a/tests/unit/src/sqlbuild/runtime/user_code/classes/test_user_code_finder.py +++ b/tests/unit/src/sqlbuild/runtime/user_code/classes/test_user_code_finder.py @@ -9,6 +9,7 @@ from sqlbuild.runtime.user_code.classes.source_only_loader import SourceOnlyLoader from sqlbuild.runtime.user_code.classes.user_code_finder import UserCodeFinder from tests.unit.src.sqlbuild.runtime.user_code.classes._test_types import ( + FinderDeletedDirectoryTestCase, FinderSpecTestCase, FinderWorkingDirectoryTestCase, ) @@ -98,5 +99,36 @@ def test_given_working_directory_entry_when_directory_changes_then_finder_follow ) == test_case.expected_source_only +@pytest.mark.parametrize( + "test_case", + [ + FinderDeletedDirectoryTestCase( + description="a project module is left to normal import", module_name="order_rules" + ), + FinderDeletedDirectoryTestCase( + description="a standard-library module is left to normal import", module_name="json" + ), + ], + ids=lambda case: case.description, +) +def test_given_deleted_working_directory_when_finding_spec_then_relative_entry_is_skipped( + test_case: FinderDeletedDirectoryTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + project.mkdir() + _ = (project / "order_rules.py").write_text("LIMIT: int = 1\n", encoding="utf-8") + deleted: Path = tmp_path / "deleted" + deleted.mkdir() + monkeypatch.setattr(sys, "path", ["", *sys.path]) + finder: UserCodeFinder = UserCodeFinder() + finder.roots.add(project.resolve()) + monkeypatch.chdir(deleted) + deleted.rmdir() + + spec: ModuleSpec | None = finder.find_spec(test_case.module_name) + + assert spec == test_case.expected_spec + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"]))