diff --git a/Cargo.lock b/Cargo.lock index 2d9d6afc03..8a45422f03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4741,8 +4741,8 @@ dependencies = [ [[package]] name = "sqlclean" -version = "0.9.6" -source = "git+https://github.com/sqlstack-io/sqlclean.git?rev=bbc94e556dc51e936e86fa4c85111c87a7915fc5#bbc94e556dc51e936e86fa4c85111c87a7915fc5" +version = "0.11.1" +source = "git+https://github.com/sqlstack-io/sqlclean.git?rev=b0728bb990b7d8ff196cb00cc20545f2fa4df555#b0728bb990b7d8ff196cb00cc20545f2fa4df555" dependencies = [ "rayon", "rustc-hash", @@ -4759,8 +4759,8 @@ dependencies = [ [[package]] name = "sqlquack-derive" -version = "0.12.1" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=75d4c8486ab262a6c9c02a6c913334856367d890#75d4c8486ab262a6c9c02a6c913334856367d890" +version = "0.22.1" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed#fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" dependencies = [ "proc-macro2", "quote", @@ -4769,8 +4769,8 @@ dependencies = [ [[package]] name = "sqlquack-diagnostics" -version = "0.12.1" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=75d4c8486ab262a6c9c02a6c913334856367d890#75d4c8486ab262a6c9c02a6c913334856367d890" +version = "0.22.1" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed#fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" dependencies = [ "serde", "serde_json", @@ -4779,8 +4779,8 @@ dependencies = [ [[package]] name = "sqlquack-fingerprint" -version = "0.12.1" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=75d4c8486ab262a6c9c02a6c913334856367d890#75d4c8486ab262a6c9c02a6c913334856367d890" +version = "0.22.1" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed#fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" dependencies = [ "sha2 0.10.9", "sqlquack-parse", @@ -4789,8 +4789,8 @@ dependencies = [ [[package]] name = "sqlquack-format" -version = "0.12.1" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=75d4c8486ab262a6c9c02a6c913334856367d890#75d4c8486ab262a6c9c02a6c913334856367d890" +version = "0.22.1" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed#fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" dependencies = [ "serde_json", "sqlquack-parse", @@ -4799,8 +4799,8 @@ dependencies = [ [[package]] name = "sqlquack-lint" -version = "0.12.1" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=75d4c8486ab262a6c9c02a6c913334856367d890#75d4c8486ab262a6c9c02a6c913334856367d890" +version = "0.22.1" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed#fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" dependencies = [ "rayon", "rustc-hash", @@ -4812,8 +4812,8 @@ dependencies = [ [[package]] name = "sqlquack-parse" -version = "0.12.1" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=75d4c8486ab262a6c9c02a6c913334856367d890#75d4c8486ab262a6c9c02a6c913334856367d890" +version = "0.22.1" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed#fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" dependencies = [ "sqlquack-syntax", "sqlquack-syntax-bigquery", @@ -4825,8 +4825,8 @@ dependencies = [ [[package]] name = "sqlquack-semantics" -version = "0.12.1" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=75d4c8486ab262a6c9c02a6c913334856367d890#75d4c8486ab262a6c9c02a6c913334856367d890" +version = "0.22.1" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed#fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" dependencies = [ "regex", "rustc-hash", @@ -4841,8 +4841,8 @@ dependencies = [ [[package]] name = "sqlquack-syntax" -version = "0.12.1" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=75d4c8486ab262a6c9c02a6c913334856367d890#75d4c8486ab262a6c9c02a6c913334856367d890" +version = "0.22.1" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed#fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" dependencies = [ "serde", "serde_json", @@ -4854,8 +4854,8 @@ dependencies = [ [[package]] name = "sqlquack-syntax-bigquery" -version = "0.12.1" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=75d4c8486ab262a6c9c02a6c913334856367d890#75d4c8486ab262a6c9c02a6c913334856367d890" +version = "0.22.1" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed#fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" dependencies = [ "sqlquack-syntax", "stacker", @@ -4863,8 +4863,8 @@ dependencies = [ [[package]] name = "sqlquack-syntax-clickhouse" -version = "0.12.1" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=75d4c8486ab262a6c9c02a6c913334856367d890#75d4c8486ab262a6c9c02a6c913334856367d890" +version = "0.22.1" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed#fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" dependencies = [ "sqlquack-syntax", "stacker", @@ -4872,8 +4872,8 @@ dependencies = [ [[package]] name = "sqlquack-syntax-databricks" -version = "0.12.1" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=75d4c8486ab262a6c9c02a6c913334856367d890#75d4c8486ab262a6c9c02a6c913334856367d890" +version = "0.22.1" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed#fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" dependencies = [ "sqlquack-syntax", "stacker", @@ -4881,8 +4881,8 @@ dependencies = [ [[package]] name = "sqlquack-syntax-duckdb" -version = "0.12.1" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=75d4c8486ab262a6c9c02a6c913334856367d890#75d4c8486ab262a6c9c02a6c913334856367d890" +version = "0.22.1" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed#fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" dependencies = [ "sqlquack-syntax", "stacker", @@ -4890,8 +4890,8 @@ dependencies = [ [[package]] name = "sqlquack-syntax-postgres" -version = "0.12.1" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=75d4c8486ab262a6c9c02a6c913334856367d890#75d4c8486ab262a6c9c02a6c913334856367d890" +version = "0.22.1" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed#fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" dependencies = [ "sqlquack-syntax", "stacker", @@ -4899,8 +4899,8 @@ dependencies = [ [[package]] name = "sqlquack-syntax-snowflake" -version = "0.12.1" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=75d4c8486ab262a6c9c02a6c913334856367d890#75d4c8486ab262a6c9c02a6c913334856367d890" +version = "0.22.1" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed#fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" dependencies = [ "serde", "serde_json", @@ -4912,8 +4912,8 @@ dependencies = [ [[package]] name = "sqlquack-syntax-sqlite" -version = "0.12.1" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=75d4c8486ab262a6c9c02a6c913334856367d890#75d4c8486ab262a6c9c02a6c913334856367d890" +version = "0.22.1" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed#fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" dependencies = [ "sqlquack-syntax", "stacker", @@ -4921,8 +4921,8 @@ dependencies = [ [[package]] name = "sqlquack-syntax-sqlserver" -version = "0.12.1" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=75d4c8486ab262a6c9c02a6c913334856367d890#75d4c8486ab262a6c9c02a6c913334856367d890" +version = "0.22.1" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed#fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" dependencies = [ "sqlquack-syntax", "stacker", diff --git a/Cargo.toml b/Cargo.toml index aac6c8e5f8..78b6c02c7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -203,22 +203,22 @@ sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } sqlbuild-state = { path = "crates/sqlbuild-state" } # SQL lint Rules, quality Rules and the formatter. -sqlclean = { git = "https://github.com/sqlstack-io/sqlclean.git", rev = "bbc94e556dc51e936e86fa4c85111c87a7915fc5" } +sqlclean = { git = "https://github.com/sqlstack-io/sqlclean.git", rev = "b0728bb990b7d8ff196cb00cc20545f2fa4df555" } # SQL change fingerprints over SQLQuack's tokenizers. -sqlquack-fingerprint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "75d4c8486ab262a6c9c02a6c913334856367d890", features = ["bigquery", "databricks", "duckdb", "postgres", "snowflake", "sqlite", "sqlserver"] } +sqlquack-fingerprint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed", features = ["bigquery", "databricks", "duckdb", "postgres", "snowflake", "sqlite", "sqlserver"] } # Scopes, name resolution, types and lineage for the native analysis route. -sqlquack-semantics = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "75d4c8486ab262a6c9c02a6c913334856367d890", features = ["bigquery", "clickhouse", "databricks", "duckdb", "postgres", "snowflake", "sqlite", "sqlserver"] } +sqlquack-semantics = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed", features = ["bigquery", "clickhouse", "databricks", "duckdb", "postgres", "snowflake", "sqlite", "sqlserver"] } # SQLQuack's Snowflake-family parser and tokenizer, reached as SQLQuack exposes them. -sqlquack-parse = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "75d4c8486ab262a6c9c02a6c913334856367d890" } +sqlquack-parse = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" } # SQLQuack's syntax tree and lossless tokens. -sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "75d4c8486ab262a6c9c02a6c913334856367d890" } -sqlquack-syntax-bigquery = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "75d4c8486ab262a6c9c02a6c913334856367d890" } -sqlquack-syntax-clickhouse = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "75d4c8486ab262a6c9c02a6c913334856367d890" } -sqlquack-syntax-databricks = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "75d4c8486ab262a6c9c02a6c913334856367d890" } -sqlquack-syntax-duckdb = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "75d4c8486ab262a6c9c02a6c913334856367d890" } -sqlquack-syntax-postgres = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "75d4c8486ab262a6c9c02a6c913334856367d890" } -sqlquack-syntax-sqlite = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "75d4c8486ab262a6c9c02a6c913334856367d890" } -sqlquack-syntax-sqlserver = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "75d4c8486ab262a6c9c02a6c913334856367d890" } +sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" } +sqlquack-syntax-bigquery = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" } +sqlquack-syntax-clickhouse = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" } +sqlquack-syntax-databricks = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" } +sqlquack-syntax-duckdb = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" } +sqlquack-syntax-postgres = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" } +sqlquack-syntax-sqlite = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" } +sqlquack-syntax-sqlserver = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "fe3d1caf5ed81981abf8abd1d00cc9d99462e6ed" } stacker = "0.1" # Counts heap allocations in growth tests, which measure work instead of time. stats_alloc = "0.1" diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/dialects/bigquery_dialect.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/dialects/bigquery_dialect.rs index 2de8b6192c..58091bd099 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/dialects/bigquery_dialect.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/dialects/bigquery_dialect.rs @@ -139,8 +139,10 @@ impl LineageSchema for BigQueryTypes<'_> { /// A type BigQuery's dry run accepts, as BigQuery names it; `None` for one it refuses. fn native_type(data_type: &DataType) -> Option { Some(match data_type { - DataType::BigInt { length: None } => "INT64".to_owned(), - DataType::Decimal { precision, scale } => { + DataType::BigInt { length: None, .. } => "INT64".to_owned(), + DataType::Decimal { + precision, scale, .. + } => { let given: Vec = [*precision, *scale].into_iter().flatten().collect(); decimal("NUMERIC", &given, NUMERIC_DIGITS)? } @@ -155,14 +157,17 @@ fn native_type(data_type: &DataType) -> Option { DataType::Time { precision: None, timezone: false, + .. } => "TIME".to_owned(), DataType::Timestamp { precision: None, timezone: false, + .. } => "DATETIME".to_owned(), DataType::Timestamp { precision: None, timezone: true, + .. } => "TIMESTAMP".to_owned(), DataType::Interval { unit: None, diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/dialects/databricks_dialect.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/dialects/databricks_dialect.rs index 23a37654d4..d83be763ce 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/dialects/databricks_dialect.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/dialects/databricks_dialect.rs @@ -130,7 +130,9 @@ fn canonical(data_type: &DataType) -> Option { DataType::String { .. } | DataType::Char { .. } | DataType::VarChar { .. } => { DATABRICKS_STRING_TYPE.to_owned() } - DataType::Decimal { precision, scale } => { + DataType::Decimal { + precision, scale, .. + } => { let (default_precision, default_scale) = DATABRICKS_DEFAULT_DECIMAL; format!( "DECIMAL({},{})", diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_bigquery_dialect.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_bigquery_dialect.rs index 595f18436e..7bad745656 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_bigquery_dialect.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_bigquery_dialect.rs @@ -401,7 +401,7 @@ fn given_sql_bigquery_refuses_when_parsing_then_the_gap_says_whether_it_is_a_syn }, ParseFailureTestCase { description: "the nesting limit is not a syntax error", - sql: format!("SELECT {}1{}", "(".repeat(300), ")".repeat(300)), + sql: format!("SELECT {}1{}", "(".repeat(600), ")".repeat(600)), expected_syntax: false, expected_offset: None, }, diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_databricks_dialect.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_databricks_dialect.rs index 3126ff3d5e..5296d4e3de 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_databricks_dialect.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_databricks_dialect.rs @@ -45,11 +45,11 @@ fn given_databricks_model_sql_when_probing_its_parser_then_only_sql_it_reads_who }, NativeGapTestCase { description: "a construct the shared tree lacks is a gap named by the parser", - sql: "VACUUM t", + sql: "REPAIR PRIVILEGES t", placeholders: &[], expected_gap: Some(LocatedGap { - construct: "vacuum".to_owned(), - message: "vacuum".to_owned(), + construct: "repair".to_owned(), + message: "repair".to_owned(), line: 1, column: 1, syntax: false, @@ -275,7 +275,7 @@ fn given_sql_databricks_refuses_when_parsing_then_the_gap_says_whether_it_is_a_s }, ParseFailureTestCase { description: "the nesting limit is not a syntax error", - sql: format!("SELECT {}1{}", "(".repeat(300), ")".repeat(300)), + sql: format!("SELECT {}1{}", "(".repeat(600), ")".repeat(600)), expected_syntax: false, expected_offset: None, }, diff --git a/crates/sqlbuild-analysis/src/parse_guard/constants.rs b/crates/sqlbuild-analysis/src/parse_guard/constants.rs index 1d65848990..ab0e6e617a 100644 --- a/crates/sqlbuild-analysis/src/parse_guard/constants.rs +++ b/crates/sqlbuild-analysis/src/parse_guard/constants.rs @@ -1,4 +1,4 @@ //! Constants of parse_guard. -/// The deepest function-call nesting a Polyglot or SQLQuack shared or Snowflake parse accepts. +/// The deepest function-call nesting that a guarded parse or a SQLClean lint accepts. pub const MAX_FUNCTION_CALL_DEPTH: usize = 256; diff --git a/crates/sqlbuild-analysis/src/parse_guard/mod.rs b/crates/sqlbuild-analysis/src/parse_guard/mod.rs index c636c77864..9a3d52ceff 100644 --- a/crates/sqlbuild-analysis/src/parse_guard/mod.rs +++ b/crates/sqlbuild-analysis/src/parse_guard/mod.rs @@ -1,4 +1,4 @@ -//! One call depth for Polyglot and SQLQuack's shared and Snowflake parsers, not per-dialect ones. +//! One call depth for Polyglot, SQLQuack's shared and Snowflake parsers and SQLClean's lint. pub mod constants; pub mod main; diff --git a/crates/sqlbuild-analysis/src/source_drift/_helpers/walk.rs b/crates/sqlbuild-analysis/src/source_drift/_helpers/walk.rs index 04078968dd..fe45723dbc 100644 --- a/crates/sqlbuild-analysis/src/source_drift/_helpers/walk.rs +++ b/crates/sqlbuild-analysis/src/source_drift/_helpers/walk.rs @@ -3,7 +3,7 @@ use std::collections::HashSet; use sqlbuild_core::text::main::python_casefold::python_casefold; -use sqlquack_syntax::expressions::queries::models::{JoinKind, Select, With}; +use sqlquack_syntax::expressions::queries::models::{Join, JoinKind, Select, With}; use sqlquack_syntax::{Expression, ExpressionWalk}; use crate::source_drift::constants::{COUNT_FUNCTION, OPAQUE_KINDS, SHARED_SCOPE_RELATIONS}; @@ -141,17 +141,14 @@ impl<'a> Walk<'a> { .map(|cte| std::ptr::from_ref(&cte.this)) .collect(); let mut local: Vec<(Option, bool)> = Vec::new(); - let relations = select - .from - .iter() - .flat_map(|from| from.expressions.iter()) - .chain(select.joins.iter().map(|join| &join.this)); + let (items, joins): (Vec<&Expression>, Vec<&Join>) = from_items_and_joins(select); + let relations = items.into_iter().chain(joins.iter().map(|join| &join.this)); for relation in relations { walked.insert(std::ptr::from_ref(relation)); local.extend(self.relation(relation, &ctes, outer)); } let drifted_here: bool = local.iter().any(|(_, exposes)| *exposes); - for join in &select.joins { + for join in joins { let natural: bool = matches!( join.kind, JoinKind::Natural @@ -338,3 +335,19 @@ fn nested_queries(expression: &Expression) -> Vec<&Expression> { } found } + +/// The select's FROM items and joins, with each unaliased joined FROM item read as its left side and joins. +fn from_items_and_joins(select: &Select) -> (Vec<&Expression>, Vec<&Join>) { + let mut items: Vec<&Expression> = Vec::new(); + let mut joins: Vec<&Join> = select.joins.iter().collect(); + for item in select.from.iter().flat_map(|from| from.expressions.iter()) { + match item { + Expression::JoinedTable(joined) if joined.alias.is_none() => { + items.push(&joined.left); + joins.extend(joined.joins.iter()); + } + _ => items.push(item), + } + } + (items, joins) +} diff --git a/crates/sqlbuild-analysis/src/source_drift/tests/test_reads_added_columns.rs b/crates/sqlbuild-analysis/src/source_drift/tests/test_reads_added_columns.rs index df91ab1e75..42035be434 100644 --- a/crates/sqlbuild-analysis/src/source_drift/tests/test_reads_added_columns.rs +++ b/crates/sqlbuild-analysis/src/source_drift/tests/test_reads_added_columns.rs @@ -1,5 +1,5 @@ use crate::source_drift::main::reads_added_columns::reads_added_columns; -use crate::source_drift::tests::test_types::AddedColumnsTestCase; +use crate::source_drift::tests::test_types::{AddedColumnsTestCase, DialectAddedColumnsTestCase}; #[test] fn given_a_source_gaining_columns_when_reading_models_then_only_column_list_readers_are_affected() { @@ -102,3 +102,86 @@ fn given_a_source_gaining_columns_when_reading_models_then_only_column_list_read ); } } + +#[test] +fn given_comma_items_with_joins_when_a_joined_source_gains_columns_then_their_keys_and_names_count() +{ + let test_cases = [ + DialectAddedColumnsTestCase { + description: "postgres: a later comma item joining the source USING a column it gains", + dialect: "postgres", + sql: r#"SELECT c.customer_id FROM __ref("customers") AS c, + __ref("products") AS p JOIN __source("orders") AS o USING (region)"#, + expected_affected: true, + }, + DialectAddedColumnsTestCase { + description: "postgres: an unqualified name in a later comma item's join ON", + dialect: "postgres", + sql: r#"SELECT c.customer_id FROM __ref("customers") AS c, + __ref("products") AS p JOIN __source("orders") AS o ON p.product_id = product_ref"#, + expected_affected: true, + }, + DialectAddedColumnsTestCase { + description: "postgres: qualified names across comma items and their joins", + dialect: "postgres", + sql: r#"SELECT c.customer_id FROM __ref("customers") AS c, + __ref("products") AS p JOIN __source("orders") AS o ON p.product_id = o.product_id"#, + expected_affected: false, + }, + DialectAddedColumnsTestCase { + description: "duckdb: a later comma item joining the source USING a column it gains", + dialect: "duckdb", + sql: r#"SELECT c.customer_id FROM __ref("customers") AS c, + __ref("products") AS p JOIN __source("orders") AS o USING (region)"#, + expected_affected: true, + }, + DialectAddedColumnsTestCase { + description: "duckdb: an unqualified name in a later comma item's join ON", + dialect: "duckdb", + sql: r#"SELECT c.customer_id FROM __ref("customers") AS c, + __ref("products") AS p JOIN __source("orders") AS o ON p.product_id = product_ref"#, + expected_affected: true, + }, + DialectAddedColumnsTestCase { + description: "duckdb: qualified names across comma items and their joins", + dialect: "duckdb", + sql: r#"SELECT c.customer_id FROM __ref("customers") AS c, + __ref("products") AS p JOIN __source("orders") AS o ON p.product_id = o.product_id"#, + expected_affected: false, + }, + DialectAddedColumnsTestCase { + description: "databricks: a later comma item joining the source USING a column it gains", + dialect: "databricks", + sql: r#"SELECT c.customer_id FROM __ref("customers") AS c, + __ref("products") AS p JOIN __source("orders") AS o USING (region)"#, + expected_affected: true, + }, + DialectAddedColumnsTestCase { + description: "databricks: an unqualified name in a later comma item's join ON", + dialect: "databricks", + sql: r#"SELECT c.customer_id FROM __ref("customers") AS c, + __ref("products") AS p JOIN __source("orders") AS o ON p.product_id = product_ref"#, + expected_affected: true, + }, + DialectAddedColumnsTestCase { + description: "databricks: qualified names across comma items and their joins", + dialect: "databricks", + sql: r#"SELECT c.customer_id FROM __ref("customers") AS c, + __ref("products") AS p JOIN __source("orders") AS o ON p.product_id = o.product_id"#, + expected_affected: false, + }, + ]; + for test_case in test_cases { + assert_eq!( + reads_added_columns( + test_case.dialect, + (test_case.sql, &[]), + &["orders".to_owned()], + &["region".to_owned()], + ), + test_case.expected_affected, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/source_drift/tests/test_types.rs b/crates/sqlbuild-analysis/src/source_drift/tests/test_types.rs index daf66cf1a3..8e6ee2b9a8 100644 --- a/crates/sqlbuild-analysis/src/source_drift/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/source_drift/tests/test_types.rs @@ -4,3 +4,10 @@ pub(super) struct AddedColumnsTestCase { pub(super) sql: &'static str, pub(super) expected_affected: bool, } + +pub(super) struct DialectAddedColumnsTestCase { + pub(super) description: &'static str, + pub(super) dialect: &'static str, + pub(super) sql: &'static str, + pub(super) expected_affected: bool, +} diff --git a/crates/sqlbuild-analysis/src/sql_tree/_helpers/node_facts.rs b/crates/sqlbuild-analysis/src/sql_tree/_helpers/node_facts.rs index 43d629debb..4e768cc433 100644 --- a/crates/sqlbuild-analysis/src/sql_tree/_helpers/node_facts.rs +++ b/crates/sqlbuild-analysis/src/sql_tree/_helpers/node_facts.rs @@ -81,12 +81,14 @@ fn marker_call<'source>(identifier: &Identifier, map: &SourceMap<'source>) -> Op fn literal_type(literal: &Literal) -> &'static str { match literal { - Literal::String(_) => "string", + Literal::String(_) | Literal::JoinedString(_, _) => "string", Literal::Number(_) => "number", Literal::HexString(_) => "hex_string", Literal::HexNumber(_) => "hex_number", Literal::BitString(_) => "bit_string", - Literal::ByteString(_) => "byte_string", + Literal::ByteString(_) | Literal::RawByteString(_) | Literal::JoinedBytes(_, _) => { + "byte_string" + } Literal::NationalString(_) => "national_string", Literal::Date(_) => "date", Literal::Time(_) => "time", diff --git a/crates/sqlbuild-analysis/src/warehouse_types/_helpers/type_trees.rs b/crates/sqlbuild-analysis/src/warehouse_types/_helpers/type_trees.rs index fe8e746a20..e489646a9f 100644 --- a/crates/sqlbuild-analysis/src/warehouse_types/_helpers/type_trees.rs +++ b/crates/sqlbuild-analysis/src/warehouse_types/_helpers/type_trees.rs @@ -3,10 +3,11 @@ use serde_json::Value; use sqlquack_parse::parse_strict; use sqlquack_syntax::Expression; +use sqlquack_syntax::expressions::types::models::DataType; use sqlquack_syntax::tokens::models::Dialect; use crate::parse_guard::main::sqlquack_parse_options::sqlquack_parse_options; -use crate::warehouse_types::constants::{CAST_KEY, CAST_TARGET_KEY}; +use crate::warehouse_types::constants::{CAST_KEY, CAST_TARGET_KEY, DATA_TYPE_KEY, NAMED_TYPE_TAG}; /// `spelling` read by `sqlquack-syntax-postgres`'s type grammar. pub(crate) fn postgres_type(spelling: &str) -> Result { @@ -40,11 +41,41 @@ fn cast(spelling: &str) -> String { fn single_cast_target(spelling: &str, statements: &[Expression]) -> Result { let tree: Value = serde_json::to_value(statements).map_err(|error| error.to_string())?; match (statements.len(), cast_targets(&tree).as_slice()) { - (1, [target]) => Ok((*target).clone()), + (1, [target]) => unnamed((*target).clone()), _ => Err(format!("`{spelling}` is not a single type name")), } } +/// `tree` with every type kept as written, such as Snowflake's `NUMBER(10, 2)`, read as the type it denotes. +fn unnamed(tree: Value) -> Result { + match tree { + Value::Object(fields) + if fields.get(DATA_TYPE_KEY).and_then(Value::as_str) == Some(NAMED_TYPE_TAG) => + { + let written: Value = Value::Object(fields); + let data_type: DataType = + serde_json::from_value(written.clone()).map_err(|error| error.to_string())?; + match data_type.unnamed() { + Some(denoted) => serde_json::to_value(denoted).map_err(|error| error.to_string()), + None => Ok(written), + } + } + Value::Object(fields) => Ok(Value::Object( + fields + .into_iter() + .map(|(key, value)| Ok((key, unnamed(value)?))) + .collect::>()?, + )), + Value::Array(items) => Ok(Value::Array( + items + .into_iter() + .map(unnamed) + .collect::>()?, + )), + other => Ok(other), + } +} + /// Every cast's target type in `tree`, in order. fn cast_targets(tree: &Value) -> Vec<&Value> { let mut targets: Vec<&Value> = Vec::new(); diff --git a/crates/sqlbuild-analysis/src/warehouse_types/constants.rs b/crates/sqlbuild-analysis/src/warehouse_types/constants.rs index 3a91139238..d64f706fbd 100644 --- a/crates/sqlbuild-analysis/src/warehouse_types/constants.rs +++ b/crates/sqlbuild-analysis/src/warehouse_types/constants.rs @@ -10,3 +10,7 @@ pub(crate) const POSTGRES_DIALECT: &str = "postgres"; pub(crate) const SNOWFLAKE_DIALECT: &str = "snowflake"; /// The dialect whose type names are parsed as a BigQuery cast's target. pub(crate) const BIGQUERY_DIALECT: &str = "bigquery"; +/// The tree key naming a type's kind. +pub(crate) const DATA_TYPE_KEY: &str = "data_type"; +/// The kind of a type SQLQuack keeps as written with its parameters in fields, such as Snowflake's `NUMBER(10, 2)`. +pub(crate) const NAMED_TYPE_TAG: &str = "named"; diff --git a/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs b/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs index 7c7e403dd8..63a8f21ee4 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_query_facts.rs @@ -21,6 +21,60 @@ fn given_dialect_sql_when_reading_query_facts_then_findings_match_the_authored_s -> Result<(), String> { install_test_identity(); let test_cases = [ + QueryFactsTestCase { + description: "a raw table joined to the first of several DuckDB FROM items is found", + dialect: "duckdb", + query_sql: "SELECT o.order_id\nFROM __ref(\"orders\") o JOIN analytics.customers c ON o.customer_id = c.customer_id,\n __ref(\"products\") p", + select: &["SQBRGRAPH102"], + enum_columns: &[], + columns: json!([]), + expected_faults: &[("SQBRGRAPH102", 2, 29, "raw qualified table")], + }, + QueryFactsTestCase { + description: "a raw table joined to a later DuckDB FROM item is found", + dialect: "duckdb", + query_sql: "SELECT o.order_id\nFROM __ref(\"orders\") o,\n __ref(\"products\") p JOIN analytics.suppliers s ON p.supplier_id = s.supplier_id", + select: &["SQBRGRAPH102"], + enum_columns: &[], + columns: json!([]), + expected_faults: &[("SQBRGRAPH102", 3, 28, "raw qualified table")], + }, + QueryFactsTestCase { + description: "a raw table joined to the first of several PostgreSQL FROM items is found", + dialect: "postgres", + query_sql: "SELECT o.order_id\nFROM __ref(\"orders\") o JOIN analytics.customers c ON o.customer_id = c.customer_id,\n __ref(\"products\") p", + select: &["SQBRGRAPH102"], + enum_columns: &[], + columns: json!([]), + expected_faults: &[("SQBRGRAPH102", 2, 29, "raw qualified table")], + }, + QueryFactsTestCase { + description: "a raw table joined to a later PostgreSQL FROM item is found", + dialect: "postgres", + query_sql: "SELECT o.order_id\nFROM __ref(\"orders\") o,\n __ref(\"products\") p JOIN analytics.suppliers s ON p.supplier_id = s.supplier_id", + select: &["SQBRGRAPH102"], + enum_columns: &[], + columns: json!([]), + expected_faults: &[("SQBRGRAPH102", 3, 28, "raw qualified table")], + }, + QueryFactsTestCase { + description: "a raw table joined to the first of several Snowflake FROM items is found", + dialect: "snowflake", + query_sql: "SELECT o.order_id\nFROM __ref(\"orders\") o JOIN analytics.customers c ON o.customer_id = c.customer_id,\n __ref(\"products\") p", + select: &["SQBRGRAPH102"], + enum_columns: &[], + columns: json!([]), + expected_faults: &[("SQBRGRAPH102", 2, 29, "raw qualified table")], + }, + QueryFactsTestCase { + description: "a raw table joined to a later Snowflake FROM item is found", + dialect: "snowflake", + query_sql: "SELECT o.order_id\nFROM __ref(\"orders\") o,\n __ref(\"products\") p JOIN analytics.suppliers s ON p.supplier_id = s.supplier_id", + select: &["SQBRGRAPH102"], + enum_columns: &[], + columns: json!([]), + expected_faults: &[("SQBRGRAPH102", 3, 28, "raw qualified table")], + }, QueryFactsTestCase { description: "a Snowflake // comment holds no dependency call", dialect: "snowflake", @@ -248,6 +302,15 @@ fn given_dialect_sql_when_reading_query_facts_then_findings_match_the_authored_s columns: json!([]), expected_faults: &[("L007", 1, 1, "cannot parse bigquery SQL for built-in rules")], }, + QueryFactsTestCase { + description: "Postgres adjacent string literals compared with an enum column read as one literal", + dialect: "postgres", + query_sql: "SELECT order_id FROM orders WHERE status = 'act'\n 'ive'", + select: ENUM_RULE, + enum_columns: STATUS, + columns: json!([]), + expected_faults: &[("SQBRDECLARATION101", 1, 35, "")], + }, QueryFactsTestCase { description: "a Postgres escape string compared with an enum column", dialect: "postgres", diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs index f3237e5385..2695b6b393 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs @@ -1,5 +1,6 @@ //! Native SQL lint request payloads and the batch engine call. +use sqlbuild_analysis::parse_guard::constants::MAX_FUNCTION_CALL_DEPTH; use std::borrow::Cow; use serde_json::{Map, Value}; @@ -153,6 +154,10 @@ pub(crate) fn lint_payload( let _ = payload.insert("version".to_string(), Value::from(LINT_API_VERSION)); let _ = payload.insert("sql".to_string(), Value::from(sql)); let _ = payload.insert("dialect".to_string(), Value::from(options.dialect.clone())); + let _ = payload.insert( + "max_function_call_depth".to_string(), + Value::from(MAX_FUNCTION_CALL_DEPTH), + ); let _ = payload.insert( "config_file".to_string(), Value::from(PROJECT_CONFIG_FILENAME), diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs index 8f347b19f3..501e0d94f9 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs @@ -37,8 +37,6 @@ const ESCAPED_QUOTE_MACRO: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR const ESCAPED_QUOTE_FORMATTED: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCONCAT(@v, 'O\\'Brien\n kept indent', 'tail')\n"; const E_STRING_MACRO: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCONCAT(\n @v,\n E'O\\'Brien\n kept indent',\n 'tail'\n)\n"; const E_STRING_FORMATTED: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCONCAT(@v, E'O\\'Brien\n kept indent', 'tail')\n"; -const NESTED_COMMENT_MACRO: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCOALESCE(\n @v, /* outer /* inner */ it's\n kept */\n 'x'\n)\n"; -const NESTED_COMMENT_FORMATTED: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCOALESCE(\n @v, /* outer /* inner */ it's\n kept */ 'x'\n)\n"; const BLOCK_COMMENT_MACRO: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCOALESCE(\n @v, /* first\n second */\n 'x'\n)\n"; const BLOCK_COMMENT_FORMATTED: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCOALESCE(\n @v, /* first\n second */ 'x'\n)\n"; const NORMALISE_NAME_FORMATTED: &str = "MACRO (\n description \"Normalise a name\",\n arguments (column_name VARCHAR),\n returns VARCHAR,\n);\n\nREGEXP_REPLACE(\n TRIM(\n UPPER(\n -- Collapse repeated spaces\n REGEXP_REPLACE(@column_name, '[[:space:]]+', ' ')\n )\n ),\n '(^| )([A-Z]) ',\n '\\\\1\\\\2'\n)\n"; @@ -1037,12 +1035,6 @@ fn given_expression_macro_bodies_when_formatting_twice_then_every_dialect_lays_t contents: E_STRING_MACRO.to_string(), expected_formatted: E_STRING_FORMATTED.to_string(), }, - ExpressionMacroFormatTestCase { - description: "a nested block comment holding an apostrophe", - dialects: &["duckdb", "databricks"], - contents: NESTED_COMMENT_MACRO.to_string(), - expected_formatted: NESTED_COMMENT_FORMATTED.to_string(), - }, ExpressionMacroFormatTestCase { description: "a block comment with indented lines", dialects: &["duckdb", "snowflake"], diff --git a/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs b/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs index ca5e1ce889..7c97427f49 100644 --- a/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs +++ b/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs @@ -801,7 +801,9 @@ fn direct_column(expression: &Expression) -> Option { fn string_literal(literal: &Literal) -> Option { match literal { - Literal::String(value) => Some(format!("'{}'", value.replace('\'', "''"))), + Literal::String(value) | Literal::JoinedString(value, _) => { + Some(format!("'{}'", value.replace('\'', "''"))) + } Literal::NationalString(value) => Some(format!("N'{}'", value.replace('\'', "''"))), Literal::TripleQuotedString(value, quote) => { let delimiter = quote.to_string().repeat(TRIPLE_QUOTE_LENGTH); diff --git a/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs b/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs index 7828a762ec..bffbe5157d 100644 --- a/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs +++ b/crates/sqlbuild-rules/src/query_facts/_helpers/flatten.rs @@ -91,7 +91,7 @@ fn literal_key(expression: &Expression) -> Option { let Expression::Literal(literal) = expression else { return None; }; - let Literal::String(path) = literal.as_ref() else { + let (Literal::String(path) | Literal::JoinedString(path, _)) = literal.as_ref() else { return None; }; let key: &str = path diff --git a/scripts/parse_guard/_helpers/scan.py b/scripts/parse_guard/_helpers/scan.py index 4693d761cf..e0dcb10ee5 100644 --- a/scripts/parse_guard/_helpers/scan.py +++ b/scripts/parse_guard/_helpers/scan.py @@ -10,9 +10,11 @@ DEFAULT_GUARD_PATTERNS, GUARD_FIELD, GUARDED_OPTION_LITERALS, + SHARED_DEPTH, SKIPPED_DIRECTORIES, SKIPPED_FILES, SOURCE_SUFFIX, + SQLCLEAN_REQUESTS, TYPE_POSITION_SUFFIXES, ) @@ -48,6 +50,13 @@ def unguarded_sites(*, label: str, text: str) -> tuple[str, ...]: for found in re.finditer(re.escape(literal), text) if _unguarded_literal(text=text, start=found.start(), literal=literal) ) + request: str + for request in SQLCLEAN_REQUESTS: + sites.extend( + f"{label}:{_line(text=text, offset=found.start())} {request}" + for found in re.finditer(request, text) + if SHARED_DEPTH not in text + ) return tuple(sites) diff --git a/scripts/parse_guard/constants.py b/scripts/parse_guard/constants.py index d2408518d6..04984db031 100644 --- a/scripts/parse_guard/constants.py +++ b/scripts/parse_guard/constants.py @@ -21,4 +21,9 @@ "ParseOptions {", ) GUARD_FIELD: str = "complexity_guard" +SQLCLEAN_REQUESTS: tuple[str, ...] = ( + r"lint_batch_json\(", + r"sqlclean::sql_lint::models::[^;]*\bFormatRequest\b", +) +SHARED_DEPTH: str = "MAX_FUNCTION_CALL_DEPTH" TYPE_POSITION_SUFFIXES: tuple[str, ...] = ("->", "struct") diff --git a/scripts/parse_guard/main/check.py b/scripts/parse_guard/main/check.py index b66ae63f97..3ea1b44026 100644 --- a/scripts/parse_guard/main/check.py +++ b/scripts/parse_guard/main/check.py @@ -14,6 +14,6 @@ def run_parse_guard_check(argv: list[str] | None = None) -> int: errors_under=get_parse_guard_errors, messages=( "Parse with sqlbuild_analysis::parse_guard's options, not a library default:", - "Polyglot and SQLQuack's shared and Snowflake parsers all use the shared guard.", + "Polyglot, SQLQuack's shared and Snowflake parsers and SQLClean use the shared guard.", ), ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/corpus.json b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/corpus.json index 701ac3f3c6..501e6324a2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/corpus.json +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/corpus.json @@ -1009,6 +1009,18 @@ "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT a.id FROM (SELECT id, status FROM __source(\"raw__orders\")) a JOIN (SELECT id FROM __source(\"raw__customers\")) b USING (status)\n" } }, + { + "description": "J05: ON references later table", + "category": "join", + "expected_exit_code": 1, + "expected_codes": [ + "B004" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON c.id = p.order_id JOIN __source(\"raw__payments\") p ON p.order_id = o.id\n" + } + }, { "description": "S04: UNION BY NAME unknown column in outer", "category": "set operation", @@ -2344,16 +2356,6 @@ "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") JOIN __source(\"raw__orders\") ON id = id\n" } }, - { - "description": "J05: ON references later table", - "category": "join", - "expected_exit_code": 1, - "expected_codes": [], - "pending_native": true, - "repo_files": { - "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON c.id = p.order_id JOIN __source(\"raw__payments\") p ON p.order_id = o.id\n" - } - }, { "description": "E01: duplicate CTE names", "category": "cte", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index f230c7979c..e32769cf54 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -54,6 +54,8 @@ class SharedDepthModelRuleCase: rule: str expected_evaluated_models: int expected_unevaluated_resources: int + path: str = "models/staging/orders.sql" + extra_files: tuple[tuple[str, str], ...] = () @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py index 7dc6388739..e81e88bcd6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py @@ -11,7 +11,7 @@ from typing import Any from scripts.rules_benchmark._helpers.custom_rules import write_custom_rules -from sqlbuild._native import rules_bulk_cache_path +from sqlbuild._native import MAX_FUNCTION_CALL_DEPTH, rules_bulk_cache_path from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import replace_project_text __all__ = ("write_custom_rules",) @@ -29,6 +29,7 @@ def write_unevaluated_rules_project( "MODEL (description 'Test model orders.', {options});\nSELECT {expression} AS order_id" ), extra_files: tuple[tuple[str, str], ...] = (), + call_depth: int = MAX_FUNCTION_CALL_DEPTH + 1, ) -> None: (project_dir / "sqlbuild_project.toml").write_text( f'name = "orders"\nadapter = "{adapter}"\n[rules]\nselect = {json.dumps(selected_rules)}\n' @@ -48,7 +49,9 @@ def write_unevaluated_rules_project( resource: Path = project_dir / resource_path resource.parent.mkdir(parents=True, exist_ok=True) resource.write_text( - resource_template.format(options=model_options, expression="ABS(" * 130 + "1" + ")" * 130) + resource_template.format( + options=model_options, expression="ABS(" * call_depth + "1" + ")" * call_depth + ) ) @@ -728,3 +731,12 @@ def duckdb_rules_project_toml(*, project_dir: Path, rules_toml: str) -> str: '[targets.dev]\nschema = "dev"\n\n' f"[rules]\n{rules_toml}" ) + + +def rules_unevaluated(items: list[dict[str, Any]]) -> list[dict[str, Any]]: + """The items that report a resource the selected Rules could not evaluate.""" + return list(filter(_reports_rules_unevaluated, items)) + + +def _reports_rules_unevaluated(item: dict[str, Any]) -> bool: + return item["code"] == "rules-unevaluated" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py index 0ac0eb5ef6..46115569c1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py @@ -7,6 +7,7 @@ import pytest +from sqlbuild._native import MAX_FUNCTION_CALL_DEPTH from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import ( EvaluatedModelRulesCase, EvaluatedPredicateCase, @@ -17,6 +18,7 @@ UnevaluatedRuleCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.rules.helpers import ( + rules_unevaluated, run_unevaluated_rules_cli, write_unevaluated_rules_project, ) @@ -36,24 +38,6 @@ "MODEL (description 'Test model orders.');\nSELECT {expression} AS order_id", 0, ), - UnevaluatedResourceCase( - "audit guard", - "models/staging/_sqlbuild/audits/singular/orders.sql", - "AUDIT ();\nSELECT {expression} AS order_id " - 'FROM __ref("orders") JOIN __ref("customers") USING (order_id)', - expected_evaluated_models=2, - extra_files=( - ( - "models/staging/customers.sql", - "MODEL (description 'Test model customers.');\nSELECT 1 AS order_id\n", - ), - ), - ), - UnevaluatedResourceCase( - "SQL-test guard", - "tests/unit/test_orders.sql", - "TEST ();\nWITH __ref__orders AS (SELECT {expression} AS order_id), __expected__orders AS (SELECT 1 AS order_id) SELECT 1", - ), ], ids=lambda case: case.description, ) @@ -112,6 +96,30 @@ def test_given_guarded_resource_when_running_rules_then_reports_failure_and_cove expected_evaluated_models=1, expected_unevaluated_resources=0, ), + SharedDepthModelRuleCase( + "a SQLClean rule lints an audit nested as deep as the compiler accepts", + "AUDIT ();\nSELECT {expression} AS order_id " + 'FROM __ref("orders") JOIN __ref("customers") USING (order_id)', + "SQLCLEAN035", + expected_evaluated_models=2, + expected_unevaluated_resources=0, + path="models/staging/_sqlbuild/audits/singular/orders.sql", + extra_files=( + ( + "models/staging/customers.sql", + "MODEL (description 'Test model customers.');\nSELECT 1 AS order_id\n", + ), + ), + ), + SharedDepthModelRuleCase( + "a SQLClean rule lints a SQL test nested as deep as the compiler accepts", + "TEST ();\nWITH __ref__orders AS (SELECT {expression} AS order_id), " + "__expected__orders AS (SELECT 1 AS order_id) SELECT 1", + "SQLCLEAN035", + expected_evaluated_models=1, + expected_unevaluated_resources=0, + path="tests/unit/test_orders.sql", + ), ], ids=lambda case: case.description, ) @@ -121,9 +129,11 @@ def test_given_calls_nested_within_the_shared_depth_when_running_model_rules_the ) -> None: write_unevaluated_rules_project( project_dir=tmp_path, - resource_path="models/staging/orders.sql", + resource_path=test_case.path, resource_template=test_case.template, + extra_files=test_case.extra_files, selected_rules=(test_case.rule,), + call_depth=MAX_FUNCTION_CALL_DEPTH, ) result: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( @@ -272,7 +282,7 @@ def test_given_explicit_escape_hatch_when_running_rules_then_honors_policy( tmp_path, "compile", "--no-cache", "--json" ) assert compiled.returncode == test_case.expected_exit, compiled.stdout + compiled.stderr - assert json.loads(compiled.stdout)["diagnostics"] == [] + assert rules_unevaluated(json.loads(compiled.stdout)["diagnostics"]) == [] @pytest.mark.parametrize( @@ -315,7 +325,7 @@ def test_given_one_failed_resource_and_multiple_rules_when_running_cli_then_grou ) assert result.returncode == 1, result.stdout + result.stderr payload: dict[str, Any] = json.loads(result.stdout) - (finding,) = payload[test_case.result_key] + (finding,) = rules_unevaluated(payload[test_case.result_key]) assert finding["code"] == "rules-unevaluated" assert finding["affected_rules"] == list(test_case.expected_rules) assert finding["message"].startswith( diff --git a/tests/unit/scripts/parse_guard/_helpers/test_scan.py b/tests/unit/scripts/parse_guard/_helpers/test_scan.py index 82952a40f4..2a575e361a 100644 --- a/tests/unit/scripts/parse_guard/_helpers/test_scan.py +++ b/tests/unit/scripts/parse_guard/_helpers/test_scan.py @@ -61,6 +61,18 @@ ), expected_sites=(), ), + UnguardedSiteTestCase( + description="a SQLClean lint request without the shared depth", + text="fn n() { lint_batch_json(&payload.to_string()) }", + expected_sites=(r"x.rs:1 lint_batch_json\(",), + ), + UnguardedSiteTestCase( + description="a SQLClean lint request that sends the shared depth", + text=( + "fn o() { insert(MAX_FUNCTION_CALL_DEPTH); lint_batch_json(&payload.to_string()) }" + ), + expected_sites=(), + ), UnguardedSiteTestCase( description="the shared options and constant", text=(