Skip to content

Commit d65aa29

Browse files
committed
ci: validate the datafusion-table-providers pin against spiceai-55-patches
The DataFusion 55 line of the fork is spiceai-55-patches.
1 parent 98800a8 commit d65aa29

6 files changed

Lines changed: 569 additions & 44 deletions

File tree

‎.github/workflows/pr-develop.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ jobs:
9898
- name: Validate datafusion-table-providers commit
9999
run: bash .github/scripts/validate_table_providers_commit.sh
100100
env:
101-
BRANCH: spiceai-54
101+
BRANCH: spiceai-55-patches
102102

103103
- name: Show sccache stats
104104
if: always() && (runner.os == 'macOS' && steps.setup-spiceio.outputs.endpoint != '')

‎.github/workflows/pr.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -646,7 +646,7 @@ jobs:
646646
- name: Validate datafusion-table-providers commit
647647
run: bash .github/scripts/validate_table_providers_commit.sh
648648
env:
649-
BRANCH: spiceai-54
649+
BRANCH: spiceai-55-patches
650650

651651
# Run Rust linting
652652
lint-rust:

‎crates/data_components/src/federation.rs‎

Lines changed: 243 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2780,4 +2780,247 @@ mod tests {
27802780
.expect("walking a logical plan cannot fail");
27812781
negated
27822782
}
2783+
2784+
/// `products AS p LEFT JOIN categories AS c`, where both relations expose a
2785+
/// column called `name`, so a projection passing both through has two outputs no
2786+
/// bare name tells apart.
2787+
fn products_left_join_categories() -> LogicalPlanBuilder {
2788+
let products = LogicalPlanBuilder::scan(
2789+
"products",
2790+
table_source(vec![
2791+
Field::new("category_id", DataType::Int32, false),
2792+
Field::new("name", DataType::Utf8, false),
2793+
Field::new("price", DataType::Int32, false),
2794+
]),
2795+
None,
2796+
)
2797+
.expect("scan products")
2798+
.alias("p")
2799+
.expect("alias products")
2800+
.build()
2801+
.expect("build products");
2802+
let categories = LogicalPlanBuilder::scan(
2803+
"categories",
2804+
table_source(vec![
2805+
Field::new("category_id", DataType::Int32, false),
2806+
Field::new("name", DataType::Utf8, false),
2807+
]),
2808+
None,
2809+
)
2810+
.expect("scan categories")
2811+
.alias("c")
2812+
.expect("alias categories")
2813+
.build()
2814+
.expect("build categories");
2815+
LogicalPlanBuilder::from(products)
2816+
.join(
2817+
categories,
2818+
JoinType::Left,
2819+
(vec!["p.category_id"], vec!["c.category_id"]),
2820+
None,
2821+
)
2822+
.expect("join products to categories")
2823+
}
2824+
2825+
/// Whether the outermost `SELECT` list of `sql` qualifies a column by
2826+
/// `relation` while the outermost `FROM` does not bring `relation` into scope.
2827+
///
2828+
/// Matched on nesting depth rather than on a rendered string, so it holds across
2829+
/// the dialects' quoting: `p.`, `"p".` and `` `p`. `` are all the same
2830+
/// reference, and `AS p`, `AS "p"` and `` AS `p` `` the same declaration.
2831+
fn outer_select_reads_an_out_of_scope_relation(sql: &str, relation: &str) -> bool {
2832+
let Some(from_at) = sql
2833+
.match_indices(" FROM ")
2834+
.map(|(at, _)| at)
2835+
.find(|at| paren_depth_at(sql, *at) == 0)
2836+
else {
2837+
panic!("expected a top-level FROM in: {sql}");
2838+
};
2839+
let spellings = [
2840+
relation.to_string(),
2841+
format!("\"{relation}\""),
2842+
format!("`{relation}`"),
2843+
];
2844+
let select_list = &sql[..from_at];
2845+
let referenced = spellings.iter().any(|spelling| {
2846+
select_list
2847+
.match_indices(&format!("{spelling}."))
2848+
.any(|(at, _)| {
2849+
select_list[..at]
2850+
.chars()
2851+
.next_back()
2852+
.is_none_or(|before| !before.is_alphanumeric() && before != '_')
2853+
})
2854+
});
2855+
let declared = spellings.iter().any(|spelling| {
2856+
sql[from_at..]
2857+
.match_indices(&format!("AS {spelling}"))
2858+
.any(|(at, _)| paren_depth_at(sql, from_at + at) == 0)
2859+
});
2860+
referenced && !declared
2861+
}
2862+
2863+
/// Prepares `sql` against an in-memory `DuckDB` holding the two tables the
2864+
/// plans below read, which binds every reference without running anything.
2865+
#[cfg(feature = "duckdb")]
2866+
fn duckdb_binds(sql: &str) -> Result<(), duckdb::Error> {
2867+
let conn = duckdb::Connection::open_in_memory()?;
2868+
conn.execute_batch(
2869+
"CREATE TABLE products (category_id INTEGER NOT NULL, name VARCHAR NOT NULL, \
2870+
price INTEGER NOT NULL); \
2871+
CREATE TABLE categories (category_id INTEGER NOT NULL, name VARCHAR NOT NULL);",
2872+
)?;
2873+
conn.prepare(sql).map(|_| ())
2874+
}
2875+
2876+
/// Regression test for the unparser's `rescope_projection_over_projection`,
2877+
/// carried by the `spiceai/datafusion` fork (upstream's
2878+
/// apache/datafusion#22961): common subexpression elimination leaves a
2879+
/// `Projection` directly over another `Projection`, and the unparser renders the
2880+
/// inner one as an unaliased derived table. The outer one's references still name
2881+
/// the relations the inner one reads — `p`, `c` — which that derived table hides,
2882+
/// so without the fix the federated statement is
2883+
/// `SELECT p."name" … FROM (SELECT … FROM products AS p LEFT JOIN …)`, and the
2884+
/// remote engine refuses it (`DuckDB`: "Referenced table p not found").
2885+
///
2886+
/// Three arms, because the fix makes three different decisions: outputs whose
2887+
/// names are unique are read from the derived table by name; two same-named
2888+
/// outputs (`p.name`, `c.name`), which no name in the derived table addresses,
2889+
/// merge the two projections so the qualifiers are in scope again; and a merge
2890+
/// that would repeat a volatile expression is refused rather than rendered.
2891+
#[test]
2892+
fn a_projection_over_a_derived_projection_reads_only_relations_in_scope() {
2893+
let unique_outputs = products_left_join_categories()
2894+
.project(vec![
2895+
(col("p.price") + lit(1)).alias("__common_expr_1"),
2896+
col("p.name"),
2897+
])
2898+
.expect("inner projection")
2899+
.project(vec![
2900+
col("p.name").alias("product_name"),
2901+
col("__common_expr_1"),
2902+
])
2903+
.expect("outer projection")
2904+
.build()
2905+
.expect("build");
2906+
let same_named_outputs = products_left_join_categories()
2907+
.project(vec![
2908+
(col("p.price") + lit(1)).alias("__common_expr_1"),
2909+
col("p.name"),
2910+
col("c.name"),
2911+
])
2912+
.expect("inner projection")
2913+
.project(vec![
2914+
col("p.name").alias("product_name"),
2915+
col("c.name").alias("category_name"),
2916+
(col("__common_expr_1") * lit(2)).alias("doubled"),
2917+
col("__common_expr_1"),
2918+
])
2919+
.expect("outer projection")
2920+
.build()
2921+
.expect("build");
2922+
2923+
for (shape, plan) in [
2924+
("unique outputs", &unique_outputs),
2925+
("same-named outputs", &same_named_outputs),
2926+
] {
2927+
for (dialect_name, dialect) in federation_dialects() {
2928+
let sql = unparse_with(dialect_name, dialect.as_ref(), plan);
2929+
for relation in ["p", "c"] {
2930+
assert!(
2931+
!outer_select_reads_an_out_of_scope_relation(&sql, relation),
2932+
"{dialect_name}/{shape}: the outer SELECT qualifies a column by '{relation}', \
2933+
which the derived table below it hides, so the remote engine cannot bind \
2934+
the statement: {sql}"
2935+
);
2936+
}
2937+
for output in ["product_name", "__common_expr_1"] {
2938+
assert!(
2939+
sql[..first_offset_of(&sql, " FROM ")].contains(output),
2940+
"{dialect_name}/{shape}: the outer SELECT lost its `{output}` output: {sql}"
2941+
);
2942+
}
2943+
}
2944+
2945+
#[cfg(feature = "duckdb")]
2946+
{
2947+
let sql = unparse_with("duckdb", &DuckDBDialect::new(), plan);
2948+
if let Err(error) = duckdb_binds(&sql) {
2949+
panic!("duckdb/{shape}: DuckDB refused the federated statement: {error}: {sql}");
2950+
}
2951+
}
2952+
}
2953+
2954+
let over_a_volatile_output = products_left_join_categories()
2955+
.project(vec![
2956+
datafusion::functions::expr_fn::random().alias("r"),
2957+
col("p.name"),
2958+
col("c.name"),
2959+
])
2960+
.expect("inner projection")
2961+
.project(vec![
2962+
col("p.name").alias("product_name"),
2963+
col("c.name").alias("category_name"),
2964+
col("r"),
2965+
])
2966+
.expect("outer projection")
2967+
.build()
2968+
.expect("build");
2969+
match federated_sql_result(&over_a_volatile_output) {
2970+
Err(DataFusionError::NotImplemented(_)) => {}
2971+
other => panic!(
2972+
"merging the projections would evaluate `random()` once per reference rather \
2973+
than once per row, and no name addresses the same-named outputs, so the plan \
2974+
has to be refused as unsupported; got {other:?}"
2975+
),
2976+
}
2977+
}
2978+
2979+
/// Regression test for `SchemaCastScanExec` forwarding its input's statistics
2980+
/// through `DataFusion` 55's `StatisticsContext`, carried by the
2981+
/// `spiceai/datafusion-federation` fork. 55 derives plan statistics through
2982+
/// `ExecutionPlan::statistics_from_inputs` and its built-in nodes no longer
2983+
/// implement the deprecated `partition_statistics`, so a cast node that still
2984+
/// forwards the latter reports an unknown row count over a scan that knows its
2985+
/// own, and join sizing and statistics-answered aggregates above a federated scan
2986+
/// lose it.
2987+
#[test]
2988+
fn a_schema_cast_scan_reports_its_inputs_statistics() {
2989+
use datafusion::arrow::array::{Int32Array, RecordBatch};
2990+
use datafusion::common::stats::Precision;
2991+
use datafusion::datasource::memory::MemorySourceConfig;
2992+
use datafusion::physical_plan::{StatisticsArgs, StatisticsContext};
2993+
use datafusion_federation::schema_cast::SchemaCastScanExec;
2994+
2995+
let input_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
2996+
let batch = RecordBatch::try_new(
2997+
Arc::clone(&input_schema),
2998+
vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
2999+
)
3000+
.expect("build the input batch");
3001+
let input = MemorySourceConfig::try_new_exec(&[vec![batch]], input_schema, None)
3002+
.expect("build the input scan");
3003+
let input_rows = StatisticsContext::new()
3004+
.compute(input.as_ref(), &StatisticsArgs::new())
3005+
.expect("the input's statistics")
3006+
.num_rows;
3007+
assert_eq!(
3008+
input_rows,
3009+
Precision::Exact(3),
3010+
"the input has to know its own row count, or this guard proves nothing"
3011+
);
3012+
3013+
let cast = SchemaCastScanExec::new(
3014+
input,
3015+
Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)])),
3016+
);
3017+
let stats = StatisticsContext::new()
3018+
.compute(&cast, &StatisticsArgs::new())
3019+
.expect("the cast node's statistics");
3020+
assert_eq!(
3021+
stats.num_rows, input_rows,
3022+
"casting changes column types, not row counts, so the cast node has to report \
3023+
its input's row count"
3024+
);
3025+
}
27833026
}

0 commit comments

Comments
 (0)