@@ -2780,4 +2780,247 @@ mod tests {
27802780 . expect ( "walking a logical plan cannot fail" ) ;
27812781 negated
27822782 }
2783+
2784+ /// `products AS p LEFT JOIN categories AS c`, where both relations expose a
2785+ /// column called `name`, so a projection passing both through has two outputs no
2786+ /// bare name tells apart.
2787+ fn products_left_join_categories ( ) -> LogicalPlanBuilder {
2788+ let products = LogicalPlanBuilder :: scan (
2789+ "products" ,
2790+ table_source ( vec ! [
2791+ Field :: new( "category_id" , DataType :: Int32 , false ) ,
2792+ Field :: new( "name" , DataType :: Utf8 , false ) ,
2793+ Field :: new( "price" , DataType :: Int32 , false ) ,
2794+ ] ) ,
2795+ None ,
2796+ )
2797+ . expect ( "scan products" )
2798+ . alias ( "p" )
2799+ . expect ( "alias products" )
2800+ . build ( )
2801+ . expect ( "build products" ) ;
2802+ let categories = LogicalPlanBuilder :: scan (
2803+ "categories" ,
2804+ table_source ( vec ! [
2805+ Field :: new( "category_id" , DataType :: Int32 , false ) ,
2806+ Field :: new( "name" , DataType :: Utf8 , false ) ,
2807+ ] ) ,
2808+ None ,
2809+ )
2810+ . expect ( "scan categories" )
2811+ . alias ( "c" )
2812+ . expect ( "alias categories" )
2813+ . build ( )
2814+ . expect ( "build categories" ) ;
2815+ LogicalPlanBuilder :: from ( products)
2816+ . join (
2817+ categories,
2818+ JoinType :: Left ,
2819+ ( vec ! [ "p.category_id" ] , vec ! [ "c.category_id" ] ) ,
2820+ None ,
2821+ )
2822+ . expect ( "join products to categories" )
2823+ }
2824+
2825+ /// Whether the outermost `SELECT` list of `sql` qualifies a column by
2826+ /// `relation` while the outermost `FROM` does not bring `relation` into scope.
2827+ ///
2828+ /// Matched on nesting depth rather than on a rendered string, so it holds across
2829+ /// the dialects' quoting: `p.`, `"p".` and `` `p`. `` are all the same
2830+ /// reference, and `AS p`, `AS "p"` and `` AS `p` `` the same declaration.
2831+ fn outer_select_reads_an_out_of_scope_relation ( sql : & str , relation : & str ) -> bool {
2832+ let Some ( from_at) = sql
2833+ . match_indices ( " FROM " )
2834+ . map ( |( at, _) | at)
2835+ . find ( |at| paren_depth_at ( sql, * at) == 0 )
2836+ else {
2837+ panic ! ( "expected a top-level FROM in: {sql}" ) ;
2838+ } ;
2839+ let spellings = [
2840+ relation. to_string ( ) ,
2841+ format ! ( "\" {relation}\" " ) ,
2842+ format ! ( "`{relation}`" ) ,
2843+ ] ;
2844+ let select_list = & sql[ ..from_at] ;
2845+ let referenced = spellings. iter ( ) . any ( |spelling| {
2846+ select_list
2847+ . match_indices ( & format ! ( "{spelling}." ) )
2848+ . any ( |( at, _) | {
2849+ select_list[ ..at]
2850+ . chars ( )
2851+ . next_back ( )
2852+ . is_none_or ( |before| !before. is_alphanumeric ( ) && before != '_' )
2853+ } )
2854+ } ) ;
2855+ let declared = spellings. iter ( ) . any ( |spelling| {
2856+ sql[ from_at..]
2857+ . match_indices ( & format ! ( "AS {spelling}" ) )
2858+ . any ( |( at, _) | paren_depth_at ( sql, from_at + at) == 0 )
2859+ } ) ;
2860+ referenced && !declared
2861+ }
2862+
2863+ /// Prepares `sql` against an in-memory `DuckDB` holding the two tables the
2864+ /// plans below read, which binds every reference without running anything.
2865+ #[ cfg( feature = "duckdb" ) ]
2866+ fn duckdb_binds ( sql : & str ) -> Result < ( ) , duckdb:: Error > {
2867+ let conn = duckdb:: Connection :: open_in_memory ( ) ?;
2868+ conn. execute_batch (
2869+ "CREATE TABLE products (category_id INTEGER NOT NULL, name VARCHAR NOT NULL, \
2870+ price INTEGER NOT NULL); \
2871+ CREATE TABLE categories (category_id INTEGER NOT NULL, name VARCHAR NOT NULL);",
2872+ ) ?;
2873+ conn. prepare ( sql) . map ( |_| ( ) )
2874+ }
2875+
2876+ /// Regression test for the unparser's `rescope_projection_over_projection`,
2877+ /// carried by the `spiceai/datafusion` fork (upstream's
2878+ /// apache/datafusion#22961): common subexpression elimination leaves a
2879+ /// `Projection` directly over another `Projection`, and the unparser renders the
2880+ /// inner one as an unaliased derived table. The outer one's references still name
2881+ /// the relations the inner one reads — `p`, `c` — which that derived table hides,
2882+ /// so without the fix the federated statement is
2883+ /// `SELECT p."name" … FROM (SELECT … FROM products AS p LEFT JOIN …)`, and the
2884+ /// remote engine refuses it (`DuckDB`: "Referenced table p not found").
2885+ ///
2886+ /// Three arms, because the fix makes three different decisions: outputs whose
2887+ /// names are unique are read from the derived table by name; two same-named
2888+ /// outputs (`p.name`, `c.name`), which no name in the derived table addresses,
2889+ /// merge the two projections so the qualifiers are in scope again; and a merge
2890+ /// that would repeat a volatile expression is refused rather than rendered.
2891+ #[ test]
2892+ fn a_projection_over_a_derived_projection_reads_only_relations_in_scope ( ) {
2893+ let unique_outputs = products_left_join_categories ( )
2894+ . project ( vec ! [
2895+ ( col( "p.price" ) + lit( 1 ) ) . alias( "__common_expr_1" ) ,
2896+ col( "p.name" ) ,
2897+ ] )
2898+ . expect ( "inner projection" )
2899+ . project ( vec ! [
2900+ col( "p.name" ) . alias( "product_name" ) ,
2901+ col( "__common_expr_1" ) ,
2902+ ] )
2903+ . expect ( "outer projection" )
2904+ . build ( )
2905+ . expect ( "build" ) ;
2906+ let same_named_outputs = products_left_join_categories ( )
2907+ . project ( vec ! [
2908+ ( col( "p.price" ) + lit( 1 ) ) . alias( "__common_expr_1" ) ,
2909+ col( "p.name" ) ,
2910+ col( "c.name" ) ,
2911+ ] )
2912+ . expect ( "inner projection" )
2913+ . project ( vec ! [
2914+ col( "p.name" ) . alias( "product_name" ) ,
2915+ col( "c.name" ) . alias( "category_name" ) ,
2916+ ( col( "__common_expr_1" ) * lit( 2 ) ) . alias( "doubled" ) ,
2917+ col( "__common_expr_1" ) ,
2918+ ] )
2919+ . expect ( "outer projection" )
2920+ . build ( )
2921+ . expect ( "build" ) ;
2922+
2923+ for ( shape, plan) in [
2924+ ( "unique outputs" , & unique_outputs) ,
2925+ ( "same-named outputs" , & same_named_outputs) ,
2926+ ] {
2927+ for ( dialect_name, dialect) in federation_dialects ( ) {
2928+ let sql = unparse_with ( dialect_name, dialect. as_ref ( ) , plan) ;
2929+ for relation in [ "p" , "c" ] {
2930+ assert ! (
2931+ !outer_select_reads_an_out_of_scope_relation( & sql, relation) ,
2932+ "{dialect_name}/{shape}: the outer SELECT qualifies a column by '{relation}', \
2933+ which the derived table below it hides, so the remote engine cannot bind \
2934+ the statement: {sql}"
2935+ ) ;
2936+ }
2937+ for output in [ "product_name" , "__common_expr_1" ] {
2938+ assert ! (
2939+ sql[ ..first_offset_of( & sql, " FROM " ) ] . contains( output) ,
2940+ "{dialect_name}/{shape}: the outer SELECT lost its `{output}` output: {sql}"
2941+ ) ;
2942+ }
2943+ }
2944+
2945+ #[ cfg( feature = "duckdb" ) ]
2946+ {
2947+ let sql = unparse_with ( "duckdb" , & DuckDBDialect :: new ( ) , plan) ;
2948+ if let Err ( error) = duckdb_binds ( & sql) {
2949+ panic ! ( "duckdb/{shape}: DuckDB refused the federated statement: {error}: {sql}" ) ;
2950+ }
2951+ }
2952+ }
2953+
2954+ let over_a_volatile_output = products_left_join_categories ( )
2955+ . project ( vec ! [
2956+ datafusion:: functions:: expr_fn:: random( ) . alias( "r" ) ,
2957+ col( "p.name" ) ,
2958+ col( "c.name" ) ,
2959+ ] )
2960+ . expect ( "inner projection" )
2961+ . project ( vec ! [
2962+ col( "p.name" ) . alias( "product_name" ) ,
2963+ col( "c.name" ) . alias( "category_name" ) ,
2964+ col( "r" ) ,
2965+ ] )
2966+ . expect ( "outer projection" )
2967+ . build ( )
2968+ . expect ( "build" ) ;
2969+ match federated_sql_result ( & over_a_volatile_output) {
2970+ Err ( DataFusionError :: NotImplemented ( _) ) => { }
2971+ other => panic ! (
2972+ "merging the projections would evaluate `random()` once per reference rather \
2973+ than once per row, and no name addresses the same-named outputs, so the plan \
2974+ has to be refused as unsupported; got {other:?}"
2975+ ) ,
2976+ }
2977+ }
2978+
2979+ /// Regression test for `SchemaCastScanExec` forwarding its input's statistics
2980+ /// through `DataFusion` 55's `StatisticsContext`, carried by the
2981+ /// `spiceai/datafusion-federation` fork. 55 derives plan statistics through
2982+ /// `ExecutionPlan::statistics_from_inputs` and its built-in nodes no longer
2983+ /// implement the deprecated `partition_statistics`, so a cast node that still
2984+ /// forwards the latter reports an unknown row count over a scan that knows its
2985+ /// own, and join sizing and statistics-answered aggregates above a federated scan
2986+ /// lose it.
2987+ #[ test]
2988+ fn a_schema_cast_scan_reports_its_inputs_statistics ( ) {
2989+ use datafusion:: arrow:: array:: { Int32Array , RecordBatch } ;
2990+ use datafusion:: common:: stats:: Precision ;
2991+ use datafusion:: datasource:: memory:: MemorySourceConfig ;
2992+ use datafusion:: physical_plan:: { StatisticsArgs , StatisticsContext } ;
2993+ use datafusion_federation:: schema_cast:: SchemaCastScanExec ;
2994+
2995+ let input_schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new( "a" , DataType :: Int32 , false ) ] ) ) ;
2996+ let batch = RecordBatch :: try_new (
2997+ Arc :: clone ( & input_schema) ,
2998+ vec ! [ Arc :: new( Int32Array :: from( vec![ 1 , 2 , 3 ] ) ) ] ,
2999+ )
3000+ . expect ( "build the input batch" ) ;
3001+ let input = MemorySourceConfig :: try_new_exec ( & [ vec ! [ batch] ] , input_schema, None )
3002+ . expect ( "build the input scan" ) ;
3003+ let input_rows = StatisticsContext :: new ( )
3004+ . compute ( input. as_ref ( ) , & StatisticsArgs :: new ( ) )
3005+ . expect ( "the input's statistics" )
3006+ . num_rows ;
3007+ assert_eq ! (
3008+ input_rows,
3009+ Precision :: Exact ( 3 ) ,
3010+ "the input has to know its own row count, or this guard proves nothing"
3011+ ) ;
3012+
3013+ let cast = SchemaCastScanExec :: new (
3014+ input,
3015+ Arc :: new ( Schema :: new ( vec ! [ Field :: new( "a" , DataType :: Int64 , false ) ] ) ) ,
3016+ ) ;
3017+ let stats = StatisticsContext :: new ( )
3018+ . compute ( & cast, & StatisticsArgs :: new ( ) )
3019+ . expect ( "the cast node's statistics" ) ;
3020+ assert_eq ! (
3021+ stats. num_rows, input_rows,
3022+ "casting changes column types, not row counts, so the cast node has to report \
3023+ its input's row count"
3024+ ) ;
3025+ }
27833026}
0 commit comments