@@ -20,7 +20,7 @@ use std::pin::Pin;
2020use std:: sync:: Arc ;
2121use std:: vec;
2222
23- use datafusion:: arrow:: array:: RecordBatch ;
23+ use datafusion:: arrow:: array:: { RecordBatch , RecordBatchOptions , new_null_array } ;
2424use datafusion:: arrow:: datatypes:: SchemaRef as ArrowSchemaRef ;
2525use datafusion:: error:: Result as DFResult ;
2626use datafusion:: execution:: { SendableRecordBatchStream , TaskContext } ;
@@ -31,7 +31,9 @@ use datafusion::physical_plan::{DisplayAs, ExecutionPlan, Partitioning, PlanProp
3131use datafusion:: prelude:: Expr ;
3232use futures:: { Stream , TryStreamExt } ;
3333use iceberg:: expr:: Predicate ;
34+ use iceberg:: metadata_columns:: is_metadata_column_name;
3435use iceberg:: table:: Table ;
36+ use iceberg:: { Error , ErrorKind } ;
3537
3638use super :: expr_to_predicate:: convert_filters_to_predicate;
3739use crate :: to_datafusion_error;
@@ -49,6 +51,9 @@ pub struct IcebergTableScan {
4951 plan_properties : PlanProperties ,
5052 /// Projection column names, None means all columns
5153 projection : Option < Vec < String > > ,
54+ /// The column names actually selected from the table: the advertised (output) columns MINUS
55+ /// any the scanned snapshot's schema does not carry. See [`IcebergTableScan::new`].
56+ scan_columns : Vec < String > ,
5257 /// Filters to apply to the table scan
5358 predicates : Option < Predicate > ,
5459 /// Optional limit on the number of rows to return
@@ -61,6 +66,23 @@ impl IcebergTableScan {
6166 /// Returns a planning error when `projection` holds an index outside `schema`
6267 /// (previously an `unwrap` panic — SAF-004). The projected column names are derived
6368 /// from the projected schema itself, so they can never index out of bounds.
69+ ///
70+ /// # The advertised schema is a contract (BUG-011)
71+ ///
72+ /// `schema` is the schema DataFusion PLANNED this query against — `projection` indexes into
73+ /// it, and every parent operator was built against its projection. That projection is therefore
74+ /// the schema this node advertises, and the batches it emits MUST match it. Two things can make
75+ /// the table disagree with it:
76+ ///
77+ /// * the caller reloaded the table between planning and scanning (the catalog-backed provider
78+ /// does exactly that), and
79+ /// * `ADD COLUMN` does not create a snapshot, so a table's CURRENT schema routinely has columns
80+ /// the scanned snapshot's schema — the schema the core scan resolves names against — lacks.
81+ ///
82+ /// So the scan never says "give me everything the table has now" (`select_all`): it selects the
83+ /// advertised columns BY NAME, minus the ones the scanned snapshot's schema cannot resolve, and
84+ /// [`conform_batch`] restores those as NULL columns at execution — which is what Java does when
85+ /// a projected field is absent from the data it reads.
6486 pub ( crate ) fn new (
6587 table : Table ,
6688 snapshot_id : Option < i64 > ,
@@ -81,6 +103,7 @@ impl IcebergTableScan {
81103 ( projected_schema, Some ( column_names) )
82104 }
83105 } ;
106+ let scan_columns = Self :: resolve_scan_columns ( & table, snapshot_id, & output_schema) ?;
84107 let plan_properties = Self :: compute_properties ( output_schema) ;
85108 let predicates = convert_filters_to_predicate ( filters) ;
86109
@@ -89,11 +112,49 @@ impl IcebergTableScan {
89112 snapshot_id,
90113 plan_properties,
91114 projection,
115+ scan_columns,
92116 predicates,
93117 limit,
94118 } )
95119 }
96120
121+ /// The advertised columns the scanned snapshot's schema can actually resolve.
122+ ///
123+ /// The acceptance test is deliberately the SAME one `TableScanBuilder::build` applies
124+ /// (`field_by_name`, with metadata columns exempt): a name this filter keeps is a name the core
125+ /// scan accepts, so the scan can never fail on a column the plan advertised. Names dropped here
126+ /// are null-filled by [`conform_batch`].
127+ ///
128+ /// When the scan has no snapshot to resolve against — an unknown snapshot id (the core scan
129+ /// reports that itself, with the better message) or a table with no snapshot at all (an empty
130+ /// scan) — every advertised name is passed through unchanged.
131+ fn resolve_scan_columns (
132+ table : & Table ,
133+ snapshot_id : Option < i64 > ,
134+ output_schema : & ArrowSchemaRef ,
135+ ) -> DFResult < Vec < String > > {
136+ let advertised = output_schema
137+ . fields ( )
138+ . iter ( )
139+ . map ( |field| field. name ( ) . clone ( ) ) ;
140+
141+ let metadata = table. metadata ( ) ;
142+ let snapshot = match snapshot_id {
143+ Some ( snapshot_id) => metadata. snapshot_by_id ( snapshot_id) ,
144+ None => metadata. current_snapshot ( ) ,
145+ } ;
146+ let Some ( snapshot) = snapshot else {
147+ return Ok ( advertised. collect ( ) ) ;
148+ } ;
149+ let snapshot_schema = snapshot. schema ( metadata) . map_err ( to_datafusion_error) ?;
150+
151+ Ok ( advertised
152+ . filter ( |name| {
153+ is_metadata_column_name ( name) || snapshot_schema. field_by_name ( name) . is_some ( )
154+ } )
155+ . collect ( ) )
156+ }
157+
97158 pub fn table ( & self ) -> & Table {
98159 & self . table
99160 }
@@ -160,10 +221,17 @@ impl ExecutionPlan for IcebergTableScan {
160221 let fut = get_batch_stream (
161222 self . table . clone ( ) ,
162223 self . snapshot_id ,
163- self . projection . clone ( ) ,
224+ self . scan_columns . clone ( ) ,
164225 self . predicates . clone ( ) ,
165226 ) ;
166- let stream = futures:: stream:: once ( fut) . try_flatten ( ) ;
227+ // Every emitted batch is conformed to the schema this node advertised — see
228+ // `IcebergTableScan::new` and `conform_batch`.
229+ let advertised_schema = self . schema ( ) ;
230+ let stream = futures:: stream:: once ( fut)
231+ . try_flatten ( )
232+ . and_then ( move |batch| {
233+ futures:: future:: ready ( conform_batch ( batch, & advertised_schema) )
234+ } ) ;
167235
168236 // Apply limit if specified
169237 let limited_stream: Pin < Box < dyn Stream < Item = DFResult < RecordBatch > > + Send > > =
@@ -219,18 +287,17 @@ impl DisplayAs for IcebergTableScan {
219287async fn get_batch_stream (
220288 table : Table ,
221289 snapshot_id : Option < i64 > ,
222- column_names : Option < Vec < String > > ,
290+ column_names : Vec < String > ,
223291 predicates : Option < Predicate > ,
224292) -> DFResult < Pin < Box < dyn Stream < Item = DFResult < RecordBatch > > + Send > > > {
225293 let scan_builder = match snapshot_id {
226294 Some ( snapshot_id) => table. scan ( ) . snapshot_id ( snapshot_id) ,
227295 None => table. scan ( ) ,
228296 } ;
229297
230- let mut scan_builder = match column_names {
231- Some ( column_names) => scan_builder. select ( column_names) ,
232- None => scan_builder. select_all ( ) ,
233- } ;
298+ // Always an explicit selection — NEVER `select_all()`, which would read whatever column set the
299+ // table happens to have now rather than the one the plan advertised (BUG-011).
300+ let mut scan_builder = scan_builder. select ( column_names) ;
234301 if let Some ( pred) = predicates {
235302 scan_builder = scan_builder. with_filter ( pred) ;
236303 }
@@ -244,6 +311,73 @@ async fn get_batch_stream(
244311 Ok ( Box :: pin ( stream) )
245312}
246313
314+ /// Coerces a scanned batch to the schema the plan advertised (BUG-011).
315+ ///
316+ /// A DataFusion operator addresses its input by ORDINAL against the schema its child advertised, so
317+ /// a batch that merely happens to carry the right columns in a different order — or an extra one —
318+ /// is silent corruption, not a nuisance. This resolves every advertised field BY NAME and rebuilds
319+ /// the batch in the advertised order:
320+ ///
321+ /// * present with the advertised type → taken as is (the common case; a batch that already equals
322+ /// the advertised schema is returned untouched);
323+ /// * absent and nullable → an all-NULL column, matching Java, whose readers null-fill a projected
324+ /// field that the data it reads does not have (e.g. a column added after the scanned snapshot);
325+ /// * absent and NOT nullable, or present with a different type → a typed error naming the column.
326+ /// Silently coercing here would hand DataFusion data that contradicts its plan.
327+ ///
328+ /// The row count is carried explicitly so a zero-column projection (`SELECT count(*)`) keeps it.
329+ fn conform_batch ( batch : RecordBatch , advertised : & ArrowSchemaRef ) -> DFResult < RecordBatch > {
330+ if batch. schema_ref ( ) == advertised {
331+ return Ok ( batch) ;
332+ }
333+
334+ let num_rows = batch. num_rows ( ) ;
335+ let mut columns = Vec :: with_capacity ( advertised. fields ( ) . len ( ) ) ;
336+ for field in advertised. fields ( ) {
337+ match batch. column_by_name ( field. name ( ) ) {
338+ Some ( column) if column. data_type ( ) == field. data_type ( ) => columns. push ( column. clone ( ) ) ,
339+ Some ( column) => {
340+ return Err ( to_datafusion_error ( Error :: new (
341+ ErrorKind :: DataInvalid ,
342+ format ! (
343+ "column '{}' was scanned as {} but the query plan expects {}: the table's \
344+ type for this column changed after the query was planned; re-plan the query",
345+ field. name( ) ,
346+ column. data_type( ) ,
347+ field. data_type( )
348+ ) ,
349+ ) ) ) ;
350+ }
351+ None if field. is_nullable ( ) => {
352+ columns. push ( new_null_array ( field. data_type ( ) , num_rows) )
353+ }
354+ None => {
355+ return Err ( to_datafusion_error ( Error :: new (
356+ ErrorKind :: DataInvalid ,
357+ format ! (
358+ "required column '{}' is absent from the scanned data and cannot be \
359+ null-filled: the table's schema changed after the query was planned; \
360+ re-plan the query",
361+ field. name( )
362+ ) ,
363+ ) ) ) ;
364+ }
365+ }
366+ }
367+
368+ RecordBatch :: try_new_with_options (
369+ advertised. clone ( ) ,
370+ columns,
371+ & RecordBatchOptions :: new ( ) . with_row_count ( Some ( num_rows) ) ,
372+ )
373+ . map_err ( |e| {
374+ datafusion:: error:: DataFusionError :: ArrowError (
375+ Box :: new ( e) ,
376+ Some ( "failed to conform a scanned batch to the schema the plan advertised" . to_string ( ) ) ,
377+ )
378+ } )
379+ }
380+
247381#[ cfg( test) ]
248382mod tests {
249383 use std:: collections:: HashMap ;
@@ -330,6 +464,135 @@ mod tests {
330464 ) ;
331465 }
332466
467+ /// BUG-011: an advertised column the scan could not read is restored as NULL, and the columns
468+ /// are returned in the ADVERTISED order — DataFusion addresses them by ordinal.
469+ #[ test]
470+ fn test_conform_batch_null_fills_and_reorders ( ) {
471+ let scanned = RecordBatch :: try_new (
472+ Arc :: new ( ArrowSchema :: new ( vec ! [
473+ ArrowField :: new( "data" , ArrowDataType :: Utf8 , false ) ,
474+ ArrowField :: new( "id" , ArrowDataType :: Int64 , false ) ,
475+ ] ) ) ,
476+ vec ! [
477+ Arc :: new( datafusion:: arrow:: array:: StringArray :: from( vec![ "a" , "b" ] ) ) ,
478+ Arc :: new( datafusion:: arrow:: array:: Int64Array :: from( vec![ 1 , 2 ] ) ) ,
479+ ] ,
480+ )
481+ . expect ( "the scanned batch must build" ) ;
482+
483+ let advertised: ArrowSchemaRef = Arc :: new ( ArrowSchema :: new ( vec ! [
484+ ArrowField :: new( "id" , ArrowDataType :: Int64 , false ) ,
485+ ArrowField :: new( "data" , ArrowDataType :: Utf8 , false ) ,
486+ ArrowField :: new( "added_later" , ArrowDataType :: Int32 , true ) ,
487+ ] ) ) ;
488+
489+ let conformed = conform_batch ( scanned, & advertised) . expect ( "the batch must conform" ) ;
490+ assert_eq ! ( conformed. schema( ) , advertised) ;
491+ assert_eq ! ( conformed. num_rows( ) , 2 ) ;
492+ assert_eq ! (
493+ conformed
494+ . column_by_name( "added_later" )
495+ . expect( "the added column must be present" )
496+ . null_count( ) ,
497+ 2 ,
498+ "a column the scan could not read must be all-NULL"
499+ ) ;
500+ assert_eq ! (
501+ conformed
502+ . column( 0 )
503+ . as_any( )
504+ . downcast_ref:: <datafusion:: arrow:: array:: Int64Array >( )
505+ . expect( "column 0 must be the advertised `id`" )
506+ . values( ) ,
507+ & [ 1 , 2 ]
508+ ) ;
509+ }
510+
511+ /// A zero-column projection (`SELECT count(*)`) must keep its row count — a rebuilt batch with
512+ /// no columns has no other way to carry it.
513+ #[ test]
514+ fn test_conform_batch_preserves_row_count_with_no_columns ( ) {
515+ let scanned = RecordBatch :: try_new (
516+ Arc :: new ( ArrowSchema :: new ( vec ! [ ArrowField :: new(
517+ "id" ,
518+ ArrowDataType :: Int64 ,
519+ false ,
520+ ) ] ) ) ,
521+ vec ! [ Arc :: new( datafusion:: arrow:: array:: Int64Array :: from( vec![
522+ 1 , 2 , 3 ,
523+ ] ) ) ] ,
524+ )
525+ . expect ( "the scanned batch must build" ) ;
526+
527+ let advertised: ArrowSchemaRef = Arc :: new ( ArrowSchema :: new ( Vec :: < ArrowField > :: new ( ) ) ) ;
528+ let conformed = conform_batch ( scanned, & advertised) . expect ( "the batch must conform" ) ;
529+ assert_eq ! ( conformed. num_columns( ) , 0 ) ;
530+ assert_eq ! (
531+ conformed. num_rows( ) ,
532+ 3 ,
533+ "the row count must survive a zero-column projection"
534+ ) ;
535+ }
536+
537+ /// A column whose type changed after planning must be a loud, named error — handing DataFusion
538+ /// data that contradicts its plan is the corruption this whole path exists to prevent.
539+ #[ test]
540+ fn test_conform_batch_rejects_a_changed_type ( ) {
541+ let scanned = RecordBatch :: try_new (
542+ Arc :: new ( ArrowSchema :: new ( vec ! [ ArrowField :: new(
543+ "id" ,
544+ ArrowDataType :: Int64 ,
545+ false ,
546+ ) ] ) ) ,
547+ vec ! [ Arc :: new( datafusion:: arrow:: array:: Int64Array :: from( vec![
548+ 1 ,
549+ ] ) ) ] ,
550+ )
551+ . expect ( "the scanned batch must build" ) ;
552+
553+ let advertised: ArrowSchemaRef = Arc :: new ( ArrowSchema :: new ( vec ! [ ArrowField :: new(
554+ "id" ,
555+ ArrowDataType :: Int32 ,
556+ false ,
557+ ) ] ) ) ;
558+
559+ let err = conform_batch ( scanned, & advertised)
560+ . expect_err ( "a type change must not be silently coerced" ) ;
561+ assert ! (
562+ err. to_string( ) . contains( "id" ) && err. to_string( ) . contains( "Int32" ) ,
563+ "the error must name the column and the expected type: {err}"
564+ ) ;
565+ }
566+
567+ /// An advertised NON-nullable column that the scan could not read cannot be null-filled, so it
568+ /// must error rather than fabricate values.
569+ #[ test]
570+ fn test_conform_batch_rejects_absent_required_column ( ) {
571+ let scanned = RecordBatch :: try_new (
572+ Arc :: new ( ArrowSchema :: new ( vec ! [ ArrowField :: new(
573+ "id" ,
574+ ArrowDataType :: Int64 ,
575+ false ,
576+ ) ] ) ) ,
577+ vec ! [ Arc :: new( datafusion:: arrow:: array:: Int64Array :: from( vec![
578+ 1 ,
579+ ] ) ) ] ,
580+ )
581+ . expect ( "the scanned batch must build" ) ;
582+
583+ let advertised: ArrowSchemaRef = Arc :: new ( ArrowSchema :: new ( vec ! [
584+ ArrowField :: new( "id" , ArrowDataType :: Int64 , false ) ,
585+ ArrowField :: new( "required_new" , ArrowDataType :: Utf8 , false ) ,
586+ ] ) ) ;
587+
588+ let err = conform_batch ( scanned, & advertised)
589+ . expect_err ( "an absent required column must not be null-filled" ) ;
590+ assert ! (
591+ err. to_string( ) . contains( "required_new" ) ,
592+ "the error must name the column: {err}"
593+ ) ;
594+ }
595+
333596 /// SAF-004 P5b (regression): a valid projection still produces the projected output schema
334597 /// and the projected column names, and no projection passes the schema through unchanged.
335598 #[ test]
0 commit comments