@@ -523,10 +523,15 @@ impl LayoutReader for StructReader {
523523 split_range : & SplitRange ,
524524 splits : & mut RowSplits ,
525525 ) -> VortexResult < ( ) > {
526- // Register splits for the validity child, if there is one
527- if let Some ( validity_ref) = self . validity ( ) ? {
528- validity_ref. register_splits ( field_mask, split_range, splits) ?;
529- }
526+ // Validity boundaries must stay hard even when they overlap a soft data-field boundary.
527+ let validity_boundaries = if let Some ( validity_ref) = self . validity ( ) ? {
528+ let mut validity_splits = RowSplits :: new_capacity ( 16 ) ;
529+ validity_ref. register_splits ( field_mask, split_range, & mut validity_splits) ?;
530+ validity_splits. into_sorted_deduped ( )
531+ } else {
532+ Vec :: new ( )
533+ } ;
534+ splits. extend_ascending ( validity_boundaries. iter ( ) . copied ( ) ) ;
530535
531536 // Each field's boundaries, kept apart so the ones only some fields have can be marked soft.
532537 let mut fields: Vec < RowSplits > = Vec :: new ( ) ;
@@ -537,24 +542,35 @@ impl LayoutReader for StructReader {
537542 fields. push ( field) ;
538543 Ok ( ( ) )
539544 } ) ?;
545+ let mut field_splits = RowSplits :: new_capacity ( 0 ) ;
540546 if let [ field] = & mut fields[ ..] {
541- splits . append ( std:: mem:: replace ( field, RowSplits :: new_capacity ( 0 ) ) ) ;
547+ field_splits = std:: mem:: replace ( field, RowSplits :: new_capacity ( 0 ) ) ;
542548 } else if !fields. is_empty ( ) {
543549 let fields: Vec < ( Vec < u64 > , Vec < u64 > ) > = fields
544550 . into_iter ( )
545551 . map ( RowSplits :: into_sorted_deduped_with_soft)
546552 . collect ( ) ;
547- let shared = |row : & u64 | {
548- fields
549- . iter ( )
550- . all ( |( boundaries, _) | boundaries. binary_search ( row) . is_ok ( ) )
551- } ;
553+ // Intersect once; checking every field for every boundary is quadratic in field count.
554+ let mut shared = fields[ 0 ] . 0 . clone ( ) ;
555+ for ( boundaries, _) in & fields[ 1 ..] {
556+ shared. retain ( |row| boundaries. binary_search ( row) . is_ok ( ) ) ;
557+ if shared. is_empty ( ) {
558+ break ;
559+ }
560+ }
552561 for ( boundaries, soft) in fields. iter ( ) {
553- splits. extend_ascending ( boundaries. iter ( ) . copied ( ) ) ;
554- splits. mark_soft ( soft. iter ( ) . copied ( ) ) ;
555- splits. mark_soft ( boundaries. iter ( ) . copied ( ) . filter ( |row| !shared ( row) ) ) ;
562+ field_splits. extend_ascending ( boundaries. iter ( ) . copied ( ) ) ;
563+ field_splits. mark_soft ( soft. iter ( ) . copied ( ) ) ;
564+ field_splits. mark_soft (
565+ boundaries
566+ . iter ( )
567+ . copied ( )
568+ . filter ( |row| shared. binary_search ( row) . is_err ( ) ) ,
569+ ) ;
556570 }
557571 }
572+ field_splits. remove_soft ( & validity_boundaries) ;
573+ splits. append ( field_splits) ;
558574
559575 // In the case of an empty struct, we need to register the end split. Pushed last so it
560576 // extends the final field's ascending run rather than starting a run of its own.
@@ -746,6 +762,7 @@ mod tests {
746762 use crate :: layouts:: table:: TableStrategy ;
747763 use crate :: layouts:: zoned:: writer:: ZonedLayoutOptions ;
748764 use crate :: layouts:: zoned:: writer:: ZonedStrategy ;
765+ use crate :: scan:: split_by:: SplitBy ;
749766 use crate :: segments:: CountingSegmentSource ;
750767 use crate :: segments:: SegmentFuture ;
751768 use crate :: segments:: SegmentId ;
@@ -851,13 +868,13 @@ mod tests {
851868 fn misaligned_struct_layout (
852869 pieces : usize ,
853870 chunked : bool ,
871+ nullable : bool ,
854872 ) -> ( Arc < dyn SegmentSource > , LayoutRef ) {
855873 let ctx = ArrayContext :: empty ( ) ;
856874 let segments = Arc :: new ( TestSegments :: default ( ) ) ;
857875 let ( ptr, eof) = SequenceId :: root ( ) . split ( ) ;
858- let table =
859- TableStrategy :: new ( Arc :: new ( FlatLayoutStrategy :: default ( ) ) , chunked_every ( 1000 ) )
860- . with_field_writer ( FieldPath :: from_name ( "b" ) , chunked_every ( 3000 ) ) ;
876+ let table = TableStrategy :: new ( chunked_every ( 2000 ) , chunked_every ( 1000 ) )
877+ . with_field_writer ( FieldPath :: from_name ( "b" ) , chunked_every ( 3000 ) ) ;
861878 let strategy: Arc < dyn LayoutStrategy > = if chunked {
862879 Arc :: new ( ChunkedLayoutStrategy :: new ( table) )
863880 } else {
@@ -868,12 +885,18 @@ mod tests {
868885 . map ( |piece| {
869886 let start = i32:: try_from ( piece * rows) . unwrap ( ) ;
870887 let end = start + i32:: try_from ( rows) . unwrap ( ) ;
871- StructArray :: from_fields (
888+ StructArray :: try_from_iter_with_validity (
872889 [
873890 ( "a" , PrimitiveArray :: from_iter ( start..end) . into_array ( ) ) ,
874891 ( "b" , PrimitiveArray :: from_iter ( start..end) . into_array ( ) ) ,
875- ]
876- . as_slice ( ) ,
892+ ] ,
893+ if nullable {
894+ Validity :: Array (
895+ BoolArray :: from_iter ( ( start..end) . map ( |row| row % 2 == 0 ) ) . into_array ( ) ,
896+ )
897+ } else {
898+ Validity :: NonNullable
899+ } ,
877900 )
878901 . unwrap ( )
879902 . into_array ( )
@@ -923,7 +946,7 @@ mod tests {
923946
924947 #[ test]
925948 fn boundaries_only_some_fields_have_are_soft ( ) {
926- let ( segments, layout) = misaligned_struct_layout ( 1 , false ) ;
949+ let ( segments, layout) = misaligned_struct_layout ( 1 , false , false ) ;
927950 let ( boundaries, soft) = boundaries_and_soft ( segments, & layout) ;
928951 // `a` chunks every 1,000 rows and `b` every 3,000, so only `b`'s are shared.
929952 assert_eq ! ( boundaries, ( 0 ..=12 ) . map( |k| k * 1000 ) . collect:: <Vec <u64 >>( ) ) ;
@@ -940,14 +963,34 @@ mod tests {
940963 fn chunk_ends_above_the_struct_stay_hard ( ) {
941964 // Two top-level chunks of 6,000 rows, each a struct whose `a` chunks every 1,000 rows and
942965 // `b` every 3,000: the top-level chunk end at 6,000 is every column's, so it stays hard.
943- let ( segments, layout) = misaligned_struct_layout ( 2 , true ) ;
966+ let ( segments, layout) = misaligned_struct_layout ( 2 , true , false ) ;
944967 let ( boundaries, soft) = boundaries_and_soft ( segments, & layout) ;
945968 assert_eq ! ( boundaries, ( 0 ..=12 ) . map( |k| k * 1000 ) . collect:: <Vec <u64 >>( ) ) ;
946969 assert ! ( !soft. contains( & 6000 ) , "the chunk end is soft: {soft:?}" ) ;
947970 assert ! ( !soft. contains( & 3000 ) && !soft. contains( & 9000 ) , "{soft:?}" ) ;
948971 assert ! ( soft. contains( & 1000 ) && soft. contains( & 7000 ) , "{soft:?}" ) ;
949972 }
950973
974+ #[ rstest]
975+ #[ case:: non_nullable( false , vec![ 0 , 3000 , 6000 , 9000 , 12000 ] ) ]
976+ #[ case:: nullable( true , vec![ 0 , 2000 , 3000 , 4000 , 6000 , 8000 , 9000 , 10000 , 12000 ] ) ]
977+ fn scan_merges_soft_boundaries_and_keeps_validity_boundaries (
978+ #[ case] nullable : bool ,
979+ #[ case] expected : Vec < u64 > ,
980+ ) -> VortexResult < ( ) > {
981+ let ( segments, layout) = misaligned_struct_layout ( 1 , false , nullable) ;
982+ let reader = layout. new_reader ( "" . into ( ) , segments, & SESSION , & Default :: default ( ) ) ?;
983+ assert_eq ! (
984+ SplitBy :: default ( ) . splits(
985+ reader. as_ref( ) ,
986+ & ( 0 ..reader. row_count( ) ) ,
987+ & [ FieldMask :: All ] ,
988+ ) ?,
989+ expected,
990+ ) ;
991+ Ok ( ( ) )
992+ }
993+
951994 #[ fixture]
952995 /// Create a chunked layout with three chunks of primitive arrays.
953996 fn null_struct_layout ( ) -> ( Arc < dyn SegmentSource > , LayoutRef ) {
0 commit comments