@@ -1487,7 +1487,7 @@ fn make_staggered_batches_binary(
14871487//
14881488// What only randomization reaches: `LeftSemi`/`LeftAnti` record matches in a shared
14891489// `AtomicUsize` watermark and emit once, from whichever streamed partition finishes last. The
1490- // streamed side below is spread round-robin over several partitions as one-row batches, so
1490+ // streamed side below is spread round-robin over several partitions in small batches, so
14911491// batches arrive in an order no static test pins down, and the counter that gates the final
14921492// pass is seeded from a partition count that deliberately disagrees with the `num_partitions`
14931493// argument. `RightSemi`/`RightAnti` take the mirror path -- every partition emits its own
@@ -1575,25 +1575,39 @@ fn pwmj_single_exec<K: PwmjFuzzKey>(
15751575 . unwrap ( )
15761576}
15771577
1578- /// Rows spread round-robin across `nparts` partitions, one row per batch, with any partition
1579- /// that draws no row left holding a single empty batch. Used for the streamed side throughout,
1580- /// and for the buffered side of the right existence joins, which place no single-partition
1581- /// requirement on it.
1578+ /// Rows spread round-robin across `nparts` partitions, with any partition that draws no row
1579+ /// left holding a single empty batch. With `stagger_seed`, each partition's rows are cut into
1580+ /// random-size batches, empty ones included, by [`stagger_batch_with_seed`]; without it, one
1581+ /// row per batch. Used for the streamed side throughout, and for the buffered side of the
1582+ /// right existence joins, which place no single-partition requirement on it.
15821583fn pwmj_parts_exec < K : PwmjFuzzKey > (
15831584 ids : & [ i32 ] ,
15841585 keys : & [ Option < K > ] ,
15851586 nparts : usize ,
1587+ stagger_seed : Option < u64 > ,
15861588) -> Arc < dyn ExecutionPlan > {
15871589 let nparts = nparts. max ( 1 ) ;
1588- let mut partitions : Vec < Vec < RecordBatch > > = vec ! [ Vec :: new ( ) ; nparts] ;
1590+ let mut part_rows : Vec < ( Vec < i32 > , Vec < Option < K > > ) > = vec ! [ Default :: default ( ) ; nparts] ;
15891591 for ( row, ( & id, key) ) in ids. iter ( ) . zip ( keys. iter ( ) ) . enumerate ( ) {
1590- partitions[ row % nparts] . push ( pwmj_kv_batch ( & [ id] , & [ * key] ) ) ;
1591- }
1592- for p in partitions. iter_mut ( ) {
1593- if p. is_empty ( ) {
1594- p. push ( pwmj_kv_batch :: < K > ( & [ ] , & [ ] ) ) ;
1595- }
1596- }
1592+ part_rows[ row % nparts] . 0 . push ( id) ;
1593+ part_rows[ row % nparts] . 1 . push ( * key) ;
1594+ }
1595+ let partitions: Vec < Vec < RecordBatch > > = part_rows
1596+ . iter ( )
1597+ . enumerate ( )
1598+ . map ( |( part, ( ids, keys) ) | match stagger_seed {
1599+ // `stagger_batch_with_seed` panics on a batch with no rows.
1600+ _ if ids. is_empty ( ) => vec ! [ pwmj_kv_batch:: <K >( & [ ] , & [ ] ) ] ,
1601+ Some ( seed) => {
1602+ stagger_batch_with_seed ( pwmj_kv_batch ( ids, keys) , seed + part as u64 )
1603+ }
1604+ None => ids
1605+ . iter ( )
1606+ . zip ( keys)
1607+ . map ( |( & id, key) | pwmj_kv_batch ( & [ id] , & [ * key] ) )
1608+ . collect ( ) ,
1609+ } )
1610+ . collect ( ) ;
15971611 MemorySourceConfig :: try_new_exec ( & partitions, pwmj_kv_schema :: < K > ( ) , None ) . unwrap ( )
15981612}
15991613
@@ -1820,6 +1834,9 @@ async fn run_pwmj_fuzz<K: PwmjFuzzKey + std::fmt::Debug>(task_ctx: &Arc<TaskCont
18201834 let left_keys = gen_keys ( left_len, & mut rng) ;
18211835 let right_ids: Vec < i32 > = ( 0 ..right_len as i32 ) . collect ( ) ;
18221836 let right_keys = gen_keys ( right_len, & mut rng) ;
1837+ // Staggered streamed batches mix rows that can and cannot match, so the classic joins
1838+ // set some aside before the sort and map the rest back to their rows.
1839+ let stagger_seed = Some ( seed) ;
18231840
18241841 for op in ops {
18251842 for join_type in join_types {
@@ -1828,7 +1845,7 @@ async fn run_pwmj_fuzz<K: PwmjFuzzKey + std::fmt::Debug>(task_ctx: &Arc<TaskCont
18281845 // single-partition shape below never reaches.
18291846 let buffered = match join_type {
18301847 JoinType :: RightSemi | JoinType :: RightAnti | JoinType :: RightMark => {
1831- pwmj_parts_exec ( & left_ids, & left_keys, buffered_nparts)
1848+ pwmj_parts_exec ( & left_ids, & left_keys, buffered_nparts, None )
18321849 }
18331850 _ => pwmj_single_exec ( & left_ids, & left_keys) ,
18341851 } ;
@@ -1840,7 +1857,12 @@ async fn run_pwmj_fuzz<K: PwmjFuzzKey + std::fmt::Debug>(task_ctx: &Arc<TaskCont
18401857 let got = pwmj_collect_mark_pairs (
18411858 pwmj_plan (
18421859 buffered,
1843- pwmj_parts_exec ( & right_ids, & right_keys, nparts) ,
1860+ pwmj_parts_exec (
1861+ & right_ids,
1862+ & right_keys,
1863+ nparts,
1864+ stagger_seed,
1865+ ) ,
18441866 op,
18451867 join_type,
18461868 ) ,
@@ -1872,7 +1894,7 @@ async fn run_pwmj_fuzz<K: PwmjFuzzKey + std::fmt::Debug>(task_ctx: &Arc<TaskCont
18721894 let got = pwmj_collect_id_pairs (
18731895 pwmj_plan (
18741896 buffered,
1875- pwmj_parts_exec ( & right_ids, & right_keys, nparts) ,
1897+ pwmj_parts_exec ( & right_ids, & right_keys, nparts, stagger_seed ) ,
18761898 op,
18771899 join_type,
18781900 ) ,
@@ -2000,7 +2022,12 @@ async fn pwmj_watermark_nulls_duplicates_multi_partition() {
20002022 let got = pwmj_collect_id_pairs (
20012023 pwmj_plan (
20022024 pwmj_single_exec ( & left_ids, & left_keys) ,
2003- pwmj_parts_exec ( & case. streamed_ids , & case. streamed_keys , nparts) ,
2025+ pwmj_parts_exec (
2026+ & case. streamed_ids ,
2027+ & case. streamed_keys ,
2028+ nparts,
2029+ None ,
2030+ ) ,
20042031 Operator :: Lt ,
20052032 join_type,
20062033 ) ,
0 commit comments