@@ -24,7 +24,9 @@ use std::sync::{Arc, LazyLock};
2424#[ cfg( feature = "extended_tests" ) ]
2525mod memory_limit_validation;
2626mod repartition_mem_limit;
27- use arrow:: array:: { ArrayRef , DictionaryArray , Int32Array , RecordBatch , StringViewArray } ;
27+ use arrow:: array:: {
28+ ArrayRef , DictionaryArray , Int32Array , Int64Array , RecordBatch , StringViewArray ,
29+ } ;
2830use arrow:: compute:: SortOptions ;
2931use arrow:: datatypes:: { Int32Type , SchemaRef } ;
3032use arrow_schema:: { DataType , Field , Schema } ;
@@ -56,6 +58,7 @@ use datafusion_physical_plan::collect as collect_batches;
5658use datafusion_physical_plan:: common:: collect;
5759use datafusion_physical_plan:: spill:: get_record_batch_memory_size;
5860use rand:: Rng ;
61+ use std:: collections:: HashSet ;
5962use test_utils:: AccessLogGenerator ;
6063
6164use async_trait:: async_trait;
@@ -1172,3 +1175,131 @@ impl TableProvider for SortedTableProvider {
11721175 Ok ( DataSourceExec :: from_data_source ( mem_conf) )
11731176 }
11741177}
1178+
1179+ // ============================================================================
1180+ // Regression tests for https://github.com/apache/datafusion/issues/20724
1181+ //
1182+ // When hash aggregation spills and switches to streaming merge,
1183+ // `group_values` must be recreated with the streaming variant.
1184+ // Otherwise `vectorized_intern` can produce non-monotonic group indices
1185+ // under hash collisions, causing `GroupOrderingFull` to prematurely
1186+ // emit groups → duplicate keys in output.
1187+ // ============================================================================
1188+
1189+ /// Helper: set up a session that forces spilling during aggregation.
1190+ async fn setup_spill_agg_context (
1191+ memory_limit : usize ,
1192+ batch_size : usize ,
1193+ ) -> Result < SessionContext > {
1194+ let disk_manager = DiskManagerBuilder :: default ( )
1195+ . with_mode ( DiskManagerMode :: OsTmpDirectory )
1196+ . build ( ) ?;
1197+
1198+ let runtime = RuntimeEnvBuilder :: new ( )
1199+ . with_memory_pool ( Arc :: new ( FairSpillPool :: new ( memory_limit) ) )
1200+ . build_arc ( )
1201+ . unwrap ( ) ;
1202+
1203+ let runtime = Arc :: new ( RuntimeEnv {
1204+ memory_pool : runtime. memory_pool . clone ( ) ,
1205+ disk_manager : Arc :: new ( disk_manager) ,
1206+ cache_manager : runtime. cache_manager . clone ( ) ,
1207+ object_store_registry : runtime. object_store_registry . clone ( ) ,
1208+ } ) ;
1209+
1210+ let config = SessionConfig :: new ( )
1211+ . with_sort_spill_reservation_bytes ( 64 * 1024 )
1212+ . with_sort_in_place_threshold_bytes ( 0 )
1213+ . with_spill_compression ( SpillCompression :: Uncompressed )
1214+ . with_batch_size ( batch_size)
1215+ . with_target_partitions ( 1 ) ;
1216+
1217+ Ok ( SessionContext :: new_with_config_rt ( config, runtime) )
1218+ }
1219+
1220+ /// Regression test for https://github.com/apache/datafusion/issues/20724
1221+ ///
1222+ /// When hash aggregation spills and switches to streaming merge,
1223+ /// `group_values` (GroupValuesColumn<false>) is not recreated with the
1224+ /// streaming variant (<true>). This means `vectorized_intern` is used
1225+ /// post-spill, which can produce non-monotonic group indices under hash
1226+ /// collisions, causing `GroupOrderingFull` to prematurely emit groups
1227+ /// and create duplicate keys in the output.
1228+ ///
1229+ /// Requirements to trigger:
1230+ /// - Two-column GROUP BY → forces `GroupValuesColumn` (not `GroupValuesPrimitive`)
1231+ /// - `force_hash_partial_collisions` feature → truncated hashes create the mix
1232+ /// of colliding/non-colliding keys needed for non-monotonic indices
1233+ /// - `batch_size=50` → not a multiple of rows-per-group in the merged stream,
1234+ /// so groups span batch boundaries and premature emission causes duplicates
1235+ #[ tokio:: test]
1236+ async fn test_no_duplicate_groups_after_spill ( ) -> Result < ( ) > {
1237+ let num_keys: i64 = 5000 ;
1238+ let rows_per_key: i64 = 4 ;
1239+ let total_rows = ( num_keys * rows_per_key) as usize ;
1240+
1241+ let schema = Arc :: new ( Schema :: new ( vec ! [
1242+ Field :: new( "key_a" , DataType :: Int64 , false ) ,
1243+ Field :: new( "key_b" , DataType :: Int64 , false ) ,
1244+ Field :: new( "value" , DataType :: Int64 , false ) ,
1245+ ] ) ) ;
1246+
1247+ let mut keys_a = Vec :: with_capacity ( total_rows) ;
1248+ let mut keys_b = Vec :: with_capacity ( total_rows) ;
1249+ let mut vals = Vec :: with_capacity ( total_rows) ;
1250+ for r in 0 ..rows_per_key {
1251+ for k in 0 ..num_keys {
1252+ keys_a. push ( k / 100 ) ;
1253+ keys_b. push ( k % 100 ) ;
1254+ vals. push ( r * num_keys + k) ;
1255+ }
1256+ }
1257+
1258+ let mut batches = Vec :: new ( ) ;
1259+ for start in ( 0 ..total_rows) . step_by ( 500 ) {
1260+ let end = ( start + 500 ) . min ( total_rows) ;
1261+ batches. push ( RecordBatch :: try_new (
1262+ Arc :: clone ( & schema) ,
1263+ vec ! [
1264+ Arc :: new( Int64Array :: from( keys_a[ start..end] . to_vec( ) ) ) ,
1265+ Arc :: new( Int64Array :: from( keys_b[ start..end] . to_vec( ) ) ) ,
1266+ Arc :: new( Int64Array :: from( vals[ start..end] . to_vec( ) ) ) ,
1267+ ] ,
1268+ ) ?) ;
1269+ }
1270+
1271+ let ctx = setup_spill_agg_context ( 128 * 1024 , 50 ) . await ?;
1272+ let table = MemTable :: try_new ( schema, vec ! [ batches] ) ?;
1273+ ctx. register_table ( "t" , Arc :: new ( table) ) ?;
1274+
1275+ let df = ctx
1276+ . sql ( "SELECT key_a, key_b, COUNT(*) as cnt FROM t GROUP BY key_a, key_b" )
1277+ . await ?;
1278+ let results =
1279+ collect_batches ( df. create_physical_plan ( ) . await ?, ctx. task_ctx ( ) ) . await ?;
1280+
1281+ let mut seen = HashSet :: new ( ) ;
1282+ for batch in & results {
1283+ let ka = batch
1284+ . column ( 0 )
1285+ . as_any ( )
1286+ . downcast_ref :: < Int64Array > ( )
1287+ . unwrap ( ) ;
1288+ let kb = batch
1289+ . column ( 1 )
1290+ . as_any ( )
1291+ . downcast_ref :: < Int64Array > ( )
1292+ . unwrap ( ) ;
1293+ for i in 0 ..batch. num_rows ( ) {
1294+ assert ! (
1295+ seen. insert( ( ka. value( i) , kb. value( i) ) ) ,
1296+ "DUPLICATE group key ({}, {}). \
1297+ Bug #20724: group_values not recreated for streaming merge.",
1298+ ka. value( i) ,
1299+ kb. value( i) ,
1300+ ) ;
1301+ }
1302+ }
1303+ assert_eq ! ( seen. len( ) , num_keys as usize ) ;
1304+ Ok ( ( ) )
1305+ }
0 commit comments