@@ -32,64 +32,6 @@ use std::time::Duration;
3232const MERGE_BYTES : usize = 64 * 1024 ;
3333const BUFFERED_BATCHES : usize = 9 ;
3434
35- #[ tokio:: test( flavor = "multi_thread" , worker_threads = 2 ) ]
36- async fn test_small_multicolumn_runs_coalesce_before_spilling ( ) -> Result < ( ) > {
37- let batches = ( 0 ..8 )
38- . map ( |batch| {
39- let rows = ( batch * 16 ..( batch + 1 ) * 16 ) . rev ( ) ;
40- RecordBatch :: try_from_iter ( [
41- (
42- "key" ,
43- Arc :: new ( Int64Array :: from_iter_values (
44- rows. clone ( ) . map ( |row| row % 5 ) ,
45- ) ) as ArrayRef ,
46- ) ,
47- (
48- "payload" ,
49- Arc :: new ( StringViewArray :: from_iter_values (
50- rows. map ( |row| format ! ( "unique-row-{row:08}" ) ) ,
51- ) ) as ArrayRef ,
52- ) ,
53- ] )
54- . map_err ( Into :: into)
55- } )
56- . collect :: < Result < Vec < _ > > > ( ) ?;
57- let schema = batches[ 0 ] . schema ( ) ;
58- let ordering: LexOrdering = [
59- PhysicalSortExpr :: new_default ( Arc :: new ( Column :: new ( "key" , 0 ) ) ) ,
60- PhysicalSortExpr :: new_default ( Arc :: new ( Column :: new ( "payload" , 1 ) ) ) ,
61- ]
62- . into ( ) ;
63- // The unique payload resolves ties, making the full output deterministic.
64- let expected = sort_batch ( & concat_batches ( & schema, & batches) ?, & ordering, None ) ?;
65- let pool = AdjustablePool :: new ( 256 * 1024 ) ;
66- let runtime = RuntimeEnvBuilder :: new ( )
67- . with_memory_pool ( Arc :: clone ( & pool) as Arc < dyn MemoryPool > )
68- . build_arc ( ) ?;
69- let mut sorter = ExternalSorter :: new (
70- 0 ,
71- Arc :: clone ( & schema) ,
72- ordering,
73- 64 ,
74- MERGE_BYTES ,
75- 0 ,
76- SpillCompression :: Uncompressed ,
77- & ExecutionPlanMetricsSet :: new ( ) ,
78- Arc :: clone ( & runtime) ,
79- ) ?;
80- for batch in batches {
81- sorter. insert_batch ( batch) . await ?;
82- }
83- sorter. sort_and_spill_in_mem_batches ( ) . await ?;
84- assert_eq ! ( sorter. spilled_rows( ) , 128 ) ;
85- let stream = sorter. sort ( ) . await ?;
86- drop ( sorter) ;
87- let output: Vec < RecordBatch > = stream. try_collect ( ) . await ?;
88- assert_eq ! ( concat_batches( & schema, & output) ?, expected) ;
89- assert_released ( & pool, & runtime) . await ;
90- Ok ( ( ) )
91- }
92-
9335#[ derive( Debug , Default ) ]
9436struct AllocationState {
9537 used : usize ,
0 commit comments