@@ -4832,6 +4832,27 @@ mod tests {
48324832 rows : usize ,
48334833 payload_bytes : usize ,
48344834 ) -> RecordBatch {
4835+ payload_rows_from ( first, & vec ! [ region; rows] , payload_bytes)
4836+ }
4837+
4838+ /// [`round_robin_batch_from`]'s rows, each with its own `payload_bytes`-byte payload.
4839+ fn round_robin_payload_batch_from (
4840+ first : usize ,
4841+ rows : usize ,
4842+ partitions : usize ,
4843+ payload_bytes : usize ,
4844+ ) -> RecordBatch {
4845+ let regions: Vec < String > = ( 0 ..rows)
4846+ . map ( |row| format ! ( "r{}" , row % partitions) )
4847+ . collect ( ) ;
4848+ let regions: Vec < & str > = regions. iter ( ) . map ( String :: as_str) . collect ( ) ;
4849+ payload_rows_from ( first, & regions, payload_bytes)
4850+ }
4851+
4852+ /// One row for each of `regions`, numbered from `first`, each with its own
4853+ /// `payload_bytes`-byte payload.
4854+ fn payload_rows_from ( first : usize , regions : & [ & str ] , payload_bytes : usize ) -> RecordBatch {
4855+ let rows = regions. len ( ) ;
48354856 let ids: Vec < i32 > = ( first..first + rows) . map ( |id| id as i32 ) . collect ( ) ;
48364857 let payloads: Vec < String > = ( first..first + rows)
48374858 . map ( |id| format ! ( "{id:0>payload_bytes$}" ) )
@@ -4844,7 +4865,7 @@ mod tests {
48444865 ] ) ) ,
48454866 vec ! [
48464867 Arc :: new( Int32Array :: from( ids) ) ,
4847- Arc :: new( StringArray :: from( vec! [ region ; rows ] ) ) ,
4868+ Arc :: new( StringArray :: from( regions . to_vec ( ) ) ) ,
48484869 Arc :: new( StringArray :: from( payloads) ) ,
48494870 ] ,
48504871 )
@@ -5114,29 +5135,31 @@ mod tests {
51145135 rows
51155136 }
51165137
5117- /// Writes `batches` into a fanout table twice: with a pool to spare, then with half the
5118- /// most that first write reserved, which it can only fit by closing partitions early.
5119- /// Returns both writes' data files and the second write's directory, after checking that
5120- /// the second fit and gave its reservation back.
5121- async fn fanout_write_in_half_the_pool (
5138+ /// Writes `batches` into a fanout table of `schema` twice: with a pool to spare, then with
5139+ /// a pool of the most that first write reserved divided by `divisor`, which it can only
5140+ /// fit by closing partitions early. Returns both writes' data files and the second write's
5141+ /// directory, after checking that the second fit and gave its reservation back.
5142+ async fn fanout_write_in_part_of_the_pool (
5143+ divisor : usize ,
5144+ schema : Schema ,
51225145 batches : impl Fn ( ) -> Vec < RecordBatch > ,
51235146 properties : impl Fn ( ) -> WriterProperties ,
51245147 ) -> ( Vec < DataFile > , Vec < DataFile > , TempDir ) {
51255148 let roomy = Arc :: new ( PeakMemoryPool :: new ( usize:: MAX ) ) ;
51265149 let ( _dir, written) = write_reserving_from (
51275150 & roomy,
5128- iceberg_user_schema ( ) ,
5151+ schema . clone ( ) ,
51295152 ProtoIcebergWriterMode :: IcebergWriterFanout ,
51305153 batches ( ) ,
51315154 properties ( ) ,
51325155 TARGET_FILE_SIZE ,
51335156 )
51345157 . await ;
51355158 let roomy_files = written. unwrap ( ) ;
5136- let tight = Arc :: new ( PeakMemoryPool :: new ( roomy. peak ( ) / 2 ) ) ;
5159+ let tight = Arc :: new ( PeakMemoryPool :: new ( roomy. peak ( ) / divisor ) ) ;
51375160 let ( dir, written) = write_reserving_from (
51385161 & tight,
5139- iceberg_user_schema ( ) ,
5162+ schema ,
51405163 ProtoIcebergWriterMode :: IcebergWriterFanout ,
51415164 batches ( ) ,
51425165 properties ( ) ,
@@ -5156,7 +5179,9 @@ mod tests {
51565179 // Four batches, each a unit for every one of 16 partitions, so a partition closed
51575180 // early gets more rows after it. One unit is one page, so every unit goes straight to
51585181 // its partition's file.
5159- let ( roomy, tight, dir) = fanout_write_in_half_the_pool (
5182+ let ( roomy, tight, dir) = fanout_write_in_part_of_the_pool (
5183+ 2 ,
5184+ iceberg_user_schema ( ) ,
51605185 || {
51615186 ( 0 ..4 )
51625187 . map ( |batch| {
@@ -5182,14 +5207,48 @@ mod tests {
51825207 /// file is open when the pool refuses: the write only fits by writing partitions out.
51835208 #[ tokio:: test]
51845209 async fn closing_a_partition_early_writes_out_the_rows_it_holds_back ( ) {
5185- let ( roomy, tight, _dir) = fanout_write_in_half_the_pool (
5210+ let ( roomy, tight, _dir) = fanout_write_in_part_of_the_pool (
5211+ 2 ,
5212+ iceberg_user_schema ( ) ,
51865213 || vec ! [ round_robin_batch( 16 * ROWS_DIVISOR , 16 ) ] ,
51875214 || WriterProperties :: builder ( ) . build ( ) ,
51885215 )
51895216 . await ;
51905217 assert_eq ! ( record_counts( & tight) , record_counts( & roomy) ) ;
51915218 }
51925219
5220+ /// Closing the partitions holding the most frees the most memory per file closed, so a
5221+ /// write short of memory closes few files. Here 16 or 64 partitions take turns through 32
5222+ /// batches, so every partition keeps getting rows, and the pool grants an eighth of what
5223+ /// the write needs. The write ends with between four and five files per partition,
5224+ /// however many partitions it has. Closing every partition at once would leave seven, and
5225+ /// closing them in the order they were first seen, or smallest first, seventeen.
5226+ #[ tokio:: test]
5227+ async fn a_fanout_write_short_of_memory_closes_the_partitions_holding_the_most ( ) {
5228+ for partitions in [ 16 , 64 ] {
5229+ let ( roomy, tight, _dir) = fanout_write_in_part_of_the_pool (
5230+ 8 ,
5231+ iceberg_payload_schema ( ) ,
5232+ || {
5233+ ( 0 ..32 )
5234+ . map ( |batch| {
5235+ round_robin_payload_batch_from ( batch * 1024 , 1024 , partitions, 100 )
5236+ } )
5237+ . collect ( )
5238+ } ,
5239+ || WriterProperties :: builder ( ) . build ( ) ,
5240+ )
5241+ . await ;
5242+ assert_eq ! ( roomy. len( ) , partitions) ;
5243+ assert_eq ! ( rows_per_partition( & tight) , rows_per_partition( & roomy) ) ;
5244+ assert ! (
5245+ tight. len( ) < 6 * partitions,
5246+ "{} files for {partitions} partitions" ,
5247+ tight. len( )
5248+ ) ;
5249+ }
5250+ }
5251+
51935252 /// A write whose open file outgrows what the pool grants, with no partition it can close
51945253 /// early, fails with the pool's error rather than holding memory nothing accounts for,
51955254 /// and deletes the files it had opened. An unpartitioned write keeps its one file open.
0 commit comments