@@ -330,10 +330,11 @@ async fn run_write_task(
330330 None ,
331331 DataFileFormat :: Parquet ,
332332 ) ;
333+ let target_file_size_bytes = common. target_file_size_bytes as usize ;
333334 let parquet_builder = ParquetWriterBuilder :: new ( writer_properties, Arc :: clone ( & iceberg_schema) ) ;
334335 let rolling_builder = RollingFileWriterBuilder :: new (
335336 parquet_builder,
336- common . target_file_size_bytes as usize ,
337+ target_file_size_bytes,
337338 file_io,
338339 location_generator,
339340 file_name_generator,
@@ -343,14 +344,16 @@ async fn run_write_task(
343344 // Build the field-id-decorated target schema once per task; every batch is cast against it.
344345 let target_schema =
345346 Arc :: new ( iceberg:: arrow:: schema_to_arrow_schema ( & iceberg_schema) . map_err ( iceberg_err) ?) ;
346- let slicer = RowSlicer :: for_schema ( & target_schema) ;
347+ let policy = PacingPolicy {
348+ slicer : RowSlicer :: for_schema ( & target_schema) ,
349+ target_file_size_bytes,
350+ } ;
347351
348352 let unpartitioned = partition_spec. is_unpartitioned ( ) ;
349353 let mut writer = match ( unpartitioned, writer_mode) {
350- ( true , ProtoIcebergWriterMode :: IcebergWriterUnpartitioned ) => InnerWriter :: Unpartitioned (
351- UnpartitionedWriter :: new ( data_file_builder) ,
352- RowPacer :: new ( slicer) ,
353- ) ,
354+ ( true , ProtoIcebergWriterMode :: IcebergWriterUnpartitioned ) => {
355+ InnerWriter :: Unpartitioned ( UnpartitionedWriter :: new ( data_file_builder) , policy. pacer ( ) )
356+ }
354357 ( false , ProtoIcebergWriterMode :: IcebergWriterFanout ) => {
355358 InnerWriter :: Fanout ( FanoutWriter :: new ( data_file_builder) , HashMap :: new ( ) )
356359 }
@@ -369,7 +372,7 @@ async fn run_write_task(
369372 InnerWriter :: Clustered ( ..) => Some ( ClusteredBatchSplitter :: try_new (
370373 Arc :: clone ( & partition_spec) ,
371374 Arc :: clone ( & iceberg_schema) ,
372- slicer,
375+ policy . slicer ,
373376 ) ?) ,
374377 _ => None ,
375378 } ;
@@ -390,7 +393,7 @@ async fn run_write_task(
390393 writer
391394 . write (
392395 decorated,
393- slicer ,
396+ policy ,
394397 fanout_splitter. as_ref ( ) ,
395398 clustered_splitter. as_ref ( ) ,
396399 )
@@ -427,7 +430,7 @@ impl InnerWriter {
427430 async fn write (
428431 & mut self ,
429432 batch : RecordBatch ,
430- slicer : RowSlicer ,
433+ policy : PacingPolicy ,
431434 fanout_splitter : Option < & RecordBatchPartitionSplitter > ,
432435 clustered_splitter : Option < & ClusteredBatchSplitter > ,
433436 ) -> DFResult < ( ) > {
@@ -447,7 +450,7 @@ impl InnerWriter {
447450 for ( key, part) in parts {
448451 let ( _, pacer) = pacers
449452 . entry ( key. data ( ) . clone ( ) )
450- . or_insert_with ( || ( key. clone ( ) , RowPacer :: new ( slicer ) ) ) ;
453+ . or_insert_with ( || ( key. clone ( ) , policy . pacer ( ) ) ) ;
451454 for unit in pacer. push ( part) ? {
452455 w. write ( key. clone ( ) , unit) . await . map_err ( iceberg_err) ?;
453456 }
@@ -468,8 +471,7 @@ impl InnerWriter {
468471 w. write ( open, rest) . await . map_err ( iceberg_err) ?;
469472 }
470473 }
471- let ( _, pacer) =
472- live. get_or_insert_with ( || ( key. clone ( ) , RowPacer :: new ( slicer) ) ) ;
474+ let ( _, pacer) = live. get_or_insert_with ( || ( key. clone ( ) , policy. pacer ( ) ) ) ;
473475 for unit in pacer. push ( part) ? {
474476 w. write ( key. clone ( ) , unit) . await . map_err ( iceberg_err) ?;
475477 }
@@ -711,27 +713,65 @@ fn gather_rows(batch: &RecordBatch, offset: usize, len: usize) -> DFResult<Recor
711713 arrow:: compute:: take_record_batch ( batch, & indices) . map_err ( DataFusionError :: from)
712714}
713715
716+ /// How this task cuts rows and when the rolling writer's size check can matter: everything needed
717+ /// to open a [`RowPacer`] for one more file.
718+ #[ derive( Clone , Copy ) ]
719+ struct PacingPolicy {
720+ slicer : RowSlicer ,
721+ target_file_size_bytes : usize ,
722+ }
723+
724+ impl PacingPolicy {
725+ fn pacer ( & self ) -> RowPacer {
726+ RowPacer :: new ( * self )
727+ }
728+ }
729+
714730/// Hands one iceberg-rust writer exactly [`ROWS_DIVISOR`] rows at a time, so the rolling writer
715- /// underneath re-checks the target file size on the same row boundaries iceberg-java's
716- /// `RollingFileWriter` checks on -- multiples of 1000 rows since the current file opened -- however
717- /// Spark happened to batch the rows.
731+ /// underneath only ever gets to roll on the row boundaries iceberg-java's `RollingFileWriter` rolls
732+ /// on -- multiples of 1000 rows since the current file opened -- however Spark happened to batch
733+ /// the rows.
718734///
719735/// Pacing, rather than just cutting each batch up, is what makes the roll point independent of the
720736/// batch shape: a task fed 800-row batches would otherwise be offered a boundary every 800 rows
721737/// and roll into 800-row files where the JVM writer produces 1000-row ones. Rows left over from a
722738/// batch wait here for the rows that complete their unit, so at most `ROWS_DIVISOR - 1` rows per
723739/// open file are held back.
740+ ///
741+ /// Pacing costs one writer call per 1000 rows instead of one per batch, which is measurable on a
742+ /// wide schema, so it stays off until it can change an outcome: see `pacing`.
724743struct RowPacer {
725- slicer : RowSlicer ,
726- /// Rows handed in but not yet handed over; fewer than [`ROWS_DIVISOR`] in total.
744+ policy : PacingPolicy ,
745+ /// Whether rows are being paced yet. While off, batches go over whole, because a batch
746+ /// boundary the writer sees can only matter if its size check might fire there -- and it cannot
747+ /// while the file is still smaller than the target.
748+ ///
749+ /// It never goes back off. A roll starts the file's size over, but the pacer cannot see rolls,
750+ /// so from the first file that reaches the target it paces for the rest of the task.
751+ pacing : bool ,
752+ /// What has been handed over so far, as an upper bound on what the open file holds: Arrow's
753+ /// in-memory footprint of a batch (allocated capacity, so it over-counts if anything) bounds
754+ /// what parquet writes for the same rows, since the writer encodes and compresses them. If that
755+ /// ever failed to hold for some schema, the only consequence is a roll up to `ROWS_DIVISOR`
756+ /// rows late -- the same order as the divergence the two size estimates already allow.
757+ handed_bytes : usize ,
758+ /// Rows of the current 1000-row block already handed over. Non-zero only for the first block
759+ /// after pacing turns on, where it is what the whole batches before it left in the open file --
760+ /// which is all of them, since nothing could have rolled yet. Carrying it over is what keeps
761+ /// the blocks aligned to the file's own row count rather than to where pacing started.
762+ block_rows : usize ,
763+ /// Rows handed in but not yet handed over; `block_rows + pending_rows < ROWS_DIVISOR`.
727764 pending : Vec < RecordBatch > ,
728765 pending_rows : usize ,
729766}
730767
731768impl RowPacer {
732- fn new ( slicer : RowSlicer ) -> Self {
769+ fn new ( policy : PacingPolicy ) -> Self {
733770 Self {
734- slicer,
771+ policy,
772+ pacing : false ,
773+ handed_bytes : 0 ,
774+ block_rows : 0 ,
735775 pending : Vec :: new ( ) ,
736776 pending_rows : 0 ,
737777 }
@@ -741,20 +781,29 @@ impl RowPacer {
741781 /// held for the next call.
742782 fn push ( & mut self , batch : RecordBatch ) -> DFResult < Vec < RecordBatch > > {
743783 debug_assert ! (
744- self . pending_rows < ROWS_DIVISOR ,
784+ self . block_rows + self . pending_rows < ROWS_DIVISOR ,
745785 "a complete unit was left pending"
746786 ) ;
747787 let rows = batch. num_rows ( ) ;
788+ if !self . pacing {
789+ self . handed_bytes += batch. get_array_memory_size ( ) ;
790+ if self . handed_bytes < self . policy . target_file_size_bytes {
791+ self . block_rows = ( self . block_rows + rows) % ROWS_DIVISOR ;
792+ return Ok ( vec ! [ batch] ) ;
793+ }
794+ self . pacing = true ;
795+ }
748796 let mut units = Vec :: with_capacity ( ( self . pending_rows + rows) / ROWS_DIVISOR ) ;
749797 let mut offset = 0 ;
750- while self . pending_rows + ( rows - offset) >= ROWS_DIVISOR {
751- let len = ROWS_DIVISOR - self . pending_rows ;
752- let piece = self . slicer . slice ( & batch, offset, len) ?;
798+ while self . block_rows + self . pending_rows + ( rows - offset) >= ROWS_DIVISOR {
799+ let len = ROWS_DIVISOR - self . block_rows - self . pending_rows ;
800+ let piece = self . policy . slicer . slice ( & batch, offset, len) ?;
753801 offset += len;
754802 units. push ( self . drain_pending_into ( piece) ?) ;
803+ self . block_rows = 0 ;
755804 }
756805 if offset < rows {
757- let rest = self . slicer . detach ( & batch, offset, rows - offset) ?;
806+ let rest = self . policy . slicer . detach ( & batch, offset, rows - offset) ?;
758807 self . pending_rows += rest. num_rows ( ) ;
759808 self . pending . push ( rest) ;
760809 }
@@ -769,8 +818,7 @@ impl RowPacer {
769818 self . concat_pending ( ) . map ( Some )
770819 }
771820
772- /// Completes a unit from the held rows plus `piece`, which together are exactly
773- /// [`ROWS_DIVISOR`] rows.
821+ /// Completes a unit from the held rows plus `piece`, which together fill the current block.
774822 fn drain_pending_into ( & mut self , piece : RecordBatch ) -> DFResult < RecordBatch > {
775823 if self . pending . is_empty ( ) {
776824 debug_assert_eq ! ( self . pending_rows, 0 , "no held rows but a non-zero count" ) ;
@@ -1167,10 +1215,18 @@ mod tests {
11671215 int_batch_from ( 0 , rows)
11681216 }
11691217
1170- /// Paces `batches` and returns the row count of every unit handed over, the trailing flush
1171- /// included.
1172- fn paced_rows ( gather : bool , batches : & [ usize ] ) -> Vec < usize > {
1173- let mut pacer = RowPacer :: new ( RowSlicer { gather } ) ;
1218+ /// A policy that paces from the first row: any batch reaches a one-byte target.
1219+ fn always_pacing ( gather : bool ) -> PacingPolicy {
1220+ PacingPolicy {
1221+ slicer : RowSlicer { gather } ,
1222+ target_file_size_bytes : 1 ,
1223+ }
1224+ }
1225+
1226+ /// Hands `batches` to a pacer and returns the row count of every unit handed over, the
1227+ /// trailing flush included.
1228+ fn units_of ( policy : PacingPolicy , batches : & [ usize ] ) -> Vec < usize > {
1229+ let mut pacer = policy. pacer ( ) ;
11741230 let mut rows = Vec :: new ( ) ;
11751231 let mut first = 0 ;
11761232 for & batch_rows in batches {
@@ -1183,6 +1239,10 @@ mod tests {
11831239 rows
11841240 }
11851241
1242+ fn paced_rows ( gather : bool , batches : & [ usize ] ) -> Vec < usize > {
1243+ units_of ( always_pacing ( gather) , batches)
1244+ }
1245+
11861246 #[ test]
11871247 fn pacer_hands_over_whole_units_whatever_the_batch_shape ( ) {
11881248 for gather in [ false , true ] {
@@ -1223,11 +1283,40 @@ mod tests {
12231283 }
12241284 }
12251285
1286+ /// Pacing is skipped while the open file cannot have reached the target, because a boundary the
1287+ /// writer sees there can never roll. Batches then go over exactly as they arrive.
1288+ #[ test]
1289+ fn pacer_hands_over_whole_batches_below_the_target ( ) {
1290+ let policy = PacingPolicy {
1291+ slicer : RowSlicer { gather : false } ,
1292+ target_file_size_bytes : usize:: MAX ,
1293+ } ;
1294+ assert_eq ! ( units_of( policy, & [ 800 ; 5 ] ) , vec![ 800 ; 5 ] ) ;
1295+ assert_eq ! ( units_of( policy, & [ 8192 , 1808 ] ) , vec![ 8192 , 1808 ] ) ;
1296+ }
1297+
1298+ /// Once the target is in reach, the blocks are aligned to the open file's row count -- not to
1299+ /// where pacing happened to start -- so the writer still only rolls on a 1000-row boundary of
1300+ /// the file. Here two 800-row batches go over whole and the third turns pacing on with 1600
1301+ /// rows already in the file, so the first unit is the 400 rows that complete that block.
1302+ #[ test]
1303+ fn pacer_aligns_the_first_paced_block_to_the_rows_already_written ( ) {
1304+ let per_batch = int_batch ( 800 ) . get_array_memory_size ( ) ;
1305+ let policy = PacingPolicy {
1306+ slicer : RowSlicer { gather : false } ,
1307+ target_file_size_bytes : 2 * per_batch + 1 ,
1308+ } ;
1309+ assert_eq ! (
1310+ units_of( policy, & [ 800 ; 5 ] ) ,
1311+ vec![ 800 , 800 , 400 , ROWS_DIVISOR , ROWS_DIVISOR ]
1312+ ) ;
1313+ }
1314+
12261315 #[ test]
12271316 fn pacer_preserves_every_row_in_order ( ) {
12281317 for gather in [ false , true ] {
12291318 let batches: Vec < usize > = vec ! [ 800 , 0 , 1 , 2500 , 999 ] ;
1230- let mut pacer = RowPacer :: new ( RowSlicer { gather } ) ;
1319+ let mut pacer = always_pacing ( gather) . pacer ( ) ;
12311320 let mut units = Vec :: new ( ) ;
12321321 let mut first = 0 ;
12331322 for batch_rows in & batches {
@@ -1678,13 +1767,14 @@ mod tests {
16781767 assert_eq ! ( rows, vec![ 1000 , 500 , 1000 , 500 ] ) ;
16791768 }
16801769
1681- /// Cutting a batch into pieces must not disturb the NaN counts the JVM carries into the
1682- /// manifest. iceberg-rust's visitor reads list children through `values()`, which ignores
1683- /// a slice's offset window, so a zero-copy slice would report every NaN in the batch once
1684- /// per piece -- three pieces here, and a `nan_value_count` that reaches `record_count`
1685- /// makes Iceberg's metrics evaluator prune the file from ordinary comparisons.
1770+ /// Cutting a batch into units must not disturb the NaN counts the JVM carries into the
1771+ /// manifest. iceberg-rust's visitor reads list children through `values()`, which ignores a
1772+ /// slice's offset window, so a zero-copy slice would report every NaN in the whole batch for
1773+ /// each unit cut from it -- and a `nan_value_count` that reaches `record_count` makes
1774+ /// Iceberg's metrics evaluator prune the file from ordinary comparisons. The one-byte target
1775+ /// is what puts the cutting in play at all: below the target the batch goes over whole.
16861776 #[ tokio:: test]
1687- async fn nan_counts_stay_exact_when_a_list_of_double_is_cut_into_pieces ( ) {
1777+ async fn nan_counts_stay_exact_when_a_list_of_double_is_cut_into_units ( ) {
16881778 use arrow:: array:: { Array , ListArray } ;
16891779 use arrow:: datatypes:: Float64Type ;
16901780 use iceberg:: spec:: ListType ;
@@ -1714,14 +1804,17 @@ mod tests {
17141804 let spec = PartitionSpec :: builder ( Arc :: new ( schema. clone ( ) ) )
17151805 . build ( )
17161806 . unwrap ( ) ;
1717- let common = common (
1718- data_location,
1719- serde_json:: to_string ( & spec) . unwrap ( ) ,
1720- serde_json:: to_string ( & schema) . unwrap ( ) ,
1721- ProtoIcebergWriterMode :: IcebergWriterUnpartitioned ,
1807+ let common = with_target_file_size (
1808+ common (
1809+ data_location,
1810+ serde_json:: to_string ( & spec) . unwrap ( ) ,
1811+ serde_json:: to_string ( & schema) . unwrap ( ) ,
1812+ ProtoIcebergWriterMode :: IcebergWriterUnpartitioned ,
1813+ ) ,
1814+ 1 ,
17221815 ) ;
17231816
1724- // 2500 rows -> three pieces; one NaN in each piece .
1817+ // 2500 rows -> three units, one NaN in each.
17251818 let rows = 2 * ROWS_DIVISOR + 500 ;
17261819 let nan_rows = [ 7usize , ROWS_DIVISOR + 7 , 2 * ROWS_DIVISOR + 7 ] ;
17271820 let vals = ListArray :: from_iter_primitive :: < Float64Type , _ , _ > ( ( 0 ..rows) . map ( |row| {
@@ -1757,13 +1850,13 @@ mod tests {
17571850 . await
17581851 . unwrap ( ) ;
17591852
1760- assert_eq ! ( record_counts ( & data_files ) , vec! [ rows as u64 ] ) ;
1761- assert_eq ! (
1762- data_files[ 0 ] . nan_value_counts ( ) . get ( & 3 ) ,
1763- Some ( & ( nan_rows . len ( ) as u64 ) ) ,
1764- "nan counts: {:?}" ,
1765- data_files [ 0 ] . nan_value_counts ( )
1766- ) ;
1853+ // A unit per file, since the one-byte target rolls on every check.
1854+ assert_eq ! ( record_counts ( & data_files ) , vec! [ 1000 , 1000 , 500 ] ) ;
1855+ let counted : Vec < u64 > = data_files
1856+ . iter ( )
1857+ . map ( |file| file . nan_value_counts ( ) . get ( & 3 ) . copied ( ) . unwrap_or ( 0 ) )
1858+ . collect ( ) ;
1859+ assert_eq ! ( counted , vec! [ 1 ; nan_rows . len ( ) ] , "per-file NaN counts" ) ;
17671860 }
17681861
17691862 #[ tokio:: test]
0 commit comments