@@ -1359,8 +1359,9 @@ impl InnerWriter {
13591359 /// shares (`open_files`, see [`MeteredParquetWriter`]) and the rows waiting in their
13601360 /// [`PartitionFeed`]s. When the pool refuses, a fanout write writes out and closes its
13611361 /// partitions, the one holding the most first, until what is left fits, and this returns how
1362- /// many it closed. The unpartitioned and clustered writers hold one file open, at most a row
1363- /// group, and cannot close it early, so the pool's refusal stands.
1362+ /// many it closed. The unpartitioned and clustered writers hold one file open and cannot close
1363+ /// it early, so the pool's refusal stands. Any other error from a resize fails the task (see
1364+ /// [`is_refusal`]).
13641365 async fn reserve (
13651366 & mut self ,
13661367 reservation : & MemoryReservation ,
@@ -1370,7 +1371,8 @@ impl InnerWriter {
13701371 let mut pending = self . pending_bytes ( ) ;
13711372 let mut refused = match reservation. try_resize ( open_files. bytes ( ) + pending) {
13721373 Ok ( ( ) ) => return Ok ( 0 ) ,
1373- Err ( refused) => refused,
1374+ Err ( e) if is_refusal ( & e) => e,
1375+ Err ( e) => return Err ( e) ,
13741376 } ;
13751377 let InnerWriter :: Fanout ( fanout, _) = self else {
13761378 return Err ( refused) ;
@@ -1380,7 +1382,8 @@ impl InnerWriter {
13801382 fanout. close_early ( i, properties) . await ?;
13811383 match reservation. try_resize ( open_files. bytes ( ) + pending) {
13821384 Ok ( ( ) ) => return Ok ( closed + 1 ) ,
1383- Err ( e) => refused = e,
1385+ Err ( e) if is_refusal ( & e) => refused = e,
1386+ Err ( e) => return Err ( e) ,
13841387 }
13851388 }
13861389 // Not reached: everything a fanout write reserves belongs to one of its partitions, so
@@ -1471,6 +1474,16 @@ impl InnerWriter {
14711474
14721475// --- helpers -------------------------------------------------------------
14731476
1477+ /// Whether `e`, from resizing a reservation, is the pool refusing the memory, which every pool
1478+ /// reports as `ResourcesExhausted` and a fanout write answers by closing partitions early. A pool
1479+ /// backed by Spark can also fail a resize with an exception Spark threw: a spill of another
1480+ /// consumer that failed, or the `InterruptedException` of a task killed while it waited for
1481+ /// memory. That comes back as another kind of error and has to fail the task. Closing partitions
1482+ /// would swallow it, because once the resize is down to a shrink it no longer calls Spark.
1483+ fn is_refusal ( e : & DataFusionError ) -> bool {
1484+ matches ! ( e. find_root( ) , DataFusionError :: ResourcesExhausted ( _) )
1485+ }
1486+
14741487fn parse_iceberg_schema ( json : & str ) -> DFResult < IcebergSchemaRef > {
14751488 let schema: IcebergSchema = serde_json:: from_str ( json) . map_err ( |e| {
14761489 DataFusionError :: Internal ( format ! ( "Failed to parse iceberg schema JSON: {e}" ) )
@@ -4899,8 +4912,8 @@ mod tests {
48994912 /// mode is unpartitioned, reserving from `pool` the way `IcebergWriteExec::execute`
49004913 /// reserves from the task's pool, and counting into `metrics`. A successful write's files
49014914 /// are left for the caller to inspect.
4902- async fn write_reserving_from (
4903- pool : & Arc < PeakMemoryPool > ,
4915+ async fn write_reserving_from < P : MemoryPool > (
4916+ pool : & Arc < P > ,
49044917 metrics : WriteMetrics ,
49054918 schema : Schema ,
49064919 writer_mode : ProtoIcebergWriterMode ,
@@ -4926,7 +4939,7 @@ mod tests {
49264939 ) ,
49274940 target_file_size_bytes,
49284941 ) ;
4929- let pool: Arc < dyn MemoryPool > = Arc :: < PeakMemoryPool > :: clone ( pool) ;
4942+ let pool: Arc < dyn MemoryPool > = Arc :: < P > :: clone ( pool) ;
49304943 let reservation = MemoryConsumer :: new ( "IcebergWriteExec[0]" ) . register ( & pool) ;
49314944 let written = run_write_task (
49324945 input_stream ( batches) ,
@@ -5349,6 +5362,111 @@ mod tests {
53495362 assert_eq ! ( tight. reserved( ) , 0 , "the failed write kept a reservation" ) ;
53505363 }
53515364
5365+ /// A pool with room to spare that refuses the first `refusals` grows it is asked for, then
5366+ /// fails the next with `error`, the way a pool backed by Spark passes on an exception
5367+ /// Spark threw, and grants every grow after that.
5368+ #[ derive( Debug ) ]
5369+ struct FailingPool {
5370+ inner : UnboundedMemoryPool ,
5371+ refusals : AtomicUsize ,
5372+ error : Mutex < Option < DataFusionError > > ,
5373+ }
5374+
5375+ impl FailingPool {
5376+ fn new ( refusals : usize , error : DataFusionError ) -> Self {
5377+ Self {
5378+ inner : UnboundedMemoryPool :: default ( ) ,
5379+ refusals : AtomicUsize :: new ( refusals) ,
5380+ error : Mutex :: new ( Some ( error) ) ,
5381+ }
5382+ }
5383+ }
5384+
5385+ impl fmt:: Display for FailingPool {
5386+ fn fmt ( & self , f : & mut fmt:: Formatter < ' _ > ) -> fmt:: Result {
5387+ write ! ( f, "FailingPool({})" , self . inner)
5388+ }
5389+ }
5390+
5391+ impl MemoryPool for FailingPool {
5392+ fn name ( & self ) -> & str {
5393+ "FailingPool"
5394+ }
5395+
5396+ fn grow ( & self , reservation : & MemoryReservation , additional : usize ) {
5397+ self . inner . grow ( reservation, additional) ;
5398+ }
5399+
5400+ fn shrink ( & self , reservation : & MemoryReservation , shrink : usize ) {
5401+ self . inner . shrink ( reservation, shrink) ;
5402+ }
5403+
5404+ fn try_grow ( & self , reservation : & MemoryReservation , additional : usize ) -> DFResult < ( ) > {
5405+ if self
5406+ . refusals
5407+ . try_update ( Ordering :: Relaxed , Ordering :: Relaxed , |left| {
5408+ left. checked_sub ( 1 )
5409+ } )
5410+ . is_ok ( )
5411+ {
5412+ return Err ( DataFusionError :: ResourcesExhausted ( format ! (
5413+ "refused {additional} bytes"
5414+ ) ) ) ;
5415+ }
5416+ if let Some ( error) = self . error . lock ( ) . unwrap ( ) . take ( ) {
5417+ return Err ( error) ;
5418+ }
5419+ self . inner . try_grow ( reservation, additional)
5420+ }
5421+
5422+ fn reserved ( & self ) -> usize {
5423+ self . inner . reserved ( )
5424+ }
5425+ }
5426+
5427+ /// Only the pool's refusal makes a fanout write close partitions early. Any other error
5428+ /// from a resize, such as an exception Spark threw because a spill failed or the task was
5429+ /// killed, fails the write, which deletes every file it created, the ones it closed early
5430+ /// included.
5431+ #[ tokio:: test]
5432+ async fn a_fanout_write_fails_on_a_resize_error_that_is_not_a_refusal ( ) {
5433+ for error in [
5434+ DataFusionError :: External ( "an exception Spark threw" . into ( ) ) ,
5435+ DataFusionError :: Execution ( "a JNI call that failed" . into ( ) ) ,
5436+ ] {
5437+ let message = error. to_string ( ) ;
5438+ // Two refusals close two partitions early, then the third resize fails.
5439+ let pool = Arc :: new ( FailingPool :: new ( 2 , error) ) ;
5440+ let ( dir, written) = write_reserving_from (
5441+ & pool,
5442+ WriteMetrics :: default ( ) ,
5443+ iceberg_user_schema ( ) ,
5444+ ProtoIcebergWriterMode :: IcebergWriterFanout ,
5445+ // A unit for each of 16 partitions, and one unit is one page, so every
5446+ // partition has a file open when the pool refuses.
5447+ vec ! [ round_robin_batch( 16 * ROWS_DIVISOR , 16 ) ] ,
5448+ WriterProperties :: builder ( )
5449+ . set_data_page_row_count_limit ( ROWS_DIVISOR )
5450+ . build ( ) ,
5451+ TARGET_FILE_SIZE ,
5452+ )
5453+ . await ;
5454+ let error =
5455+ written. expect_err ( "an error that is not a refusal must fail the write" ) ;
5456+ assert_eq ! ( error. to_string( ) , message) ;
5457+ assert_eq ! (
5458+ pool. refusals. load( Ordering :: Relaxed ) ,
5459+ 0 ,
5460+ "the write did not close partitions early before the error"
5461+ ) ;
5462+ assert_eq ! (
5463+ parquet_files_under( dir. path( ) ) ,
5464+ Vec :: <PathBuf >:: new( ) ,
5465+ "the failed write left files behind"
5466+ ) ;
5467+ }
5468+ }
5469+
53525470 /// A task that fails mid-write drops its open files without closing them, so a file's
53535471 /// share has to come back either way.
53545472 #[ tokio:: test]
0 commit comments