@@ -831,7 +831,7 @@ fn natural_exception(error: &CometError) -> JvmException<'static> {
831831 let exception = error. to_exception ( ) ;
832832 JvmException :: New {
833833 class : exception. class ,
834- message : with_spill_limit_hint ( exception. msg ) ,
834+ message : with_comet_config_hint ( exception. msg ) ,
835835 }
836836}
837837
@@ -844,11 +844,34 @@ const SPILL_LIMIT_HINT: &str = "Comet does not use `datafusion.runtime.*` settin
844844 the spill limit is `spark.comet.maxTempDirectorySize`. It applies to each Comet native plan \
845845 separately, so an executor can use several times that much disk.";
846846
847- fn with_spill_limit_hint ( message : String ) -> String {
848- if message. contains ( DATAFUSION_SPILL_LIMIT_KEY ) {
849- format ! ( "{message} {SPILL_LIMIT_HINT}" )
850- } else {
851- message
847+ /// The first config key DataFusion's external-sort OOM error tells users to change. Comet never
848+ /// uses it, and the sort is identified only by the `Context` string around `ResourcesExhausted`,
849+ /// so this quoted text is what to match on.
850+ const DATAFUSION_SORT_MEMORY_KEY : & str = "'datafusion.runtime.memory_limit'" ;
851+
852+ const SORT_MEMORY_HINT : & str = "Comet does not use `datafusion.runtime.memory_limit`, and in \
853+ production it keeps `datafusion.execution.sort_spill_reservation_bytes` at DataFusion's \
854+ default of 10 MiB. Native operators reserve memory from Spark's off-heap pool, sized by \
855+ `spark.memory.offHeap.size`, and each running task gets a share of it. Raise \
856+ `spark.memory.offHeap.size` or run fewer tasks per executor. See the Comet tuning guide: \
857+ https://datafusion.apache.org/comet/user-guide/latest/tuning.html";
858+
859+ /// DataFusion error text that names settings unavailable for production tuning in Comet,
860+ /// paired with the hint that points users at the Comet settings to change instead.
861+ const COMET_CONFIG_HINTS : & [ ( & str , & str ) ] = & [
862+ ( DATAFUSION_SPILL_LIMIT_KEY , SPILL_LIMIT_HINT ) ,
863+ ( DATAFUSION_SORT_MEMORY_KEY , SORT_MEMORY_HINT ) ,
864+ ] ;
865+
866+ /// Appends the matching hint on its own line: the DataFusion text it follows can end mid-sentence,
867+ /// for example with the `Reserved: N` of a memory pool refusal.
868+ fn with_comet_config_hint ( message : String ) -> String {
869+ match COMET_CONFIG_HINTS
870+ . iter ( )
871+ . find ( |( datafusion_key, _) | message. contains ( datafusion_key) )
872+ {
873+ Some ( ( _, hint) ) => format ! ( "{message}\n {hint}" ) ,
874+ None => message,
852875 }
853876}
854877
@@ -1566,7 +1589,7 @@ mod tests {
15661589 ] {
15671590 let error = CometError :: from ( error) ;
15681591 assert ! ( error. to_string( ) . contains( DATAFUSION_SPILL_LIMIT_KEY ) ) ;
1569- let expected = format ! ( "{error} {SPILL_LIMIT_HINT}" ) ;
1592+ let expected = format ! ( "{error}\n {SPILL_LIMIT_HINT}" ) ;
15701593 let ( _, class, message, _) = classify ( error) ;
15711594 assert_eq ! ( class, "org/apache/comet/CometNativeException" ) ;
15721595 assert_eq ! ( message, expected) ;
@@ -1578,6 +1601,76 @@ mod tests {
15781601 assert_eq ! ( message, expected) ;
15791602 }
15801603
1604+ /// DataFusion's external-sort OOM error, produced by a real `SortExec` whose memory pool
1605+ /// cannot hold a single batch, so a rewording in a DataFusion upgrade fails this test instead
1606+ /// of silently dropping the hint.
1607+ fn sort_oom_error ( ) -> DataFusionError {
1608+ use arrow:: array:: Int32Array ;
1609+ use arrow:: datatypes:: { DataType , Field , Schema } ;
1610+ use arrow:: record_batch:: RecordBatch ;
1611+ use datafusion:: datasource:: memory:: MemorySourceConfig ;
1612+ use datafusion:: execution:: memory_pool:: GreedyMemoryPool ;
1613+ use datafusion:: execution:: runtime_env:: RuntimeEnvBuilder ;
1614+ use datafusion:: execution:: TaskContext ;
1615+ use datafusion:: physical_expr:: expressions:: col;
1616+ use datafusion:: physical_expr:: { LexOrdering , PhysicalSortExpr } ;
1617+ use datafusion:: physical_plan:: sorts:: sort:: SortExec ;
1618+ use datafusion:: physical_plan:: ExecutionPlan ;
1619+ use futures:: StreamExt ;
1620+
1621+ let schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new( "a" , DataType :: Int32 , false ) ] ) ) ;
1622+ let batch = RecordBatch :: try_new (
1623+ Arc :: clone ( & schema) ,
1624+ vec ! [ Arc :: new( Int32Array :: from( vec![ 3 , 1 , 2 ] ) ) ] ,
1625+ )
1626+ . unwrap ( ) ;
1627+ let input =
1628+ MemorySourceConfig :: try_new_exec ( & [ vec ! [ batch] ] , Arc :: clone ( & schema) , None ) . unwrap ( ) ;
1629+ let ordering = LexOrdering :: new ( vec ! [ PhysicalSortExpr :: new_default(
1630+ col( "a" , & schema) . unwrap( ) ,
1631+ ) ] )
1632+ . unwrap ( ) ;
1633+ let sort = SortExec :: new ( ordering, input) ;
1634+ let runtime = RuntimeEnvBuilder :: new ( )
1635+ . with_memory_pool ( Arc :: new ( GreedyMemoryPool :: new ( 1 ) ) )
1636+ . build_arc ( )
1637+ . unwrap ( ) ;
1638+ let task_ctx = Arc :: new ( TaskContext :: default ( ) . with_runtime ( runtime) ) ;
1639+ let mut stream = sort. execute ( 0 , task_ctx) . unwrap ( ) ;
1640+ futures:: executor:: block_on ( stream. next ( ) )
1641+ . unwrap ( )
1642+ . unwrap_err ( )
1643+ }
1644+
1645+ #[ test]
1646+ fn sort_oom_error_names_the_comet_config ( ) {
1647+ assert_eq ! (
1648+ datafusion:: common:: config:: ConfigOptions :: default ( )
1649+ . execution
1650+ . sort_spill_reservation_bytes,
1651+ 10 * 1024 * 1024
1652+ ) ;
1653+
1654+ for error in [
1655+ sort_oom_error ( ) ,
1656+ DataFusionError :: Shared ( Arc :: new ( sort_oom_error ( ) ) ) ,
1657+ ] {
1658+ let error = CometError :: from ( error) ;
1659+ assert ! ( error. to_string( ) . contains( DATAFUSION_SORT_MEMORY_KEY ) ) ;
1660+ let expected = format ! ( "{error}\n {SORT_MEMORY_HINT}" ) ;
1661+ let ( _, class, message, _) = classify ( error) ;
1662+ assert_eq ! ( class, "org/apache/comet/CometNativeException" ) ;
1663+ assert_eq ! ( message, expected) ;
1664+ }
1665+
1666+ let other = CometError :: from ( DataFusionError :: ResourcesExhausted (
1667+ "Failed to acquire 1024 bytes" . to_string ( ) ,
1668+ ) ) ;
1669+ let expected = other. to_string ( ) ;
1670+ let ( _, _, message, _) = classify ( other) ;
1671+ assert_eq ! ( message, expected) ;
1672+ }
1673+
15811674 #[ test]
15821675 fn native_error_panic ( ) {
15831676 let panic = CometError :: Panic {
0 commit comments