Skip to content

Commit 6fe73d9

Browse files
authored
fix: point the external sort OOM error at Comet's memory settings (#7104)
1 parent cb19d43 commit 6fe73d9

3 files changed

Lines changed: 102 additions & 7 deletions

File tree

‎native/Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎native/jni-bridge/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,3 +45,4 @@ datafusion-comet-common = { workspace = true }
4545
[dev-dependencies]
4646
jni = { workspace = true, features = ["invocation"] }
4747
assertables = "10"
48+
futures = { workspace = true }

‎native/jni-bridge/src/errors.rs‎

Lines changed: 100 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -831,7 +831,7 @@ fn natural_exception(error: &CometError) -> JvmException<'static> {
831831
let exception = error.to_exception();
832832
JvmException::New {
833833
class: exception.class,
834-
message: with_spill_limit_hint(exception.msg),
834+
message: with_comet_config_hint(exception.msg),
835835
}
836836
}
837837

@@ -844,11 +844,34 @@ const SPILL_LIMIT_HINT: &str = "Comet does not use `datafusion.runtime.*` settin
844844
the spill limit is `spark.comet.maxTempDirectorySize`. It applies to each Comet native plan \
845845
separately, so an executor can use several times that much disk.";
846846

847-
fn with_spill_limit_hint(message: String) -> String {
848-
if message.contains(DATAFUSION_SPILL_LIMIT_KEY) {
849-
format!("{message} {SPILL_LIMIT_HINT}")
850-
} else {
851-
message
847+
/// The first config key DataFusion's external-sort OOM error tells users to change. Comet never
848+
/// uses it, and the sort is identified only by the `Context` string around `ResourcesExhausted`,
849+
/// so this quoted text is what to match on.
850+
const DATAFUSION_SORT_MEMORY_KEY: &str = "'datafusion.runtime.memory_limit'";
851+
852+
const SORT_MEMORY_HINT: &str = "Comet does not use `datafusion.runtime.memory_limit`, and in \
853+
production it keeps `datafusion.execution.sort_spill_reservation_bytes` at DataFusion's \
854+
default of 10 MiB. Native operators reserve memory from Spark's off-heap pool, sized by \
855+
`spark.memory.offHeap.size`, and each running task gets a share of it. Raise \
856+
`spark.memory.offHeap.size` or run fewer tasks per executor. See the Comet tuning guide: \
857+
https://datafusion.apache.org/comet/user-guide/latest/tuning.html";
858+
859+
/// DataFusion error text that names settings unavailable for production tuning in Comet,
860+
/// paired with the hint that points users at the Comet settings to change instead.
861+
const COMET_CONFIG_HINTS: &[(&str, &str)] = &[
862+
(DATAFUSION_SPILL_LIMIT_KEY, SPILL_LIMIT_HINT),
863+
(DATAFUSION_SORT_MEMORY_KEY, SORT_MEMORY_HINT),
864+
];
865+
866+
/// Appends the matching hint on its own line: the DataFusion text it follows can end mid-sentence,
867+
/// for example with the `Reserved: N` of a memory pool refusal.
868+
fn with_comet_config_hint(message: String) -> String {
869+
match COMET_CONFIG_HINTS
870+
.iter()
871+
.find(|(datafusion_key, _)| message.contains(datafusion_key))
872+
{
873+
Some((_, hint)) => format!("{message}\n{hint}"),
874+
None => message,
852875
}
853876
}
854877

@@ -1566,7 +1589,7 @@ mod tests {
15661589
] {
15671590
let error = CometError::from(error);
15681591
assert!(error.to_string().contains(DATAFUSION_SPILL_LIMIT_KEY));
1569-
let expected = format!("{error} {SPILL_LIMIT_HINT}");
1592+
let expected = format!("{error}\n{SPILL_LIMIT_HINT}");
15701593
let (_, class, message, _) = classify(error);
15711594
assert_eq!(class, "org/apache/comet/CometNativeException");
15721595
assert_eq!(message, expected);
@@ -1578,6 +1601,76 @@ mod tests {
15781601
assert_eq!(message, expected);
15791602
}
15801603

1604+
/// DataFusion's external-sort OOM error, produced by a real `SortExec` whose memory pool
1605+
/// cannot hold a single batch, so a rewording in a DataFusion upgrade fails this test instead
1606+
/// of silently dropping the hint.
1607+
fn sort_oom_error() -> DataFusionError {
1608+
use arrow::array::Int32Array;
1609+
use arrow::datatypes::{DataType, Field, Schema};
1610+
use arrow::record_batch::RecordBatch;
1611+
use datafusion::datasource::memory::MemorySourceConfig;
1612+
use datafusion::execution::memory_pool::GreedyMemoryPool;
1613+
use datafusion::execution::runtime_env::RuntimeEnvBuilder;
1614+
use datafusion::execution::TaskContext;
1615+
use datafusion::physical_expr::expressions::col;
1616+
use datafusion::physical_expr::{LexOrdering, PhysicalSortExpr};
1617+
use datafusion::physical_plan::sorts::sort::SortExec;
1618+
use datafusion::physical_plan::ExecutionPlan;
1619+
use futures::StreamExt;
1620+
1621+
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
1622+
let batch = RecordBatch::try_new(
1623+
Arc::clone(&schema),
1624+
vec![Arc::new(Int32Array::from(vec![3, 1, 2]))],
1625+
)
1626+
.unwrap();
1627+
let input =
1628+
MemorySourceConfig::try_new_exec(&[vec![batch]], Arc::clone(&schema), None).unwrap();
1629+
let ordering = LexOrdering::new(vec![PhysicalSortExpr::new_default(
1630+
col("a", &schema).unwrap(),
1631+
)])
1632+
.unwrap();
1633+
let sort = SortExec::new(ordering, input);
1634+
let runtime = RuntimeEnvBuilder::new()
1635+
.with_memory_pool(Arc::new(GreedyMemoryPool::new(1)))
1636+
.build_arc()
1637+
.unwrap();
1638+
let task_ctx = Arc::new(TaskContext::default().with_runtime(runtime));
1639+
let mut stream = sort.execute(0, task_ctx).unwrap();
1640+
futures::executor::block_on(stream.next())
1641+
.unwrap()
1642+
.unwrap_err()
1643+
}
1644+
1645+
#[test]
1646+
fn sort_oom_error_names_the_comet_config() {
1647+
assert_eq!(
1648+
datafusion::common::config::ConfigOptions::default()
1649+
.execution
1650+
.sort_spill_reservation_bytes,
1651+
10 * 1024 * 1024
1652+
);
1653+
1654+
for error in [
1655+
sort_oom_error(),
1656+
DataFusionError::Shared(Arc::new(sort_oom_error())),
1657+
] {
1658+
let error = CometError::from(error);
1659+
assert!(error.to_string().contains(DATAFUSION_SORT_MEMORY_KEY));
1660+
let expected = format!("{error}\n{SORT_MEMORY_HINT}");
1661+
let (_, class, message, _) = classify(error);
1662+
assert_eq!(class, "org/apache/comet/CometNativeException");
1663+
assert_eq!(message, expected);
1664+
}
1665+
1666+
let other = CometError::from(DataFusionError::ResourcesExhausted(
1667+
"Failed to acquire 1024 bytes".to_string(),
1668+
));
1669+
let expected = other.to_string();
1670+
let (_, _, message, _) = classify(other);
1671+
assert_eq!(message, expected);
1672+
}
1673+
15811674
#[test]
15821675
fn native_error_panic() {
15831676
let panic = CometError::Panic {

0 commit comments

Comments
 (0)