Skip to content

Commit 3c764f1

Browse files
Merge branch 'main' into docs/4420-docs-review-skill
2 parents 7a0962e + 383ec56 commit 3c764f1

52 files changed

Lines changed: 4639 additions & 241 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/pr_build_linux.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -543,6 +543,7 @@ jobs:
543543
org.apache.spark.sql.comet.execution.shuffle.CometNativePositionalRoundRobinSuite
544544
org.apache.spark.sql.comet.execution.shuffle.CometDiskBlockWriterSuite
545545
org.apache.spark.sql.comet.execution.shuffle.CometBlockStoreShuffleReaderSuite
546+
org.apache.spark.sql.comet.execution.shuffle.NativeBatchDecoderIteratorSuite
546547
org.apache.comet.exec.CometShuffleEncryptionSuite
547548
org.apache.comet.exec.CometShuffleManagerSuite
548549
org.apache.comet.exec.DisableAQECometShuffleSuite
@@ -569,6 +570,7 @@ jobs:
569570
org.apache.comet.exec.CometJoinSuite
570571
org.apache.comet.exec.CometTypedDatasetSuite
571572
org.apache.spark.sql.comet.CometMapInBatchSuite
573+
org.apache.spark.sql.comet.CometCollectMetricsSuite
572574
org.apache.spark.sql.comet.CometArrowPythonUdfSuite
573575
org.apache.spark.sql.execution.python.CometArrowPythonRunnerSuite
574576
org.apache.comet.CometNativeSuite

‎.github/workflows/pr_build_macos.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,7 @@ jobs:
209209
org.apache.spark.sql.comet.execution.shuffle.CometNativePositionalRoundRobinSuite
210210
org.apache.spark.sql.comet.execution.shuffle.CometDiskBlockWriterSuite
211211
org.apache.spark.sql.comet.execution.shuffle.CometBlockStoreShuffleReaderSuite
212+
org.apache.spark.sql.comet.execution.shuffle.NativeBatchDecoderIteratorSuite
212213
org.apache.comet.exec.CometShuffleEncryptionSuite
213214
org.apache.comet.exec.CometShuffleManagerSuite
214215
org.apache.comet.exec.DisableAQECometShuffleSuite
@@ -235,6 +236,7 @@ jobs:
235236
org.apache.comet.exec.CometJoinSuite
236237
org.apache.comet.exec.CometTypedDatasetSuite
237238
org.apache.spark.sql.comet.CometMapInBatchSuite
239+
org.apache.spark.sql.comet.CometCollectMetricsSuite
238240
org.apache.spark.sql.comet.CometArrowPythonUdfSuite
239241
org.apache.spark.sql.execution.python.CometArrowPythonRunnerSuite
240242
org.apache.comet.CometNativeSuite

‎docs/source/contributor-guide/expression-audits/url_funcs.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@
2525

2626
- Performance (tuned 2026-07-11, PR [#4893](https://github.com/apache/datafusion-comet/pull/4893)): cache the compiled query regex in a thread-local single-slot cache keyed by the query key, eliminating per-row `Regex::new()` recompilation in the QUERY-with-key path (the key is constant across a batch in practice). Up to 50x faster. Benchmark: `benches/parse_url.rs`.
2727

28+
- Performance (tuned 2026-10-30, PR [#7329](https://github.com/apache/datafusion-comet/pull/7329)): split raw URI components at delimiters and borrow them until Arrow constructs the output. Non-null HTTP(S) component microbenchmarks are 5.3–10.4x faster, FTP host extraction is 5.8–6.4x faster, and query-key extraction is 3.4x faster. Benchmark: `benches/parse_url.rs`.
29+
2830
## try_url_decode
2931

3032
- 4.0.1, 2026-05-05

‎docs/source/contributor-guide/ffi.md‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ implementation depends on the source of the data:
9797

9898
- `RowArrowReader`: a Spark `Iterator[InternalRow]` (row input)
9999
- `SparkColumnarArrowReader`: a non-Arrow Spark `ColumnarBatch`
100-
- `ColumnarBatchArrowReader`: an Arrow-backed `ColumnarBatch` (transfers `VectorSchemaRoot` ownership)
100+
- `ColumnarBatchArrowReader`: an Arrow-backed `ColumnarBatch` (retains its buffers before closing the input batch)
101101

102102
The exported `ArrowArrayStream`s are boxed into the `Array[Object]` that `CometExecIterator` / `CometExecRDD` pass
103103
to native `createPlan`, one slot per scan input.
@@ -166,6 +166,12 @@ Off-heap Memory: │
166166

167167
### Ownership and Lifecycle
168168

169+
For JVM columnar consumers, `CometNativeArrowSource` uses `CometArrowStream.readerBatchIter` to return batches
170+
with independent vectors that retain the reader's buffers without copying the data. Closing a batch or its
171+
columns leaves the source reader's reusable vector structure intact for its next batch. The iterator releases
172+
the batch vectors when it advances, and closes the final batch, reader and allocator at task completion. A later
173+
native consumer's `ColumnarBatchArrowReader` retains the buffers before closing its input batch.
174+
169175
The Arrow C Stream Interface transfers ownership by reference count: native takes ownership of each imported batch,
170176
so it is safe to buffer batches in operators such as `SortExec` or `ShuffleWriterExec` without a deep copy.
171177

‎docs/source/contributor-guide/memory_management.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,12 @@ the tasks of an executor that read it at the same time share one such copy. The
120120
usage log reports the allocator's total, and the part of it charged to the import allocator
121121
described below, so that the overhead can be sized for it.
122122

123+
The opt-in broadcast producer batching path also retains native decoded shuffle arrays outside
124+
the execution pool. Each group is bounded by `min(spark.comet.batchSize, 65536)` rows, 8 MiB of
125+
retained/copy estimates, and 1,024 input batches, then concatenated once before Arrow export.
126+
Inputs and output coexist during that copy, and one decoded lookahead can be retained. The grouping
127+
limit therefore does not cap total reader RSS, and oversized input batches pass through unchanged.
128+
123129
Sharing lowers the bytes that are resident, not the bytes that are reserved. Each task's native
124130
join reserves its whole build side, because DataFusion skips only the buffers that the same plan
125131
has already counted, so with N tasks sharing one copy the pool holds N times the bytes that are

‎docs/source/contributor-guide/native_shuffle.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,14 @@ read time. See [Direct Read](#direct-read-shufflescan) below for how the choice
214214
3. The native `ShuffleScanExec` decodes each block with `read_ipc_compressed` and feeds the
215215
resulting `RecordBatch` straight into the plan. No Arrow FFI export or import happens.
216216

217+
For a direct local native shuffle feeding a broadcast exchange, the opt-in
218+
`spark.comet.exec.broadcastExchange.producerCoalesce.enabled` mode uses one decoder across the task's fetched
219+
blocks. `DecodedBatchCoalescer` groups eligible small decoded batches before FFI export and
220+
`Utils.serializeBatches`. The read RDD is copied with the same dependency and AQE partition
221+
specifications, so other consumers of a reused shuffle keep their original reader. Grouping is
222+
bounded as described in [Memory Management](memory_management.md). Task completion releases
223+
pending native batches, exported Arrow batches, and the raw stream.
224+
217225
**JVM decode**, used when direct read does not apply:
218226

219227
1. `CometBlockStoreShuffleReader` fetches shuffle blocks via `ShuffleBlockFetcherIterator`.

‎docs/source/user-guide/latest/compatibility/operators.md‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,28 @@ under the License.
1919

2020
# Operator Compatibility
2121

22+
## Observations
23+
24+
`CollectMetricsExec`, used by `DataFrame.observe`, preserves Spark's observation counters while
25+
passing the original Arrow batches between native operators. Metric expressions still execute
26+
in Spark over row views of each batch; the data is not copied into a row-based pipeline.
27+
Spark retains ownership of accumulator publication, observation listeners, resets and snapshots.
28+
29+
This path requires native input and is enabled by `spark.comet.exec.collectMetrics.enabled`.
30+
Observations retain Spark execution when a consumer can stop reading mid-batch. This includes
31+
limits, joins, `Dataset.mapPartitions`, replacement sampling at a nonpositive rate, and external
32+
actions such as `foreachPartition` or `rdd.take`, which do not introduce a physical-plan node.
33+
The final row iterator is therefore
34+
treated as potentially partial, even when the eventual action reads all rows. A local sort or
35+
aggregate does not remove this restriction: an external consumer may never request its output,
36+
so Spark may leave its input unread while native stream setup fetches a batch.
37+
38+
A materializing shuffle or broadcast exchange allows columnar observations in its producer
39+
subtree, unless another short-circuiting consumer intervenes. This preserves native observation
40+
in aggregation and shuffle pipelines while keeping counters aligned with Spark's consumption.
41+
Direct observations and write pipelines without such an exchange retain Spark collection;
42+
writer nodes alone do not enable columnar observations.
43+
2244
## Empty Relations
2345

2446
On Spark 4.0 and later, Comet supports `EmptyRelationExec` as a native input. It is enabled by

‎docs/source/user-guide/latest/operators.md‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -70,11 +70,12 @@ omitted from the tables below and may be reconsidered based on demand:
7070

7171
## Projection and filtering
7272

73-
| Operator | Status | Notes |
74-
| ------------- | ------ | -------------------------------------------------------------------------------------------- |
75-
| `ProjectExec` | ✅ | |
76-
| `FilterExec` | ✅ | |
77-
| `SampleExec` | ⚠️ | Sampling without replacement only. See [Operator Compatibility](compatibility/operators.md). |
73+
| Operator | Status | Notes |
74+
| -------------------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------- |
75+
| `ProjectExec` | ✅ | |
76+
| `FilterExec` | ✅ | |
77+
| `SampleExec` | ⚠️ | Sampling without replacement only. See [Operator Compatibility](compatibility/operators.md). |
78+
| `CollectMetricsExec` | ✅ | Spark evaluates observation counters while Comet forwards Arrow batches to native consumers. See [Observations](compatibility/operators.md#observations). |
7879

7980
## Sorting and limiting
8081

‎docs/source/user-guide/latest/tuning/shuffle.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,20 @@ scalar types. Hash partitioning keys must be scalar types unless
5353
and later) map keys. That setting is disabled by default until the performance of the nested hashing paths has been
5454
measured. Columns that are not partitioning keys may contain complex types like maps, structs, and arrays.
5555

56+
### Batching shuffle inputs to broadcast
57+
58+
The experimental `spark.comet.exec.broadcastExchange.producerCoalesce.enabled` setting combines small
59+
local native shuffle batches before broadcast serialization. It is disabled by default. It
60+
can reduce Arrow export, serialization, and driver assembly work when a broadcast build reads
61+
many small shuffle blocks.
62+
63+
The broadcast must read directly from a local Comet native shuffle, including an AQE shuffle
64+
read, and have only INT, BIGINT, BOOLEAN, STRING, or BINARY columns. Grouping preserves the
65+
selected shuffle partitions and row order. Large batches pass through without a merge copy.
66+
67+
Reader buffers are outside Spark's execution memory pool. The contributor guide describes the
68+
[grouping limits and memory lifetime](../../../contributor-guide/memory_management.md).
69+
5670
### Columnar (JVM) Shuffle
5771

5872
Comet Columnar shuffle is JVM-based and supports `HashPartitioning`, `RoundRobinPartitioning`, `RangePartitioning`, and

‎native/core/src/execution/jni_api.rs‎

Lines changed: 179 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ use datafusion::{
6262
use datafusion_comet_common::decode_string_arrays;
6363
use datafusion_comet_proto::spark_expression::agg_expr::ExprStruct as AggExprStruct;
6464
use datafusion_comet_proto::spark_operator::{AggregateMode, Operator, ShuffleScan};
65+
use datafusion_comet_shuffle::decoded_batch_coalescer::DecodedBatchCoalescer;
6566
use datafusion_comet_spark_expr::url_funcs::{CometParseUrl, CometTryParseUrl};
6667
use datafusion_spark::function::array::array_contains::SparkArrayContains;
6768
use datafusion_spark::function::array::repeat::SparkArrayRepeat;
@@ -2165,6 +2166,137 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_decodeShuffleBlock(
21652166
})
21662167
}
21672168

2169+
/// Decode a local frame into owned arrays. The direct buffer can be reused after this call.
2170+
fn append_shuffle_batch(
2171+
coalescer: &mut DecodedBatchCoalescer,
2172+
block: &[u8],
2173+
num_columns: usize,
2174+
) -> CometResult<()> {
2175+
let batch = read_ipc_compressed(block)?;
2176+
if batch.num_columns() != num_columns {
2177+
return Err(CometError::Internal(
2178+
"Shuffle frame column count mismatch".into(),
2179+
));
2180+
}
2181+
coalescer.push(batch)?;
2182+
Ok(())
2183+
}
2184+
2185+
/// The reader owns this handle and serializes native calls with cleanup.
2186+
/// # Safety
2187+
/// A nonzero handle must refer to a live coalescer exclusively borrowed for this call.
2188+
unsafe fn shuffle_batch_coalescer<'a>(handle: jlong) -> CometResult<&'a mut DecodedBatchCoalescer> {
2189+
unsafe { (handle as *mut DecodedBatchCoalescer).as_mut() }
2190+
.ok_or_else(|| CometError::Internal("Closed shuffle batch coalescer".into()))
2191+
}
2192+
2193+
fn shuffle_batch_ready_columns(coalescer: &DecodedBatchCoalescer) -> jint {
2194+
coalescer
2195+
.peek_ready()
2196+
.map_or(-1, |batch| batch.num_columns() as jint)
2197+
}
2198+
2199+
#[no_mangle]
2200+
pub extern "system" fn Java_org_apache_comet_Native_createShuffleBatchCoalescer(
2201+
e: EnvUnowned,
2202+
_class: JClass,
2203+
max_rows: jint,
2204+
) -> jlong {
2205+
try_unwrap_or_throw(&e, |_| {
2206+
let coalescer = DecodedBatchCoalescer::try_new(max_rows.max(0) as usize)?;
2207+
Ok(Box::into_raw(Box::new(coalescer)) as jlong)
2208+
})
2209+
}
2210+
2211+
#[no_mangle]
2212+
/// Decode and retain one local frame, returning ready columns or -1 for more input.
2213+
/// # Safety
2214+
/// The handle must be live and exclusively owned by this reader. On error, close the reader.
2215+
pub unsafe extern "system" fn Java_org_apache_comet_Native_appendShuffleBatch(
2216+
e: EnvUnowned,
2217+
_class: JClass,
2218+
handle: jlong,
2219+
byte_buffer: JByteBuffer,
2220+
length: jint,
2221+
num_columns: jint,
2222+
tracing_enabled: jboolean,
2223+
) -> jint {
2224+
try_unwrap_or_throw(&e, |env| {
2225+
with_trace("appendShuffleBatch", tracing_enabled != JNI_FALSE, || {
2226+
if num_columns < 0 {
2227+
return Err(CometError::Internal("Invalid shuffle frame bounds".into()));
2228+
}
2229+
let block = unsafe { shuffle_frame_bytes(env, &byte_buffer, length)? };
2230+
let coalescer = unsafe { shuffle_batch_coalescer(handle)? };
2231+
append_shuffle_batch(coalescer, block, num_columns as usize)?;
2232+
Ok(shuffle_batch_ready_columns(coalescer))
2233+
})
2234+
})
2235+
}
2236+
2237+
#[no_mangle]
2238+
/// Report ready output, flushing the pending group at EOF.
2239+
/// # Safety
2240+
/// The handle must be live and exclusively owned by this reader.
2241+
pub unsafe extern "system" fn Java_org_apache_comet_Native_peekShuffleBatch(
2242+
e: EnvUnowned,
2243+
_class: JClass,
2244+
handle: jlong,
2245+
end_of_input: jboolean,
2246+
tracing_enabled: jboolean,
2247+
) -> jint {
2248+
try_unwrap_or_throw(&e, |_| {
2249+
let coalescer = unsafe { shuffle_batch_coalescer(handle)? };
2250+
if end_of_input != JNI_FALSE {
2251+
with_trace(
2252+
"coalesceShuffleBatches",
2253+
tracing_enabled != JNI_FALSE,
2254+
|| coalescer.finish(),
2255+
)?;
2256+
}
2257+
Ok(shuffle_batch_ready_columns(coalescer))
2258+
})
2259+
}
2260+
2261+
#[no_mangle]
2262+
/// Transfer one ready output using the ordinary export path, including offset repair.
2263+
/// # Safety
2264+
/// The handle and Arrow FFI output addresses must be live and exclusively owned by this reader.
2265+
/// Both address arrays must have the column count reported by `peekShuffleBatch`.
2266+
pub unsafe extern "system" fn Java_org_apache_comet_Native_takeShuffleBatch(
2267+
e: EnvUnowned,
2268+
_class: JClass,
2269+
handle: jlong,
2270+
array_addrs: JLongArray,
2271+
schema_addrs: JLongArray,
2272+
tracing_enabled: jboolean,
2273+
) -> jlong {
2274+
try_unwrap_or_throw(&e, |env| {
2275+
with_trace("exportShuffleBatch", tracing_enabled != JNI_FALSE, || {
2276+
let batch = unsafe { shuffle_batch_coalescer(handle)? }
2277+
.take_ready()
2278+
.ok_or_else(|| CometError::Internal("No decoded shuffle batch is ready".into()))?;
2279+
prepare_output(env, array_addrs, schema_addrs, batch, false)
2280+
})
2281+
})
2282+
}
2283+
2284+
#[no_mangle]
2285+
/// # Safety
2286+
/// A nonzero handle must be live, must not be in use, and must be released exactly once.
2287+
pub unsafe extern "system" fn Java_org_apache_comet_Native_closeShuffleBatchCoalescer(
2288+
e: EnvUnowned,
2289+
_class: JClass,
2290+
handle: jlong,
2291+
) {
2292+
try_unwrap_or_throw(&e, |_| {
2293+
if handle != 0 {
2294+
drop(unsafe { Box::from_raw(handle as *mut DecodedBatchCoalescer) });
2295+
}
2296+
Ok(())
2297+
})
2298+
}
2299+
21682300
#[no_mangle]
21692301
/// Parse the expected schema once for a remote shuffle iterator.
21702302
///
@@ -2251,6 +2383,22 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_decodeShuffleBlockWit
22512383
})
22522384
}
22532385

2386+
/// Borrows a codec header and IPC payload from a direct buffer.
2387+
/// # Safety
2388+
/// The buffer's backing memory must remain valid and unmodified for the borrow.
2389+
unsafe fn shuffle_frame_bytes<'a>(
2390+
env: &mut Env,
2391+
byte_buffer: &'a JByteBuffer<'_>,
2392+
length: jint,
2393+
) -> CometResult<&'a [u8]> {
2394+
let capacity = env.get_direct_buffer_capacity(byte_buffer)?;
2395+
if length < 4 || length as usize > capacity {
2396+
return Err(CometError::Internal("Invalid shuffle frame bounds".into()));
2397+
}
2398+
let pointer = env.get_direct_buffer_address(byte_buffer)?;
2399+
Ok(unsafe { std::slice::from_raw_parts(pointer, length as usize) })
2400+
}
2401+
22542402
/// Converts the JNI arguments of the `decodeShuffleBlock` entry points for
22552403
/// [`decode_shuffle_block`].
22562404
fn decode_shuffle_block_jni(
@@ -2261,9 +2409,7 @@ fn decode_shuffle_block_jni(
22612409
schema_addrs: JLongArray,
22622410
expected_types: Option<&[ArrowDataType]>,
22632411
) -> CometResult<jlong> {
2264-
let raw_pointer = env.get_direct_buffer_address(&byte_buffer)?;
2265-
let length = length as usize;
2266-
let block: &[u8] = unsafe { std::slice::from_raw_parts(raw_pointer, length) };
2412+
let block = unsafe { shuffle_frame_bytes(env, &byte_buffer, length)? };
22672413
let array_addrs = unsafe { array_addrs.get_elements(env, ReleaseMode::NoCopyBack)? };
22682414
let schema_addrs = unsafe { schema_addrs.get_elements(env, ReleaseMode::NoCopyBack)? };
22692415
unsafe { decode_shuffle_block(block, &array_addrs, &schema_addrs, expected_types) }
@@ -3398,6 +3544,36 @@ mod tests {
33983544
Ok((rows, arrow::array::make_array(data)))
33993545
}
34003546

3547+
#[test]
3548+
fn append_shuffle_batches_owns_decoded_input_and_preserves_values() {
3549+
let input = int_batch();
3550+
let mut block = shuffle_block(&input);
3551+
let mut coalescer = DecodedBatchCoalescer::try_new(16).unwrap();
3552+
append_shuffle_batch(&mut coalescer, &block, 1).unwrap();
3553+
assert_eq!(shuffle_batch_ready_columns(&coalescer), -1);
3554+
// The JVM reuses its direct input buffer before the pending group is exported.
3555+
block.fill(0);
3556+
append_shuffle_batch(&mut coalescer, &shuffle_block(&input), 1).unwrap();
3557+
coalescer.finish().unwrap();
3558+
assert_eq!(shuffle_batch_ready_columns(&coalescer), 1);
3559+
let output = coalescer.take_ready().unwrap();
3560+
assert_eq!(output.num_rows(), 6);
3561+
assert_eq!(output.slice(0, 3), input);
3562+
assert_eq!(output.slice(3, 3), input);
3563+
}
3564+
3565+
#[test]
3566+
fn append_shuffle_batch_rejects_frame_count_mismatch() {
3567+
let mut coalescer = DecodedBatchCoalescer::try_new(16).unwrap();
3568+
let error =
3569+
append_shuffle_batch(&mut coalescer, &shuffle_block(&int_batch()), 2).unwrap_err();
3570+
assert!(error
3571+
.to_string()
3572+
.contains("Shuffle frame column count mismatch"));
3573+
coalescer.finish().unwrap();
3574+
assert!(coalescer.take_ready().is_none());
3575+
}
3576+
34013577
#[test]
34023578
fn decode_shuffle_block_exports_columns() {
34033579
let batch = int_batch();

0 commit comments

Comments
 (0)