Skip to content

Commit 3f0b62a

Browse files
committed
Merge Apache main and preserve cancellation and pool cleanup
2 parents 2f27d5e + 525c05b commit 3f0b62a

87 files changed

Lines changed: 7291 additions & 951 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/actions/java-test/action.yaml‎

Lines changed: 50 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,46 @@ runs:
7676
restore-keys: |
7777
${{ runner.os }}-java-maven-
7878
79+
# Maven itself is stored outside the dependency repository. Keep its cache
80+
# independent of pom.xml changes and preserve the macOS cache workaround.
81+
- name: Restore Maven distribution
82+
id: maven-distribution
83+
if: ${{ runner.os != 'macOS' }}
84+
uses: actions/cache/restore@v5
85+
with:
86+
path: |
87+
~/.m2/wrapper/dists
88+
/root/.m2/wrapper/dists
89+
key: ${{ runner.os }}-${{ runner.arch }}-maven-wrapper-${{ hashFiles('.mvn/wrapper/maven-wrapper.properties') }}
90+
91+
# Retry only the wrapper download, never compilation or test execution.
92+
# Delays use exponential backoff (10s, 20s, 40s) plus 0-4s of random jitter.
93+
- name: Bootstrap Maven
94+
shell: bash
95+
run: |
96+
for attempt in 1 2 3 4; do
97+
if ./mvnw -B --version; then
98+
break
99+
fi
100+
if [ "$attempt" -eq 4 ]; then
101+
echo "::error::Maven bootstrap failed after $attempt attempts; tests were not started."
102+
exit 1
103+
fi
104+
delay=$((10 * (1 << (attempt - 1)) + RANDOM % 5))
105+
echo "::warning::Maven bootstrap attempt $attempt failed; retrying in ${delay}s."
106+
sleep "$delay"
107+
done
108+
109+
# Save a successful bootstrap even when the subsequent tests fail.
110+
- name: Save Maven distribution
111+
if: ${{ runner.os != 'macOS' && steps.maven-distribution.outputs.cache-hit != 'true' }}
112+
uses: actions/cache/save@v5
113+
with:
114+
path: |
115+
~/.m2/wrapper/dists
116+
/root/.m2/wrapper/dists
117+
key: ${{ steps.maven-distribution.outputs.cache-primary-key }}
118+
79119
- name: Run all tests
80120
shell: bash
81121
if: ${{ inputs.suites == '' }}
@@ -100,25 +140,32 @@ runs:
100140
with:
101141
name: crash-logs-${{ inputs.artifact_name }}
102142
path: "**/hs_err_pid*.log"
143+
if-no-files-found: ignore
103144
- name: Debug listing
104145
if: failure()
105146
shell: bash
106-
run: |
147+
run: |
107148
echo "CWD: $(pwd)"
108149
ls -lah .
109-
ls -lah target
150+
if [ -d target ]; then
151+
ls -lah target
152+
else
153+
echo "No root target directory; Maven may not have started."
154+
fi
110155
find . -name 'unit-tests.log'
111156
- name: Upload unit-tests.log
112157
if: failure()
113158
uses: actions/upload-artifact@v6
114159
with:
115160
name: unit-tests-${{ inputs.artifact_name }}
116161
path: "**/target/unit-tests.log"
162+
if-no-files-found: ignore
117163
- name: Upload test results
118-
if: ${{ inputs.upload-test-reports == 'true' }}
164+
if: ${{ !cancelled() && inputs.upload-test-reports == 'true' }}
119165
uses: actions/upload-artifact@v6
120166
with:
121167
name: java-test-reports-${{ inputs.artifact_name }}
122168
path: "**/target/surefire-reports/*.txt"
169+
if-no-files-found: ignore
123170
retention-days: 7 # 1 week for test reports
124171
overwrite: true

‎Makefile‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,11 @@ clean:
5656
bench:
5757
cd native && RUSTFLAGS="-Ctarget-cpu=native" cargo bench $(FEATURES_ARG) $(filter-out $@,$(MAKECMDGOALS))
5858
format:
59+
@if command -v prettier >/dev/null 2>&1; then \
60+
prettier -w "**/*.md"; \
61+
else \
62+
echo "prettier not found, skipping markdown formatting (npm i -g prettier)"; \
63+
fi
5964
cd native && cargo fmt
6065
./mvnw compile test-compile scalafix:scalafix -Psemanticdb $(PROFILES)
6166
./mvnw spotless:apply $(PROFILES)

‎dev/diffs/3.5.9.diff‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1446,6 +1446,30 @@ index 005e764cc30..92ec088efab 100644
14461446
}
14471447

14481448
private def checkSorts(query: String, enabledCount: Int, disabledCount: Int): Unit = {
1449+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala
1450+
index 46ed8fdfd21..4585dbab5b8 100644
1451+
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala
1452+
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala
1453+
@@ -18,6 +18,7 @@
1454+
package org.apache.spark.sql.execution
1455+
1456+
import org.apache.spark.sql.{DataFrame, QueryTest}
1457+
+import org.apache.spark.sql.comet.CometWindowGroupLimitExec
1458+
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
1459+
import org.apache.spark.sql.execution.window.WindowGroupLimitExec
1460+
import org.apache.spark.sql.functions.lit
1461+
@@ -30,7 +31,10 @@ abstract class RemoveRedundantWindowGroupLimitsSuiteBase
1462+
1463+
private def checkNumWindowGroupLimits(df: DataFrame, count: Int): Unit = {
1464+
val plan = df.queryExecution.executedPlan
1465+
- assert(collectWithSubqueries(plan) { case exec: WindowGroupLimitExec => exec }.length == count)
1466+
+ assert(collectWithSubqueries(plan) {
1467+
+ case exec: WindowGroupLimitExec => exec
1468+
+ case exec: CometWindowGroupLimitExec => exec
1469+
+ }.length == count)
1470+
}
1471+
1472+
private def checkWindowGroupLimits(query: String, count: Int): Unit = {
14491473
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala
14501474
index 47679ed7865..9ffbaecb98e 100644
14511475
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala

‎dev/diffs/4.0.4.diff‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1862,6 +1862,30 @@ index 005e764cc30..92ec088efab 100644
18621862
}
18631863

18641864
private def checkSorts(query: String, enabledCount: Int, disabledCount: Int): Unit = {
1865+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala
1866+
index 46ed8fdfd21..4585dbab5b8 100644
1867+
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala
1868+
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala
1869+
@@ -18,6 +18,7 @@
1870+
package org.apache.spark.sql.execution
1871+
1872+
import org.apache.spark.sql.{DataFrame, QueryTest}
1873+
+import org.apache.spark.sql.comet.CometWindowGroupLimitExec
1874+
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
1875+
import org.apache.spark.sql.execution.window.WindowGroupLimitExec
1876+
import org.apache.spark.sql.functions.lit
1877+
@@ -30,7 +31,10 @@ abstract class RemoveRedundantWindowGroupLimitsSuiteBase
1878+
1879+
private def checkNumWindowGroupLimits(df: DataFrame, count: Int): Unit = {
1880+
val plan = df.queryExecution.executedPlan
1881+
- assert(collectWithSubqueries(plan) { case exec: WindowGroupLimitExec => exec }.length == count)
1882+
+ assert(collectWithSubqueries(plan) {
1883+
+ case exec: WindowGroupLimitExec => exec
1884+
+ case exec: CometWindowGroupLimitExec => exec
1885+
+ }.length == count)
1886+
}
1887+
1888+
private def checkWindowGroupLimits(query: String, count: Int): Unit = {
18651889
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala
18661890
index 47679ed7865..9ffbaecb98e 100644
18671891
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala

‎dev/diffs/4.1.3.diff‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1985,6 +1985,30 @@ index 005e764cc30..92ec088efab 100644
19851985
}
19861986

19871987
private def checkSorts(query: String, enabledCount: Int, disabledCount: Int): Unit = {
1988+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala
1989+
index 46ed8fdfd21..4585dbab5b8 100644
1990+
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala
1991+
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala
1992+
@@ -18,6 +18,7 @@
1993+
package org.apache.spark.sql.execution
1994+
1995+
import org.apache.spark.sql.{DataFrame, QueryTest}
1996+
+import org.apache.spark.sql.comet.CometWindowGroupLimitExec
1997+
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
1998+
import org.apache.spark.sql.execution.window.WindowGroupLimitExec
1999+
import org.apache.spark.sql.functions.lit
2000+
@@ -30,7 +31,10 @@ abstract class RemoveRedundantWindowGroupLimitsSuiteBase
2001+
2002+
private def checkNumWindowGroupLimits(df: DataFrame, count: Int): Unit = {
2003+
val plan = df.queryExecution.executedPlan
2004+
- assert(collectWithSubqueries(plan) { case exec: WindowGroupLimitExec => exec }.length == count)
2005+
+ assert(collectWithSubqueries(plan) {
2006+
+ case exec: WindowGroupLimitExec => exec
2007+
+ case exec: CometWindowGroupLimitExec => exec
2008+
+ }.length == count)
2009+
}
2010+
2011+
private def checkWindowGroupLimits(query: String, count: Int): Unit = {
19882012
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala
19892013
index 47679ed7865..9ffbaecb98e 100644
19902014
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala

‎docs/source/contributor-guide/expression-audits/string_funcs.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,14 @@
245245
- Spark 3.5.8 (audited 2026-05-27): registry alias of `Upper`. Same support as `upper`.
246246
- Spark 4.0.1 (audited 2026-05-27): unchanged alias of `Upper`.
247247

248+
## unbase64
249+
250+
- Spark 3.4.3 (audited 2026-08-24): identical to 3.5.8.
251+
- Spark 3.5.8 (audited 2026-08-24): baseline. `doGenCode` emits `java.util.Base64.getMimeDecoder().decode(child.toString())`. The MIME decoder skips every byte outside the base64 alphabet, so CRLF-wrapped output from Spark's own `base64` round-trips cleanly; four terminal-shape errors surface as `IllegalArgumentException`. `failOnError = true` is set only when the node is constructed from `to_binary('base64')` / `try_to_binary`, which use a stricter RFC 4648 validator.
252+
- Spark 4.0.1 (audited 2026-08-24): `NullIntolerant` becomes `override def nullIntolerant: Boolean = true`; `inputTypes` widens to `StringTypeWithCollation(supportsTrimCollation = true)`. Behaviour is byte-level and collation-independent, so no divergence for `UTF8_BINARY` and no shim is needed.
253+
- Spark 4.1.1 (audited 2026-08-24): adds `contextIndependentFoldable`; no behavioural change on the decode path.
254+
- Comet native implementation (`spark_unbase64`, `native/spark-expr/src/string_funcs/unbase64.rs`) ports the JDK MIME decoder rules: 256-entry decode LUT, a reused per-batch scratch `Vec<u8>` copied into a preallocated `BinaryBuilder`, all four error messages reproduced verbatim. `CometUnBase64` handles `failOnError = false` natively; `failOnError = true` (reachable from `to_binary('base64')` / `try_to_binary`) requires strict RFC 4648 validation and is not yet implemented natively, so those cases stay on the JVM codegen dispatcher via `CodegenDispatchFallback`.
255+
248256
## upper
249257

250258
- Spark 3.4.3 (audited 2026-05-27): identical to 3.5.8.

‎docs/source/contributor-guide/native_shuffle.md‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@ Native shuffle (`CometExchange`) is selected when all of the following condition
4949
columnar output. Row-based Spark operators require JVM shuffle.
5050

5151
3. **Supported partitioning type**: Native shuffle supports:
52-
5352
- `HashPartitioning`
5453
- `RangePartitioning`
5554
- `SinglePartition`
@@ -129,7 +128,6 @@ Native shuffle (`CometExchange`) is selected when all of the following condition
129128
1. **Plan construction**: `CometNativeShuffleWriter` builds a protobuf operator tree with a
130129
`ShuffleWriter` operator at the root and `childNativeOp` as its child. `childNativeOp` takes
131130
one of two shapes:
132-
133131
- The child plan's `nativeOp` directly, when `CometShuffleExchangeExec`'s child is a
134132
`CometNativeExec` subtree. The upstream operators run inside the same `CometExecIterator`
135133
as the writer, with no JVM-to-native batch boundary between them.
@@ -142,21 +140,18 @@ Native shuffle (`CometExchange`) is selected when all of the following condition
142140
2. **Native execution**: A single `CometExecIterator` per partition runs the unified plan.
143141

144142
3. **Partitioning**: `ShuffleWriterExec` receives batches and routes to the appropriate partitioner:
145-
146143
- `MultiPartitionShuffleRepartitioner`: For hash/range/round-robin partitioning
147144
- `SinglePartitionShufflePartitioner`: For single partition (simpler path)
148145

149146
4. **Buffering and spilling**: The partitioner buffers rows per partition. When memory pressure
150147
exceeds the threshold, partitions spill to temporary files.
151148

152149
5. **Encoding**: `ShuffleBlockWriter` encodes each partition's data as compressed Arrow IPC:
153-
154150
- Writes compression type header
155151
- Writes field count header
156152
- Writes compressed IPC stream
157153

158154
6. **Output files**: Two files are produced:
159-
160155
- **Data file**: Concatenated partition data
161156
- **Index file**: Array of 8-byte little-endian offsets marking partition boundaries
162157

@@ -168,7 +163,6 @@ Native shuffle (`CometExchange`) is selected when all of the following condition
168163
1. `CometBlockStoreShuffleReader` fetches shuffle blocks via `ShuffleBlockFetcherIterator`.
169164

170165
2. For each block, `NativeBatchDecoderIterator`:
171-
172166
- Reads the 8-byte compressed length header
173167
- Reads the 8-byte field count header
174168
- Reads the compressed IPC data

‎docs/source/contributor-guide/roadmap.md‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@ Native window execution runs by default (`spark.comet.exec.window.enabled`). The
2929
`first_value`, `last_value`), and the `count`, `min`, `max`, `sum`, and `avg` aggregates are accelerated.
3030
Remaining work is to close the gaps that still fall back to Spark: statistical aggregates (`stddev`, `variance`,
3131
`corr`, `covar`) and `collect_list` / `collect_set` as window functions ([#4766]), `GROUPS` frames ([#4836]), `RANGE` frames with explicit date or
32-
decimal offsets ([#4834]), `first_value` / `last_value` on `RANGE` frames with a literal offset ([#4835]),
33-
non-literal `lag` / `lead` default values ([#4268]), and `WindowGroupLimitExec` ([#4837]). See the
32+
decimal offsets ([#4834]), `first_value` / `last_value` on `RANGE` frames with a literal offset ([#4835]), and
33+
non-literal `lag` / `lead` default values ([#4268]). See the
3434
[window function compatibility guide](../user-guide/latest/compatibility/operators.md) for the complete list of
3535
supported functions, frames, and fallback cases.
3636

@@ -39,7 +39,6 @@ supported functions, frames, and fallback cases.
3939
[#4834]: https://github.com/apache/datafusion-comet/issues/4834
4040
[#4835]: https://github.com/apache/datafusion-comet/issues/4835
4141
[#4836]: https://github.com/apache/datafusion-comet/issues/4836
42-
[#4837]: https://github.com/apache/datafusion-comet/issues/4837
4342

4443
## Native Lambda Evaluation
4544

‎docs/source/user-guide/latest/compatibility/floating-point.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,16 @@ So Comet adds additional normalization expression of NaN and zero for comparison
2727
to Spark in some cases, especially when the data contains both positive and negative zero. This is likely an edge
2828
case that is not of concern for many users. If it is a concern, setting `spark.comet.exec.strictFloatingPoint=true`
2929
will make relevant operations fall back to Spark.
30+
31+
## Ordering: signed zero (`-0.0` vs `+0.0`)
32+
33+
Spark's `ORDER BY`, `RANK`, `DENSE_RANK`, and window frame comparisons route through
34+
`SQLOrderingUtil.compareDoubles` / `compareFloats`, which explicitly define `-0.0 == 0.0`. Comet's
35+
native sort and `WindowGroupLimitExec` use the `arrow-row` row-format encoder for `ORDER BY` keys,
36+
which applies Rust's total-ordering transform to the raw IEEE-754 bits. Under that encoding `-0.0`
37+
sorts strictly less than `+0.0`, so a partition that mixes the two zeros can produce a rank
38+
distribution that differs from Spark. For example, `RANK() OVER (ORDER BY v ASC)` over
39+
`[-0.0, 0.0, 1.0]` filtered to `rk <= 1` returns two rows in Spark (both zeros tied at rank 1) but
40+
one row in Comet (`-0.0` at rank 1, `+0.0` at rank 2). If your workload materially mixes `-0.0`
41+
and `+0.0` in a ranked column, prefer Spark for that stage or normalize the column to `+0.0`
42+
upstream.

‎docs/source/user-guide/latest/compatibility/operators.md‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,8 +71,19 @@ incorrect result. When any single window expression in a `WindowExec` falls back
7171
window are not supported by Spark either.
7272
- Any `PARTITION BY` or `ORDER BY` expression that Comet cannot serialize.
7373

74-
`WindowGroupLimitExec` (window-based limit pushdown) is not yet supported and falls back to Spark
75-
([#4837](https://github.com/apache/datafusion-comet/issues/4837)).
74+
`WindowGroupLimitExec` (window-based limit pushdown for `ROW_NUMBER`, `RANK`, and `DENSE_RANK`)
75+
runs natively; it is controlled by `spark.comet.exec.windowGroupLimit.enabled` (default: true).
76+
77+
**Falls back to Spark:**
78+
79+
- Any `PARTITION BY` or `ORDER BY` key whose type carries a non-default `StringType` collation
80+
(e.g. `UTF8_LCASE`). The native operator detects partitions and order-key peer groups by
81+
comparing Arrow row-encoded keys for byte equality, which splits peers that Spark ties.
82+
83+
**Known incompatibilities:**
84+
85+
- Signed-zero ordering (`-0.0` vs `+0.0`) diverges from Spark's `RankLimitIterator`; see
86+
[floating-point ordering](./floating-point.md#ordering-signed-zero-00-vs-00).
7687

7788
## Round-Robin Partitioning
7889

0 commit comments

Comments
 (0)