Skip to content

Commit 29c6661

Browse files
committed
Merge remote-tracking branch 'apache/main' into fix-issue-258521
# Conflicts: # datafusion/sqllogictest/test_files/aggregate_memory_spill.slt
2 parents c1f5a64 + 52a8cd9 commit 29c6661

152 files changed

Lines changed: 9468 additions & 2621 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/extended.yml‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,10 +69,14 @@ jobs:
6969
- name: Show available disk space
7070
run: |
7171
df -h
72-
- name: Run tests (excluding doctests)
72+
- name: Run extended tests
7373
run: cargo xtask ci step test extended
74-
- name: Run fuzzers with more iterations
74+
- name: Run fuzz tests
7575
run: cargo xtask ci step test fuzz
76+
# Migrate it to the above unified fuzz runner over time
77+
# Tracker: https://github.com/apache/datafusion/issues/27082
78+
- name: Run unmigrated fuzzers with more iterations
79+
run: cargo xtask ci step test fuzz-legacy
7680
- parallel:
7781
- name: Verify Working Directory Clean
7882
run: git diff --exit-code

‎Cargo.lock‎

Lines changed: 17 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ members = [
2020
"datafusion/execution",
2121
"datafusion/ffi",
2222
"datafusion/ffi/integration-tests",
23+
"datafusion/fuzz",
2324
"datafusion/functions",
2425
"datafusion/functions-aggregate",
2526
"datafusion/functions-aggregate-common",

‎benchmarks/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -594,7 +594,7 @@ Iteration 2 returned 1781686 rows in 1947 ms
594594

595595
Test performance of sorting large datasets
596596

597-
This test sorts a a synthetic dataset generated during the
597+
This test sorts a synthetic dataset generated during the
598598
benchmark execution, designed to simulate sorting web server
599599
access logs. Such sorting is often done during data transformation
600600
steps.

‎benchmarks/bench.sh‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,7 @@ hj: Benchmark for simple hash joins, testing various join sc
163163
smj: Benchmark for simple sort merge joins, testing various join scenarios
164164
dict: Benchmark for dictionary-encoded group-by scenarios
165165
array_agg_distinct: 1000K-group, two-row-per-group array_agg(DISTINCT) benchmark
166+
grouped_count_distinct: Grouped count(DISTINCT) by key type, group count and distinct cardinality
166167
compile_profile: Compile and execute TPC-H across selected Cargo profiles, reporting timing and binary size
167168
168169
@@ -280,6 +281,9 @@ main() {
280281
# Data is generated inline by the suite's load SQL.
281282
echo "projection_subquery: no external data to generate"
282283
;;
284+
grouped_count_distinct)
285+
echo "grouped_count_distinct: no external data to generate"
286+
;;
283287
asof_join)
284288
data_asof_join
285289
;;
@@ -681,6 +685,9 @@ main() {
681685
array_agg_distinct)
682686
run_array_agg_distinct
683687
;;
688+
grouped_count_distinct)
689+
run_grouped_count_distinct
690+
;;
684691
compile_profile)
685692
run_compile_profile "${PROFILE_ARGS[@]}"
686693
;;
@@ -1813,6 +1820,15 @@ run_dict() {
18131820
debug_run $CARGO_COMMAND --bin dfbench -- dict --iterations 5 -o "${RESULTS_FILE}" ${QUERY_ARG} ${LATENCY_ARG}
18141821
}
18151822

1823+
# Runs grouped DISTINCT counts over inline data with varying key types and cardinalities.
1824+
run_grouped_count_distinct() {
1825+
debug_run env BENCH_NAME=grouped_count_distinct \
1826+
BENCH_RESULTS_FILE="$(sql_results_file grouped_count_distinct)" \
1827+
GROUPED_DISTINCT_ROWS="${GROUPED_DISTINCT_ROWS:-4000000}" \
1828+
${QUERY:+BENCH_QUERY="${QUERY}"} \
1829+
bash -c "$SQL_CARGO_COMMAND"
1830+
}
1831+
18161832
# Runs the data-free high-cardinality array_agg(DISTINCT) SQL benchmark.
18171833
run_array_agg_distinct() {
18181834
echo "Running array_agg_distinct benchmark..."
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
<!--
2+
SPDX-License-Identifier: Apache-2.0
3+
SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
4+
SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
5+
-->
6+
7+
# Grouped COUNT(DISTINCT)
8+
9+
Compare the default rewrite with the native accumulator opt-in at one target
10+
partition. The suite reads DataFusion settings from the environment; it does not
11+
enable the opt-in itself.
12+
13+
```bash
14+
DATAFUSION_OPTIMIZER_SINGLE_DISTINCT_PREFER_NATIVE=false cargo run --release --bin benchmark_runner -- grouped_count_distinct --iterations 10 --partitions 1 --mem-pool-type greedy --memory-limit 1G
15+
DATAFUSION_OPTIMIZER_SINGLE_DISTINCT_PREFER_NATIVE=true cargo run --release --bin benchmark_runner -- grouped_count_distinct --iterations 10 --partitions 1 --mem-pool-type greedy --memory-limit 1G
16+
```
17+
18+
Run from `benchmarks/`. Repeat at 4 and 12 target partitions to check the
19+
distributed rewrite, which remains selected with either flag value. The same
20+
environment variable works with `./bench.sh run grouped_count_distinct`.
21+
22+
Q01-Q09 vary argument and grouping-key types, group count, distinct cardinality,
23+
and skew. Q10 covers the unfavorable native shape: at the default four million
24+
rows it has one million groups, each with four copies of the same value. Each
25+
native input row probes both the group table and the per-group distinct set;
26+
the rewrite's outer aggregate only sees the deduplicated pairs.
27+
28+
Every query checks its result against an explicit distinct-pairs-then-count
29+
query before timing. Keep the memory pool and limit the same on both sides.
30+
The combined correctness query needs more memory than the measured aggregate.
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
name Q01
2+
group grouped_count_distinct
3+
4+
load sql_benchmarks/grouped_count_distinct/init/load.sql
5+
6+
assert I
7+
WITH actual AS (
8+
SELECT g AS g, count(DISTINCT x) AS n FROM grouped_distinct GROUP BY g
9+
), expected AS (
10+
SELECT g, count(x) AS n
11+
FROM (SELECT DISTINCT g AS g, x AS x FROM grouped_distinct)
12+
GROUP BY g
13+
)
14+
SELECT NOT EXISTS (SELECT * FROM actual EXCEPT SELECT * FROM expected)
15+
AND NOT EXISTS (SELECT * FROM expected EXCEPT SELECT * FROM actual);
16+
----
17+
true
18+
19+
expect_plan AggregateExec
20+
21+
run
22+
-- Int64, 2,000 groups, high distinct cardinality (issue #25936).
23+
SELECT g, count(DISTINCT x) FROM grouped_distinct GROUP BY g;
24+
25+
cleanup sql_benchmarks/grouped_count_distinct/init/cleanup.sql
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
name Q02
2+
group grouped_count_distinct
3+
4+
load sql_benchmarks/grouped_count_distinct/init/load.sql
5+
6+
assert I
7+
WITH actual AS (
8+
SELECT g AS g, count(DISTINCT low_x) AS n FROM grouped_distinct GROUP BY g
9+
), expected AS (
10+
SELECT g, count(x) AS n
11+
FROM (SELECT DISTINCT g AS g, low_x AS x FROM grouped_distinct)
12+
GROUP BY g
13+
)
14+
SELECT NOT EXISTS (SELECT * FROM actual EXCEPT SELECT * FROM expected)
15+
AND NOT EXISTS (SELECT * FROM expected EXCEPT SELECT * FROM actual);
16+
----
17+
true
18+
19+
expect_plan AggregateExec
20+
21+
run
22+
-- Int64, 2,000 groups, at most eight distinct values per group.
23+
SELECT g, count(DISTINCT low_x) FROM grouped_distinct GROUP BY g;
24+
25+
cleanup sql_benchmarks/grouped_count_distinct/init/cleanup.sql
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
name Q03
2+
group grouped_count_distinct
3+
4+
load sql_benchmarks/grouped_count_distinct/init/load.sql
5+
6+
assert I
7+
WITH actual AS (
8+
SELECT few_g AS g, count(DISTINCT x) AS n FROM grouped_distinct GROUP BY few_g
9+
), expected AS (
10+
SELECT g, count(x) AS n
11+
FROM (SELECT DISTINCT few_g AS g, x AS x FROM grouped_distinct)
12+
GROUP BY g
13+
)
14+
SELECT NOT EXISTS (SELECT * FROM actual EXCEPT SELECT * FROM expected)
15+
AND NOT EXISTS (SELECT * FROM expected EXCEPT SELECT * FROM actual);
16+
----
17+
true
18+
19+
expect_plan AggregateExec
20+
21+
run
22+
-- Int64, eight groups, high distinct cardinality.
23+
SELECT few_g, count(DISTINCT x) FROM grouped_distinct GROUP BY few_g;
24+
25+
cleanup sql_benchmarks/grouped_count_distinct/init/cleanup.sql
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
name Q04
2+
group grouped_count_distinct
3+
4+
load sql_benchmarks/grouped_count_distinct/init/load.sql
5+
6+
assert I
7+
WITH actual AS (
8+
SELECT g AS g, count(DISTINCT unsigned_x) AS n FROM grouped_distinct GROUP BY g
9+
), expected AS (
10+
SELECT g, count(x) AS n
11+
FROM (SELECT DISTINCT g AS g, unsigned_x AS x FROM grouped_distinct)
12+
GROUP BY g
13+
)
14+
SELECT NOT EXISTS (SELECT * FROM actual EXCEPT SELECT * FROM expected)
15+
AND NOT EXISTS (SELECT * FROM expected EXCEPT SELECT * FROM actual);
16+
----
17+
true
18+
19+
expect_plan AggregateExec
20+
21+
run
22+
-- UInt64, 2,000 groups, high distinct cardinality.
23+
SELECT g, count(DISTINCT unsigned_x) FROM grouped_distinct GROUP BY g;
24+
25+
cleanup sql_benchmarks/grouped_count_distinct/init/cleanup.sql

0 commit comments

Comments
 (0)