Skip to content

Commit 50f7984

Browse files
alambclaude
andcommitted
Merge remote-tracking branch 'apache/main' into update-sqlparser-0.63
Resolve conflict in datafusion/sql/src/query.rs: keep main's rewritten ORDER BY ALL handling and clone `OrderByOptions`, which is no longer `Copy` in sqlparser 0.63. Co-Authored-By: Claude Fable 5.1 <[email protected]>
2 parents e2dfc2a + a0631ed commit 50f7984

98 files changed

Lines changed: 7988 additions & 1053 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Cargo.lock‎

Lines changed: 9 additions & 8 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -283,7 +283,6 @@ enum_glob_use = "allow" # 98 hits
283283
explicit_into_iter_loop = "allow" # 55 hits
284284
explicit_iter_loop = "allow" # 189 hits
285285
float_cmp = "allow" # 8 hits; exact float comparisons are often intentional here
286-
from_iter_instead_of_collect = "allow" # 51 hits
287286
if_not_else = "allow" # 133 hits
288287
ignored_unit_patterns = "allow" # 52 hits
289288
implicit_clone = "allow" # 198 hits

‎benchmarks/README.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -923,6 +923,21 @@ cargo run --release --bin dfbench -- h2o --join-paths ./benchmarks/data/h2o/J1_1
923923

924924
# Micro-Benchmarks
925925

926+
## ASOF Join
927+
928+
This benchmark exercises ASOF joins across relative input sizes, available
929+
input ordering, equality-group cardinality and skew, match direction, and
930+
payload width. One keyed workload reads pre-sorted Parquet inputs from
931+
`benchmarks/data/asof_join` with declared ordering so it measures the join
932+
without including an input sort.
933+
934+
### Example Run
935+
936+
```bash
937+
./bench.sh data asof_join
938+
./bench.sh run asof_join
939+
```
940+
926941
## Nested Loop Join
927942

928943
This benchmark focuses on the performance of queries with nested loop joins, minimizing other overheads such as scanning data sources or evaluating predicates.

‎benchmarks/bench.sh‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@ imdb: Join Order Benchmark (JOB) using the IMDB dataset conver
160160
161161
# Micro-Benchmarks (specific operators and features)
162162
cancellation: How long cancelling a query takes
163+
asof_join: ASOF join workloads varying size, ordering, grouping, match direction, and payload width
163164
nlj: Benchmark for simple nested loop joins, testing various join scenarios
164165
hj: Benchmark for simple hash joins, testing various join scenarios
165166
smj: Benchmark for simple sort merge joins, testing various join scenarios
@@ -238,6 +239,7 @@ main() {
238239
data_clickbench_1
239240
data_clickbench_partitioned
240241
data_imdb
242+
data_asof_join
241243
# nlj uses range() function, no data generation needed
242244
;;
243245
tpch)
@@ -269,6 +271,9 @@ main() {
269271
# Data is generated inline by the suite's load SQL (COPY).
270272
echo "parquet_row_filter_skip: no external data to generate"
271273
;;
274+
asof_join)
275+
data_asof_join
276+
;;
272277
tpcds)
273278
data_tpcds
274279
;;
@@ -479,6 +484,7 @@ main() {
479484
run_h2o_join "BIG" "PARQUET" "join"
480485
run_imdb
481486
run_external_aggr
487+
run_asof_join
482488
run_nlj
483489
run_hj
484490
run_tpcds
@@ -512,6 +518,9 @@ main() {
512518
parquet_row_filter_skip)
513519
run_parquet_row_filter_skip
514520
;;
521+
asof_join)
522+
run_asof_join
523+
;;
515524
tpcds)
516525
run_tpcds
517526
;;
@@ -1632,6 +1641,51 @@ run_topk_sorted_tpch() {
16321641
$CARGO_COMMAND --bin dfbench -- sort-tpch --iterations 5 --path "${TPCH_DIR}" -o "${RESULTS_FILE}" --sorted --limit 100 ${QUERY_ARG} ${LATENCY_ARG}
16331642
}
16341643

1644+
# Generates the pre-sorted Parquet inputs for the ASOF join benchmark.
1645+
data_asof_join() {
1646+
ASOF_DIR="${DATA_DIR}/asof_join"
1647+
LEFT_FILE="${ASOF_DIR}/q07_left.parquet"
1648+
RIGHT_FILE="${ASOF_DIR}/q07_right.parquet"
1649+
1650+
if [ -f "${LEFT_FILE}" ] && [ -f "${RIGHT_FILE}" ]; then
1651+
echo "ASOF join benchmark data already exists at ${ASOF_DIR}"
1652+
return
1653+
fi
1654+
1655+
mkdir -p "${ASOF_DIR}"
1656+
echo "Generating ASOF join benchmark data at ${ASOF_DIR}..."
1657+
(
1658+
cd "${DATAFUSION_DIR}"
1659+
debug_run $CARGO_COMMAND -p datafusion-cli -- -c "
1660+
COPY (
1661+
SELECT value / 10 AS group_key,
1662+
value % 10 + 1 AS ts,
1663+
value AS payload
1664+
FROM range(100000)
1665+
ORDER BY group_key, ts
1666+
)
1667+
TO '${LEFT_FILE}' STORED AS PARQUET;
1668+
1669+
COPY (
1670+
SELECT value / 10 AS group_key,
1671+
value % 10 AS ts,
1672+
value AS payload
1673+
FROM range(100000)
1674+
ORDER BY group_key, ts
1675+
)
1676+
TO '${RIGHT_FILE}' STORED AS PARQUET;
1677+
"
1678+
)
1679+
}
1680+
1681+
# Runs the ASOF join benchmark
1682+
run_asof_join() {
1683+
RESULTS_FILE="${RESULTS_DIR}/asof_join.json"
1684+
echo "RESULTS_FILE: ${RESULTS_FILE}"
1685+
echo "Running ASOF join benchmark..."
1686+
debug_run $CARGO_COMMAND --bin benchmark_runner -- asof_join --iterations 5 --path "${DATA_DIR}" -o "${RESULTS_FILE}" ${QUERY_ARG} ${LATENCY_ARG}
1687+
}
1688+
16351689
# Runs the nlj benchmark
16361690
run_nlj() {
16371691
RESULTS_FILE="${RESULTS_DIR}/nlj.json"

‎benchmarks/sql_benchmarks/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ in the community:
2929

3030
| Benchmark Suite | Description |
3131
|-----------------------|--------------------------------------------------------------------|
32+
| `asof_join` | ASOF join benchmarks across size, ordering, grouping, match direction, and payload width |
3233
| `clickbench` | ClickBench benchmark |
3334
| `clickbench extended` | 12 additional, more complex queries against the Clickbench dataset |
3435
| `clickbench_sorted` | ClickBench benchmark using a pre-sorted hits file. |
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Workloads vary these axes:
2+
# - relative left and right input sizes
3+
# - required ordering already available or included in the measured query
4+
# - no equality key, many balanced groups, or heavily skewed groups
5+
# - latest right row at or before, or earliest right row at or after, each left row
6+
# - narrow integer or wide UTF-8 payloads
7+
8+
description = "ASOF join SQL benchmarks"
9+
10+
query_pattern = "q{QUERY_ID_PADDED}.benchmark"
11+
12+
[path_replacements]
13+
DATA_DIR = "../../data"
14+
15+
[[examples]]
16+
command = "cargo run --release --bin benchmark_runner -- asof_join"
17+
description = "Run all ASOF join queries."
18+
19+
[[examples]]
20+
command = "cargo run --release --bin benchmark_runner -- asof_join --query 7"
21+
description = "Run the pre-sorted keyed ASOF join query."
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
name Q01
2+
group asof_join
3+
4+
expect_plan AsOfJoinExec
5+
6+
run
7+
-- Left: 1M rows; right: 10K rows; no equality key; ascending timestamps.
8+
-- Matches the latest right row at or before each left timestamp (l.ts >= r.ts).
9+
WITH left_input AS (
10+
SELECT value AS ts, value AS payload FROM range(1000000)
11+
),
12+
right_input AS (
13+
SELECT value AS ts, value AS payload FROM range(10000)
14+
)
15+
SELECT l.ts, l.payload, r.payload AS right_payload
16+
FROM left_input l
17+
ASOF JOIN right_input r MATCH_CONDITION (l.ts >= r.ts);
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
name Q02
2+
group asof_join
3+
4+
expect_plan AsOfJoinExec
5+
6+
run
7+
-- Left and right: 1M rows each; 10K balanced equality groups; integer payloads.
8+
-- Inputs are not ordered by (key, ts), so required ordering is included.
9+
-- Matches the latest right row at or before each left timestamp (l.ts >= r.ts).
10+
WITH left_input AS (
11+
SELECT value % 10000 AS key,
12+
value / 10000 + 1 AS ts,
13+
value AS payload
14+
FROM range(1000000)
15+
),
16+
right_input AS (
17+
SELECT value % 10000 AS key,
18+
value / 10000 AS ts,
19+
value AS payload
20+
FROM range(1000000)
21+
)
22+
SELECT l.key, l.ts, l.payload, r.payload AS right_payload
23+
FROM left_input l
24+
ASOF JOIN right_input r MATCH_CONDITION (l.ts >= r.ts)
25+
ON l.key = r.key;
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
name Q03
2+
group asof_join
3+
4+
expect_plan AsOfJoinExec
5+
6+
run
7+
-- Left and right: 250K rows each; 10K balanced equality groups.
8+
-- Inputs are not ordered by (key, ts), so required ordering is included.
9+
-- Matches the latest right row at or before each left timestamp and materializes
10+
-- 256-byte UTF-8 payloads from both inputs.
11+
WITH left_input AS (
12+
SELECT value % 10000 AS key,
13+
value / 10000 + 1 AS ts,
14+
repeat('x', 256) AS payload
15+
FROM range(250000)
16+
),
17+
right_input AS (
18+
SELECT value % 10000 AS key,
19+
value / 10000 AS ts,
20+
repeat('y', 256) AS payload
21+
FROM range(250000)
22+
)
23+
SELECT l.key, l.ts, l.payload, r.payload AS right_payload
24+
FROM left_input l
25+
ASOF JOIN right_input r MATCH_CONDITION (l.ts >= r.ts)
26+
ON l.key = r.key;
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
name Q04
2+
group asof_join
3+
4+
expect_plan AsOfJoinExec
5+
6+
run
7+
-- Left and right: 500K rows each; no equality key; ascending source timestamps.
8+
-- Matches the earliest right row at or after each left timestamp (l.ts <= r.ts),
9+
-- so the measured query includes the required descending ordering.
10+
WITH left_input AS (
11+
SELECT value AS ts, value AS payload FROM range(500000)
12+
),
13+
right_input AS (
14+
SELECT value AS ts, value AS payload FROM range(500000)
15+
)
16+
SELECT l.ts, l.payload, r.payload AS right_payload
17+
FROM left_input l
18+
ASOF JOIN right_input r MATCH_CONDITION (l.ts <= r.ts);

0 commit comments

Comments
 (0)