Skip to content

Commit 247230b

Browse files
authored
Merge branch 'main' into asolimando/query-lifetime-stats-cache
2 parents cce8458 + b40ef06 commit 247230b

127 files changed

Lines changed: 8866 additions & 1909 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.envrc‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,5 @@
1-
use flake
1+
source_up_if_exists
2+
if command -v nix >/dev/null 2>&1; then
3+
use flake
4+
fi
5+
source_env_if_exists .envrc.local

‎.github/ISSUE_TEMPLATE/bug_report.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,12 @@ body:
2626
label: Additional context
2727
placeholder: >
2828
Add any other context about the problem here.
29+
- type: dropdown
30+
attributes:
31+
label: Willingness to contribute
32+
description: >
33+
The Apache DataFusion community encourages bug fix contributions. Would you or another member of your organization be willing to contribute a fix for this bug to the Apache DataFusion codebase?
34+
options:
35+
- I plan to contribute a fix for this bug independently
36+
- I would like to contribute a fix for this bug, but request guidance from the DataFusion community
37+
- I cannot contribute a fix for this bug at this time

‎.github/ISSUE_TEMPLATE/feature_request.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,12 @@ body:
2525
label: Additional context
2626
placeholder: >
2727
Add any other context or screenshots about the feature request here.
28+
- type: dropdown
29+
attributes:
30+
label: Willingness to contribute
31+
description: >
32+
The Apache DataFusion community encourages feature contributions. Would you or another member of your organization be willing to contribute to this feature for the Apache DataFusion codebase?
33+
options:
34+
- I plan to contribute to this feature independently
35+
- I would like to contribute to this feature, but request guidance from the DataFusion community
36+
- I cannot contribute to this feature at this time

‎.github/workflows/docs.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,10 +67,12 @@ jobs:
6767
run: |
6868
set -x
6969
cd asf-site/
70+
# don't delete old versions of the docs (see versions/ in asf-site)
7071
rsync \
7172
-a \
7273
--delete \
7374
--exclude '/.git/' \
75+
--exclude '/versions/' \
7476
../docs/build/html/ \
7577
./
7678
cp ../.asf.yaml .

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ datafusion/core/benches/data/*
7272
# Generated Nix environment hooks
7373

7474
.direnv/
75+
.envrc.local
7576

7677
# rat
7778
filtered_rat.txt
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
name Q18
2+
group nlj
3+
4+
expect_plan NestedLoopJoinExec
5+
expect_plan join_type=RightSemi
6+
7+
run
8+
-- Q18: RIGHT SEMI 4K x 4K | no matches; the maximum sum is 4096 * 4096 - 1
9+
SELECT r.value
10+
FROM range(4096) AS l
11+
RIGHT SEMI JOIN range(4096) AS r
12+
ON l.value * 4096 + r.value >= 4096 * 4096;
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
name Q19
2+
group nlj
3+
4+
expect_plan NestedLoopJoinExec
5+
expect_plan join_type=RightAnti
6+
7+
run
8+
-- Q19: RIGHT ANTI 4K x 4K | no matches; the maximum sum is 4096 * 4096 - 1
9+
SELECT r.value
10+
FROM range(4096) AS l
11+
RIGHT ANTI JOIN range(4096) AS r
12+
ON l.value * 4096 + r.value >= 4096 * 4096;
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name Q20
2+
group nlj
3+
4+
expect_plan NestedLoopJoinExec
5+
expect_plan join_type=RightSemi
6+
7+
run
8+
-- Q20: RIGHT SEMI 2K x 2K | staged matches with 260-byte filter strings
9+
-- With batch_size=8192, candidates shrink 2048 -> 1024 -> 512 -> ... -> 16.
10+
-- Under that setting, the multi-left-row path starts at 512 candidates.
11+
SELECT r.value
12+
FROM (
13+
SELECT repeat('x', 256) || lpad(CAST(
14+
CASE WHEN value < 128 THEN 1024
15+
WHEN value < 256 THEN 1536
16+
WHEN value < 384 THEN 1792
17+
WHEN value < 512 THEN 1920
18+
WHEN value < 640 THEN 1984
19+
WHEN value < 768 THEN 2016
20+
ELSE 2032 END AS VARCHAR), 4, '0') AS threshold
21+
FROM range(2048)
22+
) AS l
23+
RIGHT SEMI JOIN (
24+
SELECT value,
25+
repeat('x', 256) || lpad(CAST(value AS VARCHAR), 4, '0') AS key
26+
FROM range(2048)
27+
) AS r
28+
ON l.threshold > r.key;
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name Q21
2+
group nlj
3+
4+
expect_plan NestedLoopJoinExec
5+
expect_plan join_type=RightAnti
6+
7+
run
8+
-- Q21: RIGHT ANTI 2K x 2K | staged matches with 260-byte filter strings
9+
-- With batch_size=8192, candidates shrink 2048 -> 1024 -> 512 -> ... -> 16.
10+
-- Under that setting, the multi-left-row path starts at 512 candidates.
11+
SELECT r.value
12+
FROM (
13+
SELECT repeat('x', 256) || lpad(CAST(
14+
CASE WHEN value < 128 THEN 1024
15+
WHEN value < 256 THEN 1536
16+
WHEN value < 384 THEN 1792
17+
WHEN value < 512 THEN 1920
18+
WHEN value < 640 THEN 1984
19+
WHEN value < 768 THEN 2016
20+
ELSE 2032 END AS VARCHAR), 4, '0') AS threshold
21+
FROM range(2048)
22+
) AS l
23+
RIGHT ANTI JOIN (
24+
SELECT value,
25+
repeat('x', 256) || lpad(CAST(value AS VARCHAR), 4, '0') AS key
26+
FROM range(2048)
27+
) AS r
28+
ON l.threshold > r.key;
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
name Q22
2+
group nlj
3+
4+
expect_plan NestedLoopJoinExec
5+
expect_plan join_type=RightSemi
6+
7+
# Materialize the threshold so the filter keeps both join inputs.
8+
# Keep the left input smaller so the default optimizer preserves RightSemi.
9+
init
10+
CREATE TABLE nlj_repeated_left AS
11+
SELECT 3277 AS threshold FROM range(4096);
12+
13+
init
14+
CREATE TABLE nlj_repeated_right AS
15+
SELECT value AS id FROM range(8192);
16+
17+
run
18+
-- Repeated 40% matches: only the first probe adds matches.
19+
-- With one partition and batch_size=8192, 4915 survivors exceed the 4096-row
20+
-- compaction threshold, so every left row probes the full right batch.
21+
SELECT r.id
22+
FROM nlj_repeated_left l
23+
RIGHT SEMI JOIN nlj_repeated_right r ON l.threshold > r.id;
24+
25+
cleanup
26+
DROP TABLE nlj_repeated_left;
27+
28+
cleanup
29+
DROP TABLE nlj_repeated_right;

0 commit comments

Comments
 (0)