Skip to content

Commit 6656907

Browse files
authored
Merge branch 'main' into dev/chao/codex/skip-intermediate-replay-headroom
2 parents 78c4b7b + 991fd23 commit 6656907

149 files changed

Lines changed: 11834 additions & 3807 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.asf.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ github:
8787
- "check Cargo.toml formatting"
8888
- "check configs.md and ***_functions.md is up-to-date"
8989
- "check example README is up-to-date"
90+
- "Check datafusion crates used in datafusion-examples"
9091
- "Verify MSRV (Min Supported Rust Version)"
9192
# needs to be updated as part of the release process
9293
# .asf.yaml doesn't support wildcard branch protection rules, only exact branch names
Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Add the expected target version as a label to a merged PR.
19+
# The "target_version" is automatically updated by the dev/update_datafusion_versions.py script.
20+
21+
name: Release Version Labeler
22+
23+
concurrency:
24+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
25+
cancel-in-progress: true
26+
27+
on:
28+
pull_request_target:
29+
types:
30+
- closed
31+
branches:
32+
- main
33+
34+
jobs:
35+
label:
36+
if: github.event.pull_request.merged == true
37+
runs-on: ubuntu-slim
38+
permissions:
39+
pull-requests: write
40+
steps:
41+
- name: Add the target release version
42+
uses: actions/[email protected]
43+
with:
44+
script: |
45+
const labeler = require('./ci/scripts/release_version_labeler.js');
46+
await labeler({ github, context });

‎.github/workflows/rust.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -897,6 +897,16 @@ jobs:
897897
run: |
898898
bash ci/scripts/check_examples_docs.sh
899899
900+
# Ensures that the examples only use the main `datafusion` crate and not its subcrates, which are
901+
# already exported by `datafusion` (there are a few exceptions, like `datafusion-proto`).
902+
# See: https://github.com/apache/datafusion/pull/14305
903+
examples-check-datafusion-crates:
904+
name: Check datafusion crates used in datafusion-examples
905+
runs-on: ubuntu-slim
906+
steps:
907+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
908+
- run: python3 ci/scripts/check_examples_datafusion_crates.py
909+
900910
# Verify MSRV for the crates which are directly used by other projects:
901911
# - datafusion
902912
# - datafusion-substrait

‎AGENTS.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
- [Quick Start Setup](docs/source/contributor-guide/development_environment.md#quick-start)
66
- [Testing Quick Start](docs/source/contributor-guide/testing.md#testing-quick-start)
7+
- [Choosing What Kind of Test to Write](docs/source/contributor-guide/testing.md#choosing-what-kind-of-test-to-write)
78
- [Before Submitting a PR](docs/source/contributor-guide/index.md#before-submitting-a-pr)
89
- [Reviewing Pull Requests](docs/source/contributor-guide/pr_review.md)
910
- [Contributor Guide](docs/source/contributor-guide/index.md)
@@ -38,7 +39,10 @@ When creating a PR, you MUST follow the [PR template](.github/pull_request_templ
3839

3940
## Testing
4041

41-
If documentation files changed then run
42+
When adding tests, you MUST follow
43+
[Choosing What Kind of Test to Write](docs/source/contributor-guide/testing.md#choosing-what-kind-of-test-to-write):
44+
45+
If documentation files changed then run
4246
```bash
4347
./ci/scripts/doc_prettier_check.sh --write --allow-dirty
4448
```

‎Cargo.lock‎

Lines changed: 1 addition & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎benchmarks/sql_benchmarks/asof_join/asof_join.suite‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
# - latest right row at or before, or earliest right row at or after, each left row
66
# - narrow integer or wide UTF-8 payloads
77
# - selective left-side predicates that can run before matching
8+
# - equality-key predicates that can also prune the right input
89

910
description = "ASOF join SQL benchmarks"
1011

@@ -24,3 +25,7 @@ description = "Run the pre-sorted keyed ASOF join query."
2425
[[examples]]
2526
command = "cargo run --release --bin benchmark_runner -- asof_join --query 8"
2627
description = "Run the selective left-filter ASOF join query."
28+
29+
[[examples]]
30+
command = "cargo run --release --bin benchmark_runner -- asof_join --query 9"
31+
description = "Run the equality-key filter ASOF join query."
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
name Q09
2+
group asof_join
3+
4+
expect_plan AsOfJoinExec
5+
6+
run
7+
-- Both sides: 50M rows in 500K equality groups. The left predicate retains
8+
-- one group; mirroring it prunes the right input before sorting and broadcast.
9+
WITH left_input AS (
10+
SELECT value % 500000 AS key,
11+
value / 500000 + 1 AS ts,
12+
value AS payload
13+
FROM range(50000000)
14+
),
15+
right_input AS (
16+
SELECT value % 500000 AS key,
17+
value / 500000 AS ts,
18+
value AS payload
19+
FROM range(50000000)
20+
)
21+
SELECT l.key, l.ts, l.payload, r.payload AS right_payload
22+
FROM left_input l
23+
ASOF JOIN right_input r MATCH_CONDITION (l.ts >= r.ts)
24+
ON l.key = r.key
25+
WHERE l.key = 42;

‎benchmarks/src/statistics.rs‎

Lines changed: 141 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,10 @@ use serde::{Deserialize, Serialize};
4141

4242
/// Generate reports that compare planning statistics with runtime metrics.
4343
///
44-
/// Parser options are captured when the run starts, so `SET` statements do not
45-
/// affect parsing in later query files.
44+
/// Parser options are refreshed before each query file is parsed, so `SET`
45+
/// statements affect later files. Because each file is parsed before any of its
46+
/// statements are executed, parser-setting changes do not affect later
47+
/// statements in the same file.
4648
#[derive(Debug, Args)]
4749
#[command(verbatim_doc_comment)]
4850
pub struct RunOpt {
@@ -68,7 +70,6 @@ impl RunOpt {
6870
let mut config = SessionConfig::from_env()?.with_collect_statistics(true);
6971
config.options_mut().optimizer.prefer_hash_join = true;
7072
let ctx = SessionContext::new_with_config(config);
71-
let sql_parser_options = ctx.state().config_options().sql_parser.clone();
7273
register_parquet_files(&ctx, &self.path).await?;
7374

7475
let branch = current_branch_name();
@@ -87,48 +88,8 @@ impl RunOpt {
8788

8889
let mut reports = vec![];
8990
for query_path in query_files(&self.query_path, self.query.as_deref())? {
90-
let query = query_path
91-
.file_stem()
92-
.expect("query file has a filename")
93-
.to_string_lossy()
94-
.to_string();
95-
let sql = fs::read_to_string(query_path)?;
96-
let statements = match sql_statements(&sql, &sql_parser_options) {
97-
Ok(statements) => statements,
98-
Err(error) => {
99-
let report = QueryReport {
100-
query: query.clone(),
101-
statement: 1,
102-
operators: vec![],
103-
success: false,
104-
error: Some(error.to_string()),
105-
};
106-
print_query_report(&report, previous.as_deref());
107-
reports.push(report);
108-
continue;
109-
}
110-
};
111-
for (statement, sql) in statements.into_iter().enumerate() {
112-
let statement = statement + 1;
113-
let report = match self.report_statement(&ctx, sql).await {
114-
Ok(operators) => QueryReport {
115-
query: query.clone(),
116-
statement,
117-
operators,
118-
success: true,
119-
error: None,
120-
},
121-
Err(error) => QueryReport {
122-
query: query.clone(),
123-
statement,
124-
operators: vec![],
125-
success: false,
126-
error: Some(error.to_string()),
127-
},
128-
};
129-
print_query_report(&report, previous.as_deref());
130-
reports.push(report);
131-
}
91+
self.report_query_file(&ctx, &query_path, previous.as_deref(), &mut reports)
92+
.await?;
13293
}
13394
store_report(&result_path, &reports)?;
13495
print_q_error_summary(
@@ -140,6 +101,59 @@ impl RunOpt {
140101
Ok(())
141102
}
142103

104+
async fn report_query_file(
105+
&self,
106+
ctx: &SessionContext,
107+
query_path: &Path,
108+
previous: Option<&[QueryReport]>,
109+
reports: &mut Vec<QueryReport>,
110+
) -> Result<()> {
111+
let query = query_path
112+
.file_stem()
113+
.expect("query file has a filename")
114+
.to_string_lossy()
115+
.to_string();
116+
let sql = fs::read_to_string(query_path)?;
117+
let sql_parser_options = ctx.state().config_options().sql_parser.clone();
118+
let statements = match sql_statements(&sql, &sql_parser_options) {
119+
Ok(statements) => statements,
120+
Err(error) => {
121+
let report = QueryReport {
122+
query,
123+
statement: 1,
124+
operators: vec![],
125+
success: false,
126+
error: Some(error.to_string()),
127+
};
128+
print_query_report(&report, previous);
129+
reports.push(report);
130+
return Ok(());
131+
}
132+
};
133+
for (statement, sql) in statements.into_iter().enumerate() {
134+
let statement = statement + 1;
135+
let report = match self.report_statement(ctx, sql).await {
136+
Ok(operators) => QueryReport {
137+
query: query.clone(),
138+
statement,
139+
operators,
140+
success: true,
141+
error: None,
142+
},
143+
Err(error) => QueryReport {
144+
query: query.clone(),
145+
statement,
146+
operators: vec![],
147+
success: false,
148+
error: Some(error.to_string()),
149+
},
150+
};
151+
print_query_report(&report, previous);
152+
reports.push(report);
153+
}
154+
Ok(())
155+
}
156+
143157
fn report_path(&self, branch: &str) -> PathBuf {
144158
let report_name = self.query.as_ref().map_or_else(
145159
|| "statistics.json".to_string(),
@@ -772,6 +786,88 @@ mod tests {
772786
assert!(!reports.is_empty());
773787
}
774788

789+
async fn report_query_files(
790+
options: &RunOpt,
791+
ctx: &SessionContext,
792+
) -> Vec<QueryReport> {
793+
let mut reports = vec![];
794+
for path in query_files(&options.query_path, options.query.as_deref()).unwrap() {
795+
options
796+
.report_query_file(ctx, &path, None, &mut reports)
797+
.await
798+
.unwrap();
799+
}
800+
reports
801+
}
802+
803+
#[tokio::test]
804+
async fn refreshes_sql_dialect_between_query_files() {
805+
let directory = tempdir().unwrap();
806+
fs::write(
807+
directory.path().join("01.sql"),
808+
"SET datafusion.sql_parser.dialect = 'MySQL'",
809+
)
810+
.unwrap();
811+
fs::write(directory.path().join("02.sql"), "# MySQL comment\nSELECT 1").unwrap();
812+
fs::write(
813+
directory.path().join("03.sql"),
814+
"RESET datafusion.sql_parser.dialect",
815+
)
816+
.unwrap();
817+
fs::write(directory.path().join("04.sql"), "SELECT 1,").unwrap();
818+
let options = RunOpt {
819+
query: None,
820+
compare: None,
821+
path: directory.path().to_path_buf(),
822+
query_path: directory.path().to_path_buf(),
823+
};
824+
825+
let reports = report_query_files(&options, &SessionContext::new()).await;
826+
827+
assert_eq!(reports.len(), 4);
828+
assert!(reports.iter().all(|report| report.success));
829+
assert_eq!(reports[1].query, "02");
830+
assert!(!reports[1].operators.is_empty());
831+
assert_eq!(reports[3].query, "04");
832+
assert!(!reports[3].operators.is_empty());
833+
}
834+
835+
#[tokio::test]
836+
async fn refreshes_parser_recursion_limit_between_query_files() {
837+
let directory = tempdir().unwrap();
838+
fs::write(
839+
directory.path().join("01.sql"),
840+
"SET datafusion.sql_parser.recursion_limit = 2",
841+
)
842+
.unwrap();
843+
fs::write(
844+
directory.path().join("02.sql"),
845+
"SELECT (((((((((((1)))))))))))",
846+
)
847+
.unwrap();
848+
let options = RunOpt {
849+
query: None,
850+
compare: None,
851+
path: directory.path().to_path_buf(),
852+
query_path: directory.path().to_path_buf(),
853+
};
854+
855+
let reports = report_query_files(&options, &SessionContext::new()).await;
856+
857+
assert_eq!(reports.len(), 2);
858+
assert!(reports[0].success);
859+
assert!(!reports[1].success);
860+
assert_eq!(reports[1].query, "02");
861+
assert!(
862+
reports[1]
863+
.error
864+
.as_deref()
865+
.is_some_and(|error| error.contains("RecursionLimitExceeded")),
866+
"unexpected report: {:?}",
867+
reports[1]
868+
);
869+
}
870+
775871
#[test]
776872
fn persists_failed_reports() {
777873
let directory = tempdir().unwrap();

0 commit comments

Comments
 (0)