Skip to content

Commit 2289eee

Browse files
maxburkemaxburke
authored
Merge branch 'main' into join-avoid-concat
2 parents 6c55cb9 + 166c040 commit 2289eee

113 files changed

Lines changed: 7469 additions & 823 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/actions/setup-rust-runtime/action.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,14 @@ runs:
3131
run: |
3232
echo "RUST_BACKTRACE=1" >> $GITHUB_ENV
3333
echo "RUSTFLAGS=-C debuginfo=line-tables-only -C incremental=false" >> $GITHUB_ENV
34+
# Work around intermittent "[16] Error in the HTTP2 framing layer"
35+
# failures from curl when cargo fetches crates from crates.io.
36+
# Disabling HTTP/2 multiplexing forces cargo to serialize requests,
37+
# and raising retries makes transient network hiccups self-heal.
38+
#
39+
# Reference:
40+
# https://doc.rust-lang.org/cargo/reference/config.html?#httpmultiplexing
41+
# https://doc.rust-lang.org/cargo/reference/config.html?#netretry
42+
echo "CARGO_HTTP_MULTIPLEXING=false" >> $GITHUB_ENV
43+
echo "CARGO_NET_RETRY=10" >> $GITHUB_ENV
44+
echo "CARGO_HTTP_RETRY=10" >> $GITHUB_ENV

‎.github/workflows/audit.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,4 +51,6 @@ jobs:
5151
- name: Run audit check
5252
# Note: you can ignore specific RUSTSEC issues using the `--ignore` flag ,for example:
5353
# run: cargo audit --ignore RUSTSEC-2026-0001
54-
run: cargo audit
54+
# TODO: remove once object_store upgrades to quick-xml >= 0.41.0
55+
# https://github.com/apache/datafusion/issues/23297
56+
run: cargo audit --ignore RUSTSEC-2026-0194 --ignore RUSTSEC-2026-0195

‎.github/workflows/dependencies.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ on:
2525
push:
2626
branches-ignore:
2727
- 'gh-readonly-queue/**'
28+
- 'dependabot/**'
2829
pull_request:
2930
merge_group:
3031
# manual trigger

‎.github/workflows/dev.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ on:
2020
push:
2121
branches-ignore:
2222
- 'gh-readonly-queue/**'
23+
- 'dependabot/**'
2324
pull_request:
2425
merge_group:
2526

‎.github/workflows/docs_pr.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ on:
2626
push:
2727
paths:
2828
- "docs/**"
29+
branches:
30+
- main
31+
- branch-*
2932
pull_request:
3033
paths:
3134
- "docs/**"

‎.github/workflows/rust.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ on:
2727
push:
2828
branches-ignore:
2929
- 'gh-readonly-queue/**'
30+
- 'dependabot/**'
3031
paths-ignore:
3132
- "docs/**"
3233
- "**.md"

‎AGENTS.md‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,20 @@ When creating a PR, you MUST follow the [PR template](.github/pull_request_templ
3737

3838
## Testing
3939

40-
See the [Testing Quick Start](docs/source/contributor-guide/testing.md#testing-quick-start)
41-
for the recommended pre-PR test commands.
40+
If documentation files changed then run
41+
```bash
42+
./ci/scripts/doc_prettier_check.sh --write --allow-dirty
43+
```
44+
45+
Otherwise, run extended tests
46+
```bash
47+
RUST_BACKTRACE=1 cargo test --profile ci \
48+
--exclude datafusion-examples --exclude datafusion-benchmarks --exclude datafusion-cli \
49+
--workspace --lib --tests --bins \
50+
--features avro,json,backtrace,extended_tests,recursive_protection,parquet_encryption
51+
```
52+
53+
For modified code identify local benchmarks(if any) and run them against `main`. See [Benchmarks](benchmarks/README.md).
4254

4355
## Agent Skills
4456

‎Cargo.lock‎

Lines changed: 3 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎benchmarks/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ clap = { version = "4.6.0", features = ["derive", "env"] }
4747
criterion = { workspace = true, features = ["html_reports"] }
4848
datafusion = { workspace = true, default-features = true }
4949
datafusion-common = { workspace = true, default-features = true }
50+
datafusion-common-runtime = { workspace = true }
5051
env_logger = { workspace = true }
5152
futures = { workspace = true }
5253
libmimalloc-sys = { version = "0.1", optional = true }

‎benchmarks/benches/sql.rs‎

Lines changed: 20 additions & 239 deletions
Original file line numberDiff line numberDiff line change
@@ -22,26 +22,13 @@
2222
//! Cargo, for example: `BENCH_NAME=tpch cargo bench --bench sql`.
2323
2424
use clap::Parser;
25-
use criterion::{Criterion, SamplingMode, criterion_group, criterion_main};
26-
use datafusion::error::Result;
27-
use datafusion::prelude::SessionContext;
28-
use datafusion_benchmarks::sql_benchmark::SqlBenchmark;
29-
use datafusion_benchmarks::util::{CommonOpt, print_memory_stats};
25+
use criterion::{Criterion, criterion_group, criterion_main};
26+
use datafusion_benchmarks::sql_benchmark_runner::{
27+
BenchmarkFilter, SqlRunConfig, default_sql_benchmark_directory,
28+
run_criterion_benchmarks_impl,
29+
};
30+
use datafusion_benchmarks::util::CommonOpt;
3031
use datafusion_common::instant::Instant;
31-
use log::{debug, info};
32-
use std::collections::BTreeMap;
33-
use std::fs;
34-
use std::sync::LazyLock;
35-
use tokio::runtime::Runtime;
36-
37-
static SQL_BENCHMARK_DIRECTORY: LazyLock<String> = LazyLock::new(|| {
38-
format!(
39-
"{}{}{}",
40-
env!("CARGO_MANIFEST_DIR"),
41-
std::path::MAIN_SEPARATOR,
42-
"sql_benchmarks"
43-
)
44-
});
4532

4633
#[cfg(feature = "snmalloc")]
4734
#[global_allocator]
@@ -90,234 +77,28 @@ pub fn sql(c: &mut Criterion) {
9077

9178
let start = Instant::now();
9279
let args = EnvParser::parse();
93-
let rt = make_tokio_runtime();
80+
let config = SqlRunConfig {
81+
common: args.options,
82+
filter: BenchmarkFilter {
83+
name: args.name,
84+
subgroup: args.subgroup,
85+
query: args.query,
86+
},
87+
persist_results: args.persist_results,
88+
validate_results: args.validate,
89+
output: None,
90+
};
9491

9592
println!("Loading benchmarks...");
9693

97-
let benchmarks = rt.block_on(async {
98-
let ctx = make_ctx(&args).expect("SessionContext creation failed");
99-
100-
load_benchmarks(&args, &ctx, &SQL_BENCHMARK_DIRECTORY)
101-
.await
102-
.unwrap_or_else(|err| panic!("failed load benchmarks: {err:?}"))
103-
});
94+
run_criterion_benchmarks_impl(&default_sql_benchmark_directory(), &config, c)
95+
.unwrap_or_else(|err| panic!("failed to run SQL benchmarks: {err:?}"));
10496

10597
println!(
106-
"Loaded benchmarks in {} ms ...",
98+
"Completed benchmarks in {} ms ...",
10799
start.elapsed().as_millis()
108100
);
109-
110-
for (group, benchmarks) in benchmarks {
111-
let mut group = c.benchmark_group(group);
112-
group.sample_size(10);
113-
group.sampling_mode(SamplingMode::Flat);
114-
115-
for mut benchmark in benchmarks {
116-
// create a context
117-
let ctx = make_ctx(&args).expect("SessionContext creation failed");
118-
119-
// initialize the benchmark. This parses the benchmark file and does any pre-execution
120-
// work such as loading data into tables
121-
rt.block_on(async {
122-
benchmark
123-
.initialize(&ctx)
124-
.await
125-
.expect("initialization failed");
126-
127-
// run assertions
128-
benchmark.assert(&ctx).await.expect("assertion failed");
129-
});
130-
131-
let mut name = benchmark.name().to_string();
132-
if !benchmark.subgroup().is_empty() {
133-
name.push('_');
134-
name.push_str(benchmark.subgroup());
135-
}
136-
137-
if args.persist_results {
138-
handle_persist(&rt, &ctx, &name, &mut benchmark);
139-
} else if args.validate {
140-
handle_verify(&rt, &ctx, &name, &mut benchmark);
141-
} else {
142-
info!("Running benchmark {name} ...");
143-
144-
let name = name.clone();
145-
group.bench_function(name.clone(), |b| {
146-
b.iter(|| handle_run(&rt, &ctx, &args, &mut benchmark, &name))
147-
});
148-
149-
print_memory_stats();
150-
151-
info!("Benchmark {name} completed");
152-
}
153-
154-
// run cleanup
155-
rt.block_on(async {
156-
benchmark.cleanup(&ctx).await.expect("Cleanup failed");
157-
});
158-
}
159-
160-
group.finish();
161-
}
162-
}
163-
164-
fn handle_run(
165-
rt: &Runtime,
166-
ctx: &SessionContext,
167-
args: &EnvParser,
168-
benchmark: &mut SqlBenchmark,
169-
name: &str,
170-
) {
171-
rt.block_on(async {
172-
benchmark
173-
.run(ctx, args.validate)
174-
.await
175-
.unwrap_or_else(|err| panic!("Failed to run benchmark {name}: {err:?}"))
176-
});
177-
}
178-
179-
fn handle_persist(
180-
rt: &Runtime,
181-
ctx: &SessionContext,
182-
name: &str,
183-
benchmark: &mut SqlBenchmark,
184-
) {
185-
info!("Running benchmark {name} prior to persisting results ...");
186-
187-
rt.block_on(async {
188-
info!("Persisting benchmark {name} ...");
189-
190-
benchmark
191-
.persist(ctx)
192-
.await
193-
.expect("Failed to persist results");
194-
});
195-
196-
info!("Persisted benchmark {name} successfully");
197-
}
198-
199-
fn handle_verify(
200-
rt: &Runtime,
201-
ctx: &SessionContext,
202-
name: &str,
203-
benchmark: &mut SqlBenchmark,
204-
) {
205-
info!("Verifying benchmark {name} results ...");
206-
207-
rt.block_on(async {
208-
benchmark
209-
.run(ctx, true)
210-
.await
211-
.unwrap_or_else(|err| panic!("Failed to run benchmark {name}: {err:?}"));
212-
benchmark
213-
.verify(ctx)
214-
.await
215-
.unwrap_or_else(|err| panic!("Verification failed: {err:?}"));
216-
});
217-
218-
info!("Verified benchmark {name} results successfully");
219101
}
220102

221103
criterion_group!(benches, sql);
222104
criterion_main!(benches);
223-
224-
fn make_tokio_runtime() -> Runtime {
225-
tokio::runtime::Builder::new_multi_thread()
226-
.enable_all()
227-
.build()
228-
.unwrap()
229-
}
230-
231-
fn make_ctx(args: &EnvParser) -> Result<SessionContext> {
232-
let config = args.options.config()?;
233-
let rt = args.options.build_runtime()?;
234-
235-
Ok(SessionContext::new_with_config_rt(config, rt))
236-
}
237-
238-
/// Recursively walks the directory tree starting at `path` and
239-
/// calls the call back function for every file encountered.
240-
pub fn list_files<F>(path: &str, callback: &mut F)
241-
where
242-
F: FnMut(&str),
243-
{
244-
let mut entries: Vec<fs::DirEntry> =
245-
fs::read_dir(path).unwrap().filter_map(Result::ok).collect();
246-
entries.sort_by_key(|entry| entry.path());
247-
248-
for dir_entry in entries {
249-
let path = dir_entry.path();
250-
if path.is_dir() {
251-
// Recurse into the sub‑directory
252-
list_files(&path.to_string_lossy(), callback);
253-
} else {
254-
// For files, invoke the callback with the full path as a string
255-
let full_str = path.to_string_lossy();
256-
callback(&full_str);
257-
}
258-
}
259-
}
260-
261-
/// Loads all benchmark files in the `sql_benchmarks` directory.
262-
/// For each file ending with `.benchmark` it creates a new
263-
/// `SqlBenchmark` instance.
264-
async fn load_benchmarks(
265-
args: &EnvParser,
266-
ctx: &SessionContext,
267-
path: &str,
268-
) -> Result<BTreeMap<String, Vec<SqlBenchmark>>> {
269-
let mut benches = BTreeMap::new();
270-
let mut paths = Vec::new();
271-
272-
list_files(path, &mut |path: &str| {
273-
if path.ends_with(".benchmark") {
274-
paths.push(path.to_string());
275-
}
276-
});
277-
278-
for path in paths {
279-
debug!("Loading benchmark from {path}");
280-
281-
let benchmark = SqlBenchmark::new(ctx, &path, &*SQL_BENCHMARK_DIRECTORY).await?;
282-
let entries = benches
283-
.entry(benchmark.group().to_string())
284-
.or_insert(vec![]);
285-
286-
entries.push(benchmark);
287-
}
288-
289-
benches = filter_benchmarks(args, benches);
290-
benches.iter_mut().for_each(|(_, benchmarks)| {
291-
benchmarks.sort_by(|b1, b2| b1.name().cmp(b2.name()))
292-
});
293-
294-
Ok(benches)
295-
}
296-
297-
fn filter_benchmarks(
298-
args: &EnvParser,
299-
benchmarks: BTreeMap<String, Vec<SqlBenchmark>>,
300-
) -> BTreeMap<String, Vec<SqlBenchmark>> {
301-
match &args.name {
302-
Some(bench_name) => benchmarks
303-
.into_iter()
304-
.filter(|(key, _val)| key.eq_ignore_ascii_case(bench_name))
305-
.map(|(key, mut val)| {
306-
if let Some(subgroup) = &args.subgroup {
307-
val.retain(|bench| bench.subgroup().eq_ignore_ascii_case(subgroup));
308-
}
309-
if let Some(query) = &args.query {
310-
// Accept `1`, `01`, `6a`, `Q06a`, ... case-insensitively.
311-
// Bench names are canonical, e.g. `Q01`, `Q06a`.
312-
let q = query.trim_start_matches(['Q', 'q']);
313-
let split = q.find(|c: char| !c.is_ascii_digit()).unwrap_or(q.len());
314-
let (num, suffix) = q.split_at(split);
315-
let normalized = format!("Q{num:0>2}{suffix}");
316-
val.retain(|bench| bench.name().eq_ignore_ascii_case(&normalized));
317-
}
318-
(key, val)
319-
})
320-
.collect(),
321-
None => benchmarks,
322-
}
323-
}

0 commit comments

Comments
 (0)