Skip to content

Commit b305183

Browse files
committed
test: parallelize leaf_pushdown_fuzz with dynamic work-stealing pool (apache#26759)
1 parent 350ca76 commit b305183

1 file changed

Lines changed: 177 additions & 49 deletions

File tree

‎datafusion/core/tests/fuzz_cases/leaf_pushdown_fuzz.rs‎

Lines changed: 177 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
//! `LEAF_PUSHDOWN_FUZZ_CASES` also raises the case count of a normal run.
4040
4141
use std::sync::Arc;
42+
use std::sync::atomic::{AtomicU64, Ordering};
4243

4344
use arrow::array::{
4445
Array, ArrayRef, AsArray, Int32Array, Int64Array, RecordBatch, StringArray,
@@ -48,6 +49,8 @@ use arrow::datatypes::{DataType, Field, Fields, Int32Type, Schema};
4849
use arrow::util::pretty::pretty_format_batches;
4950
use datafusion::prelude::{ParquetReadOptions, SessionConfig, SessionContext};
5051
use datafusion_common::Result;
52+
use datafusion_common::instant::Instant;
53+
use datafusion_common_runtime::SpawnedTask;
5154
use parquet::arrow::ArrowWriter;
5255
use rand::rngs::SmallRng;
5356
use rand::{Rng, SeedableRng};
@@ -638,47 +641,170 @@ async fn check_case(
638641
}
639642
}
640643

641-
/// Runs `cases` seeded cases against the two contexts. `left` and `right` name
642-
/// the two settings in the failure report. Every case runs, so the report holds
643-
/// the share of the seeds that fail, not only the first one.
644-
async fn run(
645-
ctx_left: &SessionContext,
646-
ctx_right: &SessionContext,
647-
left: &str,
648-
right: &str,
649-
cases: usize,
650-
seed0: u64,
651-
) {
652-
let mut skipped = 0;
653-
let mut failed = 0;
654-
let mut first: Option<String> = None;
655-
for seed in seed0..seed0 + cases as u64 {
656-
let case = gen_case(&mut SmallRng::seed_from_u64(seed));
657-
let Some(detail) =
658-
check_case(ctx_left, ctx_right, left, right, &case, &mut skipped).await
659-
else {
660-
continue;
661-
};
662-
failed += 1;
663-
first.get_or_insert_with(|| {
664-
format!(
665-
" seed: {seed}\n sql: {}\n {detail}\n reproduce with \
666-
LEAF_PUSHDOWN_FUZZ_SEED={seed} LEAF_PUSHDOWN_FUZZ_CASES=1",
667-
case.sql
668-
)
669-
});
644+
/// Limit concurrent worker tasks to prevent thread thrashing
645+
/// since individual queries already execute with `target_partitions = 2`.
646+
const MAX_CONCURRENT_WORKERS: usize = 16;
647+
const DEFAULT_FALLBACK_WORKERS: usize = 1;
648+
649+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
650+
struct Seed(u64);
651+
652+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
653+
struct Cases(usize);
654+
655+
/// Couples an engine's execution context with its diagnostic label.
656+
/// Guarantees at compile time that an engine context cannot be mispaired with the wrong label.
657+
#[derive(Clone, Copy)]
658+
struct TestEngine<'a> {
659+
ctx: &'a SessionContext,
660+
label: &'a str,
661+
}
662+
663+
impl<'a> TestEngine<'a> {
664+
fn new(ctx: &'a SessionContext, label: &'a str) -> Self {
665+
Self { ctx, label }
670666
}
671-
let ratio = skipped as f64 / cases as f64;
667+
}
668+
669+
/// Aggregate execution metrics collected per worker task.
670+
#[derive(Default)]
671+
struct WorkerResult {
672+
skipped: usize,
673+
failed: usize,
674+
/// Holds `(seed, error_report)` for the lowest numerical seed that failed
675+
/// in this worker. Preserving the earliest seed guarantees 100% deterministic
676+
/// reproduction (`LEAF_PUSHDOWN_FUZZ_SEED=<seed> LEAF_PUSHDOWN_FUZZ_CASES=1`)
677+
/// even though workers finish out of order.
678+
first_failure: Option<(u64, String)>,
679+
}
680+
681+
/// Runs `cases` seeded cases against the two contexts.
682+
///
683+
/// # Concurrency & Dynamic Work-Stealing Architecture
684+
/// - Scales worker tasks to host vCPUs via `available_parallelism`, capped at
685+
/// [`MAX_CONCURRENT_WORKERS`] to prevent thread thrashing.
686+
/// - Seeds are dynamically claimed from a central [`AtomicU64`] counter with `fetch_add`,
687+
/// preventing straggler stalls when complex plans take longer to execute.
688+
/// - `SessionContext` is safely cloned across tasks since tables are pre-registered
689+
/// and differential queries only acquire read locks on `SessionState`.
690+
/// - Each worker tallies its local `skipped`, `failed`, and earliest `first_failure`.
691+
/// The coordinator reduces across all workers by minimal seed number so failure
692+
/// reporting matches sequential execution order.
693+
async fn run(left: TestEngine<'_>, right: TestEngine<'_>, cases: Cases, seed0: Seed) {
694+
let Cases(num_cases) = cases;
695+
let Seed(start_seed) = seed0;
696+
697+
if num_cases == 0 {
698+
return;
699+
}
700+
701+
let start_instant = Instant::now();
702+
703+
let num_workers = match std::thread::available_parallelism() {
704+
Ok(n) => n.get().clamp(1, MAX_CONCURRENT_WORKERS).min(num_cases),
705+
Err(e) => {
706+
eprintln!(
707+
"\n[WARNING] leaf pushdown fuzz: failed to detect CPU cores ({e}).\n\
708+
Falling back to {DEFAULT_FALLBACK_WORKERS} worker.\n"
709+
);
710+
DEFAULT_FALLBACK_WORKERS.min(num_cases)
711+
}
712+
};
713+
714+
let next_seed = Arc::new(AtomicU64::new(start_seed));
715+
let max_seed = start_seed + num_cases as u64;
716+
717+
let mut handles = Vec::with_capacity(num_workers);
718+
for _ in 0..num_workers {
719+
let ctx_left = left.ctx.clone();
720+
let ctx_right = right.ctx.clone();
721+
let left_label = left.label.to_string();
722+
let right_label = right.label.to_string();
723+
let next_seed = Arc::clone(&next_seed);
724+
725+
handles.push(SpawnedTask::spawn(async move {
726+
let mut res = WorkerResult::default();
727+
728+
loop {
729+
if next_seed.load(Ordering::Relaxed) >= max_seed {
730+
break;
731+
}
732+
let seed = next_seed.fetch_add(1, Ordering::Relaxed);
733+
if seed >= max_seed {
734+
break;
735+
}
736+
737+
let case = gen_case(&mut SmallRng::seed_from_u64(seed));
738+
let Some(detail) = check_case(
739+
&ctx_left,
740+
&ctx_right,
741+
&left_label,
742+
&right_label,
743+
&case,
744+
&mut res.skipped,
745+
)
746+
.await
747+
else {
748+
continue;
749+
};
750+
751+
res.failed += 1;
752+
if res.first_failure.is_none() {
753+
res.first_failure = Some((
754+
seed,
755+
format!(
756+
" seed: {seed}\n sql: {}\n {detail}\n reproduce with \
757+
LEAF_PUSHDOWN_FUZZ_SEED={seed} LEAF_PUSHDOWN_FUZZ_CASES=1",
758+
case.sql
759+
),
760+
));
761+
}
762+
}
763+
764+
res
765+
}));
766+
}
767+
768+
let mut total = WorkerResult::default();
769+
770+
for handle in handles {
771+
let worker_res = handle.await.expect("fuzz worker task panicked");
772+
total.skipped += worker_res.skipped;
773+
total.failed += worker_res.failed;
774+
775+
if let Some((seed, report)) = worker_res.first_failure {
776+
match &total.first_failure {
777+
Some((best_seed, _)) if seed < *best_seed => {
778+
total.first_failure = Some((seed, report));
779+
}
780+
None => {
781+
total.first_failure = Some((seed, report));
782+
}
783+
_ => {}
784+
}
785+
}
786+
}
787+
788+
let succeeded = num_cases - total.skipped - total.failed;
789+
let first = total.first_failure.map(|(_, report)| report);
790+
let ratio = total.skipped as f64 / num_cases as f64;
791+
let (left_label, right_label) = (left.label, right.label);
792+
let elapsed = start_instant.elapsed();
793+
672794
println!(
673-
"leaf pushdown fuzz: {left} vs {right}, seeds {seed0}..{}, ok {}, \
674-
skipped {skipped} ({:.1}%), failed {failed}",
675-
seed0 + cases as u64,
676-
cases - skipped - failed,
677-
ratio * 100.0
795+
"leaf pushdown fuzz: {left_label} vs {right_label}, seeds {start_seed}..{}, ok {succeeded}, \
796+
skipped {} ({:.1}%), failed {} in {:.2?}",
797+
start_seed + num_cases as u64,
798+
total.skipped,
799+
ratio * 100.0,
800+
total.failed,
801+
elapsed
678802
);
803+
679804
assert!(
680805
first.is_none(),
681-
"leaf pushdown fuzz: {failed} of {cases} seeds failed, first one:\n{}",
806+
"leaf pushdown fuzz: {} of {num_cases} seeds failed, first one:\n{}",
807+
total.failed,
682808
first.unwrap_or_default()
683809
);
684810
assert!(
@@ -695,14 +821,20 @@ fn env_usize(name: &str, default: usize) -> usize {
695821
.unwrap_or(default)
696822
}
697823

698-
fn seed() -> u64 {
699-
env_usize("LEAF_PUSHDOWN_FUZZ_SEED", 0) as u64
824+
fn seed() -> Seed {
825+
Seed(env_usize("LEAF_PUSHDOWN_FUZZ_SEED", 0) as u64)
700826
}
701827

702828
async fn run_memory(cases: usize) {
703829
let on = memory_ctx(true);
704830
let off = memory_ctx(false);
705-
run(&on, &off, "pushdown on", "pushdown off", cases, seed()).await;
831+
run(
832+
TestEngine::new(&on, "pushdown on"),
833+
TestEngine::new(&off, "pushdown off"),
834+
Cases(cases),
835+
seed(),
836+
)
837+
.await;
706838
}
707839

708840
// ---------------------------------------------------------------------------
@@ -753,11 +885,9 @@ async fn run_parquet(cases: usize) {
753885
let on = parquet_ctx(&dir, true).await;
754886
let off = parquet_ctx(&dir, false).await;
755887
run(
756-
&on,
757-
&off,
758-
"parquet pushdown on",
759-
"parquet pushdown off",
760-
cases,
888+
TestEngine::new(&on, "parquet pushdown on"),
889+
TestEngine::new(&off, "parquet pushdown off"),
890+
Cases(cases),
761891
seed(),
762892
)
763893
.await;
@@ -820,11 +950,9 @@ async fn leaf_pushdown_parquet_schema_evolution() {
820950
}
821951
check_evolved_filters(&contexts[0], &contexts[1], &wide).await;
822952
run(
823-
&contexts[0],
824-
&contexts[1],
825-
"parquet pushdown on",
826-
"parquet pushdown off",
827-
DEFAULT_CASES / 5,
953+
TestEngine::new(&contexts[0], "parquet pushdown on"),
954+
TestEngine::new(&contexts[1], "parquet pushdown off"),
955+
Cases(DEFAULT_CASES / 5),
828956
seed(),
829957
)
830958
.await;

0 commit comments

Comments
 (0)