Skip to content

Commit bc95b65

Browse files
committed
test: parallelize leaf_pushdown_fuzz with dynamic work-stealing pool (apache#26759)
1 parent 350ca76 commit bc95b65

1 file changed

Lines changed: 180 additions & 47 deletions

File tree

‎datafusion/core/tests/fuzz_cases/leaf_pushdown_fuzz.rs‎

Lines changed: 180 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
//!
3939
//! `LEAF_PUSHDOWN_FUZZ_CASES` also raises the case count of a normal run.
4040
41+
use std::sync::atomic::{AtomicU64, Ordering};
4142
use std::sync::Arc;
4243

4344
use arrow::array::{
@@ -47,7 +48,9 @@ use arrow::array::{
4748
use arrow::datatypes::{DataType, Field, Fields, Int32Type, Schema};
4849
use arrow::util::pretty::pretty_format_batches;
4950
use datafusion::prelude::{ParquetReadOptions, SessionConfig, SessionContext};
51+
use datafusion_common::instant::Instant;
5052
use datafusion_common::Result;
53+
use datafusion_common_runtime::SpawnedTask;
5154
use parquet::arrow::ArrowWriter;
5255
use rand::rngs::SmallRng;
5356
use rand::{Rng, SeedableRng};
@@ -638,47 +641,175 @@ async fn check_case(
638641
}
639642
}
640643

641-
/// Runs `cases` seeded cases against the two contexts. `left` and `right` name
642-
/// the two settings in the failure report. Every case runs, so the report holds
643-
/// the share of the seeds that fail, not only the first one.
644+
/// Limit concurrent worker tasks to prevent thread thrashing
645+
/// since individual queries already execute with `target_partitions = 2`.
646+
const MAX_CONCURRENT_WORKERS: usize = 16;
647+
const DEFAULT_FALLBACK_WORKERS: usize = 1;
648+
649+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
650+
struct Seed(u64);
651+
652+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
653+
struct Cases(usize);
654+
655+
/// Couples an engine's execution context with its diagnostic label.
656+
/// Guarantees at compile time that an engine context cannot be mispaired with the wrong label.
657+
#[derive(Clone, Copy)]
658+
struct TestEngine<'a> {
659+
ctx: &'a SessionContext,
660+
label: &'a str,
661+
}
662+
663+
impl<'a> TestEngine<'a> {
664+
fn new(ctx: &'a SessionContext, label: &'a str) -> Self {
665+
Self { ctx, label }
666+
}
667+
}
668+
669+
/// Aggregate execution metrics collected per worker task.
670+
#[derive(Default)]
671+
struct WorkerResult {
672+
skipped: usize,
673+
failed: usize,
674+
/// Holds `(seed, error_report)` for the lowest numerical seed that failed
675+
/// in this worker. Preserving the earliest seed guarantees 100% deterministic
676+
/// reproduction (`LEAF_PUSHDOWN_FUZZ_SEED=<seed> LEAF_PUSHDOWN_FUZZ_CASES=1`)
677+
/// even though workers finish out of order.
678+
first_failure: Option<(u64, String)>,
679+
}
680+
681+
/// Runs `cases` seeded cases against the two contexts.
682+
///
683+
/// # Concurrency & Dynamic Work-Stealing Architecture
684+
/// - Scales worker tasks to host vCPUs via `available_parallelism`, capped at
685+
/// [`MAX_CONCURRENT_WORKERS`] to prevent thread thrashing.
686+
/// - Seeds are dynamically claimed from a central [`AtomicU64`] counter with `fetch_add`,
687+
/// preventing straggler stalls when complex plans take longer to execute.
688+
/// - `SessionContext` is safely cloned across tasks since tables are pre-registered
689+
/// and differential queries only acquire read locks on `SessionState`.
690+
/// - Each worker tallies its local `skipped`, `failed`, and earliest `first_failure`.
691+
/// The coordinator reduces across all workers by minimal seed number so failure
692+
/// reporting matches sequential execution order.
644693
async fn run(
645-
ctx_left: &SessionContext,
646-
ctx_right: &SessionContext,
647-
left: &str,
648-
right: &str,
649-
cases: usize,
650-
seed0: u64,
694+
left: TestEngine<'_>,
695+
right: TestEngine<'_>,
696+
cases: Cases,
697+
seed0: Seed,
651698
) {
652-
let mut skipped = 0;
653-
let mut failed = 0;
654-
let mut first: Option<String> = None;
655-
for seed in seed0..seed0 + cases as u64 {
656-
let case = gen_case(&mut SmallRng::seed_from_u64(seed));
657-
let Some(detail) =
658-
check_case(ctx_left, ctx_right, left, right, &case, &mut skipped).await
659-
else {
660-
continue;
661-
};
662-
failed += 1;
663-
first.get_or_insert_with(|| {
664-
format!(
665-
" seed: {seed}\n sql: {}\n {detail}\n reproduce with \
666-
LEAF_PUSHDOWN_FUZZ_SEED={seed} LEAF_PUSHDOWN_FUZZ_CASES=1",
667-
case.sql
668-
)
669-
});
699+
let Cases(num_cases) = cases;
700+
let Seed(start_seed) = seed0;
701+
702+
if num_cases == 0 {
703+
return;
670704
}
671-
let ratio = skipped as f64 / cases as f64;
705+
706+
let start_instant = Instant::now();
707+
708+
let num_workers = match std::thread::available_parallelism() {
709+
Ok(n) => n.get().clamp(1, MAX_CONCURRENT_WORKERS).min(num_cases),
710+
Err(e) => {
711+
eprintln!(
712+
"\n[WARNING] leaf pushdown fuzz: failed to detect CPU cores ({e}).\n\
713+
Falling back to {DEFAULT_FALLBACK_WORKERS} worker.\n"
714+
);
715+
DEFAULT_FALLBACK_WORKERS.min(num_cases)
716+
}
717+
};
718+
719+
let next_seed = Arc::new(AtomicU64::new(start_seed));
720+
let max_seed = start_seed + num_cases as u64;
721+
722+
let mut handles = Vec::with_capacity(num_workers);
723+
for _ in 0..num_workers {
724+
let ctx_left = left.ctx.clone();
725+
let ctx_right = right.ctx.clone();
726+
let left_label = left.label.to_string();
727+
let right_label = right.label.to_string();
728+
let next_seed = Arc::clone(&next_seed);
729+
730+
handles.push(SpawnedTask::spawn(async move {
731+
let mut res = WorkerResult::default();
732+
733+
loop {
734+
if next_seed.load(Ordering::Relaxed) >= max_seed {
735+
break;
736+
}
737+
let seed = next_seed.fetch_add(1, Ordering::Relaxed);
738+
if seed >= max_seed {
739+
break;
740+
}
741+
742+
let case = gen_case(&mut SmallRng::seed_from_u64(seed));
743+
let Some(detail) = check_case(
744+
&ctx_left,
745+
&ctx_right,
746+
&left_label,
747+
&right_label,
748+
&case,
749+
&mut res.skipped,
750+
)
751+
.await
752+
else {
753+
continue;
754+
};
755+
756+
res.failed += 1;
757+
if res.first_failure.is_none() {
758+
res.first_failure = Some((
759+
seed,
760+
format!(
761+
" seed: {seed}\n sql: {}\n {detail}\n reproduce with \
762+
LEAF_PUSHDOWN_FUZZ_SEED={seed} LEAF_PUSHDOWN_FUZZ_CASES=1",
763+
case.sql
764+
),
765+
));
766+
}
767+
}
768+
769+
res
770+
}));
771+
}
772+
773+
let mut total = WorkerResult::default();
774+
775+
for handle in handles {
776+
let worker_res = handle.await.expect("fuzz worker task panicked");
777+
total.skipped += worker_res.skipped;
778+
total.failed += worker_res.failed;
779+
780+
if let Some((seed, report)) = worker_res.first_failure {
781+
match &total.first_failure {
782+
Some((best_seed, _)) if seed < *best_seed => {
783+
total.first_failure = Some((seed, report));
784+
}
785+
None => {
786+
total.first_failure = Some((seed, report));
787+
}
788+
_ => {}
789+
}
790+
}
791+
}
792+
793+
let succeeded = num_cases - total.skipped - total.failed;
794+
let first = total.first_failure.map(|(_, report)| report);
795+
let ratio = total.skipped as f64 / num_cases as f64;
796+
let (left_label, right_label) = (left.label, right.label);
797+
let elapsed = start_instant.elapsed();
798+
672799
println!(
673-
"leaf pushdown fuzz: {left} vs {right}, seeds {seed0}..{}, ok {}, \
674-
skipped {skipped} ({:.1}%), failed {failed}",
675-
seed0 + cases as u64,
676-
cases - skipped - failed,
677-
ratio * 100.0
800+
"leaf pushdown fuzz: {left_label} vs {right_label}, seeds {start_seed}..{}, ok {succeeded}, \
801+
skipped {} ({:.1}%), failed {} in {:.2?}",
802+
start_seed + num_cases as u64,
803+
total.skipped,
804+
ratio * 100.0,
805+
total.failed,
806+
elapsed
678807
);
808+
679809
assert!(
680810
first.is_none(),
681-
"leaf pushdown fuzz: {failed} of {cases} seeds failed, first one:\n{}",
811+
"leaf pushdown fuzz: {} of {num_cases} seeds failed, first one:\n{}",
812+
total.failed,
682813
first.unwrap_or_default()
683814
);
684815
assert!(
@@ -695,14 +826,20 @@ fn env_usize(name: &str, default: usize) -> usize {
695826
.unwrap_or(default)
696827
}
697828

698-
fn seed() -> u64 {
699-
env_usize("LEAF_PUSHDOWN_FUZZ_SEED", 0) as u64
829+
fn seed() -> Seed {
830+
Seed(env_usize("LEAF_PUSHDOWN_FUZZ_SEED", 0) as u64)
700831
}
701832

702833
async fn run_memory(cases: usize) {
703834
let on = memory_ctx(true);
704835
let off = memory_ctx(false);
705-
run(&on, &off, "pushdown on", "pushdown off", cases, seed()).await;
836+
run(
837+
TestEngine::new(&on, "pushdown on"),
838+
TestEngine::new(&off, "pushdown off"),
839+
Cases(cases),
840+
seed(),
841+
)
842+
.await;
706843
}
707844

708845
// ---------------------------------------------------------------------------
@@ -753,11 +890,9 @@ async fn run_parquet(cases: usize) {
753890
let on = parquet_ctx(&dir, true).await;
754891
let off = parquet_ctx(&dir, false).await;
755892
run(
756-
&on,
757-
&off,
758-
"parquet pushdown on",
759-
"parquet pushdown off",
760-
cases,
893+
TestEngine::new(&on, "parquet pushdown on"),
894+
TestEngine::new(&off, "parquet pushdown off"),
895+
Cases(cases),
761896
seed(),
762897
)
763898
.await;
@@ -820,11 +955,9 @@ async fn leaf_pushdown_parquet_schema_evolution() {
820955
}
821956
check_evolved_filters(&contexts[0], &contexts[1], &wide).await;
822957
run(
823-
&contexts[0],
824-
&contexts[1],
825-
"parquet pushdown on",
826-
"parquet pushdown off",
827-
DEFAULT_CASES / 5,
958+
TestEngine::new(&contexts[0], "parquet pushdown on"),
959+
TestEngine::new(&contexts[1], "parquet pushdown off"),
960+
Cases(DEFAULT_CASES / 5),
828961
seed(),
829962
)
830963
.await;

0 commit comments

Comments
 (0)