Skip to content

Commit 4e95802

Browse files
adriangbclaude
andcommitted
Use tokio::sync::watch for dynamic filter completion
Per review feedback, replace the hand-rolled Mutex<CompletionState> + Notify pair with a watch::Sender<Option<SharedResult<()>>> for the terminal completion broadcast. The mutex still guards the incremental per-partition accumulation (which needs partial writes) and the finalizer-election bool, but wait_for_completion collapses to a single rx.wait_for(|v| v.is_some()).await instead of a manual re-check loop. Net -8 lines and drops the three-state CompletionState enum. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
1 parent 9023c19 commit 4e95802

1 file changed

Lines changed: 24 additions & 32 deletions

File tree

‎datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs‎

Lines changed: 24 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ use datafusion_physical_expr::expressions::{
4242
use datafusion_physical_expr::{PhysicalExpr, PhysicalExprRef, ScalarFunctionExpr};
4343

4444
use parking_lot::Mutex;
45-
use tokio::sync::Notify;
45+
use tokio::sync::watch;
4646

4747
/// Represents the minimum and maximum values for a specific column.
4848
/// Used in dynamic filter pushdown to establish value boundaries.
@@ -216,7 +216,9 @@ fn create_bounds_predicate(
216216
pub(crate) struct SharedBuildAccumulator {
217217
/// Build-side data protected by a single mutex to avoid ordering concerns
218218
inner: Mutex<AccumulatorState>,
219-
completion_notify: Notify,
219+
/// Broadcasts the finalized filter-build result to every waiting partition.
220+
/// `None` means finalization hasn't run yet; `Some(result)` is the terminal state.
221+
completion_tx: watch::Sender<Option<SharedResult<()>>>,
220222
/// Dynamic filter for pushdown to probe side
221223
dynamic_filter: Arc<DynamicFilterPhysicalExpr>,
222224
/// Right side join expressions needed for creating filter expressions
@@ -272,15 +274,12 @@ enum AccumulatedBuildData {
272274
},
273275
}
274276

275-
enum CompletionState {
276-
Pending,
277-
Finalizing,
278-
Ready(SharedResult<()>),
279-
}
280-
281277
struct AccumulatorState {
282278
data: AccumulatedBuildData,
283-
completion: CompletionState,
279+
/// Set once a caller has been elected to run finalization. Prevents a
280+
/// second caller from repeating the filter-build work when extra reports
281+
/// arrive after the terminal condition was reached.
282+
finalizing: bool,
284283
}
285284

286285
#[derive(Clone)]
@@ -365,12 +364,13 @@ impl SharedBuildAccumulator {
365364
),
366365
};
367366

367+
let (completion_tx, _) = watch::channel(None);
368368
Self {
369369
inner: Mutex::new(AccumulatorState {
370370
data: mode_data,
371-
completion: CompletionState::Pending,
371+
finalizing: false,
372372
}),
373-
completion_notify: Notify::new(),
373+
completion_tx,
374374
dynamic_filter,
375375
on_right,
376376
repartition_random_state,
@@ -481,7 +481,7 @@ impl SharedBuildAccumulator {
481481
&self,
482482
guard: &mut AccumulatorState,
483483
) -> Option<FinalizeInput> {
484-
if !matches!(guard.completion, CompletionState::Pending) {
484+
if guard.finalizing {
485485
return None;
486486
}
487487

@@ -502,35 +502,27 @@ impl SharedBuildAccumulator {
502502
_ => None,
503503
}?;
504504

505-
guard.completion = CompletionState::Finalizing;
505+
guard.finalizing = true;
506506
Some(finalize_input)
507507
}
508508

509509
fn finish(&self, finalize_input: FinalizeInput) {
510510
let result = self.build_filter(finalize_input).map_err(Arc::new);
511511
self.dynamic_filter.mark_complete();
512-
513-
let mut guard = self.inner.lock();
514-
guard.completion = CompletionState::Ready(result);
515-
drop(guard);
516-
self.completion_notify.notify_waiters();
512+
// `send_replace` unconditionally publishes even if no receivers are
513+
// currently subscribed (e.g. every partition was canceled).
514+
self.completion_tx.send_replace(Some(result));
517515
}
518516

519517
async fn wait_for_completion(&self) -> Result<()> {
520-
loop {
521-
let notified = {
522-
let guard = self.inner.lock();
523-
match &guard.completion {
524-
CompletionState::Ready(Ok(())) => return Ok(()),
525-
CompletionState::Ready(Err(err)) => {
526-
return Err(DataFusionError::Shared(Arc::clone(err)));
527-
}
528-
CompletionState::Pending | CompletionState::Finalizing => {
529-
self.completion_notify.notified()
530-
}
531-
}
532-
};
533-
notified.await;
518+
let mut rx = self.completion_tx.subscribe();
519+
let guard = rx
520+
.wait_for(|v| v.is_some())
521+
.await
522+
.expect("completion sender is held by the accumulator");
523+
match guard.as_ref().expect("wait_for filtered out None") {
524+
Ok(()) => Ok(()),
525+
Err(err) => Err(DataFusionError::Shared(Arc::clone(err))),
534526
}
535527
}
536528

0 commit comments

Comments
 (0)