@@ -1092,22 +1092,32 @@ where
10921092/// task thread.
10931093struct BatchProducer {
10941094 batches : mpsc:: Receiver < DataFusionResult < RecordBatch > > ,
1095- /// Owns the plan's stream, and with it every reservation the stream holds.
1095+ /// The plan's stream, and with it every reservation the stream holds. The task locks it only
1096+ /// while polling it, so `stop` can take it whenever the task is waiting.
1097+ stream : Arc < Mutex < Option < SendableRecordBatchStream > > > ,
10961098 task : JoinHandle < ( ) > ,
1097- runtime : Handle ,
10981099}
10991100
11001101impl BatchProducer {
1101- fn spawn ( runtime : Handle , mut stream : SendableRecordBatchStream ) -> Self {
1102+ fn spawn ( runtime : & Handle , stream : SendableRecordBatchStream ) -> Self {
11021103 // Channel capacity of 2 allows the producer to work one batch
11031104 // ahead while the consumer processes the current one via JNI,
11041105 // without buffering excessive memory. Increasing this would
11051106 // trade memory for latency hiding if JNI/FFI overhead dominates;
11061107 // decreasing to 1 would serialize production and consumption.
11071108 let ( tx, batches) = mpsc:: channel ( 2 ) ;
1109+ let stream = Arc :: new ( Mutex :: new ( Some ( stream) ) ) ;
1110+ let polled = Arc :: clone ( & stream) ;
11081111 let task = runtime. spawn ( async move {
11091112 let result = std:: panic:: AssertUnwindSafe ( async {
1110- while let Some ( batch) = stream. next ( ) . await {
1113+ // Ends once `stop` has taken the stream.
1114+ let next = || {
1115+ poll_fn ( |cx| match polled. lock ( ) . as_mut ( ) {
1116+ Some ( stream) => stream. poll_next_unpin ( cx) ,
1117+ None => Poll :: Ready ( None ) ,
1118+ } )
1119+ } ;
1120+ while let Some ( batch) = next ( ) . await {
11111121 if tx. send ( batch) . await . is_err ( ) {
11121122 break ;
11131123 }
@@ -1133,31 +1143,30 @@ impl BatchProducer {
11331143 } ) ;
11341144 Self {
11351145 batches,
1146+ stream,
11361147 task,
1137- runtime,
11381148 }
11391149 }
11401150
1141- /// Stops the task, and parks the calling thread until it has finished, by which point it has
1142- /// dropped the plan's stream.
1151+ /// Stops the task and drops the plan's stream on the calling thread.
11431152 ///
1144- /// Closing the channel stops a task that is waiting to send a batch, and aborting it stops one
1145- /// that is waiting on the stream. A task in the middle of polling the stream stops when that
1146- /// poll returns, so this waits at most for the work the stream does between two await points.
1153+ /// The stream is dropped here rather than by cancelling the task, because running a
1154+ /// cancellation needs a free Tokio worker, and every worker can be blocked in Spark's
1155+ /// `acquireMemory` waiting for the memory this stream holds. Taking the stream waits only for
1156+ /// a poll the task is already in, so at most for the work the stream does between two await
1157+ /// points.
11471158 fn stop ( self ) -> CometResult < ( ) > {
11481159 let Self {
11491160 batches,
1161+ stream,
11501162 task,
1151- runtime,
11521163 } = self ;
11531164 drop ( batches) ;
11541165 task. abort ( ) ;
1155- match runtime. block_on ( task) {
1156- // Dropping the stream panicked, which is reported as it is for a JVM-fed plan, whose
1157- // stream `releasePlan` drops itself.
1158- Err ( e) if e. is_panic ( ) => Err ( CometError :: from ( e. into_panic ( ) ) ) ,
1159- _ => Ok ( ( ) ) ,
1160- }
1166+ let stream = stream. lock ( ) . take ( ) ;
1167+ // A panic here is reported as it is when `releasePlan` drops a JVM-fed plan's stream.
1168+ std:: panic:: catch_unwind ( std:: panic:: AssertUnwindSafe ( || drop ( stream) ) )
1169+ . map_err ( CometError :: from)
11611170 }
11621171}
11631172
@@ -1232,7 +1241,8 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_executePlan(
12321241 if exec_context. scans . is_empty ( ) && exec_context. shuffle_scans . is_empty ( ) {
12331242 // No JVM data sources — spawn onto tokio so the executor
12341243 // thread parks in blocking_recv instead of busy-polling.
1235- exec_context. batch_producer = Some ( BatchProducer :: spawn ( get_runtime ( ) , stream) ) ;
1244+ exec_context. batch_producer =
1245+ Some ( BatchProducer :: spawn ( & get_runtime ( ) , stream) ) ;
12361246 } else {
12371247 exec_context. stream = Some ( stream) ;
12381248 }
@@ -1334,7 +1344,7 @@ pub extern "system" fn Java_org_apache_comet_Native_releasePlan(
13341344 let mut execution_context: Box < ExecutionContext > =
13351345 Box :: from_raw ( exec_context as * mut ExecutionContext ) ;
13361346
1337- // The task running a plan with no JVM input owns the plan's stream, so stop it first: the
1347+ // A plan with no JVM input runs on a Tokio task. Stop it and drop its stream first, so the
13381348 // stream's memory is returned before the Spark task can end, and the metrics flushed below
13391349 // include what the stream records as it is dropped.
13401350 let producer_stopped = execution_context
@@ -2574,8 +2584,8 @@ mod tests {
25742584 ) )
25752585 }
25762586
2577- /// See issue #2453. A consumer that stops early leaves the producer waiting on the stream ,
2578- /// which only an abort interrupts .
2587+ /// See issue #2453. A consumer that stops early can leave the producer waiting on its input ,
2588+ /// still holding the stream .
25792589 #[ test]
25802590 fn stopping_a_batch_producer_drops_its_stream_while_it_waits_on_input ( ) {
25812591 let runtime = single_worker_runtime ( ) ;
@@ -2592,7 +2602,7 @@ mod tests {
25922602 }
25932603 } ) ;
25942604
2595- let mut producer = BatchProducer :: spawn ( runtime. handle ( ) . clone ( ) , plan_stream ( stream) ) ;
2605+ let mut producer = BatchProducer :: spawn ( runtime. handle ( ) , plan_stream ( stream) ) ;
25962606 assert ! ( producer. batches. blocking_recv( ) . unwrap( ) . is_ok( ) ) ;
25972607 producer. stop ( ) . unwrap ( ) ;
25982608 assert_eq ! ( pool. reserved( ) , 0 , "the stream outlived its producer" ) ;
@@ -2620,7 +2630,7 @@ mod tests {
26202630 Poll :: Ready ( Some ( Ok ( empty_batch ( ) ) ) )
26212631 } ) ;
26222632
2623- let mut producer = BatchProducer :: spawn ( runtime. handle ( ) . clone ( ) , plan_stream ( stream) ) ;
2633+ let mut producer = BatchProducer :: spawn ( runtime. handle ( ) , plan_stream ( stream) ) ;
26242634 assert ! ( producer. batches. blocking_recv( ) . unwrap( ) . is_ok( ) ) ;
26252635 polling_rx. recv ( ) . unwrap ( ) ;
26262636 let finished = Arc :: new ( AtomicBool :: new ( false ) ) ;
@@ -2658,7 +2668,7 @@ mod tests {
26582668 let _guard = & guard;
26592669 Poll :: < Option < DataFusionResult < RecordBatch > > > :: Pending
26602670 } ) ;
2661- let producer = BatchProducer :: spawn ( runtime. handle ( ) . clone ( ) , plan_stream ( stream) ) ;
2671+ let producer = BatchProducer :: spawn ( runtime. handle ( ) , plan_stream ( stream) ) ;
26622672 match producer. stop ( ) {
26632673 Err ( CometError :: Panic { msg } ) => assert ! ( msg. contains( "the stream failed to drop" ) ) ,
26642674 other => panic ! ( "expected the panic, got {other:?}" ) ,
@@ -2670,7 +2680,7 @@ mod tests {
26702680 /// memory, and `releasePlan` also waits on the plan's pool.
26712681 #[ test]
26722682 fn a_task_the_plan_spawned_keeps_its_memory_until_it_next_yields ( ) {
2673- // One worker runs the spawned task, which blocks it, and the other runs the producer .
2683+ // One worker runs the spawned task, which blocks it, and the other polls the plan's stream .
26742684 let runtime = tokio:: runtime:: Builder :: new_multi_thread ( )
26752685 . worker_threads ( 2 )
26762686 . build ( )
@@ -2701,7 +2711,7 @@ mod tests {
27012711 Poll :: < Option < DataFusionResult < RecordBatch > > > :: Pending
27022712 } ) ;
27032713
2704- BatchProducer :: spawn ( runtime. handle ( ) . clone ( ) , plan_stream ( stream) )
2714+ BatchProducer :: spawn ( runtime. handle ( ) , plan_stream ( stream) )
27052715 . stop ( )
27062716 . unwrap ( ) ;
27072717 assert_eq ! (
@@ -2715,4 +2725,47 @@ mod tests {
27152725 0
27162726 ) ;
27172727 }
2728+
2729+ /// Spark's `acquireMemory` blocks the calling thread until other tasks release memory, and it
2730+ /// is called on Tokio workers. When every worker is waiting on memory the stopped plan holds,
2731+ /// stopping the producer must not need a free worker, or neither task ever finishes.
2732+ #[ test]
2733+ fn stopping_a_batch_producer_does_not_need_a_free_worker ( ) {
2734+ let runtime = single_worker_runtime ( ) ;
2735+ let pool: Arc < dyn MemoryPool > = Arc :: new ( UnboundedMemoryPool :: default ( ) ) ;
2736+ let reservation = MemoryConsumer :: new ( "sort" ) . register ( & pool) ;
2737+ reservation. grow ( 4096 ) ;
2738+ let mut first = Some ( empty_batch ( ) ) ;
2739+ let stream = futures:: stream:: poll_fn ( move |_| {
2740+ let _held = & reservation;
2741+ match first. take ( ) {
2742+ Some ( batch) => Poll :: Ready ( Some ( Ok ( batch) ) ) ,
2743+ None => Poll :: Pending ,
2744+ }
2745+ } ) ;
2746+ let mut producer = BatchProducer :: spawn ( runtime. handle ( ) , plan_stream ( stream) ) ;
2747+ assert ! ( producer. batches. blocking_recv( ) . unwrap( ) . is_ok( ) ) ;
2748+
2749+ // Holds the only worker until the stopped plan's memory comes back, as another task
2750+ // waiting in `acquireMemory` does. It gives up eventually so a failure cannot hang the
2751+ // suite.
2752+ let ( waiting_tx, waiting_rx) = std:: sync:: mpsc:: channel ( ) ;
2753+ let waiting_on = Arc :: clone ( & pool) ;
2754+ runtime. spawn ( async move {
2755+ waiting_tx. send ( ( ) ) . unwrap ( ) ;
2756+ let give_up = Instant :: now ( ) + Duration :: from_secs ( 20 ) ;
2757+ while waiting_on. reserved ( ) > 0 && Instant :: now ( ) < give_up {
2758+ std:: thread:: sleep ( Duration :: from_millis ( 1 ) ) ;
2759+ }
2760+ } ) ;
2761+ waiting_rx. recv ( ) . unwrap ( ) ;
2762+
2763+ let ( stopped_tx, stopped_rx) = std:: sync:: mpsc:: channel ( ) ;
2764+ std:: thread:: spawn ( move || stopped_tx. send ( producer. stop ( ) ) . unwrap ( ) ) ;
2765+ stopped_rx
2766+ . recv_timeout ( Duration :: from_secs ( 10 ) )
2767+ . expect ( "stopping the producer waited for a free worker" )
2768+ . unwrap ( ) ;
2769+ assert_eq ! ( pool. reserved( ) , 0 , "the stream outlived its producer" ) ;
2770+ }
27182771}
0 commit comments