Skip to content

Commit 6e1c407

Browse files
committed
Support API with filter
1 parent bdfd0d0 commit 6e1c407

2 files changed

Lines changed: 39 additions & 2 deletions

File tree

‎datafusion/physical-plan/src/coalesce/mod.rs‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,44 @@ impl LimitedBatchCoalescer {
120120
Ok(PushBatchStatus::Continue)
121121
}
122122

123+
pub fn push_batch_with_filter(
124+
&mut self,
125+
batch: RecordBatch,
126+
filter: &arrow::array::BooleanArray,
127+
) -> Result<PushBatchStatus> {
128+
assert_or_internal_err!(
129+
!self.finished,
130+
"LimitedBatchCoalescer: cannot push batch after finish"
131+
);
132+
133+
// if we are at the limit, return LimitReached
134+
if let Some(fetch) = self.fetch {
135+
// limit previously reached
136+
if self.total_rows >= fetch {
137+
return Ok(PushBatchStatus::LimitReached);
138+
}
139+
}
140+
141+
let count = filter.true_count();
142+
143+
// if we are at the limit after filtering, return LimitReached
144+
if let Some(fetch) = self.fetch {
145+
// limit now reached
146+
if self.total_rows + count as usize >= fetch {
147+
// TODO: limit at remaining rows
148+
self.total_rows += count as usize;
149+
self.inner.push_batch_with_filter(batch, filter)?;
150+
return Ok(PushBatchStatus::LimitReached);
151+
}
152+
}
153+
154+
// Limit not reached, push the entire filtered batch
155+
self.total_rows += count;
156+
self.inner.push_batch_with_filter(batch, filter)?;
157+
158+
Ok(PushBatchStatus::Continue)
159+
}
160+
123161
/// Return true if there is no data buffered
124162
pub fn is_empty(&self) -> bool {
125163
self.inner.is_empty()

‎datafusion/physical-plan/src/filter.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -800,8 +800,7 @@ impl Stream for FilterExecStream {
800800
Ok(filter_array) => {
801801
self.metrics.selectivity.add_total(batch.num_rows());
802802
// TODO: support push_batch_with_filter in LimitedBatchCoalescer
803-
let batch = filter_record_batch(&batch, filter_array)?;
804-
let state = self.batch_coalescer.push_batch(batch)?;
803+
let state = self.batch_coalescer.push_batch_with_filter(batch, filter_array)?;
805804
Ok(state)
806805
}
807806
Err(_) => {

0 commit comments

Comments
 (0)