Skip to content

Commit d50a09a

Browse files
committed
feat: remove extra Rows from ReusableRows
Fixes: apache#23801 It will only work after apache#23619 is merged, until then test_round_robin_tie_breaker_success will fail: Error: Internal("Rows from RowCursorStream is still in use by consumer") test sorts::sort_preserving_merge::tests::test_round_robin_tie_breaker_success ... FAILED The failure is triggered by prev_cursors from SortPreservingMergeStream keeping the previous Cursor alive for round robin tie breaking purposes. The optimization from apache#23619 only keeps the last Row, so there's no longer a need to keep two Rows cached in ReusableRows.
1 parent 186f96f commit d50a09a

1 file changed

Lines changed: 8 additions & 18 deletions

File tree

‎datafusion/physical-plan/src/sorts/stream.rs‎

Lines changed: 8 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -96,36 +96,28 @@ impl FusedStreams {
9696
/// A pair of `Arc<Rows>` that can be reused
9797
#[derive(Debug)]
9898
struct ReusableRows {
99-
// inner[stream_idx] holds a two Arcs:
100-
// at start of a new poll
101-
// .0 is the rows from the previous poll (at start),
102-
// .1 is the one that is being written to
103-
// at end of a poll, .0 will be swapped with .1,
104-
inner: Vec<[Option<Arc<Rows>>; 2]>,
99+
inner: Vec<Option<Arc<Rows>>>,
105100
}
106101

107102
impl ReusableRows {
108103
// return a Rows for writing,
109104
// does not clone if the existing rows can be reused
110105
fn take_next(&mut self, stream_idx: usize) -> Result<Rows> {
111-
Arc::try_unwrap(self.inner[stream_idx][1].take().unwrap()).map_err(|_| {
106+
Arc::try_unwrap(self.inner[stream_idx].take().unwrap()).map_err(|_| {
112107
internal_datafusion_err!(
113108
"Rows from RowCursorStream is still in use by consumer"
114109
)
115110
})
116111
}
117112
// save the Rows
118113
fn save(&mut self, stream_idx: usize, rows: &Arc<Rows>) {
119-
self.inner[stream_idx][1] = Some(Arc::clone(rows));
120-
// swap the current with the previous one, so that the next poll can reuse the Rows from the previous poll
121-
let [a, b] = &mut self.inner[stream_idx];
122-
mem::swap(a, b);
114+
self.inner[stream_idx] = Some(Arc::clone(rows));
123115
}
124116
}
125117

126118
/// A [`PartitionedStream`] that wraps a set of [`SendableRecordBatchStream`]
127119
/// and computes [`RowValues`] based on the provided [`PhysicalSortExpr`]
128-
/// Note: the stream returns an error if the consumer buffers more than one RowValues (i.e. holds on to two RowValues
120+
/// Note: the stream returns an error if the consumer buffers even one RowValues (i.e. holds on to one RowValues
129121
/// from the same partition at the same time).
130122
#[derive(Debug)]
131123
pub struct RowCursorStream {
@@ -137,8 +129,9 @@ pub struct RowCursorStream {
137129
streams: FusedStreams,
138130
/// Tracks the memory used by `converter`
139131
reservation: MemoryReservation,
140-
/// Allocated rows for each partition, we keep two to allow for buffering one
141-
/// in the consumer of the stream
132+
/// Reused `Rows` allocation for each partition. The consumer must not
133+
/// buffer the `RowValues` returned for a partition, since the old
134+
/// `Arc<Rows>` must be dropped before that partition can be polled again.
142135
rows: ReusableRows,
143136
}
144137

@@ -162,10 +155,7 @@ impl RowCursorStream {
162155
let mut rows = Vec::with_capacity(streams.len());
163156
for _ in &streams {
164157
// Initialize each stream with an empty Rows
165-
rows.push([
166-
Some(Arc::new(converter.empty_rows(0, 0))),
167-
Some(Arc::new(converter.empty_rows(0, 0))),
168-
]);
158+
rows.push(Some(Arc::new(converter.empty_rows(0, 0))));
169159
}
170160
Ok(Self {
171161
converter,

0 commit comments

Comments
 (0)