Skip to content

Commit 454fd16

Browse files
antiguruclaude
andcommitted
compute: drop per-bucket temporal bucket staging column
The time-major staging column was a field of every bucket and only ever cleared, so each bucket that received a large push kept a push-sized allocation until it was peeled. That memory sat outside the spill pool and the batcher's size accounting. Build the staging container per push instead and hand it to the chunker, which takes its body out by value. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 23f08ba commit 454fd16

1 file changed

Lines changed: 5 additions & 9 deletions

File tree

‎src/compute/src/extensions/temporal_bucket.rs‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -351,8 +351,6 @@ where
351351
{
352352
logger: Option<differential_dataflow::logging::Logger>,
353353
operator_id: usize,
354-
/// Time-major copy of the updates on their way into the chunker.
355-
staging: Column<((T, D), T, R)>,
356354
chunker: ChunkChunker<(T, D), T, R, ComputeSpill>,
357355
inner: AccountedChunkBatcher<(T, D), T, R, ComputeSpill>,
358356
}
@@ -372,7 +370,6 @@ where
372370
Self {
373371
logger: logger.clone(),
374372
operator_id,
375-
staging: Column::default(),
376373
chunker: ChunkChunker::default(),
377374
inner: AccountedChunkBatcher::new(logger, operator_id),
378375
}
@@ -386,18 +383,17 @@ where
386383
if buffer.is_empty() {
387384
return;
388385
}
389-
// `staging` is only ever cleared, which keeps it typed.
390-
let Column::Typed(staging) = &mut self.staging else {
391-
unreachable!("staging column is typed");
392-
};
386+
// A fresh time-major copy per push. Retaining it would keep a
387+
// push-sized allocation in every bucket of the chain, outside the
388+
// pool and the batcher's size accounting.
389+
let mut staging = <((T, D), T, R) as Columnar>::Container::default();
393390
let (data, time, diff) = buffer.borrow();
394391
staging.0.0.extend_from_self(time, 0..time.len());
395392
staging.0.1.extend_from_self(data, 0..data.len());
396393
staging.1.extend_from_self(time, 0..time.len());
397394
staging.2.extend_from_self(diff, 0..diff.len());
398395
buffer.clear();
399-
self.chunker.push_into(&mut self.staging);
400-
self.staging.clear();
396+
self.chunker.push_into(&mut Column::Typed(staging));
401397
while let Some(chunk) = self.chunker.extract() {
402398
self.inner.push_into(std::mem::take(chunk));
403399
}

0 commit comments

Comments
 (0)