Skip to content

Commit abf5807

Browse files
committed
fix: bound retained zstd context memory outside encode bursts
SessionOnly reset preserves zstd's allocated window, so a retained context grows to the largest workspace it has seen (~128 MiB for the decoder after one wide-window frame, ~834 MiB for the encoder at level 22) and stays there. Cap retained contexts at 8 MiB -- covering the commonly configured levels -- and drop anything larger after each decode (errors included) and each local block encode; higher levels fall back to per-frame creation, the pre-existing cost. The local writer also releases its context when a spill event or the final flush completes, so nothing is retained between write phases.
1 parent 8da6e6b commit abf5807

7 files changed

Lines changed: 227 additions & 6 deletions

File tree

‎native/shuffle/src/codec_context.rs‎

Lines changed: 42 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,20 @@ use arrow::ipc::writer::CompressionContext;
1919
use std::io;
2020
use zstd::zstd_safe::{CCtx, CParameter, DCtx, ResetDirective};
2121

22+
/// Largest zstd workspace worth caching between frames. Covers the commonly configured
23+
/// levels; higher levels (tens to hundreds of MiB of window) fall back to a fresh context
24+
/// per frame, which is what per-block encoding paid anyway.
25+
const MAX_RETAINED_ZSTD_CONTEXT_BYTES: usize = 8 * 1024 * 1024;
26+
2227
/// Reusable compression state for encoding shuffle blocks.
2328
///
2429
/// A zstd context costs about a megabyte and real setup time, so a task shares one across all
2530
/// the blocks it encodes instead of paying per block. Keep ownership task-scoped, never
26-
/// per-output-partition -- a shuffle can have thousands of partitions. Local shuffle holds the
27-
/// zstd context for the whole task; the remote (RSS) path frees it after each admitted encode
28-
/// via [`Self::release_zstd`], since its memory accounting only reserves the workspace per
29-
/// invocation.
31+
/// per-output-partition -- a shuffle can have thousands of partitions. Local shuffle reuses
32+
/// the zstd context between blocks but bounds what it retains
33+
/// ([`Self::release_zstd_if_oversized`]) and drops it at spill/finish boundaries; the remote
34+
/// (RSS) path frees it after each admitted encode via [`Self::release_zstd`], since its
35+
/// memory accounting only reserves the workspace per invocation.
3036
#[derive(Default)]
3137
pub struct ShuffleCodecContext {
3238
/// Arrow's per-message IPC compression scratch, reused across encodes.
@@ -66,6 +72,19 @@ impl ShuffleCodecContext {
6672
self.zstd = None;
6773
}
6874

75+
/// Drops the cached zstd context when its workspace outgrew
76+
/// [`MAX_RETAINED_ZSTD_CONTEXT_BYTES`] (a session reset keeps the allocation); the next
77+
/// encode re-creates it lazily.
78+
pub(crate) fn release_zstd_if_oversized(&mut self) {
79+
if self
80+
.zstd
81+
.as_ref()
82+
.is_some_and(|cctx| cctx.sizeof() > MAX_RETAINED_ZSTD_CONTEXT_BYTES)
83+
{
84+
self.zstd = None;
85+
}
86+
}
87+
6988
/// Test hook for the release-vs-retain contract of the two encode paths.
7089
#[cfg(test)]
7190
pub(crate) fn holds_zstd_cctx(&self) -> bool {
@@ -95,6 +114,25 @@ impl ShuffleDecodeContext {
95114
.map_err(map_zstd_error)?;
96115
Ok(dctx)
97116
}
117+
118+
/// Drops the cached zstd context when its workspace outgrew
119+
/// [`MAX_RETAINED_ZSTD_CONTEXT_BYTES`]: one frame advertising a large window grows the
120+
/// context past 100 MiB, and a session reset keeps that allocation.
121+
pub(crate) fn release_zstd_if_oversized(&mut self) {
122+
if self
123+
.zstd
124+
.as_ref()
125+
.is_some_and(|dctx| dctx.sizeof() > MAX_RETAINED_ZSTD_CONTEXT_BYTES)
126+
{
127+
self.zstd = None;
128+
}
129+
}
130+
131+
/// Test hook for the retained-workspace bound on the decode path.
132+
#[cfg(test)]
133+
pub(crate) fn holds_zstd_dctx(&self) -> bool {
134+
self.zstd.is_some()
135+
}
98136
}
99137

100138
fn map_zstd_error(code: usize) -> io::Error {

‎native/shuffle/src/ipc.rs‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,19 @@ fn read_ipc_compressed_impl(
6060
decode_context: &mut ShuffleDecodeContext,
6161
bytes: &[u8],
6262
validate: bool,
63+
) -> Result<RecordBatch> {
64+
let result = decode_shuffle_frame(decode_context, bytes, validate);
65+
// Decoding a frame with a large advertised window grows the context past 100 MiB, and
66+
// contexts here are long-lived (thread-local per executor thread). Bound what survives
67+
// the frame, whether it decoded or not.
68+
decode_context.release_zstd_if_oversized();
69+
result
70+
}
71+
72+
fn decode_shuffle_frame(
73+
decode_context: &mut ShuffleDecodeContext,
74+
bytes: &[u8],
75+
validate: bool,
6376
) -> Result<RecordBatch> {
6477
let codec = bytes.get(..4).ok_or_else(|| {
6578
DataFusionError::Execution("Failed to decode batch: truncated compression codec".to_owned())
@@ -323,6 +336,61 @@ mod tests {
323336
}
324337
}
325338

339+
/// ZSTD shuffle frame whose header advertises the full level-22 window (streaming
340+
/// encode with no pledged source size), so decoding it demands a >100 MiB workspace.
341+
fn encode_zstd_wide_window(payload: &[u8]) -> Vec<u8> {
342+
let mut bytes = b"ZSTD".to_vec();
343+
let mut writer = zstd::Encoder::new(&mut bytes, 22).unwrap();
344+
writer.write_all(payload).unwrap();
345+
writer.finish().unwrap();
346+
bytes
347+
}
348+
349+
/// Small-window frames keep the context cached for reuse; a frame that inflates the
350+
/// workspace far past its usual size must not leave it pinned in a long-lived context.
351+
#[test]
352+
#[cfg_attr(miri, ignore)] // Miri cannot call Zstd's C FFI.
353+
fn decode_context_drops_oversized_zstd_workspace() {
354+
let mut ctx = ShuffleDecodeContext::default();
355+
let small = encode(b"ZSTD", &ipc_stream(1));
356+
assert_eq!(
357+
read_ipc_compressed_with(&mut ctx, &small)
358+
.unwrap()
359+
.num_rows(),
360+
3
361+
);
362+
assert!(
363+
ctx.holds_zstd_dctx(),
364+
"a small-window frame must keep the context cached for reuse"
365+
);
366+
367+
let wide = encode_zstd_wide_window(&ipc_stream(1));
368+
assert_eq!(
369+
read_ipc_compressed_with(&mut ctx, &wide)
370+
.unwrap()
371+
.num_rows(),
372+
3
373+
);
374+
assert!(
375+
!ctx.holds_zstd_dctx(),
376+
"a wide-window frame must not leave its workspace cached"
377+
);
378+
}
379+
380+
/// The workspace bound must hold on the error path too: a failed decode of a
381+
/// wide-window frame cannot leave the inflated context behind.
382+
#[test]
383+
#[cfg_attr(miri, ignore)] // Miri cannot call Zstd's C FFI.
384+
fn decode_error_drops_oversized_zstd_workspace() {
385+
let mut ctx = ShuffleDecodeContext::default();
386+
let wide = encode_zstd_wide_window(&ipc_stream(1));
387+
assert!(read_ipc_compressed_with(&mut ctx, &wide[..wide.len() - 7]).is_err());
388+
assert!(
389+
!ctx.holds_zstd_dctx(),
390+
"a failed wide-window decode must not leave its workspace cached"
391+
);
392+
}
393+
326394
/// A truncated frame must not poison the context for the next valid one.
327395
#[test]
328396
#[cfg_attr(miri, ignore)] // Miri cannot call Zstd's C FFI.

‎native/shuffle/src/partitioners/multi_partition.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -577,6 +577,7 @@ impl<T: PartitionWriter> MultiPartitionShuffleRepartitioner<T> {
577577
)
578578
})
579579
};
580+
self.partition_writer.write_burst_complete();
580581

581582
let memory_spilled_bytes = self
582583
.reservation

‎native/shuffle/src/shuffle_writer.rs‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -566,6 +566,56 @@ mod test {
566566
repartitioner.insert_batch(batch.clone()).await.unwrap();
567567
}
568568

569+
/// The zstd context is reused within one encode burst but must not survive past it: a
570+
/// spill event and the final shuffle write each end with the context released.
571+
#[tokio::test]
572+
#[cfg_attr(miri, ignore)] // miri can't call foreign function `ZSTD_createCCtx`
573+
async fn local_writer_releases_zstd_context_at_burst_boundaries() {
574+
let batch = create_batch(900);
575+
let num_partitions = 2;
576+
let runtime_env = create_runtime(512 * 1024);
577+
let metrics_set = ExecutionPlanMetricsSet::new();
578+
let dir = tempfile::tempdir().unwrap();
579+
let shuffle_block_writer =
580+
ShuffleBlockWriter::try_new(batch.schema().as_ref(), CompressionCodec::Zstd(1))
581+
.unwrap();
582+
let local_partition_writer = LocalPartitionWriter::try_new(
583+
dir.path().join("data.out").to_str().unwrap().to_string(),
584+
dir.path().join("index.out").to_str().unwrap().to_string(),
585+
shuffle_block_writer,
586+
num_partitions,
587+
1024,
588+
1024 * 1024,
589+
Arc::clone(&runtime_env),
590+
)
591+
.unwrap();
592+
let mut repartitioner = MultiPartitionShuffleRepartitioner::try_new(
593+
0,
594+
local_partition_writer,
595+
CometPartitioning::Hash(vec![Arc::new(Column::new("a", 0))], num_partitions),
596+
ShufflePartitionerMetrics::new(&metrics_set, 0),
597+
runtime_env,
598+
1024,
599+
false,
600+
None,
601+
)
602+
.unwrap();
603+
604+
repartitioner.insert_batch(batch.clone()).await.unwrap();
605+
repartitioner.spill(0).unwrap();
606+
assert!(
607+
!repartitioner.partition_writer().holds_zstd_cctx(),
608+
"a finished spill burst must not keep the zstd context cached"
609+
);
610+
611+
repartitioner.insert_batch(batch.clone()).await.unwrap();
612+
repartitioner.shuffle_write().unwrap();
613+
assert!(
614+
!repartitioner.partition_writer().holds_zstd_cctx(),
615+
"finish_all must release the zstd context"
616+
);
617+
}
618+
569619
#[tokio::test]
570620
async fn shuffle_partitioner_charges_shared_buffer_once() {
571621
// `insert_batch` slices a large batch into batch_size chunks that all share one backing

‎native/shuffle/src/writers/local/local_partition_writer.rs‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,8 @@ pub(crate) struct LocalPartitionWriter {
6666
output_index_file: String,
6767
data_output: DataOutput,
6868
/// Compression state shared by every block this task writes; the per-partition
69-
/// `BufBatchWriter`s borrow it (see [`ShuffleCodecContext`]).
69+
/// `BufBatchWriter`s borrow it (see [`ShuffleCodecContext`]). Retention is bounded:
70+
/// released at spill/finish boundaries and whenever its workspace is oversized.
7071
codec_context: ShuffleCodecContext,
7172
/// Start offset of each partition in the data file, plus a trailing entry
7273
/// with the total length so partition sizes are simple offset differences.
@@ -134,6 +135,11 @@ impl LocalPartitionWriter {
134135
})
135136
}
136137

138+
#[cfg(test)]
139+
pub(crate) fn holds_zstd_cctx(&self) -> bool {
140+
self.codec_context.holds_zstd_cctx()
141+
}
142+
137143
#[cfg(test)]
138144
pub(crate) fn get_spill_writers(&self) -> &Vec<SpillWriter> {
139145
match &self.data_output {
@@ -319,6 +325,15 @@ impl PartitionWriter for LocalPartitionWriter {
319325
output_index.flush()?;
320326
write_timer.stop();
321327

328+
// The shuffle output is complete; nothing else encodes through this context.
329+
self.codec_context.release_zstd();
330+
322331
Ok(())
323332
}
333+
334+
fn write_burst_complete(&mut self) {
335+
// A spill burst just ended and the next encode may be a long time coming; the zstd
336+
// workspace is native memory no reservation tracks, so don't sit on it.
337+
self.codec_context.release_zstd();
338+
}
324339
}

‎native/shuffle/src/writers/partition_writer.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,4 +68,9 @@ pub(crate) trait PartitionWriter: Send + Sync {
6868
/// [`finish_partition`](PartitionWriter::finish_partition).
6969
fn finish_all(&mut self, metrics: &ShufflePartitionerMetrics)
7070
-> datafusion::common::Result<()>;
71+
72+
/// Marks the end of one burst of [`write`](PartitionWriter::write) calls (a spill
73+
/// event), letting the writer drop transient encode state. Staging more batches
74+
/// afterwards is still allowed.
75+
fn write_burst_complete(&mut self) {}
7176
}

‎native/shuffle/src/writers/shuffle_block_writer.rs‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -279,8 +279,12 @@ impl ShuffleBlockWriter {
279279
// RSS charges the zstd workspace (rss_codec_workspace) to each admitted encode
280280
// and releases the charge when it ends, success or not. Free the workspace inside
281281
// that window -- kept alive it would be native memory the reservation system no
282-
// longer tracks. Local shuffle keeps the context for the whole task.
282+
// longer tracks.
283283
codec_context.release_zstd();
284+
} else {
285+
// Local shuffle reuses the context across blocks, but nothing reserves its
286+
// memory: a high-level workspace (hundreds of MiB) must not outlive the block.
287+
codec_context.release_zstd_if_oversized();
284288
}
285289
encode_result?;
286290

@@ -452,6 +456,46 @@ mod tests {
452456
assert_eq!(sizes[1], sizes[3], "same writer, same input, same level");
453457
}
454458

459+
/// Common zstd levels stay cached between local blocks; a high level allocates a
460+
/// workspace of hundreds of MiB that must be dropped as soon as its block is done.
461+
#[test]
462+
#[cfg_attr(miri, ignore)] // miri can't call foreign function `ZSTD_createCCtx`
463+
fn local_write_drops_oversized_zstd_context() {
464+
let batch = test_batch(3, 100);
465+
let mut ctx = ShuffleCodecContext::default();
466+
467+
let fast = ShuffleBlockWriter::try_new(&test_schema(), CompressionCodec::Zstd(1)).unwrap();
468+
let mut fast_out = vec![];
469+
fast.write_batch(
470+
&batch,
471+
&mut Cursor::new(&mut fast_out),
472+
&mut ctx,
473+
&Time::default(),
474+
)
475+
.unwrap();
476+
assert!(
477+
ctx.holds_zstd_cctx(),
478+
"a common-level workspace must stay cached for reuse"
479+
);
480+
481+
let slow = ShuffleBlockWriter::try_new(&test_schema(), CompressionCodec::Zstd(22)).unwrap();
482+
let mut slow_out = vec![];
483+
slow.write_batch(
484+
&batch,
485+
&mut Cursor::new(&mut slow_out),
486+
&mut ctx,
487+
&Time::default(),
488+
)
489+
.unwrap();
490+
assert!(
491+
!ctx.holds_zstd_cctx(),
492+
"a level-22 workspace must not stay cached past its block"
493+
);
494+
495+
assert_eq!(read_ipc_compressed(&fast_out[16..]).unwrap(), batch);
496+
assert_eq!(read_ipc_compressed(&slow_out[16..]).unwrap(), batch);
497+
}
498+
455499
/// Accepts a fixed number of bytes, then fails every write.
456500
struct FailingSink {
457501
inner: Cursor<Vec<u8>>,

0 commit comments

Comments
 (0)