Skip to content

Commit e9d8f76

Browse files
committed
test: check the files_closed_early count in the fanout early-close tests
The Rust tests passed `WriteMetrics::default()` and never read it back, so a miscount in `reserve`'s return went unnoticed. `write_reserving_from` now takes the metrics, and `fanout_write_in_part_of_the_pool` gives its tight write a real `WriteMetrics` and checks `closed_early <= files <= closed_early + partitions`: no file rolls at the target size, so every file was closed either early or when the task ended, at most one per partition. Counting one file fewer or one more per refusal fails `a_fanout_write_short_of_memory_closes_the_partitions_holding_the_most`.
1 parent c29bb0a commit e9d8f76

1 file changed

Lines changed: 26 additions & 4 deletions

File tree

‎native/core/src/execution/operators/iceberg_write.rs‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4897,10 +4897,11 @@ mod tests {
48974897

48984898
/// Writes `batches` into a fresh table of `schema`, partitioned by `region` unless the
48994899
/// mode is unpartitioned, reserving from `pool` the way `IcebergWriteExec::execute`
4900-
/// reserves from the task's pool. A successful write's files are left for the caller to
4901-
/// inspect.
4900+
/// reserves from the task's pool, and counting into `metrics`. A successful write's files
4901+
/// are left for the caller to inspect.
49024902
async fn write_reserving_from(
49034903
pool: &Arc<PeakMemoryPool>,
4904+
metrics: WriteMetrics,
49044905
schema: Schema,
49054906
writer_mode: ProtoIcebergWriterMode,
49064907
batches: Vec<RecordBatch>,
@@ -4936,7 +4937,7 @@ mod tests {
49364937
writer_properties,
49374938
Some(0),
49384939
Some(0),
4939-
WriteMetrics::default(),
4940+
metrics,
49404941
reservation,
49414942
)
49424943
.await
@@ -4962,6 +4963,7 @@ mod tests {
49624963
.build();
49634964
let (_dir, written) = write_reserving_from(
49644965
&pool,
4966+
WriteMetrics::default(),
49654967
iceberg_user_schema(),
49664968
writer_mode,
49674969
vec![batch],
@@ -5002,6 +5004,7 @@ mod tests {
50025004
let pool = Arc::new(PeakMemoryPool::new(usize::MAX));
50035005
let (_dir, written) = write_reserving_from(
50045006
&pool,
5007+
WriteMetrics::default(),
50055008
iceberg_user_schema(),
50065009
ProtoIcebergWriterMode::IcebergWriterFanout,
50075010
vec![round_robin_batch(
@@ -5084,6 +5087,7 @@ mod tests {
50845087
let pool = Arc::new(PeakMemoryPool::new(usize::MAX));
50855088
let (_dir, written) = write_reserving_from(
50865089
&pool,
5090+
WriteMetrics::default(),
50875091
iceberg_user_schema(),
50885092
ProtoIcebergWriterMode::IcebergWriterUnpartitioned,
50895093
batches,
@@ -5126,6 +5130,7 @@ mod tests {
51265130
let pool = Arc::new(PeakMemoryPool::new(8 * row_group_bytes));
51275131
let (_dir, written) = write_reserving_from(
51285132
&pool,
5133+
WriteMetrics::default(),
51295134
iceberg_payload_schema(),
51305135
ProtoIcebergWriterMode::IcebergWriterFanout,
51315136
batches,
@@ -5161,7 +5166,8 @@ mod tests {
51615166
/// Writes `batches` into a fanout table of `schema` twice: with a pool to spare, then with
51625167
/// a pool of the most that first write reserved divided by `divisor`, which it can only
51635168
/// fit by closing partitions early. Returns both writes' data files and the second write's
5164-
/// directory, after checking that the second fit and gave its reservation back.
5169+
/// directory, after checking that the second fit, gave its reservation back and counted
5170+
/// the files it closed early.
51655171
async fn fanout_write_in_part_of_the_pool(
51665172
divisor: usize,
51675173
schema: Schema,
@@ -5171,6 +5177,7 @@ mod tests {
51715177
let roomy = Arc::new(PeakMemoryPool::new(usize::MAX));
51725178
let (_dir, written) = write_reserving_from(
51735179
&roomy,
5180+
WriteMetrics::default(),
51745181
schema.clone(),
51755182
ProtoIcebergWriterMode::IcebergWriterFanout,
51765183
batches(),
@@ -5180,8 +5187,10 @@ mod tests {
51805187
.await;
51815188
let roomy_files = written.unwrap();
51825189
let tight = Arc::new(PeakMemoryPool::new(roomy.peak() / divisor));
5190+
let metrics = WriteMetrics::default();
51835191
let (dir, written) = write_reserving_from(
51845192
&tight,
5193+
metrics.clone(),
51855194
schema,
51865195
ProtoIcebergWriterMode::IcebergWriterFanout,
51875196
batches(),
@@ -5191,6 +5200,17 @@ mod tests {
51915200
.await;
51925201
let tight_files = written.expect("closing partitions early lets the write fit");
51935202
assert_eq!(tight.reserved(), 0, "the write kept a reservation");
5203+
// No file rolls at `TARGET_FILE_SIZE`, so each file was closed either early, and
5204+
// counted, or when the task ended, which closes at most one file per partition.
5205+
let closed_early = metrics.files_closed_early.value();
5206+
let partitions = rows_per_partition(&roomy_files).len();
5207+
assert!(
5208+
closed_early > 0
5209+
&& closed_early <= tight_files.len()
5210+
&& tight_files.len() <= closed_early + partitions,
5211+
"{closed_early} files counted as closed early, of {} for {partitions} partitions",
5212+
tight_files.len()
5213+
);
51945214
(roomy_files, tight_files, dir)
51955215
}
51965216

@@ -5293,6 +5313,7 @@ mod tests {
52935313
let roomy = Arc::new(PeakMemoryPool::new(usize::MAX));
52945314
let (dir, written) = write_reserving_from(
52955315
&roomy,
5316+
WriteMetrics::default(),
52965317
iceberg_user_schema(),
52975318
ProtoIcebergWriterMode::IcebergWriterUnpartitioned,
52985319
batches(),
@@ -5306,6 +5327,7 @@ mod tests {
53065327
let tight = Arc::new(PeakMemoryPool::new(roomy.peak() / 2));
53075328
let (dir, written) = write_reserving_from(
53085329
&tight,
5330+
WriteMetrics::default(),
53095331
iceberg_user_schema(),
53105332
ProtoIcebergWriterMode::IcebergWriterUnpartitioned,
53115333
batches(),

0 commit comments

Comments
 (0)