Skip to content

Commit 6c3ef48

Browse files
authored
fix tpchgen row group count (#10725)
#10691 introduced a data generation regression in which parquet files have had much more row groups they should, which resulted in incorrect benchmark results. Revert this, and cap the number of row groups per file to the ones we had before Signed-off-by: Mikhail Kot <[email protected]>
1 parent a1fdb64 commit 6c3ef48

1 file changed

Lines changed: 22 additions & 6 deletions

File tree

‎vortex-bench/src/tpch/tpchgen.rs‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,19 @@ impl TableGenerator {
210210
}
211211
}
212212

213+
fn estimated_rows(self, scale_factor: f64) -> i64 {
214+
match self {
215+
Self::Nation => 25,
216+
Self::Region => 5,
217+
Self::Part => PartGenerator::calculate_row_count(scale_factor, 1, 1),
218+
Self::Supplier => SupplierGenerator::calculate_row_count(scale_factor, 1, 1),
219+
Self::Customer => CustomerGenerator::calculate_row_count(scale_factor, 1, 1),
220+
Self::PartSupp => PartSuppGenerator::calculate_row_count(scale_factor, 1, 1),
221+
Self::Orders => OrderGenerator::calculate_row_count(scale_factor, 1, 1),
222+
Self::LineItem => 4 * OrderGenerator::calculate_row_count(scale_factor, 1, 1),
223+
}
224+
}
225+
213226
/// Number of files to split the table into at `scale_factor`.
214227
fn num_parts(self, scale_factor: f64) -> Result<usize> {
215228
let Some(bytes_sf1) = self.compressed_bytes_sf1() else {
@@ -292,11 +305,12 @@ fn generate_table_file(
292305
}
293306

294307
/// Number of parts generated in parallel for a table written as one Parquet file.
295-
fn table_num_parts(generator: TableGenerator) -> usize {
296-
match generator {
297-
TableGenerator::Nation | TableGenerator::Region => 1,
298-
_ => get_available_parallelism().unwrap_or(1),
299-
}
308+
fn table_num_parts(generator: TableGenerator, scale_factor: f64) -> usize {
309+
let rows = usize::try_from(generator.estimated_rows(scale_factor)).unwrap_or(1);
310+
get_available_parallelism()
311+
.unwrap_or(1)
312+
.min(rows.div_ceil(ROW_GROUP_SIZE))
313+
.max(1)
300314
}
301315

302316
/// Write one part of a table as Parquet.
@@ -319,7 +333,9 @@ async fn write_parquet(
319333
.await?;
320334
}
321335

322-
write_parquet_parts(generator, options, table_num_parts(generator), output).await
336+
let scale_factor = options.scale_factor.parse::<f64>()?;
337+
let gen_parts = table_num_parts(generator, scale_factor);
338+
write_parquet_parts(generator, options, gen_parts, output).await
323339
}
324340

325341
async fn write_parquet_parts(

0 commit comments

Comments
 (0)