Skip to content

Commit 56d6f22

Browse files
Split TPC-H and TPC-DS benchmark tables into ~200MB files (#10519)
## Summary TPC-H and TPC-DS benchmark tables were each written as a single file per format, so larger scale factors produced multi-GB files (e.g. SF=10 `lineitem` ≈1.5GB, `store_sales` ≈1GB Parquet). This splits large tables into files of about 200MB compressed, with identical row ranges per file across Parquet, Vortex and Vortex Compact, so formats are compared on the same partitioning.
1 parent d29435b commit 56d6f22

7 files changed

Lines changed: 362 additions & 104 deletions

File tree

‎vortex-bench/src/datasets/tpch_l_comment.rs‎

Lines changed: 22 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -23,25 +23,30 @@ use crate::SESSION;
2323
use crate::datasets::Dataset;
2424
use crate::tpch::tpchgen::TpchGenOptions;
2525
use crate::tpch::tpchgen::generate_tpch_tables;
26+
use crate::utils::split;
2627

2728
pub struct TPCHLCommentChunked;
2829

29-
async fn ensure_tpch_parquet() -> Result<PathBuf> {
30-
let base_path = "tpch".to_data_path();
31-
let scale_factor_dir = base_path.join("1.0");
32-
let data_dir = scale_factor_dir.join(Format::Parquet.name());
33-
34-
// Generate TPC-H parquet data if it doesn't exist
35-
if !data_dir.exists() {
36-
let options =
37-
TpchGenOptions::new("1.0".to_string(), scale_factor_dir).with_format(Format::Parquet);
30+
/// Path of TPC-H SF=1 `lineitem` in `format`, generating the TPC-H tables if it is missing.
31+
///
32+
/// At SF=1 `lineitem` fits in a single file (see [`split::TARGET_FILE_COMPRESSED_BYTES`]), so this
33+
/// reads the TPC-H benchmark's own data.
34+
async fn ensure_lineitem(format: Format) -> Result<PathBuf> {
35+
let scale_factor_dir = "tpch".to_data_path().join("1.0");
36+
let format_dir = scale_factor_dir.join(format.name());
37+
let path = format_dir.join(split::part_file_name("lineitem", 1, format.ext()));
38+
if !path.exists() {
39+
let options = TpchGenOptions::new("1.0".to_string(), scale_factor_dir).with_format(format);
3840
generate_tpch_tables(options).await?;
3941
}
40-
41-
let path = data_dir.join("lineitem.parquet");
42-
path.exists()
43-
.then_some(path)
44-
.ok_or_else(|| anyhow::anyhow!("No lineitem parquet file found"))
42+
anyhow::ensure!(path.exists(), "No lineitem {format} file found");
43+
anyhow::ensure!(
44+
!format_dir
45+
.join(split::part_file_name("lineitem", 2, format.ext()))
46+
.exists(),
47+
"TPC-H SF=1 lineitem is split into several {format} files"
48+
);
49+
Ok(path)
4550
}
4651

4752
#[async_trait]
@@ -51,20 +56,7 @@ impl Dataset for TPCHLCommentChunked {
5156
}
5257

5358
async fn to_vortex_array(&self, ctx: &mut ExecutionCtx) -> Result<ArrayRef> {
54-
let base_path = "tpch".to_data_path();
55-
let scale_factor_dir = base_path.join("1.0");
56-
let data_dir = scale_factor_dir.join(Format::OnDiskVortex.name());
57-
58-
// Generate TPC-H Vortex data if it doesn't exist
59-
if !data_dir.exists() {
60-
// Use blocking call like TPC-H benchmark does
61-
let options = TpchGenOptions::new("1.0".to_string(), scale_factor_dir)
62-
.with_format(Format::OnDiskVortex);
63-
64-
futures::executor::block_on(generate_tpch_tables(options))?;
65-
}
66-
67-
let path = data_dir.join("lineitem.vortex");
59+
let path = ensure_lineitem(Format::OnDiskVortex).await?;
6860
let file = SESSION.open_options().open_path(path).await?;
6961
let projection = pack(vec![("l_comment", col("l_comment"))], NonNullable)
7062
.bind(file.dtype())?
@@ -88,7 +80,7 @@ impl Dataset for TPCHLCommentChunked {
8880
}
8981

9082
async fn to_parquet_path(&self) -> Result<PathBuf> {
91-
ensure_tpch_parquet().await
83+
ensure_lineitem(Format::Parquet).await
9284
}
9385
}
9486

@@ -107,6 +99,6 @@ impl Dataset for TPCHLCommentCanonical {
10799
}
108100

109101
async fn to_parquet_path(&self) -> Result<PathBuf> {
110-
ensure_tpch_parquet().await
102+
ensure_lineitem(Format::Parquet).await
111103
}
112104
}

‎vortex-bench/src/tpcds/tpcds_benchmark.rs‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ use crate::IdempotentPath;
1616
use crate::TableSpec;
1717
use crate::tpcds::tpcds_queries;
1818
use crate::tpcds::tpcdsgen::generate_tpcds;
19+
use crate::utils::split;
1920

2021
/// TPC-DS benchmark implementation
2122
pub struct TpcDsBenchmark {
@@ -128,12 +129,7 @@ impl Benchmark for TpcDsBenchmark {
128129
]
129130
}
130131

131-
#[expect(clippy::expect_used)]
132132
fn pattern(&self, table_name: &str, format: Format) -> Option<Pattern> {
133-
Some(
134-
format!("{}.{}", table_name, format.ext())
135-
.parse()
136-
.expect("valid glob pattern"),
137-
)
133+
Some(split::part_pattern(table_name, format.ext()))
138134
}
139135
}

‎vortex-bench/src/tpcds/tpcdsgen.rs‎

Lines changed: 154 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ use std::fs;
77
use std::fs::File;
88
use std::path::Path;
99
use std::path::PathBuf;
10+
use std::sync::Arc;
1011

1112
use anyhow::Result;
1213
use arrow_array::RecordBatchReader;
@@ -16,6 +17,7 @@ use parquet::arrow::ArrowWriter;
1617
use tpcdsgen::config::CompatMode;
1718
use tpcdsgen::config::Session;
1819
use tpcdsgen::config::SessionBuilder;
20+
use tpcdsgen::config::Table;
1921
use tpcdsgen_arrow::CallCenterArrow;
2022
use tpcdsgen_arrow::CatalogPageArrow;
2123
use tpcdsgen_arrow::CatalogReturnsArrow;
@@ -44,19 +46,22 @@ use tracing::info;
4446
use vortex::utils::parallelism::get_available_parallelism;
4547

4648
use crate::Format;
47-
use crate::utils::file::idempotent;
49+
use crate::utils::file::temp_download_filepath;
4850
use crate::utils::parquet::parquet_writer_properties;
51+
use crate::utils::split;
4952

5053
const BATCH_SIZE: usize = 8192 * 16;
5154

5255
/// Matches the row group size of DuckDB's Parquet writer, which previously generated this data.
5356
const ROW_GROUP_SIZE: usize = 122_880;
5457

55-
/// Generate every TPC-DS table as a single Parquet file per table under `base_dir/parquet`.
58+
/// Generate every TPC-DS table as Parquet under `base_dir/parquet`.
5659
///
57-
/// The files hold the same data and schema as `tpcgen-cli tpcds parquet` of the same `tpcdsgen`
58-
/// version, which generates the SQL logic test data. Each table is generated on its own blocking
59-
/// thread. Already generated tables are skipped.
60+
/// Large tables are split into files of about [`split::TARGET_FILE_COMPRESSED_BYTES`], named by
61+
/// [`split::part_file_name`]; every other table is a single `{table}_0001.parquet`. The files hold
62+
/// the same data and schema as `tpcgen-cli tpcds parquet` of the same `tpcdsgen` version, which
63+
/// generates the SQL logic test data. Each table is generated on its own blocking thread. Already
64+
/// generated tables are skipped.
6065
pub async fn generate_tpcds(base_dir: PathBuf, scale_factor: String) -> Result<PathBuf> {
6166
let output_dir = base_dir.join(Format::Parquet.name());
6267
fs::create_dir_all(&output_dir)?;
@@ -71,12 +76,21 @@ pub async fn generate_tpcds(base_dir: PathBuf, scale_factor: String) -> Result<P
7176
let parallelism = get_available_parallelism().unwrap_or(1);
7277
let mut tasks = stream::iter(TABLES.map(|table| {
7378
let session = session.clone();
74-
let path = output_dir.join(format!("{table}.{}", Format::Parquet.ext()));
79+
let output_dir = output_dir.clone();
7580
tokio::task::spawn_blocking(move || {
76-
idempotent(&path, |tmp| {
77-
info!(table, scale_factor, "Generating TPC-DS table");
78-
write_parquet(tmp, table_batches(table, session))
79-
})
81+
let first_file =
82+
output_dir.join(split::part_file_name(table, 1, Format::Parquet.ext()));
83+
if first_file.exists() {
84+
return Ok(());
85+
}
86+
let layout = FileLayout::new(table, &session);
87+
info!(
88+
table,
89+
scale_factor,
90+
files = layout.num_files,
91+
"Generating TPC-DS table"
92+
);
93+
write_parquet_files(&output_dir, table, table_batches(table, session), layout)
8094
})
8195
}))
8296
.buffer_unordered(parallelism);
@@ -116,6 +130,72 @@ const TABLES: [&str; 24] = [
116130
"web_site",
117131
];
118132

133+
/// How a TPC-DS table's size scales, used to split it into files.
134+
struct TableSize {
135+
/// The table whose `tpcdsgen` row count drives generation. Returns tables are generated
136+
/// from their sales tickets.
137+
source: Table,
138+
/// Generated rows per source row, such as line items per ticket.
139+
rows_per_source_row: f64,
140+
/// Compressed bytes per row: the mean of the Parquet and default Vortex sizes.
141+
compressed_bytes_per_row: f64,
142+
}
143+
144+
/// Sizes of the tables that can outgrow one file, measured at scale factor 10. The other tables
145+
/// stay well below [`split::TARGET_FILE_COMPRESSED_BYTES`] even at scale factor 1000.
146+
fn table_size(table: &str) -> Option<TableSize> {
147+
let (source, rows_per_source_row, compressed_bytes_per_row) = match table {
148+
"store_sales" => (Table::StoreSales, 12.0004, 37.283),
149+
"store_returns" => (Table::StoreSales, 1.1981, 43.643),
150+
"catalog_sales" => (Table::CatalogSales, 9.0008, 54.241),
151+
"catalog_returns" => (Table::CatalogSales, 0.8998, 56.303),
152+
"web_sales" => (Table::WebSales, 11.9959, 52.890),
153+
"web_returns" => (Table::WebSales, 1.1987, 55.432),
154+
"inventory" => (Table::Inventory, 1.0, 2.762),
155+
"customer" => (Table::Customer, 1.0, 41.164),
156+
"customer_address" => (Table::CustomerAddress, 1.0, 15.311),
157+
"item" => (Table::Item, 1.0, 74.525),
158+
_ => return None,
159+
};
160+
Some(TableSize {
161+
source,
162+
rows_per_source_row,
163+
compressed_bytes_per_row,
164+
})
165+
}
166+
167+
/// How the rows of one table are spread over its files.
168+
#[derive(Clone, Copy)]
169+
struct FileLayout {
170+
num_files: usize,
171+
/// Rows in every file but the last, which also takes any rows beyond the estimate.
172+
rows_per_file: usize,
173+
}
174+
175+
impl FileLayout {
176+
#[expect(
177+
clippy::cast_possible_truncation,
178+
clippy::cast_precision_loss,
179+
clippy::cast_sign_loss,
180+
reason = "row counts are far below the f64 and usize limits"
181+
)]
182+
fn new(table: &str, session: &Session) -> Self {
183+
let Some(size) = table_size(table) else {
184+
return Self {
185+
num_files: 1,
186+
rows_per_file: usize::MAX,
187+
};
188+
};
189+
let rows =
190+
session.get_scaling().get_row_count(size.source) as f64 * size.rows_per_source_row;
191+
let num_files = split::num_files(rows * size.compressed_bytes_per_row);
192+
Self {
193+
num_files,
194+
rows_per_file: (rows / num_files as f64).ceil().max(1.0) as usize,
195+
}
196+
}
197+
}
198+
119199
fn table_batches(table: &str, session: Session) -> Box<dyn RecordBatchReader + Send> {
120200
match table {
121201
"call_center" => Box::new(CallCenterArrow::new(session).with_batch_size(BATCH_SIZE)),
@@ -154,14 +234,74 @@ fn table_batches(table: &str, session: Session) -> Box<dyn RecordBatchReader + S
154234
}
155235
}
156236

157-
fn write_parquet(path: &Path, batches: Box<dyn RecordBatchReader + Send>) -> Result<()> {
237+
/// Write `batches` as the Parquet files of `table` in `output_dir`, split by `layout`.
238+
///
239+
/// Files are written to temporary paths and renamed into place last part first, so the first
240+
/// part only exists once the whole table does.
241+
fn write_parquet_files(
242+
output_dir: &Path,
243+
table: &str,
244+
batches: Box<dyn RecordBatchReader + Send>,
245+
layout: FileLayout,
246+
) -> Result<()> {
247+
let schema = batches.schema();
158248
let props = parquet_writer_properties()
159249
.set_max_row_group_row_count(Some(ROW_GROUP_SIZE))
160250
.build();
161-
let mut writer = ArrowWriter::try_new(File::create(path)?, batches.schema(), Some(props))?;
251+
252+
let mut files = Vec::with_capacity(layout.num_files);
253+
let mut writer: Option<ArrowWriter<File>> = None;
254+
let mut rows_in_file = 0;
162255
for batch in batches {
163-
writer.write(&batch?)?;
256+
let mut batch = batch?;
257+
while batch.num_rows() > 0 {
258+
let current = match writer.as_mut() {
259+
Some(current) => current,
260+
None => {
261+
let path = temp_download_filepath();
262+
let file = File::create(&path)?;
263+
files.push(path);
264+
rows_in_file = 0;
265+
writer.insert(ArrowWriter::try_new(
266+
file,
267+
Arc::clone(&schema),
268+
Some(props.clone()),
269+
)?)
270+
}
271+
};
272+
let is_last_file = files.len() == layout.num_files;
273+
let len = if is_last_file {
274+
batch.num_rows()
275+
} else {
276+
batch.num_rows().min(layout.rows_per_file - rows_in_file)
277+
};
278+
current.write(&batch.slice(0, len))?;
279+
rows_in_file += len;
280+
batch = batch.slice(len, batch.num_rows() - len);
281+
if !is_last_file
282+
&& rows_in_file == layout.rows_per_file
283+
&& let Some(full) = writer.take()
284+
{
285+
full.close()?;
286+
}
287+
}
288+
}
289+
match writer {
290+
Some(writer) => {
291+
writer.close()?;
292+
}
293+
// An empty table still gets one file carrying its schema.
294+
None if files.is_empty() => {
295+
let path = temp_download_filepath();
296+
ArrowWriter::try_new(File::create(&path)?, schema, Some(props))?.close()?;
297+
files.push(path);
298+
}
299+
None => {}
300+
}
301+
302+
for (idx, path) in files.iter().enumerate().rev() {
303+
let name = split::part_file_name(table, idx + 1, Format::Parquet.ext());
304+
fs::rename(path, output_dir.join(name))?;
164305
}
165-
writer.close()?;
166306
Ok(())
167307
}

‎vortex-bench/src/tpch/benchmark.rs‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ use crate::tpch::schema::SUPPLIER;
2626
use crate::tpch::tpch_queries;
2727
use crate::tpch::tpchgen;
2828
use crate::tpch::tpchgen::TpchGenOptions;
29+
use crate::utils::split;
2930

3031
/// TPCH benchmark implementation
3132
pub struct TpcHBenchmark {
@@ -139,12 +140,7 @@ impl Benchmark for TpcHBenchmark {
139140
]
140141
}
141142

142-
#[expect(clippy::expect_used)]
143143
fn pattern(&self, table_name: &str, format: Format) -> Option<Pattern> {
144-
Some(
145-
format!("{}.{}", table_name, format.ext())
146-
.parse()
147-
.expect("valid glob pattern"),
148-
)
144+
Some(split::part_pattern(table_name, format.ext()))
149145
}
150146
}

0 commit comments

Comments
 (0)