Skip to content

Commit 999acbb

Browse files
joseph-isaacsclaude
authored andcommitted
Read TPC-H l_comment datasets from the shared split files
At SF=1 lineitem fits in a single `lineitem_0001` file, so the l_comment compress datasets read the TPC-H benchmark's own data instead of a separate unsplit `tpch-unsplit` directory. This drops the `split_files` option from `TpchGenOptions`, and the datasets fail loudly if lineitem is ever split at SF=1. Signed-off-by: Joe Isaacs <[email protected]> Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01XxMYCv7MTA9RdzqMVUZVuX
1 parent e8eea77 commit 999acbb

2 files changed

Lines changed: 24 additions & 55 deletions

File tree

‎vortex-bench/src/datasets/tpch_l_comment.rs‎

Lines changed: 22 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -23,30 +23,30 @@ use crate::SESSION;
2323
use crate::datasets::Dataset;
2424
use crate::tpch::tpchgen::TpchGenOptions;
2525
use crate::tpch::tpchgen::generate_tpch_tables;
26+
use crate::utils::split;
2627

2728
pub struct TPCHLCommentChunked;
2829

29-
/// Unsplit TPC-H data, kept apart from the TPC-H benchmark's split files so `lineitem` stays a
30-
/// single file.
31-
const DATA_DIR: &str = "tpch-unsplit";
32-
33-
async fn ensure_tpch_parquet() -> Result<PathBuf> {
34-
let base_path = DATA_DIR.to_data_path();
35-
let scale_factor_dir = base_path.join("1.0");
36-
let data_dir = scale_factor_dir.join(Format::Parquet.name());
37-
38-
// Generate TPC-H parquet data if it doesn't exist
39-
if !data_dir.exists() {
40-
let options = TpchGenOptions::new("1.0".to_string(), scale_factor_dir)
41-
.with_format(Format::Parquet)
42-
.with_split_files(false);
30+
/// Path of TPC-H SF=1 `lineitem` in `format`, generating the TPC-H tables if it is missing.
31+
///
32+
/// At SF=1 `lineitem` fits in a single file (see [`split::TARGET_FILE_COMPRESSED_BYTES`]), so this
33+
/// reads the TPC-H benchmark's own data.
34+
async fn ensure_lineitem(format: Format) -> Result<PathBuf> {
35+
let scale_factor_dir = "tpch".to_data_path().join("1.0");
36+
let format_dir = scale_factor_dir.join(format.name());
37+
let path = format_dir.join(split::part_file_name("lineitem", 1, format.ext()));
38+
if !path.exists() {
39+
let options = TpchGenOptions::new("1.0".to_string(), scale_factor_dir).with_format(format);
4340
generate_tpch_tables(options).await?;
4441
}
45-
46-
let path = data_dir.join("lineitem.parquet");
47-
path.exists()
48-
.then_some(path)
49-
.ok_or_else(|| anyhow::anyhow!("No lineitem parquet file found"))
42+
anyhow::ensure!(path.exists(), "No lineitem {format} file found");
43+
anyhow::ensure!(
44+
!format_dir
45+
.join(split::part_file_name("lineitem", 2, format.ext()))
46+
.exists(),
47+
"TPC-H SF=1 lineitem is split into several {format} files"
48+
);
49+
Ok(path)
5050
}
5151

5252
#[async_trait]
@@ -56,21 +56,7 @@ impl Dataset for TPCHLCommentChunked {
5656
}
5757

5858
async fn to_vortex_array(&self, ctx: &mut ExecutionCtx) -> Result<ArrayRef> {
59-
let base_path = DATA_DIR.to_data_path();
60-
let scale_factor_dir = base_path.join("1.0");
61-
let data_dir = scale_factor_dir.join(Format::OnDiskVortex.name());
62-
63-
// Generate TPC-H Vortex data if it doesn't exist
64-
if !data_dir.exists() {
65-
// Use blocking call like TPC-H benchmark does
66-
let options = TpchGenOptions::new("1.0".to_string(), scale_factor_dir)
67-
.with_format(Format::OnDiskVortex)
68-
.with_split_files(false);
69-
70-
futures::executor::block_on(generate_tpch_tables(options))?;
71-
}
72-
73-
let path = data_dir.join("lineitem.vortex");
59+
let path = ensure_lineitem(Format::OnDiskVortex).await?;
7460
let file = SESSION.open_options().open_path(path).await?;
7561
let projection = pack(vec![("l_comment", col("l_comment"))], NonNullable)
7662
.bind(file.dtype())?
@@ -94,7 +80,7 @@ impl Dataset for TPCHLCommentChunked {
9480
}
9581

9682
async fn to_parquet_path(&self) -> Result<PathBuf> {
97-
ensure_tpch_parquet().await
83+
ensure_lineitem(Format::Parquet).await
9884
}
9985
}
10086

@@ -113,6 +99,6 @@ impl Dataset for TPCHLCommentCanonical {
11399
}
114100

115101
async fn to_parquet_path(&self) -> Result<PathBuf> {
116-
ensure_tpch_parquet().await
102+
ensure_lineitem(Format::Parquet).await
117103
}
118104
}

‎vortex-bench/src/tpch/tpchgen.rs‎

Lines changed: 2 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,6 @@ pub struct TpchGenOptions {
7575
pub format: Format,
7676
/// Batch size for streaming
7777
pub batch_size: usize,
78-
/// Split each table into files of about [`split::TARGET_FILE_COMPRESSED_BYTES`] named by
79-
/// [`split::part_file_name`]. Otherwise each table is a single `{table}.{ext}` file.
80-
pub split_files: bool,
8178
}
8279

8380
impl Default for TpchGenOptions {
@@ -87,7 +84,6 @@ impl Default for TpchGenOptions {
8784
output_dir: "tpch".to_data_path(),
8885
format: Format::Parquet,
8986
batch_size: 8192 * 64,
90-
split_files: true,
9187
}
9288
}
9389
}
@@ -110,11 +106,6 @@ impl TpchGenOptions {
110106
self.batch_size = batch_size;
111107
self
112108
}
113-
114-
pub fn with_split_files(mut self, split_files: bool) -> Self {
115-
self.split_files = split_files;
116-
self
117-
}
118109
}
119110

120111
/// Generate all TPC-H tables for a single scale factor
@@ -124,11 +115,7 @@ pub async fn generate_tpch_tables(options: TpchGenOptions) -> Result<()> {
124115
let scale_factor = options.scale_factor.parse::<f64>()?;
125116
let mut all_futures = Vec::new();
126117
for generator in TableGenerator::ALL {
127-
let num_parts = if options.split_files {
128-
generator.num_parts(scale_factor)?
129-
} else {
130-
1
131-
};
118+
let num_parts = generator.num_parts(scale_factor)?;
132119
info!(
133120
scale_factor = options.scale_factor,
134121
format = %options.format,
@@ -258,11 +245,7 @@ fn generate_table_file(
258245
fs::create_dir_all(&output_dir)?;
259246

260247
let table_name = generator.name();
261-
let output_file = if options.split_files {
262-
output_dir.join(split::part_file_name(table_name, part, write_format.ext()))
263-
} else {
264-
output_dir.join(format!("{table_name}.{}", write_format.ext()))
265-
};
248+
let output_file = output_dir.join(split::part_file_name(table_name, part, write_format.ext()));
266249

267250
let future = async move {
268251
let of = output_file.clone();

0 commit comments

Comments
 (0)