@@ -7,6 +7,7 @@ use std::fs;
77use std:: fs:: File ;
88use std:: path:: Path ;
99use std:: path:: PathBuf ;
10+ use std:: sync:: Arc ;
1011
1112use anyhow:: Result ;
1213use arrow_array:: RecordBatchReader ;
@@ -16,6 +17,7 @@ use parquet::arrow::ArrowWriter;
1617use tpcdsgen:: config:: CompatMode ;
1718use tpcdsgen:: config:: Session ;
1819use tpcdsgen:: config:: SessionBuilder ;
20+ use tpcdsgen:: config:: Table ;
1921use tpcdsgen_arrow:: CallCenterArrow ;
2022use tpcdsgen_arrow:: CatalogPageArrow ;
2123use tpcdsgen_arrow:: CatalogReturnsArrow ;
@@ -44,19 +46,22 @@ use tracing::info;
4446use vortex:: utils:: parallelism:: get_available_parallelism;
4547
4648use crate :: Format ;
47- use crate :: utils:: file:: idempotent ;
49+ use crate :: utils:: file:: temp_download_filepath ;
4850use crate :: utils:: parquet:: parquet_writer_properties;
51+ use crate :: utils:: split;
4952
5053const BATCH_SIZE : usize = 8192 * 16 ;
5154
5255/// Matches the row group size of DuckDB's Parquet writer, which previously generated this data.
5356const ROW_GROUP_SIZE : usize = 122_880 ;
5457
55- /// Generate every TPC-DS table as a single Parquet file per table under `base_dir/parquet`.
58+ /// Generate every TPC-DS table as Parquet under `base_dir/parquet`.
5659///
57- /// The files hold the same data and schema as `tpcgen-cli tpcds parquet` of the same `tpcdsgen`
58- /// version, which generates the SQL logic test data. Each table is generated on its own blocking
59- /// thread. Already generated tables are skipped.
60+ /// Large tables are split into files of about [`split::TARGET_FILE_COMPRESSED_BYTES`], named by
61+ /// [`split::part_file_name`]; every other table is a single `{table}_0001.parquet`. The files hold
62+ /// the same data and schema as `tpcgen-cli tpcds parquet` of the same `tpcdsgen` version, which
63+ /// generates the SQL logic test data. Each table is generated on its own blocking thread. Already
64+ /// generated tables are skipped.
6065pub async fn generate_tpcds ( base_dir : PathBuf , scale_factor : String ) -> Result < PathBuf > {
6166 let output_dir = base_dir. join ( Format :: Parquet . name ( ) ) ;
6267 fs:: create_dir_all ( & output_dir) ?;
@@ -71,12 +76,21 @@ pub async fn generate_tpcds(base_dir: PathBuf, scale_factor: String) -> Result<P
7176 let parallelism = get_available_parallelism ( ) . unwrap_or ( 1 ) ;
7277 let mut tasks = stream:: iter ( TABLES . map ( |table| {
7378 let session = session. clone ( ) ;
74- let path = output_dir. join ( format ! ( "{table}.{}" , Format :: Parquet . ext ( ) ) ) ;
79+ let output_dir = output_dir. clone ( ) ;
7580 tokio:: task:: spawn_blocking ( move || {
76- idempotent ( & path, |tmp| {
77- info ! ( table, scale_factor, "Generating TPC-DS table" ) ;
78- write_parquet ( tmp, table_batches ( table, session) )
79- } )
81+ let first_file =
82+ output_dir. join ( split:: part_file_name ( table, 1 , Format :: Parquet . ext ( ) ) ) ;
83+ if first_file. exists ( ) {
84+ return Ok ( ( ) ) ;
85+ }
86+ let layout = FileLayout :: new ( table, & session) ;
87+ info ! (
88+ table,
89+ scale_factor,
90+ files = layout. num_files,
91+ "Generating TPC-DS table"
92+ ) ;
93+ write_parquet_files ( & output_dir, table, table_batches ( table, session) , layout)
8094 } )
8195 } ) )
8296 . buffer_unordered ( parallelism) ;
@@ -116,6 +130,72 @@ const TABLES: [&str; 24] = [
116130 "web_site" ,
117131] ;
118132
133+ /// How a TPC-DS table's size scales, used to split it into files.
134+ struct TableSize {
135+ /// The table whose `tpcdsgen` row count drives generation. Returns tables are generated
136+ /// from their sales tickets.
137+ source : Table ,
138+ /// Generated rows per source row, such as line items per ticket.
139+ rows_per_source_row : f64 ,
140+ /// Compressed bytes per row: the mean of the Parquet and default Vortex sizes.
141+ compressed_bytes_per_row : f64 ,
142+ }
143+
144+ /// Sizes of the tables that can outgrow one file, measured at scale factor 10. The other tables
145+ /// stay well below [`split::TARGET_FILE_COMPRESSED_BYTES`] even at scale factor 1000.
146+ fn table_size ( table : & str ) -> Option < TableSize > {
147+ let ( source, rows_per_source_row, compressed_bytes_per_row) = match table {
148+ "store_sales" => ( Table :: StoreSales , 12.0004 , 37.283 ) ,
149+ "store_returns" => ( Table :: StoreSales , 1.1981 , 43.643 ) ,
150+ "catalog_sales" => ( Table :: CatalogSales , 9.0008 , 54.241 ) ,
151+ "catalog_returns" => ( Table :: CatalogSales , 0.8998 , 56.303 ) ,
152+ "web_sales" => ( Table :: WebSales , 11.9959 , 52.890 ) ,
153+ "web_returns" => ( Table :: WebSales , 1.1987 , 55.432 ) ,
154+ "inventory" => ( Table :: Inventory , 1.0 , 2.762 ) ,
155+ "customer" => ( Table :: Customer , 1.0 , 41.164 ) ,
156+ "customer_address" => ( Table :: CustomerAddress , 1.0 , 15.311 ) ,
157+ "item" => ( Table :: Item , 1.0 , 74.525 ) ,
158+ _ => return None ,
159+ } ;
160+ Some ( TableSize {
161+ source,
162+ rows_per_source_row,
163+ compressed_bytes_per_row,
164+ } )
165+ }
166+
167+ /// How the rows of one table are spread over its files.
168+ #[ derive( Clone , Copy ) ]
169+ struct FileLayout {
170+ num_files : usize ,
171+ /// Rows in every file but the last, which also takes any rows beyond the estimate.
172+ rows_per_file : usize ,
173+ }
174+
175+ impl FileLayout {
176+ #[ expect(
177+ clippy:: cast_possible_truncation,
178+ clippy:: cast_precision_loss,
179+ clippy:: cast_sign_loss,
180+ reason = "row counts are far below the f64 and usize limits"
181+ ) ]
182+ fn new ( table : & str , session : & Session ) -> Self {
183+ let Some ( size) = table_size ( table) else {
184+ return Self {
185+ num_files : 1 ,
186+ rows_per_file : usize:: MAX ,
187+ } ;
188+ } ;
189+ let rows =
190+ session. get_scaling ( ) . get_row_count ( size. source ) as f64 * size. rows_per_source_row ;
191+ let num_files = split:: num_files ( rows * size. compressed_bytes_per_row ) ;
192+ Self {
193+ num_files,
194+ rows_per_file : ( rows / num_files as f64 ) . ceil ( ) . max ( 1.0 ) as usize ,
195+ }
196+ }
197+ }
198+
119199fn table_batches ( table : & str , session : Session ) -> Box < dyn RecordBatchReader + Send > {
120200 match table {
121201 "call_center" => Box :: new ( CallCenterArrow :: new ( session) . with_batch_size ( BATCH_SIZE ) ) ,
@@ -154,14 +234,74 @@ fn table_batches(table: &str, session: Session) -> Box<dyn RecordBatchReader + S
154234 }
155235}
156236
157- fn write_parquet ( path : & Path , batches : Box < dyn RecordBatchReader + Send > ) -> Result < ( ) > {
237+ /// Write `batches` as the Parquet files of `table` in `output_dir`, split by `layout`.
238+ ///
239+ /// Files are written to temporary paths and renamed into place last part first, so the first
240+ /// part only exists once the whole table does.
241+ fn write_parquet_files (
242+ output_dir : & Path ,
243+ table : & str ,
244+ batches : Box < dyn RecordBatchReader + Send > ,
245+ layout : FileLayout ,
246+ ) -> Result < ( ) > {
247+ let schema = batches. schema ( ) ;
158248 let props = parquet_writer_properties ( )
159249 . set_max_row_group_row_count ( Some ( ROW_GROUP_SIZE ) )
160250 . build ( ) ;
161- let mut writer = ArrowWriter :: try_new ( File :: create ( path) ?, batches. schema ( ) , Some ( props) ) ?;
251+
252+ let mut files = Vec :: with_capacity ( layout. num_files ) ;
253+ let mut writer: Option < ArrowWriter < File > > = None ;
254+ let mut rows_in_file = 0 ;
162255 for batch in batches {
163- writer. write ( & batch?) ?;
256+ let mut batch = batch?;
257+ while batch. num_rows ( ) > 0 {
258+ let current = match writer. as_mut ( ) {
259+ Some ( current) => current,
260+ None => {
261+ let path = temp_download_filepath ( ) ;
262+ let file = File :: create ( & path) ?;
263+ files. push ( path) ;
264+ rows_in_file = 0 ;
265+ writer. insert ( ArrowWriter :: try_new (
266+ file,
267+ Arc :: clone ( & schema) ,
268+ Some ( props. clone ( ) ) ,
269+ ) ?)
270+ }
271+ } ;
272+ let is_last_file = files. len ( ) == layout. num_files ;
273+ let len = if is_last_file {
274+ batch. num_rows ( )
275+ } else {
276+ batch. num_rows ( ) . min ( layout. rows_per_file - rows_in_file)
277+ } ;
278+ current. write ( & batch. slice ( 0 , len) ) ?;
279+ rows_in_file += len;
280+ batch = batch. slice ( len, batch. num_rows ( ) - len) ;
281+ if !is_last_file
282+ && rows_in_file == layout. rows_per_file
283+ && let Some ( full) = writer. take ( )
284+ {
285+ full. close ( ) ?;
286+ }
287+ }
288+ }
289+ match writer {
290+ Some ( writer) => {
291+ writer. close ( ) ?;
292+ }
293+ // An empty table still gets one file carrying its schema.
294+ None if files. is_empty ( ) => {
295+ let path = temp_download_filepath ( ) ;
296+ ArrowWriter :: try_new ( File :: create ( & path) ?, schema, Some ( props) ) ?. close ( ) ?;
297+ files. push ( path) ;
298+ }
299+ None => { }
300+ }
301+
302+ for ( idx, path) in files. iter ( ) . enumerate ( ) . rev ( ) {
303+ let name = split:: part_file_name ( table, idx + 1 , Format :: Parquet . ext ( ) ) ;
304+ fs:: rename ( path, output_dir. join ( name) ) ?;
164305 }
165- writer. close ( ) ?;
166306 Ok ( ( ) )
167307}
0 commit comments