@@ -14,7 +14,12 @@ use arrow_schema::SchemaRef;
1414use futures:: FutureExt ;
1515use futures:: StreamExt ;
1616use futures:: future:: BoxFuture ;
17- use parquet:: arrow:: AsyncArrowWriter ;
17+ use parquet:: arrow:: ArrowWriter ;
18+ use parquet:: column:: writer:: ColumnCloseResult ;
19+ use parquet:: file:: metadata:: PageIndexPolicy ;
20+ use parquet:: file:: metadata:: ParquetMetaDataReader ;
21+ use parquet:: file:: properties:: WriterProperties ;
22+ use parquet:: file:: writer:: SerializedFileWriter ;
1823use tokio:: fs:: File as TokioFile ;
1924use tokio:: sync:: Semaphore ;
2025use tokio:: sync:: mpsc;
@@ -33,6 +38,7 @@ use vortex::array::ArrayRef;
3338use vortex:: array:: stream:: ArrayStreamAdapter ;
3439use vortex:: error:: VortexExpect ;
3540use vortex:: file:: WriteOptionsSessionExt ;
41+ use vortex:: utils:: parallelism:: get_available_parallelism;
3642use vortex_arrow:: ArrowSessionExt ;
3743
3844use crate :: CompactionStrategy ;
@@ -190,14 +196,15 @@ fn generate_table_file(
190196 of. to_string_lossy( )
191197 ) ;
192198
193- let iter = create_batch_iterator ( generator, & options ) ? ;
199+ let num_parts = table_num_parts ( generator) ;
194200
195- // Create generator and process batches in streaming fashion
196- let schema = iter. schema ( ) ;
201+ if matches ! ( write_format, Format :: Parquet ) {
202+ return write_parquet_parts ( generator, & options, num_parts, & path) . await ;
203+ }
204+
205+ let schema = create_batch_iterator ( generator, & options, 1 , 1 ) ?. schema ( ) ;
197206
198- // Create writer based on format
199207 let mut writer: Box < dyn FileWriter + Send > = match write_format {
200- Format :: Parquet => Box :: new ( ParquetWriter :: new ( path, schema) . await ?) ,
201208 Format :: OnDiskVortex => Box :: new ( VortexWriter :: new (
202209 path,
203210 schema,
@@ -211,6 +218,7 @@ fn generate_table_file(
211218 _ => unreachable ! ( ) ,
212219 } ;
213220
221+ let iter = create_batch_iterator ( generator, & options, 1 , 1 ) ?;
214222 for batch in iter {
215223 writer. write_batch ( & batch?) . await ?;
216224 }
@@ -226,14 +234,106 @@ fn generate_table_file(
226234 Ok ( future)
227235}
228236
229- /// Create a batch iterator for one full table
237+ fn table_num_parts ( generator : TableGenerator ) -> usize {
238+ match generator {
239+ TableGenerator :: Nation | TableGenerator :: Region => 1 ,
240+ _ => get_available_parallelism ( ) . unwrap_or ( 1 ) ,
241+ }
242+ }
243+
244+ async fn write_parquet_parts (
245+ generator : TableGenerator ,
246+ options : & TpchGenOptions ,
247+ num_parts : usize ,
248+ output : & Path ,
249+ ) -> Result < ( ) > {
250+ let mut tasks = Vec :: with_capacity ( num_parts) ;
251+ for part in 1 ..=num_parts {
252+ let options = options. clone ( ) ;
253+ let part_path = output. with_extension ( format ! ( "part{part}" ) ) ;
254+ tasks. push ( tokio:: task:: spawn_blocking ( move || {
255+ let iter = create_batch_iterator ( generator, & options, part, num_parts) ?;
256+ let schema = iter. schema ( ) ;
257+ let props = parquet_writer_properties ( )
258+ . set_max_row_group_row_count ( Some ( ROW_GROUP_SIZE ) )
259+ . build ( ) ;
260+ let mut writer =
261+ ArrowWriter :: try_new ( fs:: File :: create ( & part_path) ?, schema, Some ( props) ) ?;
262+ for batch in iter {
263+ writer. write ( & batch?) ?;
264+ }
265+ writer. close ( ) ?;
266+ Ok :: < _ , anyhow:: Error > ( part_path)
267+ } ) ) ;
268+ }
269+
270+ let mut part_paths = Vec :: with_capacity ( num_parts) ;
271+ for task in tasks {
272+ part_paths. push ( task. await ??) ;
273+ }
274+
275+ let output = output. to_path_buf ( ) ;
276+ tokio:: task:: spawn_blocking ( move || concat_parquet_parts ( & part_paths, & output) ) . await ?
277+ }
278+
279+ fn concat_parquet_parts ( parts : & [ PathBuf ] , output : & Path ) -> Result < ( ) > {
280+ let inputs = parts
281+ . iter ( )
282+ . map ( |part| -> Result < _ > {
283+ let reader = fs:: File :: open ( part) ?;
284+ let metadata = ParquetMetaDataReader :: new ( )
285+ . with_page_index_policy ( PageIndexPolicy :: Optional )
286+ . parse_and_finish ( & reader) ?;
287+ Ok ( ( reader, metadata) )
288+ } )
289+ . collect :: < Result < Vec < _ > > > ( ) ?;
290+
291+ let root_schema = inputs[ 0 ] . 1 . file_metadata ( ) . schema_descr ( ) . root_schema_ptr ( ) ;
292+ let mut writer = SerializedFileWriter :: new (
293+ fs:: File :: create ( output) ?,
294+ root_schema,
295+ Arc :: new ( WriterProperties :: builder ( ) . build ( ) ) ,
296+ ) ?;
297+
298+ for ( input, metadata) in & inputs {
299+ let column_indexes = metadata. column_index ( ) ;
300+ let offset_indexes = metadata. offset_index ( ) ;
301+ for ( rg_idx, rg) in metadata. row_groups ( ) . iter ( ) . enumerate ( ) {
302+ let rg_column_indexes = column_indexes. and_then ( |ci| ci. get ( rg_idx) ) ;
303+ let rg_offset_indexes = offset_indexes. and_then ( |oi| oi. get ( rg_idx) ) ;
304+ let mut rg_out = writer. next_row_group ( ) ?;
305+ for ( col_idx, column) in rg. columns ( ) . iter ( ) . enumerate ( ) {
306+ let result = ColumnCloseResult {
307+ bytes_written : column. compressed_size ( ) as _ ,
308+ rows_written : rg. num_rows ( ) as _ ,
309+ metadata : column. clone ( ) ,
310+ bloom_filter : None ,
311+ column_index : rg_column_indexes. and_then ( |row| row. get ( col_idx) ) . cloned ( ) ,
312+ offset_index : rg_offset_indexes. and_then ( |row| row. get ( col_idx) ) . cloned ( ) ,
313+ } ;
314+ rg_out. append_column ( input, result) ?;
315+ }
316+ rg_out. close ( ) ?;
317+ }
318+ }
319+ writer. close ( ) ?;
320+
321+ for part in parts {
322+ fs:: remove_file ( part) ?;
323+ }
324+
325+ Ok ( ( ) )
326+ }
327+
230328fn create_batch_iterator (
231329 generator : TableGenerator ,
232330 options : & TpchGenOptions ,
331+ part : usize ,
332+ num_parts : usize ,
233333) -> Result < Box < dyn RecordBatchReader + Send > > {
234334 let scale_factor = options. scale_factor . parse :: < f64 > ( ) ?;
235- let part = 1 ;
236- let num_parts = 1 ;
335+ let part = i32 :: try_from ( part ) ? ;
336+ let num_parts = i32 :: try_from ( num_parts ) ? ;
237337 let batch_size = options. batch_size ;
238338
239339 let iterator: Box < dyn RecordBatchReader + Send > = match generator {
@@ -297,34 +397,6 @@ trait FileWriter {
297397 async fn finalize ( self : Box < Self > ) -> Result < ( ) > ;
298398}
299399
300- /// Parquet writer for streaming TPC-H data
301- struct ParquetWriter {
302- writer : AsyncArrowWriter < TokioFile > ,
303- }
304-
305- impl ParquetWriter {
306- async fn new ( path : PathBuf , schema : SchemaRef ) -> Result < Self > {
307- let file = TokioFile :: create ( & path) . await ?;
308- let props = parquet_writer_properties ( )
309- . set_max_row_group_row_count ( Some ( ROW_GROUP_SIZE ) )
310- . build ( ) ;
311- let writer = AsyncArrowWriter :: try_new ( file, schema, Some ( props) ) ?;
312- Ok ( Self { writer } )
313- }
314- }
315-
316- #[ async_trait:: async_trait]
317- impl FileWriter for ParquetWriter {
318- async fn write_batch ( & mut self , batch : & RecordBatch ) -> Result < ( ) > {
319- self . writer . write ( batch) . await . map_err ( |e| anyhow ! ( e) )
320- }
321-
322- async fn finalize ( self : Box < Self > ) -> Result < ( ) > {
323- self . writer . close ( ) . await ?;
324- Ok ( ( ) )
325- }
326- }
327-
328400/// Vortex writer for streaming TPC-H data
329401struct VortexWriter {
330402 sender : Option < mpsc:: Sender < vortex:: error:: VortexResult < ArrayRef > > > ,
0 commit comments