Skip to content

Commit 31027fc

Browse files
authored
Parallelize tpch parquet data generation (#10691)
Data-gen now uses all cores to generate parquet data --------- Signed-off-by: Mikhail Kot <[email protected]>
1 parent 86d2845 commit 31027fc

1 file changed

Lines changed: 109 additions & 37 deletions

File tree

‎vortex-bench/src/tpch/tpchgen.rs‎

Lines changed: 109 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,12 @@ use arrow_schema::SchemaRef;
1414
use futures::FutureExt;
1515
use futures::StreamExt;
1616
use futures::future::BoxFuture;
17-
use parquet::arrow::AsyncArrowWriter;
17+
use parquet::arrow::ArrowWriter;
18+
use parquet::column::writer::ColumnCloseResult;
19+
use parquet::file::metadata::PageIndexPolicy;
20+
use parquet::file::metadata::ParquetMetaDataReader;
21+
use parquet::file::properties::WriterProperties;
22+
use parquet::file::writer::SerializedFileWriter;
1823
use tokio::fs::File as TokioFile;
1924
use tokio::sync::Semaphore;
2025
use tokio::sync::mpsc;
@@ -33,6 +38,7 @@ use vortex::array::ArrayRef;
3338
use vortex::array::stream::ArrayStreamAdapter;
3439
use vortex::error::VortexExpect;
3540
use vortex::file::WriteOptionsSessionExt;
41+
use vortex::utils::parallelism::get_available_parallelism;
3642
use vortex_arrow::ArrowSessionExt;
3743

3844
use crate::CompactionStrategy;
@@ -190,14 +196,15 @@ fn generate_table_file(
190196
of.to_string_lossy()
191197
);
192198

193-
let iter = create_batch_iterator(generator, &options)?;
199+
let num_parts = table_num_parts(generator);
194200

195-
// Create generator and process batches in streaming fashion
196-
let schema = iter.schema();
201+
if matches!(write_format, Format::Parquet) {
202+
return write_parquet_parts(generator, &options, num_parts, &path).await;
203+
}
204+
205+
let schema = create_batch_iterator(generator, &options, 1, 1)?.schema();
197206

198-
// Create writer based on format
199207
let mut writer: Box<dyn FileWriter + Send> = match write_format {
200-
Format::Parquet => Box::new(ParquetWriter::new(path, schema).await?),
201208
Format::OnDiskVortex => Box::new(VortexWriter::new(
202209
path,
203210
schema,
@@ -211,6 +218,7 @@ fn generate_table_file(
211218
_ => unreachable!(),
212219
};
213220

221+
let iter = create_batch_iterator(generator, &options, 1, 1)?;
214222
for batch in iter {
215223
writer.write_batch(&batch?).await?;
216224
}
@@ -226,14 +234,106 @@ fn generate_table_file(
226234
Ok(future)
227235
}
228236

229-
/// Create a batch iterator for one full table
237+
fn table_num_parts(generator: TableGenerator) -> usize {
238+
match generator {
239+
TableGenerator::Nation | TableGenerator::Region => 1,
240+
_ => get_available_parallelism().unwrap_or(1),
241+
}
242+
}
243+
244+
async fn write_parquet_parts(
245+
generator: TableGenerator,
246+
options: &TpchGenOptions,
247+
num_parts: usize,
248+
output: &Path,
249+
) -> Result<()> {
250+
let mut tasks = Vec::with_capacity(num_parts);
251+
for part in 1..=num_parts {
252+
let options = options.clone();
253+
let part_path = output.with_extension(format!("part{part}"));
254+
tasks.push(tokio::task::spawn_blocking(move || {
255+
let iter = create_batch_iterator(generator, &options, part, num_parts)?;
256+
let schema = iter.schema();
257+
let props = parquet_writer_properties()
258+
.set_max_row_group_row_count(Some(ROW_GROUP_SIZE))
259+
.build();
260+
let mut writer =
261+
ArrowWriter::try_new(fs::File::create(&part_path)?, schema, Some(props))?;
262+
for batch in iter {
263+
writer.write(&batch?)?;
264+
}
265+
writer.close()?;
266+
Ok::<_, anyhow::Error>(part_path)
267+
}));
268+
}
269+
270+
let mut part_paths = Vec::with_capacity(num_parts);
271+
for task in tasks {
272+
part_paths.push(task.await??);
273+
}
274+
275+
let output = output.to_path_buf();
276+
tokio::task::spawn_blocking(move || concat_parquet_parts(&part_paths, &output)).await?
277+
}
278+
279+
fn concat_parquet_parts(parts: &[PathBuf], output: &Path) -> Result<()> {
280+
let inputs = parts
281+
.iter()
282+
.map(|part| -> Result<_> {
283+
let reader = fs::File::open(part)?;
284+
let metadata = ParquetMetaDataReader::new()
285+
.with_page_index_policy(PageIndexPolicy::Optional)
286+
.parse_and_finish(&reader)?;
287+
Ok((reader, metadata))
288+
})
289+
.collect::<Result<Vec<_>>>()?;
290+
291+
let root_schema = inputs[0].1.file_metadata().schema_descr().root_schema_ptr();
292+
let mut writer = SerializedFileWriter::new(
293+
fs::File::create(output)?,
294+
root_schema,
295+
Arc::new(WriterProperties::builder().build()),
296+
)?;
297+
298+
for (input, metadata) in &inputs {
299+
let column_indexes = metadata.column_index();
300+
let offset_indexes = metadata.offset_index();
301+
for (rg_idx, rg) in metadata.row_groups().iter().enumerate() {
302+
let rg_column_indexes = column_indexes.and_then(|ci| ci.get(rg_idx));
303+
let rg_offset_indexes = offset_indexes.and_then(|oi| oi.get(rg_idx));
304+
let mut rg_out = writer.next_row_group()?;
305+
for (col_idx, column) in rg.columns().iter().enumerate() {
306+
let result = ColumnCloseResult {
307+
bytes_written: column.compressed_size() as _,
308+
rows_written: rg.num_rows() as _,
309+
metadata: column.clone(),
310+
bloom_filter: None,
311+
column_index: rg_column_indexes.and_then(|row| row.get(col_idx)).cloned(),
312+
offset_index: rg_offset_indexes.and_then(|row| row.get(col_idx)).cloned(),
313+
};
314+
rg_out.append_column(input, result)?;
315+
}
316+
rg_out.close()?;
317+
}
318+
}
319+
writer.close()?;
320+
321+
for part in parts {
322+
fs::remove_file(part)?;
323+
}
324+
325+
Ok(())
326+
}
327+
230328
fn create_batch_iterator(
231329
generator: TableGenerator,
232330
options: &TpchGenOptions,
331+
part: usize,
332+
num_parts: usize,
233333
) -> Result<Box<dyn RecordBatchReader + Send>> {
234334
let scale_factor = options.scale_factor.parse::<f64>()?;
235-
let part = 1;
236-
let num_parts = 1;
335+
let part = i32::try_from(part)?;
336+
let num_parts = i32::try_from(num_parts)?;
237337
let batch_size = options.batch_size;
238338

239339
let iterator: Box<dyn RecordBatchReader + Send> = match generator {
@@ -297,34 +397,6 @@ trait FileWriter {
297397
async fn finalize(self: Box<Self>) -> Result<()>;
298398
}
299399

300-
/// Parquet writer for streaming TPC-H data
301-
struct ParquetWriter {
302-
writer: AsyncArrowWriter<TokioFile>,
303-
}
304-
305-
impl ParquetWriter {
306-
async fn new(path: PathBuf, schema: SchemaRef) -> Result<Self> {
307-
let file = TokioFile::create(&path).await?;
308-
let props = parquet_writer_properties()
309-
.set_max_row_group_row_count(Some(ROW_GROUP_SIZE))
310-
.build();
311-
let writer = AsyncArrowWriter::try_new(file, schema, Some(props))?;
312-
Ok(Self { writer })
313-
}
314-
}
315-
316-
#[async_trait::async_trait]
317-
impl FileWriter for ParquetWriter {
318-
async fn write_batch(&mut self, batch: &RecordBatch) -> Result<()> {
319-
self.writer.write(batch).await.map_err(|e| anyhow!(e))
320-
}
321-
322-
async fn finalize(self: Box<Self>) -> Result<()> {
323-
self.writer.close().await?;
324-
Ok(())
325-
}
326-
}
327-
328400
/// Vortex writer for streaming TPC-H data
329401
struct VortexWriter {
330402
sender: Option<mpsc::Sender<vortex::error::VortexResult<ArrayRef>>>,

0 commit comments

Comments
 (0)