@@ -23,30 +23,30 @@ use crate::SESSION;
2323use crate :: datasets:: Dataset ;
2424use crate :: tpch:: tpchgen:: TpchGenOptions ;
2525use crate :: tpch:: tpchgen:: generate_tpch_tables;
26+ use crate :: utils:: split;
2627
2728pub struct TPCHLCommentChunked ;
2829
29- /// Unsplit TPC-H data, kept apart from the TPC-H benchmark's split files so `lineitem` stays a
30- /// single file.
31- const DATA_DIR : & str = "tpch-unsplit" ;
32-
33- async fn ensure_tpch_parquet ( ) -> Result < PathBuf > {
34- let base_path = DATA_DIR . to_data_path ( ) ;
35- let scale_factor_dir = base_path. join ( "1.0" ) ;
36- let data_dir = scale_factor_dir. join ( Format :: Parquet . name ( ) ) ;
37-
38- // Generate TPC-H parquet data if it doesn't exist
39- if !data_dir. exists ( ) {
40- let options = TpchGenOptions :: new ( "1.0" . to_string ( ) , scale_factor_dir)
41- . with_format ( Format :: Parquet )
42- . with_split_files ( false ) ;
30+ /// Path of TPC-H SF=1 `lineitem` in `format`, generating the TPC-H tables if it is missing.
31+ ///
32+ /// At SF=1 `lineitem` fits in a single file (see [`split::TARGET_FILE_COMPRESSED_BYTES`]), so this
33+ /// reads the TPC-H benchmark's own data.
34+ async fn ensure_lineitem ( format : Format ) -> Result < PathBuf > {
35+ let scale_factor_dir = "tpch" . to_data_path ( ) . join ( "1.0" ) ;
36+ let format_dir = scale_factor_dir. join ( format. name ( ) ) ;
37+ let path = format_dir. join ( split:: part_file_name ( "lineitem" , 1 , format. ext ( ) ) ) ;
38+ if !path. exists ( ) {
39+ let options = TpchGenOptions :: new ( "1.0" . to_string ( ) , scale_factor_dir) . with_format ( format) ;
4340 generate_tpch_tables ( options) . await ?;
4441 }
45-
46- let path = data_dir. join ( "lineitem.parquet" ) ;
47- path. exists ( )
48- . then_some ( path)
49- . ok_or_else ( || anyhow:: anyhow!( "No lineitem parquet file found" ) )
42+ anyhow:: ensure!( path. exists( ) , "No lineitem {format} file found" ) ;
43+ anyhow:: ensure!(
44+ !format_dir
45+ . join( split:: part_file_name( "lineitem" , 2 , format. ext( ) ) )
46+ . exists( ) ,
47+ "TPC-H SF=1 lineitem is split into several {format} files"
48+ ) ;
49+ Ok ( path)
5050}
5151
5252#[ async_trait]
@@ -56,21 +56,7 @@ impl Dataset for TPCHLCommentChunked {
5656 }
5757
5858 async fn to_vortex_array ( & self , ctx : & mut ExecutionCtx ) -> Result < ArrayRef > {
59- let base_path = DATA_DIR . to_data_path ( ) ;
60- let scale_factor_dir = base_path. join ( "1.0" ) ;
61- let data_dir = scale_factor_dir. join ( Format :: OnDiskVortex . name ( ) ) ;
62-
63- // Generate TPC-H Vortex data if it doesn't exist
64- if !data_dir. exists ( ) {
65- // Use blocking call like TPC-H benchmark does
66- let options = TpchGenOptions :: new ( "1.0" . to_string ( ) , scale_factor_dir)
67- . with_format ( Format :: OnDiskVortex )
68- . with_split_files ( false ) ;
69-
70- futures:: executor:: block_on ( generate_tpch_tables ( options) ) ?;
71- }
72-
73- let path = data_dir. join ( "lineitem.vortex" ) ;
59+ let path = ensure_lineitem ( Format :: OnDiskVortex ) . await ?;
7460 let file = SESSION . open_options ( ) . open_path ( path) . await ?;
7561 let projection = pack ( vec ! [ ( "l_comment" , col( "l_comment" ) ) ] , NonNullable )
7662 . bind ( file. dtype ( ) ) ?
@@ -94,7 +80,7 @@ impl Dataset for TPCHLCommentChunked {
9480 }
9581
9682 async fn to_parquet_path ( & self ) -> Result < PathBuf > {
97- ensure_tpch_parquet ( ) . await
83+ ensure_lineitem ( Format :: Parquet ) . await
9884 }
9985}
10086
@@ -113,6 +99,6 @@ impl Dataset for TPCHLCommentCanonical {
11399 }
114100
115101 async fn to_parquet_path ( & self ) -> Result < PathBuf > {
116- ensure_tpch_parquet ( ) . await
102+ ensure_lineitem ( Format :: Parquet ) . await
117103 }
118104}
0 commit comments