2626//! select * from data limit 10;
2727//! ```
2828
29+ use std:: sync:: Arc ;
2930use std:: time:: Instant ;
3031
3132use arrow:: compute:: concat_batches;
@@ -42,6 +43,7 @@ use test_utils::AccessLogGenerator;
4243
4344/// how many rows of generated data to write to our parquet file (arbitrary)
4445const NUM_ROWS : usize = 53819 ;
46+ const ROW_LIMIT : usize = 4096 ;
4547
4648#[ cfg( test) ]
4749#[ ctor:: ctor]
@@ -51,45 +53,49 @@ fn init() {
5153}
5254
5355#[ cfg( not( target_family = "windows" ) ) ]
54- #[ tokio:: test]
56+ // Use multi-threaded executor as this test consumes CPU
57+ #[ tokio:: test( flavor = "multi_thread" ) ]
5558async fn single_file ( ) {
5659 // Only create the parquet file once as it is fairly large
5760
5861 let tempdir = TempDir :: new ( ) . unwrap ( ) ;
5962
60- let generator = AccessLogGenerator :: new ( ) . with_row_limit ( NUM_ROWS ) ;
63+ let generator = AccessLogGenerator :: new ( )
64+ . with_row_limit ( NUM_ROWS )
65+ . with_max_batch_size ( ROW_LIMIT ) ;
6166
6267 // default properties
6368 let props = WriterProperties :: builder ( ) . build ( ) ;
6469 let file = tempdir. path ( ) . join ( "data.parquet" ) ;
6570
6671 let start = Instant :: now ( ) ;
6772 println ! ( "Writing test data to {:?}" , file) ;
68- let test_parquet_file = TestParquetFile :: try_new ( file, props, generator) . unwrap ( ) ;
73+ let test_parquet_file =
74+ Arc :: new ( TestParquetFile :: try_new ( file, props, generator) . unwrap ( ) ) ;
6975 println ! (
7076 "Completed generating test data in {:?}" ,
7177 Instant :: now( ) - start
7278 ) ;
7379
74- TestCase :: new ( & test_parquet_file)
80+ let mut set = tokio:: task:: JoinSet :: new ( ) ;
81+
82+ let case = TestCase :: new ( test_parquet_file. clone ( ) )
7583 . with_name ( "selective" )
7684 // request_method = 'GET'
7785 . with_filter ( col ( "request_method" ) . eq ( lit ( "GET" ) ) )
7886 . with_pushdown_expected ( PushdownExpected :: Some )
79- . with_expected_rows ( 8886 )
80- . run ( )
81- . await ;
87+ . with_expected_rows ( 8875 ) ;
88+ set. spawn ( async move { case. run ( ) . await } ) ;
8289
83- TestCase :: new ( & test_parquet_file)
90+ let case = TestCase :: new ( test_parquet_file. clone ( ) )
8491 . with_name ( "non_selective" )
8592 // request_method != 'GET'
8693 . with_filter ( col ( "request_method" ) . not_eq ( lit ( "GET" ) ) )
8794 . with_pushdown_expected ( PushdownExpected :: Some )
88- . with_expected_rows ( 44933 )
89- . run ( )
90- . await ;
95+ . with_expected_rows ( 44944 ) ;
96+ set. spawn ( async move { case. run ( ) . await } ) ;
9197
92- TestCase :: new ( & test_parquet_file)
98+ let case = TestCase :: new ( test_parquet_file. clone ( ) )
9399 . with_name ( "basic_conjunction" )
94100 // request_method = 'POST' AND
95101 // response_status = 503
@@ -101,49 +107,44 @@ async fn single_file() {
101107 . unwrap ( ) ,
102108 )
103109 . with_pushdown_expected ( PushdownExpected :: Some )
104- . with_expected_rows ( 1729 )
105- . run ( )
106- . await ;
110+ . with_expected_rows ( 1731 ) ;
111+ set. spawn ( async move { case. run ( ) . await } ) ;
107112
108- TestCase :: new ( & test_parquet_file)
113+ let case = TestCase :: new ( test_parquet_file. clone ( ) )
109114 . with_name ( "everything" )
110115 // filter filters everything (no row has this status)
111116 // response_status = 429
112117 . with_filter ( col ( "response_status" ) . eq ( lit ( 429_u16 ) ) )
113118 . with_pushdown_expected ( PushdownExpected :: Some )
114- . with_expected_rows ( 0 )
115- . run ( )
116- . await ;
119+ . with_expected_rows ( 0 ) ;
120+ set. spawn ( async move { case. run ( ) . await } ) ;
117121
118- TestCase :: new ( & test_parquet_file)
122+ let case = TestCase :: new ( test_parquet_file. clone ( ) )
119123 . with_name ( "nothing" )
120124 // No rows are filtered out -- all are returned
121125 // response_status > 0
122126 . with_filter ( col ( "response_status" ) . gt ( lit ( 0_u16 ) ) )
123127 . with_pushdown_expected ( PushdownExpected :: None )
124- . with_expected_rows ( NUM_ROWS )
125- . run ( )
126- . await ;
128+ . with_expected_rows ( NUM_ROWS ) ;
129+ set. spawn ( async move { case. run ( ) . await } ) ;
127130
128- TestCase :: new ( & test_parquet_file)
131+ let case = TestCase :: new ( test_parquet_file. clone ( ) )
129132 . with_name ( "dict_selective" )
130133 // container = 'backend_container_0'
131134 . with_filter ( col ( "container" ) . eq ( lit ( "backend_container_0" ) ) )
132135 . with_pushdown_expected ( PushdownExpected :: Some )
133- . with_expected_rows ( 37856 )
134- . run ( )
135- . await ;
136+ . with_expected_rows ( 15911 ) ;
137+ set. spawn ( async move { case. run ( ) . await } ) ;
136138
137- TestCase :: new ( & test_parquet_file)
139+ let case = TestCase :: new ( test_parquet_file. clone ( ) )
138140 . with_name ( "not eq" )
139141 // container != 'backend_container_0'
140142 . with_filter ( col ( "container" ) . not_eq ( lit ( "backend_container_0" ) ) )
141143 . with_pushdown_expected ( PushdownExpected :: Some )
142- . with_expected_rows ( 15963 )
143- . run ( )
144- . await ;
144+ . with_expected_rows ( 37908 ) ;
145+ set. spawn ( async move { case. run ( ) . await } ) ;
145146
146- TestCase :: new ( & test_parquet_file)
147+ let case = TestCase :: new ( test_parquet_file. clone ( ) )
147148 . with_name ( "dict_conjunction" )
148149 // container == 'backend_container_0' AND
149150 // pod = 'aqcathnxqsphdhgjtgvxsfyiwbmhlmg'
@@ -155,11 +156,10 @@ async fn single_file() {
155156 . unwrap ( ) ,
156157 )
157158 . with_pushdown_expected ( PushdownExpected :: Some )
158- . with_expected_rows ( 3052 )
159- . run ( )
160- . await ;
159+ . with_expected_rows ( 3052 ) ;
160+ set. spawn ( async move { case. run ( ) . await } ) ;
161161
162- TestCase :: new ( & test_parquet_file)
162+ let case = TestCase :: new ( test_parquet_file. clone ( ) )
163163 . with_name ( "dict_very_selective" )
164164 // request_bytes > 2B AND
165165 // container == 'backend_container_0' AND
@@ -173,11 +173,10 @@ async fn single_file() {
173173 . unwrap ( ) ,
174174 )
175175 . with_pushdown_expected ( PushdownExpected :: Some )
176- . with_expected_rows ( 88 )
177- . run ( )
178- . await ;
176+ . with_expected_rows ( 88 ) ;
177+ set. spawn ( async move { case. run ( ) . await } ) ;
179178
180- TestCase :: new ( & test_parquet_file)
179+ let case = TestCase :: new ( test_parquet_file. clone ( ) )
181180 . with_name ( "dict_very_selective2" )
182181 // picks only 2 rows
183182 // client_addr = '204.47.29.82' AND
@@ -192,11 +191,10 @@ async fn single_file() {
192191 . unwrap ( ) ,
193192 )
194193 . with_pushdown_expected ( PushdownExpected :: Some )
195- . with_expected_rows ( 88 )
196- . run ( )
197- . await ;
194+ . with_expected_rows ( 88 ) ;
195+ set. spawn ( async move { case. run ( ) . await } ) ;
198196
199- TestCase :: new ( & test_parquet_file)
197+ let case = TestCase :: new ( test_parquet_file. clone ( ) )
200198 . with_name ( "dict_disjunction" )
201199 // container = 'backend_container_0' OR
202200 // pod = 'aqcathnxqsphdhgjtgvxsfyiwbmhlmg'
@@ -207,12 +205,11 @@ async fn single_file() {
207205 ] )
208206 . unwrap ( ) ,
209207 )
210- . with_pushdown_expected ( PushdownExpected :: Some )
211- . with_expected_rows ( 39982 )
212- . run ( )
213- . await ;
208+ . with_pushdown_expected ( PushdownExpected :: None )
209+ . with_expected_rows ( 16955 ) ;
210+ set. spawn ( async move { case. run ( ) . await } ) ;
214211
215- TestCase :: new ( & test_parquet_file)
212+ let case = TestCase :: new ( test_parquet_file. clone ( ) )
216213 . with_name ( "dict_disjunction3" )
217214 // request_method != 'GET' OR
218215 // response_status = 400 OR
@@ -225,10 +222,14 @@ async fn single_file() {
225222 ] )
226223 . unwrap ( ) ,
227224 )
228- . with_pushdown_expected ( PushdownExpected :: None )
229- . with_expected_rows ( NUM_ROWS )
230- . run ( )
231- . await ;
225+ . with_pushdown_expected ( PushdownExpected :: Some )
226+ . with_expected_rows ( 48919 ) ;
227+ set. spawn ( async move { case. run ( ) . await } ) ;
228+
229+ // Join all the cases.
230+ while let Some ( result) = set. join_next ( ) . await {
231+ result. unwrap ( )
232+ }
232233}
233234
234235#[ cfg( not( target_family = "windows" ) ) ]
@@ -247,7 +248,8 @@ async fn single_file_small_data_pages() {
247248
248249 let start = Instant :: now ( ) ;
249250 println ! ( "Writing test data to {:?}" , file) ;
250- let test_parquet_file = TestParquetFile :: try_new ( file, props, generator) . unwrap ( ) ;
251+ let test_parquet_file =
252+ Arc :: new ( TestParquetFile :: try_new ( file, props, generator) . unwrap ( ) ) ;
251253 println ! (
252254 "Completed generating test data in {:?}" ,
253255 Instant :: now( ) - start
@@ -267,7 +269,7 @@ async fn single_file_small_data_pages() {
267269 // page 4: DLE:RLE RLE:RLE VLE:RLE_DICTIONARY ST:[min: fktdcgtmzvoedpwhfevcvvrtaurzgex, max: fwtdpgtxwqkkgtgvthhwycrvjiizdifyp, num_nulls not defined] CRC:[none] SZ:7 VC:9216
268270 // page 5: DLE:RLE RLE:RLE VLE:RLE_DICTIONARY ST:[min: fwtdpgtxwqkkgtgvthhwycrvjiizdifyp, max: iadnalqpdzthpifrvewossmpqibgtsuin, num_nulls not defined] CRC:[none] SZ:7 VC:7739
269271
270- TestCase :: new ( & test_parquet_file)
272+ TestCase :: new ( test_parquet_file. clone ( ) )
271273 . with_name ( "selective" )
272274 // predicate is chosen carefully to prune pages 0, 1, 2, 3, 4
273275 // pod = 'iadnalqpdzthpifrvewossmpqibgtsuin'
@@ -286,7 +288,7 @@ async fn single_file_small_data_pages() {
286288 // page 3: DLE:RLE RLE:RLE VLE:RLE_DICTIONARY ST:[min: 1970-01-01T00:00:00.000000000, max: 1970-01-01T00:00:00.004269056, num_nulls not defined] CRC:[none] SZ:14996 VC:9216
287289 // page 4: DLE:RLE RLE:RLE VLE:RLE_DICTIONARY ST:[min: 1970-01-01T00:00:00.000000000, max: 1970-01-01T00:00:00.007261184, num_nulls not defined] CRC:[none] SZ:14996 VC:9216
288290 // page 5: DLE:RLE RLE:RLE VLE:RLE_DICTIONARY ST:[min: 1970-01-01T00:00:00.000000000, max: 1970-01-01T00:00:00.005330944, num_nulls not defined] CRC:[none] SZ:12601 VC:7739
289- TestCase :: new ( & test_parquet_file)
291+ TestCase :: new ( test_parquet_file. clone ( ) )
290292 . with_name ( "selective" )
291293 // predicate is chosen carefully to prune pages 1, 2, 4, and 5
292294 // time > 1970-01-01T00:00:00.004300000
@@ -314,7 +316,7 @@ async fn single_file_small_data_pages() {
314316 // offset compressed size first row index
315317 // page-0 5581636 147517 0
316318 // page-1 5729153 147517 9216
317- TestCase :: new ( & test_parquet_file)
319+ TestCase :: new ( test_parquet_file. clone ( ) )
318320 . with_name ( "selective_on_decimal" )
319321 // predicate is chosen carefully to prune pages 1, 2, 3, 4, and 5
320322 // decimal_price < 9200
@@ -345,8 +347,8 @@ enum PageIndexFilteringExpected {
345347}
346348
347349/// parameters for running a test
348- struct TestCase < ' a > {
349- test_parquet_file : & ' a TestParquetFile ,
350+ struct TestCase {
351+ test_parquet_file : Arc < TestParquetFile > ,
350352 /// Human readable name to help debug failures
351353 name : String ,
352354 /// The filter to apply
@@ -361,8 +363,8 @@ struct TestCase<'a> {
361363 expected_rows : usize ,
362364}
363365
364- impl < ' a > TestCase < ' a > {
365- fn new ( test_parquet_file : & ' a TestParquetFile ) -> Self {
366+ impl TestCase {
367+ fn new ( test_parquet_file : Arc < TestParquetFile > ) -> Self {
366368 Self {
367369 test_parquet_file,
368370 name : "<NOT SPECIFIED>" . into ( ) ,
@@ -533,12 +535,13 @@ impl<'a> TestCase<'a> {
533535
534536 match pushdown_expected {
535537 PushdownExpected :: None => {
536- assert_eq ! ( pushdown_rows_filtered, 0 ) ;
538+ assert_eq ! ( pushdown_rows_filtered, 0 , "{}" , self . name ) ;
537539 }
538540 PushdownExpected :: Some => {
539541 assert ! (
540542 pushdown_rows_filtered > 0 ,
541- "Expected to filter rows via pushdown, but none were"
543+ "{}: Expected to filter rows via pushdown, but none were" ,
544+ self . name
542545 ) ;
543546 }
544547 } ;
0 commit comments