Skip to content

Commit 38a24c0

Browse files
waynexiaalamb
andauthored
refactor: parallelize parquet_exec test case single_file (#4735)
* refactor: parallelize test case Signed-off-by: Ruihang Xia <[email protected]> * Apply suggestions from code review Co-authored-by: Andrew Lamb <[email protected]> * format code Signed-off-by: Ruihang Xia <[email protected]> * change row limit Signed-off-by: Ruihang Xia <[email protected]> Signed-off-by: Ruihang Xia <[email protected]> Co-authored-by: Andrew Lamb <[email protected]>
1 parent 734c211 commit 38a24c0

1 file changed

Lines changed: 66 additions & 63 deletions

File tree

‎datafusion/core/tests/parquet/filter_pushdown.rs‎

Lines changed: 66 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
//! select * from data limit 10;
2727
//! ```
2828
29+
use std::sync::Arc;
2930
use std::time::Instant;
3031

3132
use arrow::compute::concat_batches;
@@ -42,6 +43,7 @@ use test_utils::AccessLogGenerator;
4243

4344
/// how many rows of generated data to write to our parquet file (arbitrary)
4445
const NUM_ROWS: usize = 53819;
46+
const ROW_LIMIT: usize = 4096;
4547

4648
#[cfg(test)]
4749
#[ctor::ctor]
@@ -51,45 +53,49 @@ fn init() {
5153
}
5254

5355
#[cfg(not(target_family = "windows"))]
54-
#[tokio::test]
56+
// Use multi-threaded executor as this test consumes CPU
57+
#[tokio::test(flavor = "multi_thread")]
5558
async fn single_file() {
5659
// Only create the parquet file once as it is fairly large
5760

5861
let tempdir = TempDir::new().unwrap();
5962

60-
let generator = AccessLogGenerator::new().with_row_limit(NUM_ROWS);
63+
let generator = AccessLogGenerator::new()
64+
.with_row_limit(NUM_ROWS)
65+
.with_max_batch_size(ROW_LIMIT);
6166

6267
// default properties
6368
let props = WriterProperties::builder().build();
6469
let file = tempdir.path().join("data.parquet");
6570

6671
let start = Instant::now();
6772
println!("Writing test data to {:?}", file);
68-
let test_parquet_file = TestParquetFile::try_new(file, props, generator).unwrap();
73+
let test_parquet_file =
74+
Arc::new(TestParquetFile::try_new(file, props, generator).unwrap());
6975
println!(
7076
"Completed generating test data in {:?}",
7177
Instant::now() - start
7278
);
7379

74-
TestCase::new(&test_parquet_file)
80+
let mut set = tokio::task::JoinSet::new();
81+
82+
let case = TestCase::new(test_parquet_file.clone())
7583
.with_name("selective")
7684
// request_method = 'GET'
7785
.with_filter(col("request_method").eq(lit("GET")))
7886
.with_pushdown_expected(PushdownExpected::Some)
79-
.with_expected_rows(8886)
80-
.run()
81-
.await;
87+
.with_expected_rows(8875);
88+
set.spawn(async move { case.run().await });
8289

83-
TestCase::new(&test_parquet_file)
90+
let case = TestCase::new(test_parquet_file.clone())
8491
.with_name("non_selective")
8592
// request_method != 'GET'
8693
.with_filter(col("request_method").not_eq(lit("GET")))
8794
.with_pushdown_expected(PushdownExpected::Some)
88-
.with_expected_rows(44933)
89-
.run()
90-
.await;
95+
.with_expected_rows(44944);
96+
set.spawn(async move { case.run().await });
9197

92-
TestCase::new(&test_parquet_file)
98+
let case = TestCase::new(test_parquet_file.clone())
9399
.with_name("basic_conjunction")
94100
// request_method = 'POST' AND
95101
// response_status = 503
@@ -101,49 +107,44 @@ async fn single_file() {
101107
.unwrap(),
102108
)
103109
.with_pushdown_expected(PushdownExpected::Some)
104-
.with_expected_rows(1729)
105-
.run()
106-
.await;
110+
.with_expected_rows(1731);
111+
set.spawn(async move { case.run().await });
107112

108-
TestCase::new(&test_parquet_file)
113+
let case = TestCase::new(test_parquet_file.clone())
109114
.with_name("everything")
110115
// filter filters everything (no row has this status)
111116
// response_status = 429
112117
.with_filter(col("response_status").eq(lit(429_u16)))
113118
.with_pushdown_expected(PushdownExpected::Some)
114-
.with_expected_rows(0)
115-
.run()
116-
.await;
119+
.with_expected_rows(0);
120+
set.spawn(async move { case.run().await });
117121

118-
TestCase::new(&test_parquet_file)
122+
let case = TestCase::new(test_parquet_file.clone())
119123
.with_name("nothing")
120124
// No rows are filtered out -- all are returned
121125
// response_status > 0
122126
.with_filter(col("response_status").gt(lit(0_u16)))
123127
.with_pushdown_expected(PushdownExpected::None)
124-
.with_expected_rows(NUM_ROWS)
125-
.run()
126-
.await;
128+
.with_expected_rows(NUM_ROWS);
129+
set.spawn(async move { case.run().await });
127130

128-
TestCase::new(&test_parquet_file)
131+
let case = TestCase::new(test_parquet_file.clone())
129132
.with_name("dict_selective")
130133
// container = 'backend_container_0'
131134
.with_filter(col("container").eq(lit("backend_container_0")))
132135
.with_pushdown_expected(PushdownExpected::Some)
133-
.with_expected_rows(37856)
134-
.run()
135-
.await;
136+
.with_expected_rows(15911);
137+
set.spawn(async move { case.run().await });
136138

137-
TestCase::new(&test_parquet_file)
139+
let case = TestCase::new(test_parquet_file.clone())
138140
.with_name("not eq")
139141
// container != 'backend_container_0'
140142
.with_filter(col("container").not_eq(lit("backend_container_0")))
141143
.with_pushdown_expected(PushdownExpected::Some)
142-
.with_expected_rows(15963)
143-
.run()
144-
.await;
144+
.with_expected_rows(37908);
145+
set.spawn(async move { case.run().await });
145146

146-
TestCase::new(&test_parquet_file)
147+
let case = TestCase::new(test_parquet_file.clone())
147148
.with_name("dict_conjunction")
148149
// container == 'backend_container_0' AND
149150
// pod = 'aqcathnxqsphdhgjtgvxsfyiwbmhlmg'
@@ -155,11 +156,10 @@ async fn single_file() {
155156
.unwrap(),
156157
)
157158
.with_pushdown_expected(PushdownExpected::Some)
158-
.with_expected_rows(3052)
159-
.run()
160-
.await;
159+
.with_expected_rows(3052);
160+
set.spawn(async move { case.run().await });
161161

162-
TestCase::new(&test_parquet_file)
162+
let case = TestCase::new(test_parquet_file.clone())
163163
.with_name("dict_very_selective")
164164
// request_bytes > 2B AND
165165
// container == 'backend_container_0' AND
@@ -173,11 +173,10 @@ async fn single_file() {
173173
.unwrap(),
174174
)
175175
.with_pushdown_expected(PushdownExpected::Some)
176-
.with_expected_rows(88)
177-
.run()
178-
.await;
176+
.with_expected_rows(88);
177+
set.spawn(async move { case.run().await });
179178

180-
TestCase::new(&test_parquet_file)
179+
let case = TestCase::new(test_parquet_file.clone())
181180
.with_name("dict_very_selective2")
182181
// picks only 2 rows
183182
// client_addr = '204.47.29.82' AND
@@ -192,11 +191,10 @@ async fn single_file() {
192191
.unwrap(),
193192
)
194193
.with_pushdown_expected(PushdownExpected::Some)
195-
.with_expected_rows(88)
196-
.run()
197-
.await;
194+
.with_expected_rows(88);
195+
set.spawn(async move { case.run().await });
198196

199-
TestCase::new(&test_parquet_file)
197+
let case = TestCase::new(test_parquet_file.clone())
200198
.with_name("dict_disjunction")
201199
// container = 'backend_container_0' OR
202200
// pod = 'aqcathnxqsphdhgjtgvxsfyiwbmhlmg'
@@ -207,12 +205,11 @@ async fn single_file() {
207205
])
208206
.unwrap(),
209207
)
210-
.with_pushdown_expected(PushdownExpected::Some)
211-
.with_expected_rows(39982)
212-
.run()
213-
.await;
208+
.with_pushdown_expected(PushdownExpected::None)
209+
.with_expected_rows(16955);
210+
set.spawn(async move { case.run().await });
214211

215-
TestCase::new(&test_parquet_file)
212+
let case = TestCase::new(test_parquet_file.clone())
216213
.with_name("dict_disjunction3")
217214
// request_method != 'GET' OR
218215
// response_status = 400 OR
@@ -225,10 +222,14 @@ async fn single_file() {
225222
])
226223
.unwrap(),
227224
)
228-
.with_pushdown_expected(PushdownExpected::None)
229-
.with_expected_rows(NUM_ROWS)
230-
.run()
231-
.await;
225+
.with_pushdown_expected(PushdownExpected::Some)
226+
.with_expected_rows(48919);
227+
set.spawn(async move { case.run().await });
228+
229+
// Join all the cases.
230+
while let Some(result) = set.join_next().await {
231+
result.unwrap()
232+
}
232233
}
233234

234235
#[cfg(not(target_family = "windows"))]
@@ -247,7 +248,8 @@ async fn single_file_small_data_pages() {
247248

248249
let start = Instant::now();
249250
println!("Writing test data to {:?}", file);
250-
let test_parquet_file = TestParquetFile::try_new(file, props, generator).unwrap();
251+
let test_parquet_file =
252+
Arc::new(TestParquetFile::try_new(file, props, generator).unwrap());
251253
println!(
252254
"Completed generating test data in {:?}",
253255
Instant::now() - start
@@ -267,7 +269,7 @@ async fn single_file_small_data_pages() {
267269
// page 4: DLE:RLE RLE:RLE VLE:RLE_DICTIONARY ST:[min: fktdcgtmzvoedpwhfevcvvrtaurzgex, max: fwtdpgtxwqkkgtgvthhwycrvjiizdifyp, num_nulls not defined] CRC:[none] SZ:7 VC:9216
268270
// page 5: DLE:RLE RLE:RLE VLE:RLE_DICTIONARY ST:[min: fwtdpgtxwqkkgtgvthhwycrvjiizdifyp, max: iadnalqpdzthpifrvewossmpqibgtsuin, num_nulls not defined] CRC:[none] SZ:7 VC:7739
269271

270-
TestCase::new(&test_parquet_file)
272+
TestCase::new(test_parquet_file.clone())
271273
.with_name("selective")
272274
// predicate is chosen carefully to prune pages 0, 1, 2, 3, 4
273275
// pod = 'iadnalqpdzthpifrvewossmpqibgtsuin'
@@ -286,7 +288,7 @@ async fn single_file_small_data_pages() {
286288
// page 3: DLE:RLE RLE:RLE VLE:RLE_DICTIONARY ST:[min: 1970-01-01T00:00:00.000000000, max: 1970-01-01T00:00:00.004269056, num_nulls not defined] CRC:[none] SZ:14996 VC:9216
287289
// page 4: DLE:RLE RLE:RLE VLE:RLE_DICTIONARY ST:[min: 1970-01-01T00:00:00.000000000, max: 1970-01-01T00:00:00.007261184, num_nulls not defined] CRC:[none] SZ:14996 VC:9216
288290
// page 5: DLE:RLE RLE:RLE VLE:RLE_DICTIONARY ST:[min: 1970-01-01T00:00:00.000000000, max: 1970-01-01T00:00:00.005330944, num_nulls not defined] CRC:[none] SZ:12601 VC:7739
289-
TestCase::new(&test_parquet_file)
291+
TestCase::new(test_parquet_file.clone())
290292
.with_name("selective")
291293
// predicate is chosen carefully to prune pages 1, 2, 4, and 5
292294
// time > 1970-01-01T00:00:00.004300000
@@ -314,7 +316,7 @@ async fn single_file_small_data_pages() {
314316
// offset compressed size first row index
315317
// page-0 5581636 147517 0
316318
// page-1 5729153 147517 9216
317-
TestCase::new(&test_parquet_file)
319+
TestCase::new(test_parquet_file.clone())
318320
.with_name("selective_on_decimal")
319321
// predicate is chosen carefully to prune pages 1, 2, 3, 4, and 5
320322
// decimal_price < 9200
@@ -345,8 +347,8 @@ enum PageIndexFilteringExpected {
345347
}
346348

347349
/// parameters for running a test
348-
struct TestCase<'a> {
349-
test_parquet_file: &'a TestParquetFile,
350+
struct TestCase {
351+
test_parquet_file: Arc<TestParquetFile>,
350352
/// Human readable name to help debug failures
351353
name: String,
352354
/// The filter to apply
@@ -361,8 +363,8 @@ struct TestCase<'a> {
361363
expected_rows: usize,
362364
}
363365

364-
impl<'a> TestCase<'a> {
365-
fn new(test_parquet_file: &'a TestParquetFile) -> Self {
366+
impl TestCase {
367+
fn new(test_parquet_file: Arc<TestParquetFile>) -> Self {
366368
Self {
367369
test_parquet_file,
368370
name: "<NOT SPECIFIED>".into(),
@@ -533,12 +535,13 @@ impl<'a> TestCase<'a> {
533535

534536
match pushdown_expected {
535537
PushdownExpected::None => {
536-
assert_eq!(pushdown_rows_filtered, 0);
538+
assert_eq!(pushdown_rows_filtered, 0, "{}", self.name);
537539
}
538540
PushdownExpected::Some => {
539541
assert!(
540542
pushdown_rows_filtered > 0,
541-
"Expected to filter rows via pushdown, but none were"
543+
"{}: Expected to filter rows via pushdown, but none were",
544+
self.name
542545
);
543546
}
544547
};

0 commit comments

Comments
 (0)