@@ -267,10 +267,7 @@ suite("test_parquet_lazy_mat_profile", "p0,external,hive,external_docker,externa
267267
268268
269269
270- def test_true_true = {
271- sql """ set enable_parquet_filter_by_min_max = true; """
272- sql """ set enable_parquet_lazy_materialization = true; """
273-
270+ def checkV2Profile = {
274271 def metrics = q1()
275272 logger. info(" metrics = ${ metrics} " )
276273 assertEquals (" 99" , metrics[" FilteredRowsByGroup" ])
@@ -336,7 +333,7 @@ suite("test_parquet_lazy_mat_profile", "p0,external,hive,external_docker,externa
336333 assertEquals (" 0" , metrics[" FilteredRowsByGroup" ])
337334 assertEquals (" 1" , metrics[" FilteredRowsByLazyRead" ])
338335 assertEquals (" 0" , metrics[" FilteredRowsByPage" ])
339- assertTrue ( metrics[" RawRowsRead" ]. contains( " 7300 " ))
336+ assertEquals ( 7300L , metricValueAsLong( metrics[" RawRowsRead" ]))
340337 assertEquals (" 0" , metrics[" RowGroupsFiltered" ])
341338 assertEquals (" 0" , metrics[" RowGroupsFilteredByBloomFilter" ])
342339 assertEquals (" 0" , metrics[" RowGroupsFilteredByMinMax" ])
@@ -357,284 +354,17 @@ suite("test_parquet_lazy_mat_profile", "p0,external,hive,external_docker,externa
357354 }
358355
359356
360- def test_true_false = {
361- sql """ set enable_parquet_filter_by_min_max = true; """
362- sql """ set enable_parquet_lazy_materialization = false; """
363- // Versioned Parquet requires V2, whose lazy materialization stays enabled.
364- sql """ set enable_file_scanner_v2=false; """
365-
366- def metrics = q1()
367- logger. info(" metrics = ${ metrics} " )
368- assertEquals (" 99" , metrics[" FilteredRowsByGroup" ])
369- assertEquals (0L , metricValueAsLong(metrics[" FilteredRowsByLazyRead" ]))
370- assertEquals (" 0" , metrics[" FilteredRowsByPage" ])
371- assertEquals (" 1" , metrics[" RawRowsRead" ])
372- assertEquals (" 99" , metrics[" RowGroupsFiltered" ])
373- assertEquals (" 0" , metrics[" RowGroupsFilteredByBloomFilter" ])
374- assertEquals (" 99" , metrics[" RowGroupsFilteredByMinMax" ])
375- assertEquals (" 1" , metrics[" RowGroupsReadNum" ])
376- assertEquals (" 100" , metrics[" RowGroupsTotalNum" ])
377-
378- metrics = q2()
379- logger. info(" metrics = ${ metrics} " )
380- assertEquals (" 99" , metrics[" FilteredRowsByGroup" ])
381- assertEquals (0L , metricValueAsLong(metrics[" FilteredRowsByLazyRead" ]))
382- assertEquals (" 0" , metrics[" FilteredRowsByPage" ])
383- assertEquals (" 1" , metrics[" RawRowsRead" ])
384- assertEquals (" 99" , metrics[" RowGroupsFiltered" ])
385- assertEquals (" 0" , metrics[" RowGroupsFilteredByBloomFilter" ])
386- assertEquals (" 99" , metrics[" RowGroupsFilteredByMinMax" ])
387- assertEquals (" 1" , metrics[" RowGroupsReadNum" ])
388- assertEquals (" 100" , metrics[" RowGroupsTotalNum" ])
389-
390- metrics = q3()
391- logger. info(" metrics = ${ metrics} " )
392- assertEquals (" 100" , metrics[" FilteredRowsByGroup" ])
393- assertEquals (0L , metricValueAsLong(metrics[" FilteredRowsByLazyRead" ]))
394- assertEquals (" 0" , metrics[" FilteredRowsByPage" ])
395- assertEquals (" 0" , metrics[" RawRowsRead" ])
396- assertEquals (" 100" , metrics[" RowGroupsFiltered" ])
397- assertEquals (" 0" , metrics[" RowGroupsFilteredByBloomFilter" ])
398- assertEquals (" 100" , metrics[" RowGroupsFilteredByMinMax" ])
399- assertEquals (" 0" , metrics[" RowGroupsReadNum" ])
400- assertEquals (" 100" , metrics[" RowGroupsTotalNum" ])
401-
402- metrics = q4()
403- logger. info(" metrics = ${ metrics} " )
404- assertEquals (" 0" , metrics[" FilteredRowsByGroup" ])
405- assertEquals (20L , metricValueAsLong(metrics[" FilteredRowsByLazyRead" ]))
406- assertEquals (" 7.279K (7279)" , metrics[" FilteredRowsByPage" ])
407- assertEquals (" 21" , metrics[" RawRowsRead" ])
408- assertEquals (" 0" , metrics[" RowGroupsFiltered" ])
409- assertEquals (" 0" , metrics[" RowGroupsFilteredByBloomFilter" ])
410- assertEquals (" 0" , metrics[" RowGroupsFilteredByMinMax" ])
411- assertEquals (" 1" , metrics[" RowGroupsReadNum" ])
412- assertEquals (" 1" , metrics[" RowGroupsTotalNum" ])
413-
414- metrics = q5()
415- logger. info(" metrics = ${ metrics} " )
416- assertEquals (" 0" , metrics[" FilteredRowsByGroup" ])
417- assertEquals (28L , metricValueAsLong(metrics[" FilteredRowsByLazyRead" ]))
418- assertEquals (" 7.258K (7258)" , metrics[" FilteredRowsByPage" ])
419- assertEquals (" 42" , metrics[" RawRowsRead" ])
420- assertEquals (" 0" , metrics[" RowGroupsFiltered" ])
421- assertEquals (" 0" , metrics[" RowGroupsFilteredByBloomFilter" ])
422- assertEquals (" 0" , metrics[" RowGroupsFilteredByMinMax" ])
423- assertEquals (" 1" , metrics[" RowGroupsReadNum" ])
424- assertEquals (" 1" , metrics[" RowGroupsTotalNum" ])
425-
426- metrics = q6()
427- logger. info(" metrics = ${ metrics} " )
428- assertEquals (" 0" , metrics[" FilteredRowsByGroup" ])
429- assertEquals (1L , metricValueAsLong(metrics[" FilteredRowsByLazyRead" ]))
430- assertEquals (" 0" , metrics[" FilteredRowsByPage" ])
431- assertTrue (metrics[" RawRowsRead" ]. contains(" 7300" ))
432- assertEquals (" 0" , metrics[" RowGroupsFiltered" ])
433- assertEquals (" 0" , metrics[" RowGroupsFilteredByBloomFilter" ])
434- assertEquals (" 0" , metrics[" RowGroupsFilteredByMinMax" ])
435- assertEquals (" 1" , metrics[" RowGroupsReadNum" ])
436- assertEquals (" 1" , metrics[" RowGroupsTotalNum" ])
437-
438- metrics = q7()
439- logger. info(" metrics = ${ metrics} " )
440- assertEquals (" 0" , metrics[" FilteredRowsByGroup" ])
441- assertEquals (19L , metricValueAsLong(metrics[" FilteredRowsByLazyRead" ]))
442- assertEquals (" 7.279K (7279)" , metrics[" FilteredRowsByPage" ])
443- assertEquals (" 21" , metrics[" RawRowsRead" ])
444- assertEquals (" 0" , metrics[" RowGroupsFiltered" ])
445- assertEquals (" 0" , metrics[" RowGroupsFilteredByBloomFilter" ])
446- assertEquals (" 0" , metrics[" RowGroupsFilteredByMinMax" ])
447- assertEquals (" 1" , metrics[" RowGroupsReadNum" ])
448- assertEquals (" 1" , metrics[" RowGroupsTotalNum" ])
449- }
450-
451-
452- def test_false_false = {
453- sql """ set enable_parquet_filter_by_min_max = false; """
454- sql """ set enable_parquet_lazy_materialization = false; """
455-
456- def metrics = q1()
457- logger. info(" metrics = ${ metrics} " )
458- assertEquals (" 0" , metrics[" FilteredRowsByGroup" ])
459- assertEquals (99L , metricValueAsLong(metrics[" FilteredRowsByLazyRead" ]))
460- assertEquals (" 0" , metrics[" FilteredRowsByPage" ])
461- assertEquals (" 100" , metrics[" RawRowsRead" ])
462- assertEquals (" 0" , metrics[" RowGroupsFiltered" ])
463- assertEquals (" 0" , metrics[" RowGroupsFilteredByBloomFilter" ])
464- assertEquals (" 0" , metrics[" RowGroupsFilteredByMinMax" ])
465- assertEquals (" 100" , metrics[" RowGroupsReadNum" ])
466- assertEquals (" 100" , metrics[" RowGroupsTotalNum" ])
467-
468- metrics = q2()
469- logger. info(" metrics = ${ metrics} " )
470- assertEquals (" 0" , metrics[" FilteredRowsByGroup" ])
471- assertEquals (99L , metricValueAsLong(metrics[" FilteredRowsByLazyRead" ]))
472- assertEquals (" 0" , metrics[" FilteredRowsByPage" ])
473- assertEquals (" 100" , metrics[" RawRowsRead" ])
474- assertEquals (" 0" , metrics[" RowGroupsFiltered" ])
475- assertEquals (" 0" , metrics[" RowGroupsFilteredByBloomFilter" ])
476- assertEquals (" 0" , metrics[" RowGroupsFilteredByMinMax" ])
477- assertEquals (" 100" , metrics[" RowGroupsReadNum" ])
478- assertEquals (" 100" , metrics[" RowGroupsTotalNum" ])
479-
480- metrics = q3()
481- logger. info(" metrics = ${ metrics} " )
482- assertEquals (" 0" , metrics[" FilteredRowsByGroup" ])
483- assertEquals (100L , metricValueAsLong(metrics[" FilteredRowsByLazyRead" ]))
484- assertEquals (" 0" , metrics[" FilteredRowsByPage" ])
485- assertEquals (" 100" , metrics[" RawRowsRead" ])
486- assertEquals (" 0" , metrics[" RowGroupsFiltered" ])
487- assertEquals (" 0" , metrics[" RowGroupsFilteredByBloomFilter" ])
488- assertEquals (" 0" , metrics[" RowGroupsFilteredByMinMax" ])
489- assertEquals (" 100" , metrics[" RowGroupsReadNum" ])
490- assertEquals (" 100" , metrics[" RowGroupsTotalNum" ])
491-
492- metrics = q4()
493- logger. info(" metrics = ${ metrics} " )
494- assertEquals (" 0" , metrics[" FilteredRowsByGroup" ])
495- assertEquals (7299L , metricValueAsLong(metrics[" FilteredRowsByLazyRead" ]))
496- assertEquals (" 0" , metrics[" FilteredRowsByPage" ])
497- assertEquals (" 7.3K (7300)" , metrics[" RawRowsRead" ])
498- assertEquals (" 0" , metrics[" RowGroupsFiltered" ])
499- assertEquals (" 0" , metrics[" RowGroupsFilteredByBloomFilter" ])
500- assertEquals (" 0" , metrics[" RowGroupsFilteredByMinMax" ])
501- assertEquals (" 1" , metrics[" RowGroupsReadNum" ])
502- assertEquals (" 1" , metrics[" RowGroupsTotalNum" ])
503-
504- metrics = q5()
505- logger. info(" metrics = ${ metrics} " )
506- assertEquals (" 0" , metrics[" FilteredRowsByGroup" ])
507- assertEquals (7286L , metricValueAsLong(metrics[" FilteredRowsByLazyRead" ]))
508- assertEquals (" 0" , metrics[" FilteredRowsByPage" ])
509- assertEquals (" 7.3K (7300)" , metrics[" RawRowsRead" ])
510- assertEquals (" 0" , metrics[" RowGroupsFiltered" ])
511- assertEquals (" 0" , metrics[" RowGroupsFilteredByBloomFilter" ])
512- assertEquals (" 0" , metrics[" RowGroupsFilteredByMinMax" ])
513- assertEquals (" 1" , metrics[" RowGroupsReadNum" ])
514- assertEquals (" 1" , metrics[" RowGroupsTotalNum" ])
515-
516- metrics = q6()
517- logger. info(" metrics = ${ metrics} " )
518- assertEquals (" 0" , metrics[" FilteredRowsByGroup" ])
519- assertEquals (1L , metricValueAsLong(metrics[" FilteredRowsByLazyRead" ]))
520- assertEquals (" 0" , metrics[" FilteredRowsByPage" ])
521- assertEquals (" 7.3K (7300)" , metrics[" RawRowsRead" ])
522- assertEquals (" 0" , metrics[" RowGroupsFiltered" ])
523- assertEquals (" 0" , metrics[" RowGroupsFilteredByBloomFilter" ])
524- assertEquals (" 0" , metrics[" RowGroupsFilteredByMinMax" ])
525- assertEquals (" 1" , metrics[" RowGroupsReadNum" ])
526- assertEquals (" 1" , metrics[" RowGroupsTotalNum" ])
527-
528- metrics = q7()
529- logger. info(" metrics = ${ metrics} " )
530- assertEquals (" 0" , metrics[" FilteredRowsByGroup" ])
531- assertEquals (7298L , metricValueAsLong(metrics[" FilteredRowsByLazyRead" ]))
532- assertEquals (" 0" , metrics[" FilteredRowsByPage" ])
533- assertEquals (" 7.3K (7300)" , metrics[" RawRowsRead" ])
534- assertEquals (" 0" , metrics[" RowGroupsFiltered" ])
535- assertEquals (" 0" , metrics[" RowGroupsFilteredByBloomFilter" ])
536- assertEquals (" 0" , metrics[" RowGroupsFilteredByMinMax" ])
537- assertEquals (" 1" , metrics[" RowGroupsReadNum" ])
538- assertEquals (" 1" , metrics[" RowGroupsTotalNum" ])
539- }
540-
541-
542- def test_false_true = {
543- sql """ set enable_parquet_filter_by_min_max = false; """
544- sql """ set enable_parquet_lazy_materialization = true; """
545-
546- def metrics = q1()
547- logger. info(" metrics = ${ metrics} " )
548- assertEquals (" 0" , metrics[" FilteredRowsByGroup" ])
549- assertEquals (" 99" , metrics[" FilteredRowsByLazyRead" ])
550- assertEquals (" 0" , metrics[" FilteredRowsByPage" ])
551- assertEquals (" 100" , metrics[" RawRowsRead" ])
552- assertEquals (" 0" , metrics[" RowGroupsFiltered" ])
553- assertEquals (" 0" , metrics[" RowGroupsFilteredByBloomFilter" ])
554- assertEquals (" 0" , metrics[" RowGroupsFilteredByMinMax" ])
555- assertEquals (" 100" , metrics[" RowGroupsReadNum" ])
556- assertEquals (" 100" , metrics[" RowGroupsTotalNum" ])
557-
558- metrics = q2()
559- logger. info(" metrics = ${ metrics} " )
560- assertEquals (" 0" , metrics[" FilteredRowsByGroup" ])
561- assertEquals (" 99" , metrics[" FilteredRowsByLazyRead" ])
562- assertEquals (" 0" , metrics[" FilteredRowsByPage" ])
563- assertEquals (" 100" , metrics[" RawRowsRead" ])
564- assertEquals (" 0" , metrics[" RowGroupsFiltered" ])
565- assertEquals (" 0" , metrics[" RowGroupsFilteredByBloomFilter" ])
566- assertEquals (" 0" , metrics[" RowGroupsFilteredByMinMax" ])
567- assertEquals (" 100" , metrics[" RowGroupsReadNum" ])
568- assertEquals (" 100" , metrics[" RowGroupsTotalNum" ])
569-
570- metrics = q3()
571- logger. info(" metrics = ${ metrics} " )
572- assertEquals (" 0" , metrics[" FilteredRowsByGroup" ])
573- assertEquals (" 100" , metrics[" FilteredRowsByLazyRead" ])
574- assertEquals (" 0" , metrics[" FilteredRowsByPage" ])
575- assertEquals (" 100" , metrics[" RawRowsRead" ])
576- assertEquals (" 0" , metrics[" RowGroupsFiltered" ])
577- assertEquals (" 0" , metrics[" RowGroupsFilteredByBloomFilter" ])
578- assertEquals (" 0" , metrics[" RowGroupsFilteredByMinMax" ])
579- assertEquals (" 100" , metrics[" RowGroupsReadNum" ])
580- assertEquals (" 100" , metrics[" RowGroupsTotalNum" ])
581-
582- metrics = q4()
583- logger. info(" metrics = ${ metrics} " )
584- assertEquals (" 0" , metrics[" FilteredRowsByGroup" ])
585- assertTrue (metrics[" FilteredRowsByLazyRead" ]. contains(" 7299" ))
586- assertEquals (" 0" , metrics[" FilteredRowsByPage" ])
587- assertTrue (metrics[" RawRowsRead" ]. contains(" 7300" ))
588- assertEquals (" 0" , metrics[" RowGroupsFiltered" ])
589- assertEquals (" 0" , metrics[" RowGroupsFilteredByBloomFilter" ])
590- assertEquals (" 0" , metrics[" RowGroupsFilteredByMinMax" ])
591- assertEquals (" 1" , metrics[" RowGroupsReadNum" ])
592- assertEquals (" 1" , metrics[" RowGroupsTotalNum" ])
593-
594- metrics = q5()
595- logger. info(" metrics = ${ metrics} " )
596- assertEquals (" 0" , metrics[" FilteredRowsByGroup" ])
597- assertTrue (metrics[" FilteredRowsByLazyRead" ]. contains(" 7286" ))
598- assertEquals (" 0" , metrics[" FilteredRowsByPage" ])
599- assertTrue (metrics[" RawRowsRead" ]. contains(" 7300" ))
600- assertEquals (" 0" , metrics[" RowGroupsFiltered" ])
601- assertEquals (" 0" , metrics[" RowGroupsFilteredByBloomFilter" ])
602- assertEquals (" 0" , metrics[" RowGroupsFilteredByMinMax" ])
603- assertEquals (" 1" , metrics[" RowGroupsReadNum" ])
604- assertEquals (" 1" , metrics[" RowGroupsTotalNum" ])
605-
606- metrics = q6()
607- logger. info(" metrics = ${ metrics} " )
608- assertEquals (" 0" , metrics[" FilteredRowsByGroup" ])
609- assertTrue (metrics[" FilteredRowsByLazyRead" ]. contains(" 1" ))
610- assertEquals (" 0" , metrics[" FilteredRowsByPage" ])
611- assertTrue (metrics[" RawRowsRead" ]. contains(" 7300" ))
612- assertEquals (" 0" , metrics[" RowGroupsFiltered" ])
613- assertEquals (" 0" , metrics[" RowGroupsFilteredByBloomFilter" ])
614- assertEquals (" 0" , metrics[" RowGroupsFilteredByMinMax" ])
615- assertEquals (" 1" , metrics[" RowGroupsReadNum" ])
616- assertEquals (" 1" , metrics[" RowGroupsTotalNum" ])
617-
618- metrics = q7()
619- logger. info(" metrics = ${ metrics} " )
620- assertEquals (" 0" , metrics[" FilteredRowsByGroup" ])
621- assertTrue (metrics[" FilteredRowsByLazyRead" ]. contains(" 7298" ))
622- assertEquals (" 0" , metrics[" FilteredRowsByPage" ])
623- assertTrue (metrics[" RawRowsRead" ]. contains(" 7300" ))
624- assertEquals (" 0" , metrics[" RowGroupsFiltered" ])
625- assertEquals (" 0" , metrics[" RowGroupsFilteredByBloomFilter" ])
626- assertEquals (" 0" , metrics[" RowGroupsFilteredByMinMax" ])
627- assertEquals (" 1" , metrics[" RowGroupsReadNum" ])
628- assertEquals (" 1" , metrics[" RowGroupsTotalNum" ])
629- }
630-
631- // Versioned Parquet forces V2 even when these session preferences are disabled. Keep
632- // checking exact pruning counters; disabling lazy materialization cannot select V1.
357+ // The versioned Parquet contract requires V2 even when its session preference is off.
358+ // V2 always applies safe ZoneMap pruning and lazy materialization, so the legacy
359+ // min/max and lazy-read switches must not change these exact profile counters.
633360 sql """ set enable_file_scanner_v2=false; """
634- test_true_true();
635- test_true_false();
636- test_false_false();
637- test_false_true();
361+ for (boolean minMaxEnabled : [true , false ]) {
362+ for (boolean lazyReadEnabled : [true , false ]) {
363+ sql """ set enable_parquet_filter_by_min_max = ${ minMaxEnabled} ; """
364+ sql """ set enable_parquet_lazy_materialization = ${ lazyReadEnabled} ; """
365+ checkV2Profile()
366+ }
367+ }
638368 q8();
639369
640370
0 commit comments