Skip to content

Commit 4b04c28

Browse files
committed
[test](parquet) Align profile counters with required V2 pruning
1 parent 2811a36 commit 4b04c28

1 file changed

Lines changed: 12 additions & 282 deletions

File tree

‎regression-test/suites/external_table_p0/hive/test_parquet_lazy_mat_profile.groovy‎

Lines changed: 12 additions & 282 deletions
Original file line numberDiff line numberDiff line change
@@ -267,10 +267,7 @@ suite("test_parquet_lazy_mat_profile", "p0,external,hive,external_docker,externa
267267

268268

269269

270-
def test_true_true = {
271-
sql """ set enable_parquet_filter_by_min_max = true; """
272-
sql """ set enable_parquet_lazy_materialization = true; """
273-
270+
def checkV2Profile = {
274271
def metrics = q1()
275272
logger.info("metrics = ${metrics}")
276273
assertEquals("99", metrics["FilteredRowsByGroup"])
@@ -336,7 +333,7 @@ suite("test_parquet_lazy_mat_profile", "p0,external,hive,external_docker,externa
336333
assertEquals("0", metrics["FilteredRowsByGroup"])
337334
assertEquals("1", metrics["FilteredRowsByLazyRead"])
338335
assertEquals("0", metrics["FilteredRowsByPage"])
339-
assertTrue(metrics["RawRowsRead"].contains("7300"))
336+
assertEquals(7300L, metricValueAsLong(metrics["RawRowsRead"]))
340337
assertEquals("0", metrics["RowGroupsFiltered"])
341338
assertEquals("0", metrics["RowGroupsFilteredByBloomFilter"])
342339
assertEquals("0", metrics["RowGroupsFilteredByMinMax"])
@@ -357,284 +354,17 @@ suite("test_parquet_lazy_mat_profile", "p0,external,hive,external_docker,externa
357354
}
358355

359356

360-
def test_true_false = {
361-
sql """ set enable_parquet_filter_by_min_max = true; """
362-
sql """ set enable_parquet_lazy_materialization = false; """
363-
// Versioned Parquet requires V2, whose lazy materialization stays enabled.
364-
sql """ set enable_file_scanner_v2=false; """
365-
366-
def metrics = q1()
367-
logger.info("metrics = ${metrics}")
368-
assertEquals("99", metrics["FilteredRowsByGroup"])
369-
assertEquals(0L, metricValueAsLong(metrics["FilteredRowsByLazyRead"]))
370-
assertEquals("0", metrics["FilteredRowsByPage"])
371-
assertEquals("1", metrics["RawRowsRead"])
372-
assertEquals("99", metrics["RowGroupsFiltered"])
373-
assertEquals("0", metrics["RowGroupsFilteredByBloomFilter"])
374-
assertEquals("99", metrics["RowGroupsFilteredByMinMax"])
375-
assertEquals("1", metrics["RowGroupsReadNum"])
376-
assertEquals("100", metrics["RowGroupsTotalNum"])
377-
378-
metrics = q2()
379-
logger.info("metrics = ${metrics}")
380-
assertEquals("99", metrics["FilteredRowsByGroup"])
381-
assertEquals(0L, metricValueAsLong(metrics["FilteredRowsByLazyRead"]))
382-
assertEquals("0", metrics["FilteredRowsByPage"])
383-
assertEquals("1", metrics["RawRowsRead"])
384-
assertEquals("99", metrics["RowGroupsFiltered"])
385-
assertEquals("0", metrics["RowGroupsFilteredByBloomFilter"])
386-
assertEquals("99", metrics["RowGroupsFilteredByMinMax"])
387-
assertEquals("1", metrics["RowGroupsReadNum"])
388-
assertEquals("100", metrics["RowGroupsTotalNum"])
389-
390-
metrics = q3()
391-
logger.info("metrics = ${metrics}")
392-
assertEquals("100", metrics["FilteredRowsByGroup"])
393-
assertEquals(0L, metricValueAsLong(metrics["FilteredRowsByLazyRead"]))
394-
assertEquals("0", metrics["FilteredRowsByPage"])
395-
assertEquals("0", metrics["RawRowsRead"])
396-
assertEquals("100", metrics["RowGroupsFiltered"])
397-
assertEquals("0", metrics["RowGroupsFilteredByBloomFilter"])
398-
assertEquals("100", metrics["RowGroupsFilteredByMinMax"])
399-
assertEquals("0", metrics["RowGroupsReadNum"])
400-
assertEquals("100", metrics["RowGroupsTotalNum"])
401-
402-
metrics = q4()
403-
logger.info("metrics = ${metrics}")
404-
assertEquals("0", metrics["FilteredRowsByGroup"])
405-
assertEquals(20L, metricValueAsLong(metrics["FilteredRowsByLazyRead"]))
406-
assertEquals("7.279K (7279)", metrics["FilteredRowsByPage"])
407-
assertEquals("21", metrics["RawRowsRead"])
408-
assertEquals("0", metrics["RowGroupsFiltered"])
409-
assertEquals("0", metrics["RowGroupsFilteredByBloomFilter"])
410-
assertEquals("0", metrics["RowGroupsFilteredByMinMax"])
411-
assertEquals("1", metrics["RowGroupsReadNum"])
412-
assertEquals("1", metrics["RowGroupsTotalNum"])
413-
414-
metrics = q5()
415-
logger.info("metrics = ${metrics}")
416-
assertEquals("0", metrics["FilteredRowsByGroup"])
417-
assertEquals(28L, metricValueAsLong(metrics["FilteredRowsByLazyRead"]))
418-
assertEquals("7.258K (7258)", metrics["FilteredRowsByPage"])
419-
assertEquals("42", metrics["RawRowsRead"])
420-
assertEquals("0", metrics["RowGroupsFiltered"])
421-
assertEquals("0", metrics["RowGroupsFilteredByBloomFilter"])
422-
assertEquals("0", metrics["RowGroupsFilteredByMinMax"])
423-
assertEquals("1", metrics["RowGroupsReadNum"])
424-
assertEquals("1", metrics["RowGroupsTotalNum"])
425-
426-
metrics = q6()
427-
logger.info("metrics = ${metrics}")
428-
assertEquals("0", metrics["FilteredRowsByGroup"])
429-
assertEquals(1L, metricValueAsLong(metrics["FilteredRowsByLazyRead"]))
430-
assertEquals("0", metrics["FilteredRowsByPage"])
431-
assertTrue(metrics["RawRowsRead"].contains("7300"))
432-
assertEquals("0", metrics["RowGroupsFiltered"])
433-
assertEquals("0", metrics["RowGroupsFilteredByBloomFilter"])
434-
assertEquals("0", metrics["RowGroupsFilteredByMinMax"])
435-
assertEquals("1", metrics["RowGroupsReadNum"])
436-
assertEquals("1", metrics["RowGroupsTotalNum"])
437-
438-
metrics = q7()
439-
logger.info("metrics = ${metrics}")
440-
assertEquals("0", metrics["FilteredRowsByGroup"])
441-
assertEquals(19L, metricValueAsLong(metrics["FilteredRowsByLazyRead"]))
442-
assertEquals("7.279K (7279)", metrics["FilteredRowsByPage"])
443-
assertEquals("21", metrics["RawRowsRead"])
444-
assertEquals("0", metrics["RowGroupsFiltered"])
445-
assertEquals("0", metrics["RowGroupsFilteredByBloomFilter"])
446-
assertEquals("0", metrics["RowGroupsFilteredByMinMax"])
447-
assertEquals("1", metrics["RowGroupsReadNum"])
448-
assertEquals("1", metrics["RowGroupsTotalNum"])
449-
}
450-
451-
452-
def test_false_false = {
453-
sql """ set enable_parquet_filter_by_min_max = false; """
454-
sql """ set enable_parquet_lazy_materialization = false; """
455-
456-
def metrics = q1()
457-
logger.info("metrics = ${metrics}")
458-
assertEquals("0", metrics["FilteredRowsByGroup"])
459-
assertEquals(99L, metricValueAsLong(metrics["FilteredRowsByLazyRead"]))
460-
assertEquals("0", metrics["FilteredRowsByPage"])
461-
assertEquals("100", metrics["RawRowsRead"])
462-
assertEquals("0", metrics["RowGroupsFiltered"])
463-
assertEquals("0", metrics["RowGroupsFilteredByBloomFilter"])
464-
assertEquals("0", metrics["RowGroupsFilteredByMinMax"])
465-
assertEquals("100", metrics["RowGroupsReadNum"])
466-
assertEquals("100", metrics["RowGroupsTotalNum"])
467-
468-
metrics = q2()
469-
logger.info("metrics = ${metrics}")
470-
assertEquals("0", metrics["FilteredRowsByGroup"])
471-
assertEquals(99L, metricValueAsLong(metrics["FilteredRowsByLazyRead"]))
472-
assertEquals("0", metrics["FilteredRowsByPage"])
473-
assertEquals("100", metrics["RawRowsRead"])
474-
assertEquals("0", metrics["RowGroupsFiltered"])
475-
assertEquals("0", metrics["RowGroupsFilteredByBloomFilter"])
476-
assertEquals("0", metrics["RowGroupsFilteredByMinMax"])
477-
assertEquals("100", metrics["RowGroupsReadNum"])
478-
assertEquals("100", metrics["RowGroupsTotalNum"])
479-
480-
metrics = q3()
481-
logger.info("metrics = ${metrics}")
482-
assertEquals("0", metrics["FilteredRowsByGroup"])
483-
assertEquals(100L, metricValueAsLong(metrics["FilteredRowsByLazyRead"]))
484-
assertEquals("0", metrics["FilteredRowsByPage"])
485-
assertEquals("100", metrics["RawRowsRead"])
486-
assertEquals("0", metrics["RowGroupsFiltered"])
487-
assertEquals("0", metrics["RowGroupsFilteredByBloomFilter"])
488-
assertEquals("0", metrics["RowGroupsFilteredByMinMax"])
489-
assertEquals("100", metrics["RowGroupsReadNum"])
490-
assertEquals("100", metrics["RowGroupsTotalNum"])
491-
492-
metrics = q4()
493-
logger.info("metrics = ${metrics}")
494-
assertEquals("0", metrics["FilteredRowsByGroup"])
495-
assertEquals(7299L, metricValueAsLong(metrics["FilteredRowsByLazyRead"]))
496-
assertEquals("0", metrics["FilteredRowsByPage"])
497-
assertEquals("7.3K (7300)", metrics["RawRowsRead"])
498-
assertEquals("0", metrics["RowGroupsFiltered"])
499-
assertEquals("0", metrics["RowGroupsFilteredByBloomFilter"])
500-
assertEquals("0", metrics["RowGroupsFilteredByMinMax"])
501-
assertEquals("1", metrics["RowGroupsReadNum"])
502-
assertEquals("1", metrics["RowGroupsTotalNum"])
503-
504-
metrics = q5()
505-
logger.info("metrics = ${metrics}")
506-
assertEquals("0", metrics["FilteredRowsByGroup"])
507-
assertEquals(7286L, metricValueAsLong(metrics["FilteredRowsByLazyRead"]))
508-
assertEquals("0", metrics["FilteredRowsByPage"])
509-
assertEquals("7.3K (7300)", metrics["RawRowsRead"])
510-
assertEquals("0", metrics["RowGroupsFiltered"])
511-
assertEquals("0", metrics["RowGroupsFilteredByBloomFilter"])
512-
assertEquals("0", metrics["RowGroupsFilteredByMinMax"])
513-
assertEquals("1", metrics["RowGroupsReadNum"])
514-
assertEquals("1", metrics["RowGroupsTotalNum"])
515-
516-
metrics = q6()
517-
logger.info("metrics = ${metrics}")
518-
assertEquals("0", metrics["FilteredRowsByGroup"])
519-
assertEquals(1L, metricValueAsLong(metrics["FilteredRowsByLazyRead"]))
520-
assertEquals("0", metrics["FilteredRowsByPage"])
521-
assertEquals("7.3K (7300)", metrics["RawRowsRead"])
522-
assertEquals("0", metrics["RowGroupsFiltered"])
523-
assertEquals("0", metrics["RowGroupsFilteredByBloomFilter"])
524-
assertEquals("0", metrics["RowGroupsFilteredByMinMax"])
525-
assertEquals("1", metrics["RowGroupsReadNum"])
526-
assertEquals("1", metrics["RowGroupsTotalNum"])
527-
528-
metrics = q7()
529-
logger.info("metrics = ${metrics}")
530-
assertEquals("0", metrics["FilteredRowsByGroup"])
531-
assertEquals(7298L, metricValueAsLong(metrics["FilteredRowsByLazyRead"]))
532-
assertEquals("0", metrics["FilteredRowsByPage"])
533-
assertEquals("7.3K (7300)", metrics["RawRowsRead"])
534-
assertEquals("0", metrics["RowGroupsFiltered"])
535-
assertEquals("0", metrics["RowGroupsFilteredByBloomFilter"])
536-
assertEquals("0", metrics["RowGroupsFilteredByMinMax"])
537-
assertEquals("1", metrics["RowGroupsReadNum"])
538-
assertEquals("1", metrics["RowGroupsTotalNum"])
539-
}
540-
541-
542-
def test_false_true = {
543-
sql """ set enable_parquet_filter_by_min_max = false; """
544-
sql """ set enable_parquet_lazy_materialization = true; """
545-
546-
def metrics = q1()
547-
logger.info("metrics = ${metrics}")
548-
assertEquals("0", metrics["FilteredRowsByGroup"])
549-
assertEquals("99", metrics["FilteredRowsByLazyRead"])
550-
assertEquals("0", metrics["FilteredRowsByPage"])
551-
assertEquals("100", metrics["RawRowsRead"])
552-
assertEquals("0", metrics["RowGroupsFiltered"])
553-
assertEquals("0", metrics["RowGroupsFilteredByBloomFilter"])
554-
assertEquals("0", metrics["RowGroupsFilteredByMinMax"])
555-
assertEquals("100", metrics["RowGroupsReadNum"])
556-
assertEquals("100", metrics["RowGroupsTotalNum"])
557-
558-
metrics = q2()
559-
logger.info("metrics = ${metrics}")
560-
assertEquals("0", metrics["FilteredRowsByGroup"])
561-
assertEquals("99", metrics["FilteredRowsByLazyRead"])
562-
assertEquals("0", metrics["FilteredRowsByPage"])
563-
assertEquals("100", metrics["RawRowsRead"])
564-
assertEquals("0", metrics["RowGroupsFiltered"])
565-
assertEquals("0", metrics["RowGroupsFilteredByBloomFilter"])
566-
assertEquals("0", metrics["RowGroupsFilteredByMinMax"])
567-
assertEquals("100", metrics["RowGroupsReadNum"])
568-
assertEquals("100", metrics["RowGroupsTotalNum"])
569-
570-
metrics = q3()
571-
logger.info("metrics = ${metrics}")
572-
assertEquals("0", metrics["FilteredRowsByGroup"])
573-
assertEquals("100", metrics["FilteredRowsByLazyRead"])
574-
assertEquals("0", metrics["FilteredRowsByPage"])
575-
assertEquals("100", metrics["RawRowsRead"])
576-
assertEquals("0", metrics["RowGroupsFiltered"])
577-
assertEquals("0", metrics["RowGroupsFilteredByBloomFilter"])
578-
assertEquals("0", metrics["RowGroupsFilteredByMinMax"])
579-
assertEquals("100", metrics["RowGroupsReadNum"])
580-
assertEquals("100", metrics["RowGroupsTotalNum"])
581-
582-
metrics = q4()
583-
logger.info("metrics = ${metrics}")
584-
assertEquals("0", metrics["FilteredRowsByGroup"])
585-
assertTrue(metrics["FilteredRowsByLazyRead"].contains("7299"))
586-
assertEquals("0", metrics["FilteredRowsByPage"])
587-
assertTrue(metrics["RawRowsRead"].contains("7300"))
588-
assertEquals("0", metrics["RowGroupsFiltered"])
589-
assertEquals("0", metrics["RowGroupsFilteredByBloomFilter"])
590-
assertEquals("0", metrics["RowGroupsFilteredByMinMax"])
591-
assertEquals("1", metrics["RowGroupsReadNum"])
592-
assertEquals("1", metrics["RowGroupsTotalNum"])
593-
594-
metrics = q5()
595-
logger.info("metrics = ${metrics}")
596-
assertEquals("0", metrics["FilteredRowsByGroup"])
597-
assertTrue(metrics["FilteredRowsByLazyRead"].contains("7286"))
598-
assertEquals("0", metrics["FilteredRowsByPage"])
599-
assertTrue(metrics["RawRowsRead"].contains("7300"))
600-
assertEquals("0", metrics["RowGroupsFiltered"])
601-
assertEquals("0", metrics["RowGroupsFilteredByBloomFilter"])
602-
assertEquals("0", metrics["RowGroupsFilteredByMinMax"])
603-
assertEquals("1", metrics["RowGroupsReadNum"])
604-
assertEquals("1", metrics["RowGroupsTotalNum"])
605-
606-
metrics = q6()
607-
logger.info("metrics = ${metrics}")
608-
assertEquals("0", metrics["FilteredRowsByGroup"])
609-
assertTrue(metrics["FilteredRowsByLazyRead"].contains("1"))
610-
assertEquals("0", metrics["FilteredRowsByPage"])
611-
assertTrue(metrics["RawRowsRead"].contains("7300"))
612-
assertEquals("0", metrics["RowGroupsFiltered"])
613-
assertEquals("0", metrics["RowGroupsFilteredByBloomFilter"])
614-
assertEquals("0", metrics["RowGroupsFilteredByMinMax"])
615-
assertEquals("1", metrics["RowGroupsReadNum"])
616-
assertEquals("1", metrics["RowGroupsTotalNum"])
617-
618-
metrics = q7()
619-
logger.info("metrics = ${metrics}")
620-
assertEquals("0", metrics["FilteredRowsByGroup"])
621-
assertTrue(metrics["FilteredRowsByLazyRead"].contains("7298"))
622-
assertEquals("0", metrics["FilteredRowsByPage"])
623-
assertTrue(metrics["RawRowsRead"].contains("7300"))
624-
assertEquals("0", metrics["RowGroupsFiltered"])
625-
assertEquals("0", metrics["RowGroupsFilteredByBloomFilter"])
626-
assertEquals("0", metrics["RowGroupsFilteredByMinMax"])
627-
assertEquals("1", metrics["RowGroupsReadNum"])
628-
assertEquals("1", metrics["RowGroupsTotalNum"])
629-
}
630-
631-
// Versioned Parquet forces V2 even when these session preferences are disabled. Keep
632-
// checking exact pruning counters; disabling lazy materialization cannot select V1.
357+
// The versioned Parquet contract requires V2 even when its session preference is off.
358+
// V2 always applies safe ZoneMap pruning and lazy materialization, so the legacy
359+
// min/max and lazy-read switches must not change these exact profile counters.
633360
sql """ set enable_file_scanner_v2=false; """
634-
test_true_true();
635-
test_true_false();
636-
test_false_false();
637-
test_false_true();
361+
for (boolean minMaxEnabled : [true, false]) {
362+
for (boolean lazyReadEnabled : [true, false]) {
363+
sql """ set enable_parquet_filter_by_min_max = ${minMaxEnabled}; """
364+
sql """ set enable_parquet_lazy_materialization = ${lazyReadEnabled}; """
365+
checkV2Profile()
366+
}
367+
}
638368
q8();
639369

640370

0 commit comments

Comments
 (0)