@@ -715,48 +715,48 @@ NULL
715715statement ok
716716DROP TABLE dict_nulls_order;
717717
718- # Aggregate output is emitted in `batch_size` slices that keep every group's
719- # dictionary value, so these batches have many more dictionary values than
720- # rows. Hashing them and running string functions on them must give the same
721- # results as on the decoded values.
718+ # Aggregate output is emitted in `batch_size` slices of one array that holds
719+ # every group's dictionary value. Hashing the slices and running string
720+ # functions on them must give the same results as on the decoded values.
722721statement ok
723- set datafusion.execution.batch_size = 1 ;
722+ set datafusion.execution.batch_size = 2 ;
724723
725724statement ok
726725set datafusion.execution.target_partitions = 2;
727726
728727statement ok
729728create table padded_groups as
730729select arrow_cast(
731- case when value % 13 = 0 then null else ' Grp' || (value % 100 ) || ' ' end,
730+ case when value % 13 = 0 then null else ' Grp' || (value % 400 ) || ' ' end,
732731 'Dictionary(Int32, Utf8)'
733732) as p
734733from generate_series(1, 1000);
735734
736735query I
737736select count(*) from (select p from padded_groups group by p);
738737----
739- 101
738+ 401
740739
741740# Each function on the grouped dictionary must match the same function on the
742- # decoded value
743- query I
744- select count(*)
745- from (
746- select p, arrow_cast(p, 'Utf8') as u
747- from (select p from padded_groups group by p)
748- )
749- where lower(p) is distinct from lower(u)
741+ # decoded value. These are projected, not filtered on: a filter on the group
742+ # key is pushed below the aggregate, where a batch has very few values.
743+ query B
744+ select distinct
745+ lower(p) is distinct from lower(u)
750746 or upper(p) is distinct from upper(u)
751747 or reverse(p) is distinct from reverse(u)
752748 or character_length(p) is distinct from character_length(u)
753749 or ascii(p) is distinct from ascii(u)
754750 or initcap(p) is distinct from initcap(u)
755751 or btrim(p) is distinct from btrim(u)
756752 or ltrim(p) is distinct from ltrim(u)
757- or rtrim(p) is distinct from rtrim(u);
753+ or rtrim(p) is distinct from rtrim(u)
754+ from (
755+ select p, arrow_cast(p, 'Utf8') as u
756+ from (select p from padded_groups group by p)
757+ );
758758----
759- 0
759+ false
760760
761761query T
762762select distinct arrow_typeof(lower(p)) from (select p from padded_groups group by p);
@@ -766,6 +766,36 @@ Dictionary(Int32, Utf8)
766766statement ok
767767drop table padded_groups;
768768
769+ # A filter keeps the whole dictionary, so these batches have many more
770+ # dictionary values than rows.
771+ statement ok
772+ set datafusion.execution.batch_size = 8192;
773+
774+ statement ok
775+ create table filtered_groups as
776+ select value as id, arrow_cast(' Grp' || (value % 400) || ' ', 'Dictionary(Int32, Utf8)') as p
777+ from generate_series(1, 1000);
778+
779+ query B
780+ select distinct
781+ lower(p) is distinct from lower(u)
782+ or upper(p) is distinct from upper(u)
783+ or reverse(p) is distinct from reverse(u)
784+ or character_length(p) is distinct from character_length(u)
785+ or ascii(p) is distinct from ascii(u)
786+ or initcap(p) is distinct from initcap(u)
787+ or btrim(p) is distinct from btrim(u)
788+ or ltrim(p) is distinct from ltrim(u)
789+ or rtrim(p) is distinct from rtrim(u)
790+ from (
791+ select p, arrow_cast(p, 'Utf8') as u from filtered_groups where id % 200 = 0
792+ );
793+ ----
794+ false
795+
796+ statement ok
797+ drop table filtered_groups;
798+
769799# The SLT runner sets `target_partitions` to 4 instead of using the default, so
770800# reset it explicitly.
771801statement ok
0 commit comments