Skip to content

Commit 4326f45

Browse files
committed
[fix](fe) Write the elements of an array literal with their own boundaries
### What problem does this solve? Problem Summary: the text of a literal was the text of its value, and the value of an array is a list whose text separates its elements with an apostrophe a value may carry: the two-element array `['a','b']` and the one-element array `['a', 'b']` (value `a', 'b`) rendered one `lit([a', 'b]:ARRAY<TEXT>)`. A filter over `ARRAY_OVERLAP(t.arr, <the literal>)` keeps a different number of rows for the two arrays, so an exact pin of one applied to the other on an unchanged scan baseline. The elements of a composite literal are literals themselves and are now rendered by the same rule as every literal of a canonical (`lit(<value>:<type>)`, separated by a comma), so an apostrophe inside an element cannot be taken for the boundary between two elements. A scalar literal, and therefore every existing key, is unchanged. ### Release note Two plans which differ only by the elements of an array literal no longer share an hbo statistics entry. ### Check List (For Author) - Test: Unit Test - Two corpus cases pin the two arrays above and a test asserts their exact keys differ; the regenerated file only gained those cases and its count line - no existing key moved - and 194 cases of the stats package plus the HBO command suite pass. - Behavior changed: Yes - the exact key of a plan with an array literal moved once. - Does this need documentation: No
1 parent fdce05c commit 4326f45

3 files changed

Lines changed: 86 additions & 1 deletion

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/stats/GroupStructInfo.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import org.apache.doris.nereids.trees.expressions.functions.Udf;
4141
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
4242
import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda;
43+
import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral;
4344
import org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
4445
import org.apache.doris.nereids.trees.expressions.literal.Literal;
4546
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
@@ -2540,6 +2541,23 @@ private static String literalText(Literal literal) {
25402541
if (literal instanceof DateLiteral) {
25412542
return ((DateLiteral) literal).getStringValue();
25422543
}
2544+
if (literal instanceof ArrayLiteral) {
2545+
// the elements of a composite literal are literals themselves, so they are rendered by the
2546+
// same rule: the text of the value a list holds separates its elements with an apostrophe
2547+
// which an element may carry, and `['a','b']` would be one text with `['a', 'b']` as a
2548+
// single element - the boundary each element is written with removes that ambiguity
2549+
StringBuilder text = new StringBuilder("[");
2550+
List<Literal> elements = ((ArrayLiteral) literal).getValue();
2551+
for (int i = 0; i < elements.size(); i++) {
2552+
if (i > 0) {
2553+
text.append(',');
2554+
}
2555+
Literal element = elements.get(i);
2556+
text.append("lit(").append(literalText(element)).append(':')
2557+
.append(element.getDataType()).append(')');
2558+
}
2559+
return text.append(']').toString();
2560+
}
25432561
return String.valueOf(literal.getValue());
25442562
}
25452563

‎fe/fe-core/src/test/java/org/apache/doris/nereids/stats/GroupStructInfoGoldenTest.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,10 @@
6060
import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdf;
6161
import org.apache.doris.nereids.trees.expressions.functions.window.Ntile;
6262
import org.apache.doris.nereids.trees.expressions.functions.window.RowNumber;
63+
import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral;
6364
import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal;
6465
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
66+
import org.apache.doris.nereids.trees.expressions.literal.Literal;
6567
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
6668
import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
6769
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
@@ -449,6 +451,18 @@ void testAFractionalSecondIsPartOfTheKey() {
449451
"the fraction of the second is not in the key: " + two.getCanonical(WITH_LITERAL));
450452
}
451453

454+
@Test
455+
void testTheElementsOfAnArrayLiteralKeepTheirBoundaries() {
456+
// the two-element array and the one-element array whose value is `a', 'b` hold different rows but
457+
// their raw text is one and the same: the elements are written with their own boundaries
458+
GroupStructInfo twoElements = built("filter_with_an_array_literal");
459+
GroupStructInfo oneElement = built("filter_with_an_ambiguous_array_literal");
460+
Assertions.assertNotEquals(twoElements.getFingerprint(WITH_LITERAL),
461+
oneElement.getFingerprint(WITH_LITERAL), "two array literals share an exact key");
462+
Assertions.assertTrue(twoElements.getCanonical(WITH_LITERAL).contains("%5Blit"),
463+
"the elements of the array are not delimited: " + twoElements.getCanonical(WITH_LITERAL));
464+
}
465+
452466
@Test
453467
void testANullAndTheStringNullDoNotShareAKey() {
454468
// `CAST(NULL AS STRING)` and `'null'` have one data type and one text without the marker: a
@@ -1225,6 +1239,12 @@ private static List<PlanCase> planCases() {
12251239
GroupStructInfoGoldenTest::generateOutputsSwapped));
12261240
cases.add(new PlanCase("repeat_grouping_sets", "repeat of the grouping sets ((a), ()) over a scan",
12271241
GroupStructInfoGoldenTest::repeatGroupingSets));
1242+
cases.add(new PlanCase("filter_with_an_array_literal",
1243+
"a filter over an array literal of two elements",
1244+
GroupStructInfoGoldenTest::filterWithAnArrayLiteral));
1245+
cases.add(new PlanCase("filter_with_an_ambiguous_array_literal",
1246+
"the same filter over one element whose text looks like the two elements of the first",
1247+
GroupStructInfoGoldenTest::filterWithAnAmbiguousArrayLiteral));
12281248
cases.add(new PlanCase("filter_with_a_null_argument",
12291249
"a filter whose argument is the NULL literal",
12301250
GroupStructInfoGoldenTest::filterWithANullArgument));
@@ -1673,6 +1693,29 @@ private static GroupStructInfo joinLeftOuter() {
16731693
new EqualTo(t1.getOutput().get(0), t2.getOutput().get(0))), t1, t2, null));
16741694
}
16751695

1696+
private static GroupStructInfo filterWithAnArrayLiteral() {
1697+
return filterWithAnArrayLiteral("a", "b");
1698+
}
1699+
1700+
private static GroupStructInfo filterWithAnArrayLiteral(String... values) {
1701+
LogicalOlapScan t1 = scan(0, "t1");
1702+
List<Literal> elements = new ArrayList<>();
1703+
for (String value : values) {
1704+
elements.add(new StringLiteral(value));
1705+
}
1706+
Expression array = new Cast(t1.getOutput().get(1), ArrayType.of(StringType.INSTANCE));
1707+
return describe(new LogicalFilter<>(ImmutableSet.of(
1708+
new EqualTo(array, new ArrayLiteral(elements))), t1));
1709+
}
1710+
1711+
/**
1712+
* A filter over a one-element array whose element carries the text of two: the text of the value a
1713+
* list holds (`['a', 'b']`) is the same for `['a', 'b']` and for one element `a', 'b`.
1714+
*/
1715+
private static GroupStructInfo filterWithAnAmbiguousArrayLiteral() {
1716+
return filterWithAnArrayLiteral("a', 'b");
1717+
}
1718+
16761719
private static GroupStructInfo filterWithANullArgument() {
16771720
LogicalOlapScan t1 = scan(0, "t1");
16781721
return describe(new LogicalFilter<>(ImmutableSet.of(new EqualTo(

‎fe/fe-core/src/test/resources/hbo-struct-info-golden.txt‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
# [TEXT <kind>/<name>] a pure function case: `in =` is the argument, `out =` / `err =`
3939
# the result (an exception is recorded as its class and message)
4040
#
41-
# plan cases: 78, text cases: 51 (the PLAN cases are sorted by name, the TEXT cases by kind and then by name)
41+
# plan cases: 80, text cases: 51 (the PLAN cases are sorted by name, the TEXT cases by kind and then by name)
4242

4343
[PLAN a_column_whose_name_carries_a_dot]
4444
desc = a filter on the column `a.b` of the table t
@@ -474,6 +474,30 @@ NO_LITERAL.shape = F{EqualTo(col(db.t1.name),lit(*))}(S{null.null.t1#ti:0})
474474
NO_LITERAL.fingerprint = 68b397abfeabd376f1ed9e8219ad77f83588cdbbcbf6c51d9c23a70a8323e9bb
475475
scans = 1
476476
scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity=
477+
[PLAN filter_with_an_ambiguous_array_literal]
478+
desc = the same filter over one element whose text looks like the two elements of the first
479+
valid = true
480+
skipReason =
481+
WITH_LITERAL.canonical = F{EqualTo(Cast(col(db.t1.name)),lit(%5Blit%28a'%2C 'b%3ATEXT%29%5D:ARRAY<TEXT>))}(S{null.null.t1#ti:0,v1,r0})
482+
WITH_LITERAL.shape = F{EqualTo(Cast(col(db.t1.name)),lit(%5Blit%28a'%2C 'b%3ATEXT%29%5D:ARRAY<TEXT>))}(S{null.null.t1#ti:0})
483+
WITH_LITERAL.fingerprint = 0d60950278001d4a8bfbc37f820bf00a238a18238acea88f6afba32b9ef8a81e
484+
NO_LITERAL.canonical = F{EqualTo(Cast(col(db.t1.name)),lit(*))}(S{null.null.t1#ti:0,v1,r0})
485+
NO_LITERAL.shape = F{EqualTo(Cast(col(db.t1.name)),lit(*))}(S{null.null.t1#ti:0})
486+
NO_LITERAL.fingerprint = bd4e7abb554baac77f8799bf5950b249e3adafc2928424e8ecef5278894fbeb2
487+
scans = 1
488+
scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity=
489+
[PLAN filter_with_an_array_literal]
490+
desc = a filter over an array literal of two elements
491+
valid = true
492+
skipReason =
493+
WITH_LITERAL.canonical = F{EqualTo(Cast(col(db.t1.name)),lit(%5Blit%28a%3ATEXT%29%2Clit%28b%3ATEXT%29%5D:ARRAY<TEXT>))}(S{null.null.t1#ti:0,v1,r0})
494+
WITH_LITERAL.shape = F{EqualTo(Cast(col(db.t1.name)),lit(%5Blit%28a%3ATEXT%29%2Clit%28b%3ATEXT%29%5D:ARRAY<TEXT>))}(S{null.null.t1#ti:0})
495+
WITH_LITERAL.fingerprint = e21ad2f91cbe38fd8764f623ecb5ec914ece792d0abcb60052beeabfa853d5f3
496+
NO_LITERAL.canonical = F{EqualTo(Cast(col(db.t1.name)),lit(*))}(S{null.null.t1#ti:0,v1,r0})
497+
NO_LITERAL.shape = F{EqualTo(Cast(col(db.t1.name)),lit(*))}(S{null.null.t1#ti:0})
498+
NO_LITERAL.fingerprint = bd4e7abb554baac77f8799bf5950b249e3adafc2928424e8ecef5278894fbeb2
499+
scans = 1
500+
scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity=
477501
[PLAN filter_with_the_string_null_argument]
478502
desc = the same filter with the string `null`: the two arguments cannot share a key
479503
valid = true

0 commit comments

Comments
 (0)