Skip to content

Commit b6ad0f2

Browse files
committed
[fix](fe) Preserve nullable-key matches in connector MERGE
### What problem does this PR solve? Issue Number: N/A Related PR: #69343 Problem Summary: With Paimon 2.1 nullable primary keys, a NULL-safe MERGE can join a target row whose first primary key is NULL. Changelog branch selection then treated that row as absent and silently dropped matched UPDATE and DELETE changes. Treat every surviving INNER JOIN row as matched for MATCHED-only MERGE. For MERGE with INSERT branches, project a non-null presence marker on the target before the outer join and inspect its null-extended output by ExprId. Keep private marker slots outside the user expression-binding scope so same-named source columns retain their values. Add nullable single/composite-key SQL coverage, branch and cardinality checks, marker-name collision coverage, and focused planner unit tests. ### Release note Fix matched MERGE UPDATE and DELETE silently omitting Paimon rows with NULL primary keys under NULL-safe equality. ### Check List (For Author) - Test: Unit Test: 38 related FE tests passed. Regression test: 9 Paimon suites passed, including nullable-key MERGE with generated/replayed expected output and all six existing write suites containing MERGE. Manual test: FE build and before/after NULL-key UPDATE/DELETE reproduction. - Behavior changed: Yes, matched NULL-key rows are updated/deleted and target presence is independent of key values - Does this need documentation: No
1 parent 9337b7c commit b6ad0f2

4 files changed

Lines changed: 327 additions & 7 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ConnectorChangelogPlanBuilder.java‎

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.apache.doris.nereids.trees.expressions.Cast;
3030
import org.apache.doris.nereids.trees.expressions.EqualTo;
3131
import org.apache.doris.nereids.trees.expressions.Expression;
32+
import org.apache.doris.nereids.trees.expressions.IsNull;
3233
import org.apache.doris.nereids.trees.expressions.LessThanEqual;
3334
import org.apache.doris.nereids.trees.expressions.NamedExpression;
3435
import org.apache.doris.nereids.trees.expressions.Not;
@@ -38,6 +39,7 @@
3839
import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
3940
import org.apache.doris.nereids.trees.expressions.functions.scalar.AssertTrue;
4041
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
42+
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
4143
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
4244
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
4345
import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
@@ -218,6 +220,7 @@ private static final class MergeBuilder {
218220
private final LogicalPlan child;
219221
private final CascadesContext context;
220222
private final ExpressionAnalyzer analyzer;
223+
private final Expression targetPresent;
221224

222225
private MergeBuilder(List<Column> schema, ExternalTable targetTable,
223226
List<String> primaryKeys,
@@ -229,8 +232,31 @@ private MergeBuilder(List<Column> schema, ExternalTable targetTable,
229232
this.primaryKeys = primaryKeys;
230233
this.mode = mode;
231234
this.merge = merge;
232-
this.child = child;
233235
this.context = context;
236+
if (merge.getNotMatchedClauses().isEmpty()) {
237+
// MATCHED-only MERGE uses an INNER JOIN, including matches on NULL-safe keys.
238+
this.child = child;
239+
this.targetPresent = BooleanLiteral.TRUE;
240+
} else {
241+
if (!(child instanceof LogicalJoin)) {
242+
throw new AnalysisException("Connector MERGE input must be a logical join");
243+
}
244+
LogicalJoin<?, ?> join = (LogicalJoin<?, ?>) child;
245+
// A nullable primary key cannot distinguish a real target row from the RIGHT
246+
// OUTER JOIN's null-extension. Add a non-null marker before the join instead.
247+
Alias present = new Alias(new TinyIntLiteral((byte) 1), "__DORIS_CHANGELOG_TARGET_PRESENT__");
248+
List<NamedExpression> targetOutput = new ArrayList<>(join.left().getOutput());
249+
targetOutput.add(present);
250+
this.child = join.withChildren(ImmutableList.of(
251+
new LogicalProject<>(targetOutput, join.left()), join.right()));
252+
// Use the join output's nullability, not the marker's non-null pre-join slot.
253+
Slot marker = this.child.getOutput().stream()
254+
.filter(slot -> slot.getExprId().equals(present.getExprId())).findFirst()
255+
.orElseThrow(() -> new IllegalStateException("Connector MERGE lost its target marker"));
256+
this.targetPresent = new Not(new IsNull(marker));
257+
}
258+
// Resolve user expressions in the original scope so the private marker cannot shadow
259+
// a user column with the same name. The marker expression already uses its bound slot.
234260
this.analyzer = analyzer(child, context);
235261
}
236262

@@ -284,9 +310,6 @@ private void validateNotMatchedPrimaryKeys(List<List<Expression>> branches) {
284310
Set<Slot> targetSlots = child.getOutput().stream()
285311
.filter(slot -> qualifierEndsWith(slot.getQualifier(), merge.getTargetNameInPlan()))
286312
.collect(ImmutableSet.toImmutableSet());
287-
if (!(child instanceof LogicalJoin)) {
288-
throw new AnalysisException("Connector MERGE input must be a logical join");
289-
}
290313
Expression onClause = ((LogicalJoin<?, ?>) child).getOnClauseCondition()
291314
.orElseThrow(() -> new AnalysisException("Connector MERGE requires an ON condition"));
292315
Map<String, Expression> sourceKeys = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
@@ -398,8 +421,6 @@ private AnalysisException invalidNotMatchedKeyCondition() {
398421
}
399422

400423
private Alias bindBranchLabel() {
401-
Expression targetPresent = new Not(new org.apache.doris.nereids.trees.expressions.IsNull(
402-
findTargetSlot(primaryKeys.get(0))));
403424
Expression matched = new NullLiteral(IntegerType.INSTANCE);
404425
for (int i = merge.getMatchedClauses().size() - 1; i >= 0; i--) {
405426
MergeMatchedClause clause = merge.getMatchedClauses().get(i);

‎fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/ConnectorChangelogPlanBuilderTest.java‎

Lines changed: 84 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,27 +25,39 @@
2525
import org.apache.doris.nereids.CascadesContext;
2626
import org.apache.doris.nereids.analyzer.UnboundSlot;
2727
import org.apache.doris.nereids.exceptions.AnalysisException;
28+
import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE;
29+
import org.apache.doris.nereids.trees.expressions.Alias;
2830
import org.apache.doris.nereids.trees.expressions.Default;
2931
import org.apache.doris.nereids.trees.expressions.EqualTo;
32+
import org.apache.doris.nereids.trees.expressions.Expression;
3033
import org.apache.doris.nereids.trees.expressions.NamedExpression;
34+
import org.apache.doris.nereids.trees.expressions.Slot;
3135
import org.apache.doris.nereids.trees.expressions.SlotReference;
3236
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
37+
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
38+
import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
3339
import org.apache.doris.nereids.trees.plans.RelationId;
3440
import org.apache.doris.nereids.trees.plans.commands.info.ConnectorChangelogRowChangeSpec;
3541
import org.apache.doris.nereids.trees.plans.commands.merge.MergeMatchedClause;
42+
import org.apache.doris.nereids.trees.plans.commands.merge.MergeNotMatchedClause;
43+
import org.apache.doris.nereids.trees.plans.commands.merge.MergeUtils;
3644
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
3745
import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
46+
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
3847
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
3948
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
4049
import org.apache.doris.nereids.types.IntegerType;
50+
import org.apache.doris.nereids.util.ExpressionUtils;
4151
import org.apache.doris.nereids.util.MemoTestUtils;
4252

4353
import com.google.common.collect.ImmutableList;
4454
import org.junit.jupiter.api.Assertions;
4555
import org.junit.jupiter.api.Test;
4656
import org.mockito.Mockito;
4757

58+
import java.util.HashMap;
4859
import java.util.List;
60+
import java.util.Map;
4961
import java.util.Optional;
5062

5163
class ConnectorChangelogPlanBuilderTest {
@@ -158,6 +170,73 @@ void matchedMergeUpdateResolvesDefaultFromPinnedWriteSchema() {
158170
Assertions.assertFalse(projections.contains("DEFAULT"));
159171
}
160172

173+
@Test
174+
void matchedMergeAcceptsNullPrimaryKeyForUpdateAndDelete() {
175+
for (boolean delete : new boolean[] {false, true}) {
176+
LogicalPlan child = targetRow(true);
177+
MergeMatchedClause clause = new MergeMatchedClause(Optional.empty(), delete ? ImmutableList.of()
178+
: ImmutableList.of(new EqualTo(new UnboundSlot("value"), new IntegerLiteral(99))), delete);
179+
ConnectorChangelogRowChangeSpec.Merge spec = new ConnectorChangelogRowChangeSpec.Merge(
180+
ImmutableList.of("target"), ImmutableList.of(clause), ImmutableList.of());
181+
LogicalPlan result = ConnectorChangelogPlanBuilder.build(
182+
SCHEMA, TARGET_TABLE, ImmutableList.of("id"), MODE, spec, child, context(child));
183+
Map<Expression, Expression> nullRow = new HashMap<>();
184+
child.getOutput().forEach(slot -> nullRow.put(slot, new NullLiteral(slot.getDataType())));
185+
186+
Assertions.assertEquals(new IntegerLiteral(0), FoldConstantRuleOnFE.evaluateWithoutContext(
187+
ExpressionUtils.replace(branchLabel(result), nullRow)));
188+
}
189+
}
190+
191+
@Test
192+
void mergeDistinguishesNullTargetValuesFromNullExtendedRows() {
193+
LogicalPlan target = targetRow(true);
194+
LogicalPlan source = new LogicalEmptyRelation(new RelationId(2), ImmutableList.of(
195+
new SlotReference("id", IntegerType.INSTANCE, true, ImmutableList.of("source")),
196+
new SlotReference("__DORIS_CHANGELOG_TARGET_PRESENT__", IntegerType.INSTANCE, true,
197+
ImmutableList.of("source"))));
198+
LogicalPlan child = MergeUtils.buildMergeJoin(target, source,
199+
new EqualTo(target.getOutput().get(0), source.getOutput().get(0)), true);
200+
MergeMatchedClause update = new MergeMatchedClause(Optional.empty(), ImmutableList.of(
201+
new EqualTo(new UnboundSlot("value"),
202+
new UnboundSlot("__DORIS_CHANGELOG_TARGET_PRESENT__"))), false);
203+
MergeNotMatchedClause insert = new MergeNotMatchedClause(Optional.empty(), ImmutableList.of(),
204+
ImmutableList.copyOf(source.getOutput()));
205+
ConnectorChangelogRowChangeSpec.Merge spec = new ConnectorChangelogRowChangeSpec.Merge(
206+
ImmutableList.of("target"), ImmutableList.of(update), ImmutableList.of(insert));
207+
LogicalPlan result = ConnectorChangelogPlanBuilder.build(
208+
SCHEMA, TARGET_TABLE, ImmutableList.of("id"), MODE, spec, child, context(child));
209+
LogicalJoin<?, ?> join = result.<LogicalJoin<?, ?>>collectToList(LogicalJoin.class::isInstance).get(0);
210+
Slot marker = join.getOutput().get(target.getOutput().size());
211+
Assertions.assertTrue(marker.nullable(), "the marker must inherit outer-join null-extension");
212+
Map<Expression, Expression> nullRow = new HashMap<>();
213+
join.getOutput().forEach(slot -> nullRow.put(slot, new NullLiteral(slot.getDataType())));
214+
Expression label = branchLabel(result);
215+
216+
Assertions.assertEquals(new IntegerLiteral(1), FoldConstantRuleOnFE.evaluateWithoutContext(
217+
ExpressionUtils.replace(label, nullRow)));
218+
nullRow.put(marker, new TinyIntLiteral((byte) 1));
219+
Assertions.assertEquals(new IntegerLiteral(0), FoldConstantRuleOnFE.evaluateWithoutContext(
220+
ExpressionUtils.replace(label, nullRow)));
221+
// A source column sharing the private marker's name must still supply the UPDATE value.
222+
nullRow.put(source.getOutput().get(1), new IntegerLiteral(555));
223+
Expression value = result.<LogicalProject<?>>collectToList(LogicalProject.class::isInstance).stream()
224+
.flatMap(project -> project.getProjects().stream())
225+
.filter(project -> project instanceof Alias && project.getName().equals("value"))
226+
.findFirst().get().child(0);
227+
value.getInputSlots().stream().filter(slot -> slot.getName().equals("__DORIS_CHANGELOG_BRANCH__"))
228+
.forEach(slot -> nullRow.put(slot, new IntegerLiteral(0)));
229+
Assertions.assertEquals(new IntegerLiteral(555), FoldConstantRuleOnFE.evaluateWithoutContext(
230+
ExpressionUtils.replace(value, nullRow)));
231+
}
232+
233+
private Expression branchLabel(LogicalPlan plan) {
234+
return plan.<LogicalProject<?>>collectToList(LogicalProject.class::isInstance).stream()
235+
.flatMap(project -> project.getProjects().stream())
236+
.filter(project -> project.getName().equals("__DORIS_CHANGELOG_BRANCH__"))
237+
.findFirst().get().child(0);
238+
}
239+
161240
private List<Column> schemaWithValueDefault(String defaultValue) {
162241
return ImmutableList.of(
163242
new Column("id", ScalarType.createType(PrimitiveType.INT)),
@@ -166,7 +245,11 @@ private List<Column> schemaWithValueDefault(String defaultValue) {
166245
}
167246

168247
private LogicalPlan targetRow() {
169-
SlotReference id = new SlotReference("id", IntegerType.INSTANCE, false,
248+
return targetRow(false);
249+
}
250+
251+
private LogicalPlan targetRow(boolean nullableKey) {
252+
SlotReference id = new SlotReference("id", IntegerType.INSTANCE, nullableKey,
170253
ImmutableList.of("target"));
171254
SlotReference value = new SlotReference("value", IntegerType.INSTANCE, true,
172255
ImmutableList.of("target"));
Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !null_key_update --
3+
\N 99
4+
1 200
5+
3 300
6+
7+
-- !null_key_equal_no_match --
8+
\N 99
9+
1 200
10+
3 300
11+
12+
-- !matched_and_insert --
13+
\N 99
14+
1 111
15+
2 222
16+
3 300
17+
18+
-- !null_key_delete --
19+
1 111
20+
2 222
21+
3 300
22+
23+
-- !missing_null_key_no_match --
24+
1 111
25+
2 222
26+
3 300
27+
28+
-- !insert_null_key --
29+
\N 77
30+
1 111
31+
2 222
32+
3 300
33+
34+
-- !null_key_branch_priority --
35+
\N 88
36+
1 111
37+
2 222
38+
3 300
39+
40+
-- !null_key_false_condition --
41+
\N 88
42+
1 111
43+
2 222
44+
3 300
45+
46+
-- !null_key_duplicate_unchanged --
47+
\N 88
48+
1 111
49+
2 222
50+
3 300
51+
52+
-- !duplicate_null_insert_unchanged --
53+
\N 88
54+
1 111
55+
2 222
56+
3 300
57+
58+
-- !insert_only --
59+
\N 88
60+
1 111
61+
2 222
62+
3 300
63+
4 444
64+
65+
-- !marker_name_collision --
66+
\N 88
67+
1 111
68+
2 222
69+
3 300
70+
4 444
71+
5 555
72+
73+
-- !composite_update --
74+
\N \N 130
75+
\N 1 110
76+
1 \N 120
77+
1 1 140
78+
79+
-- !composite_delete --
80+
\N 1 110
81+
1 \N 120
82+
1 1 140
83+

0 commit comments

Comments
 (0)