Skip to content

Commit 794d514

Browse files
authored
[feature](hive) Support static partition overwrite for hive tables (#65991)
Add Hive static partition regression and FE plan tests, and fix explicit-column INSERT cases to use the supported PARTITION-before-column-list syntax.
1 parent 8305fe7 commit 794d514

7 files changed

Lines changed: 639 additions & 18 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundHiveTableSink.java‎

Lines changed: 40 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,30 +19,49 @@
1919

2020
import org.apache.doris.nereids.memo.GroupExpression;
2121
import org.apache.doris.nereids.properties.LogicalProperties;
22+
import org.apache.doris.nereids.trees.expressions.Expression;
2223
import org.apache.doris.nereids.trees.plans.Plan;
2324
import org.apache.doris.nereids.trees.plans.PlanType;
2425
import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType;
2526
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
2627

2728
import com.google.common.base.Preconditions;
2829
import com.google.common.collect.ImmutableList;
30+
import com.google.common.collect.ImmutableMap;
2931

3032
import java.util.List;
33+
import java.util.Map;
3134
import java.util.Optional;
3235

3336
/**
3437
* Represent a hive table sink plan node that has not been bound.
3538
*/
3639
public class UnboundHiveTableSink<CHILD_TYPE extends Plan> extends UnboundBaseExternalTableSink<CHILD_TYPE> {
3740

41+
// Static partition key-value pairs for INSERT [OVERWRITE] ... PARTITION
42+
// (col='val', ...)
43+
private final Map<String, Expression> staticPartitionKeyValues;
44+
3845
public UnboundHiveTableSink(List<String> nameParts, List<String> colNames, List<String> hints,
3946
List<String> partitions, CHILD_TYPE child) {
4047
this(nameParts, colNames, hints, partitions, DMLCommandType.NONE,
41-
Optional.empty(), Optional.empty(), child);
48+
Optional.empty(), Optional.empty(), child, null);
49+
}
50+
51+
public UnboundHiveTableSink(List<String> nameParts,
52+
List<String> colNames,
53+
List<String> hints,
54+
List<String> partitions,
55+
DMLCommandType dmlCommandType,
56+
Optional<GroupExpression> groupExpression,
57+
Optional<LogicalProperties> logicalProperties,
58+
CHILD_TYPE child) {
59+
this(nameParts, colNames, hints, partitions, dmlCommandType,
60+
groupExpression, logicalProperties, child, null);
4261
}
4362

4463
/**
45-
* constructor
64+
* constructor with static partition
4665
*/
4766
public UnboundHiveTableSink(List<String> nameParts,
4867
List<String> colNames,
@@ -51,34 +70,46 @@ public UnboundHiveTableSink(List<String> nameParts,
5170
DMLCommandType dmlCommandType,
5271
Optional<GroupExpression> groupExpression,
5372
Optional<LogicalProperties> logicalProperties,
54-
CHILD_TYPE child) {
73+
CHILD_TYPE child,
74+
Map<String, Expression> staticPartitionKeyValues) {
5575
super(nameParts, PlanType.LOGICAL_UNBOUND_HIVE_TABLE_SINK, ImmutableList.of(), groupExpression,
5676
logicalProperties, colNames, dmlCommandType, child, hints, partitions);
77+
this.staticPartitionKeyValues = staticPartitionKeyValues != null
78+
? ImmutableMap.copyOf(staticPartitionKeyValues)
79+
: null;
5780
}
5881

59-
@Override
60-
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
61-
return visitor.visitUnboundHiveTableSink(this, context);
82+
public Map<String, Expression> getStaticPartitionKeyValues() {
83+
return staticPartitionKeyValues;
84+
}
85+
86+
public boolean hasStaticPartition() {
87+
return staticPartitionKeyValues != null && !staticPartitionKeyValues.isEmpty();
6288
}
6389

6490
@Override
6591
public Plan withChildren(List<Plan> children) {
6692
Preconditions.checkArgument(children.size() == 1,
6793
"UnboundHiveTableSink only accepts one child");
6894
return new UnboundHiveTableSink<>(nameParts, colNames, hints, partitions,
69-
dmlCommandType, groupExpression, Optional.empty(), children.get(0));
95+
dmlCommandType, groupExpression, Optional.empty(), children.get(0), staticPartitionKeyValues);
96+
}
97+
98+
@Override
99+
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
100+
return visitor.visitUnboundHiveTableSink(this, context);
70101
}
71102

72103
@Override
73104
public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
74105
return new UnboundHiveTableSink<>(nameParts, colNames, hints, partitions,
75-
dmlCommandType, groupExpression, Optional.of(getLogicalProperties()), child());
106+
dmlCommandType, groupExpression, Optional.of(getLogicalProperties()), child(), staticPartitionKeyValues);
76107
}
77108

78109
@Override
79110
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
80111
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
81112
return new UnboundHiveTableSink<>(nameParts, colNames, hints, partitions,
82-
dmlCommandType, groupExpression, logicalProperties, children.get(0));
113+
dmlCommandType, groupExpression, logicalProperties, children.get(0), staticPartitionKeyValues);
83114
}
84115
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSinkCreator.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ public static LogicalSink<? extends Plan> createUnboundTableSink(List<String> na
8383
}
8484

8585
/**
86-
* create unbound sink for DML plan with static partition support for Iceberg.
86+
* create unbound sink for DML plan with static partition support for Iceberg / Hive / MaxCompute.
8787
*/
8888
public static LogicalSink<? extends Plan> createUnboundTableSink(List<String> nameParts,
8989
List<String> colNames, List<String> hints, boolean temporaryPartition, List<String> partitions,
@@ -98,7 +98,7 @@ public static LogicalSink<? extends Plan> createUnboundTableSink(List<String> na
9898
Optional.empty(), plan);
9999
} else if (curCatalog instanceof HMSExternalCatalog) {
100100
return new UnboundHiveTableSink<>(nameParts, colNames, hints, partitions,
101-
dmlCommandType, Optional.empty(), Optional.empty(), plan);
101+
dmlCommandType, Optional.empty(), Optional.empty(), plan, staticPartitionKeyValues);
102102
} else if (curCatalog instanceof IcebergExternalCatalog) {
103103
return new UnboundIcebergTableSink<>(nameParts, colNames, hints, partitions,
104104
dmlCommandType, Optional.empty(), Optional.empty(), plan, staticPartitionKeyValues, false);
@@ -114,8 +114,7 @@ public static LogicalSink<? extends Plan> createUnboundTableSink(List<String> na
114114

115115
/**
116116
* create unbound sink for DML plan with auto detect overwrite partition enable
117-
* and static partition support for Iceberg.
118-
* TODO: staticPartitionKeyValues is only used for Iceberg, support other catalog types in future.
117+
* and static partition support for Iceberg / Hive / MaxCompute.
119118
*/
120119
public static LogicalSink<? extends Plan> createUnboundTableSinkMaybeOverwrite(List<String> nameParts,
121120
List<String> colNames, List<String> hints, boolean temporaryPartition, List<String> partitions,
@@ -139,7 +138,7 @@ public static LogicalSink<? extends Plan> createUnboundTableSinkMaybeOverwrite(L
139138
Optional.empty(), plan);
140139
} else if (curCatalog instanceof HMSExternalCatalog && !isAutoDetectPartition) {
141140
return new UnboundHiveTableSink<>(nameParts, colNames, hints, partitions,
142-
dmlCommandType, Optional.empty(), Optional.empty(), plan);
141+
dmlCommandType, Optional.empty(), Optional.empty(), plan, staticPartitionKeyValues);
143142
} else if (curCatalog instanceof IcebergExternalCatalog && !isAutoDetectPartition) {
144143
return new UnboundIcebergTableSink<>(nameParts, colNames, hints, partitions,
145144
dmlCommandType, Optional.empty(), Optional.empty(), plan, staticPartitionKeyValues, false);

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java‎

Lines changed: 91 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@
123123
import java.util.ArrayList;
124124
import java.util.HashMap;
125125
import java.util.List;
126+
import java.util.Locale;
126127
import java.util.Map;
127128
import java.util.Optional;
128129
import java.util.Set;
@@ -690,11 +691,44 @@ private Plan bindHiveTableSink(MatchingContext<UnboundHiveTableSink<Plan>> ctx)
690691
+ "(input format: " + inputFormat + "). LZO tables are read-only in Doris.");
691692
}
692693

694+
// Get static partition columns if present:
695+
// INSERT [OVERWRITE] TABLE t PARTITION(col='val', ...) SELECT ...
696+
Map<String, Expression> staticPartitions = sink.getStaticPartitionKeyValues();
697+
Set<String> staticPartitionColNames = staticPartitions != null
698+
? staticPartitions.keySet()
699+
: Sets.newHashSet();
700+
// Hive column/partition names are case-insensitive (stored lowercase in HMS), so
701+
// e.g. PARTITION(dt='21', DT='22') refers to the same column twice. Detect such
702+
// case-insensitive duplicates here and fail fast, instead of silently letting a
703+
// later value overwrite an earlier one in the case-insensitive columnToOutput map.
704+
// Use Locale.ROOT to keep the folding locale-independent (matching how the Hive
705+
// schema is loaded), otherwise e.g. the Turkish locale would fold 'I' to 'ı' and
706+
// fail to match a metadata column named 'id'.
707+
Set<String> lowerStaticPartitionColNames = Sets.newHashSet();
708+
for (String name : staticPartitionColNames) {
709+
if (!lowerStaticPartitionColNames.add(name.toLowerCase(Locale.ROOT))) {
710+
throw new AnalysisException("Duplicate partition column: " + name);
711+
}
712+
}
713+
714+
// Validate static partition columns against the hive table's partition columns
715+
if (sink.hasStaticPartition()) {
716+
validateHiveStaticPartition(staticPartitions, table);
717+
}
718+
719+
// Build bindColumns: exclude static partition columns from the columns that
720+
// need to come from SELECT, because their values come from the PARTITION clause.
693721
List<Column> bindColumns;
694722
if (sink.getColNames().isEmpty()) {
695-
bindColumns = table.getBaseSchema(true).stream().collect(ImmutableList.toImmutableList());
723+
bindColumns = table.getBaseSchema(true).stream()
724+
.filter(col -> !lowerStaticPartitionColNames.contains(col.getName().toLowerCase(Locale.ROOT)))
725+
.collect(ImmutableList.toImmutableList());
696726
} else {
697727
bindColumns = sink.getColNames().stream().map(cn -> {
728+
if (lowerStaticPartitionColNames.contains(cn.toLowerCase(Locale.ROOT))) {
729+
throw new AnalysisException(String.format(
730+
"column %s is a static partition column, should not be in the insert column list", cn));
731+
}
698732
Column column = table.getColumn(cn);
699733
if (column == null) {
700734
throw new AnalysisException(String.format("column %s is not found in table %s",
@@ -715,15 +749,71 @@ private Plan bindHiveTableSink(MatchingContext<UnboundHiveTableSink<Plan>> ctx)
715749
Optional.empty(),
716750
child);
717751
// we need to insert all the columns of the target table
752+
// (except the static partition columns which are supplied by the PARTITION clause)
718753
if (boundSink.getCols().size() != child.getOutput().size()) {
719754
throw new AnalysisException("insert into cols should be corresponding to the query output");
720755
}
721756
Map<String, NamedExpression> columnToOutput = getColumnToOutput(ctx, table, false, false,
722757
boundSink, child);
758+
759+
// For static partition columns, add constant expressions from the PARTITION clause.
760+
// This ensures every written row carries the static partition value, so all rows
761+
// land in the specified partition (and INSERT OVERWRITE only overwrites that partition).
762+
if (!staticPartitionColNames.isEmpty()) {
763+
for (Map.Entry<String, Expression> entry : staticPartitions.entrySet()) {
764+
Expression valueExpr = entry.getValue();
765+
Column column = table.getColumn(entry.getKey());
766+
if (column != null) {
767+
Expression castExpr = TypeCoercionUtils.castIfNotSameType(
768+
valueExpr, DataType.fromCatalogType(column.getType()));
769+
// Use the canonical (lowercase) column name from the table schema as both the
770+
// map key and the alias name, so it aligns with getOutputProjectByCoercion which
771+
// looks up columnToOutput by table.getFullSchema() column names.
772+
columnToOutput.put(column.getName(), new Alias(castExpr, column.getName()));
773+
}
774+
}
775+
}
776+
723777
LogicalProject<?> fullOutputProject = getOutputProjectByCoercion(table.getFullSchema(), child, columnToOutput);
724778
return boundSink.withChildAndUpdateOutput(fullOutputProject);
725779
}
726780

781+
/**
782+
* Validate static partition specification for Hive table.
783+
* The specified columns must be partition columns of the target hive table,
784+
* and each partition value must be a literal.
785+
*/
786+
private void validateHiveStaticPartition(Map<String, Expression> staticPartitions, HMSExternalTable table) {
787+
if (staticPartitions == null || staticPartitions.isEmpty()) {
788+
return;
789+
}
790+
Set<String> partitionColNames = table.getPartitionColumnNames();
791+
if (partitionColNames.isEmpty()) {
792+
throw new AnalysisException(
793+
String.format("Table %s is not a partitioned table, cannot use static partition syntax",
794+
table.getName()));
795+
}
796+
// build a case-insensitive view of partition column names
797+
Set<String> lowerPartitionColNames = Sets.newHashSet();
798+
for (String name : partitionColNames) {
799+
lowerPartitionColNames.add(name.toLowerCase());
800+
}
801+
for (Map.Entry<String, Expression> entry : staticPartitions.entrySet()) {
802+
String partitionColName = entry.getKey();
803+
Expression partitionValue = entry.getValue();
804+
if (!lowerPartitionColNames.contains(partitionColName.toLowerCase())) {
805+
throw new AnalysisException(
806+
String.format("Unknown partition column '%s' in table '%s'. Available partition columns: %s",
807+
partitionColName, table.getName(), partitionColNames));
808+
}
809+
if (!(partitionValue instanceof Literal)) {
810+
throw new AnalysisException(
811+
String.format("Partition value for column '%s' must be a literal, but got: %s",
812+
partitionColName, partitionValue));
813+
}
814+
}
815+
}
816+
727817
private Plan bindIcebergTableSink(MatchingContext<UnboundIcebergTableSink<Plan>> ctx) {
728818
UnboundIcebergTableSink<?> sink = ctx.root;
729819
Pair<IcebergExternalDatabase, IcebergExternalTable> pair = bind(ctx.cascadesContext, sink);

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -375,7 +375,8 @@ private void insertIntoPartitions(ConnectContext ctx, StmtExecutor executor, Lis
375375
false,
376376
TPartialUpdateNewRowPolicy.APPEND,
377377
sink.getDMLCommandType(),
378-
(LogicalPlan) (sink.child(0)));
378+
(LogicalPlan) (sink.child(0)),
379+
sink.getStaticPartitionKeyValues());
379380
insertCtx = new HiveInsertCommandContext();
380381
((HiveInsertCommandContext) insertCtx).setOverwrite(true);
381382
} else if (logicalQuery instanceof UnboundIcebergTableSink) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@
9999

100100
import java.util.ArrayList;
101101
import java.util.List;
102+
import java.util.Locale;
102103
import java.util.Map;
103104
import java.util.Optional;
104105
import java.util.Set;
@@ -379,12 +380,24 @@ private static Plan normalizePlanWithoutLock(LogicalPlan plan, TableIf table,
379380
staticPartitions = ((UnboundIcebergTableSink<?>) unboundLogicalSink).getStaticPartitionKeyValues();
380381
} else if (unboundLogicalSink instanceof UnboundMaxComputeTableSink) {
381382
staticPartitions = ((UnboundMaxComputeTableSink<?>) unboundLogicalSink).getStaticPartitionKeyValues();
383+
} else if (unboundLogicalSink instanceof UnboundHiveTableSink) {
384+
staticPartitions = ((UnboundHiveTableSink<?>) unboundLogicalSink).getStaticPartitionKeyValues();
382385
}
383386
if (staticPartitions != null && !staticPartitions.isEmpty()
384387
&& CollectionUtils.isEmpty(unboundLogicalSink.getColNames())) {
385-
Set<String> staticPartitionColNames = staticPartitions.keySet();
388+
// Static partition columns get their values from the PARTITION clause instead of the
389+
// inline VALUES list, so they must be excluded from the implicit target schema before
390+
// the column-count check below. Otherwise e.g. Hive table (v, dt) with
391+
// PARTITION(dt='x') VALUES (1) would be checked against 2 columns and fail with
392+
// "Column count doesn't match value count".
393+
// Match case-insensitively via Locale.ROOT because partition column names are
394+
// case-insensitive (Hive stores them lowercase in HMS), avoiding locale-dependent
395+
// folding (e.g. the Turkish 'I').
396+
Set<String> staticPartitionColNames = staticPartitions.keySet().stream()
397+
.map(name -> name.toLowerCase(Locale.ROOT))
398+
.collect(Collectors.toSet());
386399
columns = columns.stream()
387-
.filter(column -> !staticPartitionColNames.contains(column.getName()))
400+
.filter(column -> !staticPartitionColNames.contains(column.getName().toLowerCase(Locale.ROOT)))
388401
.collect(ImmutableList.toImmutableList());
389402
}
390403

0 commit comments

Comments
 (0)