Skip to content

Commit 6efd95c

Browse files
wuwenchimorningman
authored andcommitted
[opt](iceberg) support insert to branch (apache#52751)
Related Issue: apache#48285 Problem Summary: support insert to branch, like : ```sql insert into tb@branch(br1); ```
1 parent e685b48 commit 6efd95c

15 files changed

Lines changed: 908 additions & 39 deletions

File tree

‎fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,9 +130,13 @@ constraintStatement
130130
| SHOW CONSTRAINTS FROM table=multipartIdentifier #showConstraint
131131
;
132132

133+
optSpecBranch
134+
: ATSIGN BRANCH LEFT_PAREN name=identifier RIGHT_PAREN
135+
;
136+
133137
supportedDmlStatement
134138
: explain? cte? INSERT (INTO | OVERWRITE TABLE)
135-
(tableName=multipartIdentifier | DORIS_INTERNAL_TABLE_ID LEFT_PAREN tableId=INTEGER_VALUE RIGHT_PAREN)
139+
(tableName=multipartIdentifier (optSpecBranch)? | DORIS_INTERNAL_TABLE_ID LEFT_PAREN tableId=INTEGER_VALUE RIGHT_PAREN)
136140
partitionSpec? // partition define
137141
(WITH LABEL labelName=identifier)? cols=identifierList? // label and columns define
138142
(LEFT_BRACKET hints=identifierSeq RIGHT_BRACKET)? // hint define

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java‎

Lines changed: 39 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
import org.apache.doris.datasource.ExternalTable;
2525
import org.apache.doris.datasource.NameMapping;
2626
import org.apache.doris.datasource.iceberg.helper.IcebergWriterHelper;
27-
import org.apache.doris.nereids.trees.plans.commands.insert.BaseExternalTableInsertCommandContext;
27+
import org.apache.doris.nereids.trees.plans.commands.insert.IcebergInsertCommandContext;
2828
import org.apache.doris.nereids.trees.plans.commands.insert.InsertCommandContext;
2929
import org.apache.doris.thrift.TIcebergCommitData;
3030
import org.apache.doris.thrift.TUpdateMode;
@@ -38,6 +38,7 @@
3838
import org.apache.iceberg.OverwriteFiles;
3939
import org.apache.iceberg.PartitionSpec;
4040
import org.apache.iceberg.ReplacePartitions;
41+
import org.apache.iceberg.SnapshotRef;
4142
import org.apache.iceberg.Table;
4243
import org.apache.iceberg.io.CloseableIterable;
4344
import org.apache.iceberg.io.WriteResult;
@@ -61,6 +62,9 @@ public class IcebergTransaction implements Transaction {
6162
private org.apache.iceberg.Transaction transaction;
6263
private final List<TIcebergCommitData> commitDataList = Lists.newArrayList();
6364

65+
private IcebergInsertCommandContext insertCtx;
66+
private String branchName;
67+
6468
public IcebergTransaction(IcebergMetadataOps ops) {
6569
this.ops = ops;
6670
}
@@ -71,29 +75,43 @@ public void updateIcebergCommitData(List<TIcebergCommitData> commitDataList) {
7175
}
7276
}
7377

74-
public void beginInsert(ExternalTable dorisTable) throws UserException {
78+
public void beginInsert(ExternalTable dorisTable, Optional<InsertCommandContext> ctx) throws UserException {
79+
ctx.ifPresent(c -> this.insertCtx = (IcebergInsertCommandContext) c);
7580
try {
7681
ops.getPreExecutionAuthenticator().execute(() -> {
7782
// create and start the iceberg transaction
7883
this.table = IcebergUtils.getIcebergTable(dorisTable);
84+
// check branch
85+
if (insertCtx != null && insertCtx.getBranchName().isPresent()) {
86+
this.branchName = insertCtx.getBranchName().get();
87+
SnapshotRef branchRef = table.refs().get(branchName);
88+
if (branchRef == null) {
89+
throw new RuntimeException(branchName + " is not founded in " + dorisTable.getName());
90+
} else if (!branchRef.isBranch()) {
91+
throw new RuntimeException(
92+
branchName
93+
+ " is a tag, not a branch. Tags cannot be targets for producing snapshots");
94+
}
95+
}
7996
this.transaction = table.newTransaction();
8097
});
8198
} catch (Exception e) {
82-
throw new UserException("Failed to begin insert for iceberg table " + dorisTable.getName(), e);
99+
throw new UserException("Failed to begin insert for iceberg table " + dorisTable.getName()
100+
+ "because: " + e.getMessage(), e);
83101
}
84102

85103
}
86104

87-
public void finishInsert(NameMapping nameMapping, Optional<InsertCommandContext> insertCtx) {
105+
public void finishInsert(NameMapping nameMapping) {
88106
if (LOG.isDebugEnabled()) {
89107
LOG.info("iceberg table {} insert table finished!", nameMapping.getFullLocalName());
90108
}
91109
try {
92110
ops.getPreExecutionAuthenticator().execute(() -> {
93111
//create and start the iceberg transaction
94112
TUpdateMode updateMode = TUpdateMode.APPEND;
95-
if (insertCtx.isPresent()) {
96-
updateMode = ((BaseExternalTableInsertCommandContext) insertCtx.get()).isOverwrite()
113+
if (insertCtx != null) {
114+
updateMode = insertCtx.isOverwrite()
97115
? TUpdateMode.OVERWRITE
98116
: TUpdateMode.APPEND;
99117
}
@@ -146,6 +164,9 @@ public long getUpdateCnt() {
146164
private void commitAppendTxn(List<WriteResult> pendingResults) {
147165
// commit append files.
148166
AppendFiles appendFiles = transaction.newAppend().scanManifestsWith(ops.getThreadPoolWithPreAuth());
167+
if (branchName != null) {
168+
appendFiles = appendFiles.toBranch(branchName);
169+
}
149170
for (WriteResult result : pendingResults) {
150171
Preconditions.checkState(result.referencedDataFiles().length == 0,
151172
"Should have no referenced data files for append.");
@@ -161,11 +182,14 @@ private void commitReplaceTxn(List<WriteResult> pendingResults) {
161182
// 1. if dst_tb is a partitioned table, it will return directly.
162183
// 2. if dst_tb is an unpartitioned table, the `dst_tb` table will be emptied.
163184
if (!transaction.table().spec().isPartitioned()) {
164-
OverwriteFiles overwriteFiles = transaction
165-
.newOverwrite()
166-
.scanManifestsWith(ops.getThreadPoolWithPreAuth());
185+
OverwriteFiles overwriteFiles = transaction.newOverwrite();
186+
if (branchName != null) {
187+
overwriteFiles = overwriteFiles.toBranch(branchName);
188+
}
189+
overwriteFiles = overwriteFiles.scanManifestsWith(ops.getThreadPoolWithPreAuth());
167190
try (CloseableIterable<FileScanTask> fileScanTasks = table.newScan().planFiles()) {
168-
fileScanTasks.forEach(f -> overwriteFiles.deleteFile(f.file()));
191+
OverwriteFiles finalOverwriteFiles = overwriteFiles;
192+
fileScanTasks.forEach(f -> finalOverwriteFiles.deleteFile(f.file()));
169193
} catch (IOException e) {
170194
throw new RuntimeException(e);
171195
}
@@ -175,9 +199,11 @@ private void commitReplaceTxn(List<WriteResult> pendingResults) {
175199
}
176200

177201
// commit replace partitions
178-
ReplacePartitions appendPartitionOp = transaction
179-
.newReplacePartitions()
180-
.scanManifestsWith(ops.getThreadPoolWithPreAuth());
202+
ReplacePartitions appendPartitionOp = transaction.newReplacePartitions();
203+
if (branchName != null) {
204+
appendPartitionOp = appendPartitionOp.toBranch(branchName);
205+
}
206+
appendPartitionOp = appendPartitionOp.scanManifestsWith(ops.getThreadPoolWithPreAuth());
181207
for (WriteResult result : pendingResults) {
182208
Preconditions.checkState(result.referencedDataFiles().length == 0,
183209
"Should have no referenced data files.");

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -610,6 +610,10 @@ public LogicalPlan visitInsertTable(InsertTableContext ctx) {
610610
throw new ParseException("tableName and tableId cannot both be null");
611611
}
612612
Optional<String> labelName = (ctx.labelName == null) ? Optional.empty() : Optional.of(ctx.labelName.getText());
613+
Optional<String> branchName = Optional.empty();
614+
if (ctx.optSpecBranch() != null) {
615+
branchName = Optional.of(ctx.optSpecBranch().name.getText());
616+
}
613617
List<String> colNames = ctx.cols == null ? ImmutableList.of() : visitIdentifierList(ctx.cols);
614618
// TODO visit partitionSpecCtx
615619
LogicalPlan plan = visitQuery(ctx.query());
@@ -638,7 +642,7 @@ public LogicalPlan visitInsertTable(InsertTableContext ctx) {
638642
}
639643
LogicalPlan command;
640644
if (isOverwrite) {
641-
command = new InsertOverwriteTableCommand(sink, labelName, cte);
645+
command = new InsertOverwriteTableCommand(sink, labelName, cte, branchName);
642646
} else {
643647
if (ConnectContext.get() != null && ConnectContext.get().isTxnModel()
644648
&& sink.child() instanceof InlineTable
@@ -648,7 +652,7 @@ public LogicalPlan visitInsertTable(InsertTableContext ctx) {
648652
// Now handle it as `insert into select`(a separate load job), should fix it as the legacy.
649653
command = new BatchInsertIntoTableCommand(sink);
650654
} else {
651-
command = new InsertIntoTableCommand(sink, labelName, Optional.empty(), cte);
655+
command = new InsertIntoTableCommand(sink, labelName, Optional.empty(), cte, true, branchName);
652656
}
653657
}
654658
return withExplain(command, ctx.explain());

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -176,8 +176,8 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
176176
ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), query);
177177
try {
178178
if (!FeConstants.runningUnitTest) {
179-
new InsertIntoTableCommand(query, Optional.empty(), Optional.empty(), Optional.empty()).run(
180-
ctx, executor);
179+
new InsertIntoTableCommand(query, Optional.empty(), Optional.empty(),
180+
Optional.empty(), true, Optional.empty()).run(ctx, executor);
181181
}
182182
if (ctx.getState().getStateType() == MysqlStateType.ERR) {
183183
handleFallbackFailedCtas(ctx);

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromUsingCommand.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
5454
}
5555
// NOTE: delete from using command is executed as insert command, so txn insert can support it
5656
new InsertIntoTableCommand(completeQueryPlan(ctx, logicalQuery), Optional.empty(), Optional.empty(),
57-
Optional.empty()).run(ctx, executor);
57+
Optional.empty(), true, Optional.empty()).run(ctx, executor);
5858
}
5959

6060
@Override

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ public UpdateCommand(List<String> nameParts, @Nullable String tableAlias, List<E
9797
public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
9898
// NOTE: update command is executed as insert command, so txn insert can support it
9999
new InsertIntoTableCommand(completeQueryPlan(ctx, logicalQuery), Optional.empty(), Optional.empty(),
100-
Optional.empty()).run(ctx, executor);
100+
Optional.empty(), true, Optional.empty()).run(ctx, executor);
101101
}
102102

103103
/**

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ public class UpdateMvByPartitionCommand extends InsertOverwriteTableCommand {
8686
private static final Logger LOG = LogManager.getLogger(UpdateMvByPartitionCommand.class);
8787

8888
private UpdateMvByPartitionCommand(LogicalPlan logicalQuery) {
89-
super(logicalQuery, Optional.empty(), Optional.empty());
89+
super(logicalQuery, Optional.empty(), Optional.empty(), Optional.empty());
9090
}
9191

9292
@Override

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergInsertCommandContext.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,19 @@
1717

1818
package org.apache.doris.nereids.trees.plans.commands.insert;
1919

20+
import java.util.Optional;
21+
2022
/**
2123
* For iceberg External Table
2224
*/
2325
public class IcebergInsertCommandContext extends BaseExternalTableInsertCommandContext {
26+
private Optional<String> branchName = Optional.empty();
27+
28+
public Optional<String> getBranchName() {
29+
return branchName;
30+
}
31+
32+
public void setBranchName(Optional<String> branchName) {
33+
this.branchName = branchName;
34+
}
2435
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergInsertExecutor.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ public IcebergInsertExecutor(ConnectContext ctx, IcebergExternalTable table,
4949
@Override
5050
protected void beforeExec() throws UserException {
5151
IcebergTransaction transaction = (IcebergTransaction) transactionManager.getTransaction(txnId);
52-
transaction.beginInsert((IcebergExternalTable) table);
52+
transaction.beginInsert((IcebergExternalTable) table, insertCtx);
5353
}
5454

5555
@Override
@@ -60,7 +60,7 @@ protected void doBeforeCommit() throws UserException {
6060
dorisTable.getRemoteDbName(), dorisTable.getRemoteName());
6161
IcebergTransaction transaction = (IcebergTransaction) transactionManager.getTransaction(txnId);
6262
this.loadedRows = transaction.getUpdateCnt();
63-
transaction.finishInsert(nameMapping, insertCtx);
63+
transaction.finishInsert(nameMapping);
6464
}
6565

6666
@Override

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ public class InsertIntoTableCommand extends Command implements ForwardWithSync,
9292
private LogicalPlan originLogicalQuery;
9393
private Optional<LogicalPlan> logicalQuery;
9494
private Optional<String> labelName;
95+
private Optional<String> branchName;
9596
/**
9697
* When source it's from job scheduler,it will be set.
9798
*/
@@ -102,22 +103,23 @@ public class InsertIntoTableCommand extends Command implements ForwardWithSync,
102103

103104
public InsertIntoTableCommand(LogicalPlan logicalQuery, Optional<String> labelName,
104105
Optional<InsertCommandContext> insertCtx, Optional<LogicalPlan> cte) {
105-
this(logicalQuery, labelName, insertCtx, cte, true);
106+
this(logicalQuery, labelName, insertCtx, cte, true, Optional.empty());
106107
}
107108

108109
/**
109110
* constructor
110111
*/
111112
public InsertIntoTableCommand(LogicalPlan logicalQuery, Optional<String> labelName,
112113
Optional<InsertCommandContext> insertCtx, Optional<LogicalPlan> cte,
113-
boolean needNormalizePlan) {
114+
boolean needNormalizePlan, Optional<String> branchName) {
114115
super(PlanType.INSERT_INTO_TABLE_COMMAND);
115116
this.originLogicalQuery = Objects.requireNonNull(logicalQuery, "logicalQuery should not be null");
116117
this.labelName = Objects.requireNonNull(labelName, "labelName should not be null");
117118
this.logicalQuery = Optional.empty();
118119
this.insertCtx = insertCtx;
119120
this.cte = cte;
120121
this.needNormalizePlan = needNormalizePlan;
122+
this.branchName = branchName;
121123
}
122124

123125
public LogicalPlan getLogicalQuery() {
@@ -284,6 +286,11 @@ private ExecutorFactory selectInsertExecutorFactory(
284286
String label = this.labelName.orElse(
285287
ctx.isTxnModel() ? null : String.format("label_%x_%x", ctx.queryId().hi, ctx.queryId().lo));
286288

289+
// check branch
290+
if (branchName.isPresent() && !(physicalSink instanceof PhysicalIcebergTableSink)) {
291+
throw new AnalysisException("Only support insert data into iceberg table's branch");
292+
}
293+
287294
if (physicalSink instanceof PhysicalOlapTableSink) {
288295
boolean emptyInsert = childIsEmptyRelation(physicalSink);
289296
OlapTable olapTable = (OlapTable) targetTableIf;
@@ -339,12 +346,16 @@ private ExecutorFactory selectInsertExecutorFactory(
339346
} else if (physicalSink instanceof PhysicalIcebergTableSink) {
340347
boolean emptyInsert = childIsEmptyRelation(physicalSink);
341348
IcebergExternalTable icebergExternalTable = (IcebergExternalTable) targetTableIf;
349+
IcebergInsertCommandContext icebergInsertCtx = insertCtx
350+
.map(insertCommandContext -> (IcebergInsertCommandContext) insertCommandContext)
351+
.orElseGet(IcebergInsertCommandContext::new);
352+
branchName.ifPresent(notUsed -> icebergInsertCtx.setBranchName(branchName));
342353
return ExecutorFactory.from(
343354
planner,
344355
dataSink,
345356
physicalSink,
346357
() -> new IcebergInsertExecutor(ctx, icebergExternalTable, label, planner,
347-
Optional.of(insertCtx.orElse((new BaseExternalTableInsertCommandContext()))),
358+
Optional.of(icebergInsertCtx),
348359
emptyInsert
349360
)
350361
);

0 commit comments

Comments
 (0)