Skip to content

Commit fd06842

Browse files
committed
[feature](paimon) Add connector transaction and write planning
Issue Number: close apache#65086 Related PR: apache#65868, apache#66498, apache#66685 Problem Summary: Forward-port the connector transaction, commit-message bridge, connector-neutral write distribution contract, and Paimon write-plan selection needed to execute table writes on master. The plan preserves native Paimon fixed-bucket routing and exposes row-change capabilities for subsequent DML planning. Support planning and committing Paimon table writes through the connector framework. - Test: Manual test - FE checkstyle passed; final compilation is deferred until all forward-port picks are complete. - Behavior changed: Yes, Paimon writes now use connector transactions and native write distribution planning. - Does this need documentation: No
1 parent 5216ba7 commit fd06842

7 files changed

Lines changed: 923 additions & 1 deletion

File tree

‎fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.apache.doris.connector.spi.ConnectorValidationContext;
3636
import org.apache.doris.connector.spi.handle.ConnectorTableHandle;
3737
import org.apache.doris.connector.spi.scan.ConnectorScanPlanProvider;
38+
import org.apache.doris.connector.spi.write.ConnectorWritePlanProvider;
3839
import org.apache.doris.filesystem.Location;
3940
import org.apache.doris.filesystem.properties.StorageProperties;
4041
import org.apache.doris.kerberos.AuthType;
@@ -355,11 +356,18 @@ public ConnectorScanPlanProvider getScanPlanProvider() {
355356
context, schemaAtMemo);
356357
}
357358

359+
@Override
360+
public ConnectorWritePlanProvider getWritePlanProvider() {
361+
return new PaimonWritePlanProvider(catalogProps,
362+
new PaimonCatalogOps.CatalogBackedPaimonCatalogOps(ensureCatalog(), tableOptions),
363+
context);
364+
}
365+
358366
/**
359367
* Declares the E5 read-path capabilities paimon supports: MVCC snapshot pinning. The B5 fe-core
360368
* MvccTable wiring keys off this to call {@link PaimonConnectorMetadata#beginQuerySnapshot} /
361369
* {@code resolveTimeTravel}.
362-
* No write capability is declared: paimon write is not migrated.
370+
* Write support is declared by {@link #getWritePlanProvider()}.
363371
*/
364372
@Override
365373
public Set<ConnectorCapability> getCapabilities() {

‎fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.apache.doris.connector.spi.ddl.ConnectorCreateTableRequest;
3232
import org.apache.doris.connector.spi.handle.ConnectorColumnHandle;
3333
import org.apache.doris.connector.spi.handle.ConnectorTableHandle;
34+
import org.apache.doris.connector.spi.handle.ConnectorTransaction;
3435
import org.apache.doris.connector.spi.mvcc.ConnectorMvccSnapshot;
3536
import org.apache.doris.connector.spi.mvcc.ConnectorTimeTravelSpec;
3637
import org.apache.doris.connector.spi.pushdown.ConnectorExpression;
@@ -167,6 +168,27 @@ public PaimonConnectorMetadata(PaimonCatalogOps catalogOps, PaimonCatalogPropert
167168
this.partitionViewCache = partitionViewCache;
168169
}
169170

171+
@Override
172+
public ConnectorTransaction beginTransaction(ConnectorSession session) {
173+
return new PaimonConnectorTransaction(session.allocateTransactionId(), context);
174+
}
175+
176+
@Override
177+
public void validateStaticPartitionColumns(ConnectorSession session, ConnectorTableHandle handle,
178+
List<String> staticPartitionColumnNames) {
179+
PaimonTableHandle paimonHandle = (PaimonTableHandle) handle;
180+
Set<String> partitionNames = new HashSet<>();
181+
for (String name : paimonHandle.getPartitionKeys()) {
182+
partitionNames.add(name.toLowerCase(java.util.Locale.ROOT));
183+
}
184+
for (String name : staticPartitionColumnNames) {
185+
if (!partitionNames.contains(name.toLowerCase(java.util.Locale.ROOT))) {
186+
throw new DorisConnectorException("Column '" + name
187+
+ "' is not a partition column of Paimon table");
188+
}
189+
}
190+
}
191+
170192
@Override
171193
public List<String> listDatabaseNames(ConnectorSession session) {
172194
// M-11: wrap the remote read in executeAuthenticated so the FE-injected Kerberos UGI applies (legacy
Lines changed: 326 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,326 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.connector.paimon;
19+
20+
import org.apache.doris.connector.spi.ConnectorContext;
21+
import org.apache.doris.connector.spi.DorisConnectorException;
22+
import org.apache.doris.connector.spi.handle.ConnectorTransaction;
23+
import org.apache.doris.thrift.TPaimonCommitMessage;
24+
25+
import com.google.common.base.Preconditions;
26+
import org.apache.logging.log4j.LogManager;
27+
import org.apache.logging.log4j.Logger;
28+
import org.apache.paimon.io.DataInputDeserializer;
29+
import org.apache.paimon.table.sink.CommitMessage;
30+
import org.apache.paimon.table.sink.CommitMessageSerializer;
31+
import org.apache.paimon.table.sink.InnerTableCommit;
32+
import org.apache.thrift.TDeserializer;
33+
import org.apache.thrift.TException;
34+
import org.apache.thrift.protocol.TBinaryProtocol;
35+
36+
import java.io.IOException;
37+
import java.util.ArrayList;
38+
import java.util.Arrays;
39+
import java.util.Collections;
40+
import java.util.HashMap;
41+
import java.util.HashSet;
42+
import java.util.List;
43+
import java.util.Map;
44+
import java.util.Set;
45+
46+
/** Connector-SPI transaction for one Paimon write statement. */
47+
public class PaimonConnectorTransaction implements ConnectorTransaction {
48+
49+
private static final Logger LOG = LogManager.getLogger(PaimonConnectorTransaction.class);
50+
private static final int COMMIT_HEADER_SIZE = 12;
51+
private static final byte[] COMMIT_MAGIC = new byte[] {'D', 'P', 'C', 'M'};
52+
53+
enum CommitState {
54+
PREPARED,
55+
COMMITTING,
56+
COMMITTED,
57+
OUTCOME_UNKNOWN
58+
}
59+
60+
private final long transactionId;
61+
private final String commitUser;
62+
private final ConnectorContext context;
63+
private final List<byte[]> commitPayloads = new ArrayList<>();
64+
private final Set<CommitPayloadKey> commitPayloadSet = new HashSet<>();
65+
private PaimonWriteBinding binding;
66+
private CommitState state = CommitState.PREPARED;
67+
68+
PaimonConnectorTransaction(long transactionId, ConnectorContext context) {
69+
Preconditions.checkArgument(transactionId > 0, "Paimon transaction id must be positive");
70+
this.transactionId = transactionId;
71+
this.context = Preconditions.checkNotNull(context, "Paimon connector context must not be null");
72+
this.commitUser = commitUser(context.getClusterId(), transactionId);
73+
}
74+
75+
synchronized void bind(PaimonWriteBinding writeBinding) {
76+
Preconditions.checkNotNull(writeBinding, "Paimon write binding must not be null");
77+
Preconditions.checkState(binding == null, "Paimon transaction is already bound");
78+
Preconditions.checkState(state == CommitState.PREPARED,
79+
"Paimon transaction can only be bound while prepared");
80+
binding = writeBinding;
81+
}
82+
83+
String getCommitUser() {
84+
return commitUser;
85+
}
86+
87+
static String commitUser(int clusterId, long transactionId) {
88+
return "doris_cluster_" + clusterId + "_txn_" + transactionId;
89+
}
90+
91+
@Override
92+
public long getTransactionId() {
93+
return transactionId;
94+
}
95+
96+
@Override
97+
public void addCommitData(byte[] commitFragment) {
98+
TPaimonCommitMessage message = new TPaimonCommitMessage();
99+
try {
100+
new TDeserializer(new TBinaryProtocol.Factory()).deserialize(message, commitFragment);
101+
} catch (TException e) {
102+
throw new DorisConnectorException("Failed to deserialize Paimon commit message", e);
103+
}
104+
Preconditions.checkState(message.isSetPayload(), "Paimon commit message payload is missing");
105+
byte[] payload = message.getPayload();
106+
Preconditions.checkState(payload.length > 0, "Paimon commit message payload is empty");
107+
synchronized (this) {
108+
if (commitPayloadSet.add(new CommitPayloadKey(payload))) {
109+
commitPayloads.add(payload);
110+
}
111+
}
112+
}
113+
114+
@Override
115+
public void commit() {
116+
PaimonWriteBinding writeBinding = requireBinding();
117+
List<byte[]> payloads = snapshotPayloads();
118+
if (payloads.isEmpty() && !writeBinding.isOverwrite()) {
119+
markPreparedTransactionCommitted();
120+
return;
121+
}
122+
try {
123+
List<CommitMessage> messages = deserializePayloads(payloads);
124+
doCommitWithReconciliation(writeBinding, messages);
125+
} catch (Exception e) {
126+
throw new DorisConnectorException("Failed to commit Paimon transaction on FE", e);
127+
}
128+
}
129+
130+
@Override
131+
public void rollback() {
132+
CommitState current = getState();
133+
if (current == CommitState.COMMITTED) {
134+
return;
135+
}
136+
if (current == CommitState.COMMITTING || current == CommitState.OUTCOME_UNKNOWN) {
137+
LOG.warn("Skip rollback for Paimon transaction in state {}, txnId={}, table={}",
138+
current, transactionId, tableName());
139+
return;
140+
}
141+
List<byte[]> payloads = snapshotPayloads();
142+
if (payloads.isEmpty()) {
143+
return;
144+
}
145+
try {
146+
PaimonWriteBinding writeBinding = requireBinding();
147+
List<CommitMessage> messages = deserializePayloads(payloads);
148+
context.executeAuthenticated(() -> {
149+
try (InnerTableCommit committer = writeBinding.getTable().newCommit(commitUser)) {
150+
committer.abort(messages);
151+
}
152+
return null;
153+
});
154+
} catch (Exception e) {
155+
LOG.warn("Failed to rollback Paimon transaction, txnId={}, table={}",
156+
transactionId, tableName(), e);
157+
}
158+
}
159+
160+
@Override
161+
public void close() {
162+
// The Paimon committer is scoped to commit/rollback and closed there.
163+
}
164+
165+
@Override
166+
public String profileLabel() {
167+
return "PAIMON";
168+
}
169+
170+
private void doCommitWithReconciliation(PaimonWriteBinding writeBinding,
171+
List<CommitMessage> messages) throws Exception {
172+
Exception firstFailure;
173+
try {
174+
doCommit(writeBinding, messages);
175+
return;
176+
} catch (Exception e) {
177+
if (getState() == CommitState.COMMITTED) {
178+
return;
179+
}
180+
if (getState() == CommitState.PREPARED) {
181+
throw e;
182+
}
183+
firstFailure = e;
184+
}
185+
186+
Exception retryFailure;
187+
try {
188+
doCommit(writeBinding, messages);
189+
return;
190+
} catch (Exception e) {
191+
if (getState() == CommitState.COMMITTED) {
192+
return;
193+
}
194+
retryFailure = e;
195+
}
196+
197+
try {
198+
boolean committed = context.executeAuthenticated(() -> !writeBinding.getTable()
199+
.snapshotManager().findSnapshotsForIdentifiers(
200+
commitUser, Collections.singletonList(transactionId)).isEmpty());
201+
if (committed) {
202+
markCommitted();
203+
return;
204+
}
205+
} catch (Exception reconciliationFailure) {
206+
retryFailure.addSuppressed(reconciliationFailure);
207+
}
208+
retryFailure.addSuppressed(firstFailure);
209+
markOutcomeUnknown();
210+
throw retryFailure;
211+
}
212+
213+
private void doCommit(PaimonWriteBinding writeBinding, List<CommitMessage> messages)
214+
throws Exception {
215+
context.executeAuthenticated(() -> {
216+
try (InnerTableCommit committer = writeBinding.getTable().newCommit(commitUser)) {
217+
if (writeBinding.isOverwrite()) {
218+
committer.withOverwrite(writeBinding.getStaticPartition());
219+
}
220+
Map<Long, List<CommitMessage>> commitMap = new HashMap<>();
221+
commitMap.put(transactionId, messages);
222+
markCommitting();
223+
committer.filterAndCommit(commitMap);
224+
markCommitted();
225+
}
226+
return null;
227+
});
228+
}
229+
230+
private synchronized PaimonWriteBinding requireBinding() {
231+
Preconditions.checkState(binding != null,
232+
"Missing Paimon write binding for transaction " + transactionId);
233+
return binding;
234+
}
235+
236+
private synchronized CommitState getState() {
237+
return state;
238+
}
239+
240+
private synchronized void markPreparedTransactionCommitted() {
241+
Preconditions.checkState(state == CommitState.PREPARED,
242+
"Only a prepared Paimon transaction can complete without a commit");
243+
state = CommitState.COMMITTED;
244+
}
245+
246+
private synchronized void markCommitting() {
247+
Preconditions.checkState(state == CommitState.PREPARED || state == CommitState.COMMITTING,
248+
"Cannot enter Paimon commit from state " + state);
249+
state = CommitState.COMMITTING;
250+
}
251+
252+
private synchronized void markCommitted() {
253+
Preconditions.checkState(state == CommitState.COMMITTING,
254+
"Only a committing Paimon transaction can be committed");
255+
state = CommitState.COMMITTED;
256+
}
257+
258+
private synchronized void markOutcomeUnknown() {
259+
Preconditions.checkState(state == CommitState.COMMITTING,
260+
"Only a committing Paimon transaction can have an unknown outcome");
261+
state = CommitState.OUTCOME_UNKNOWN;
262+
}
263+
264+
private synchronized List<byte[]> snapshotPayloads() {
265+
return new ArrayList<>(commitPayloads);
266+
}
267+
268+
private String tableName() {
269+
synchronized (this) {
270+
return binding == null ? "unbound" : binding.tableName();
271+
}
272+
}
273+
274+
static List<CommitMessage> deserializePayloads(List<byte[]> payloads) throws IOException {
275+
List<CommitMessage> messages = new ArrayList<>();
276+
for (byte[] payload : payloads) {
277+
Preconditions.checkState(payload.length >= COMMIT_HEADER_SIZE && hasMagic(payload),
278+
"Invalid Paimon commit message payload header");
279+
int version = readInt(payload, 4);
280+
int length = readInt(payload, 8);
281+
Preconditions.checkState(length >= 0 && payload.length == COMMIT_HEADER_SIZE + length,
282+
"Invalid Paimon commit message payload length");
283+
byte[] raw = Arrays.copyOfRange(payload, COMMIT_HEADER_SIZE, payload.length);
284+
messages.addAll(new CommitMessageSerializer().deserializeList(
285+
version, new DataInputDeserializer(raw)));
286+
}
287+
return messages;
288+
}
289+
290+
private static boolean hasMagic(byte[] payload) {
291+
for (int i = 0; i < COMMIT_MAGIC.length; i++) {
292+
if (payload[i] != COMMIT_MAGIC[i]) {
293+
return false;
294+
}
295+
}
296+
return true;
297+
}
298+
299+
private static int readInt(byte[] payload, int offset) {
300+
return ((payload[offset] & 0xFF) << 24)
301+
| ((payload[offset + 1] & 0xFF) << 16)
302+
| ((payload[offset + 2] & 0xFF) << 8)
303+
| (payload[offset + 3] & 0xFF);
304+
}
305+
306+
private static final class CommitPayloadKey {
307+
private final byte[] payload;
308+
private final int hashCode;
309+
310+
private CommitPayloadKey(byte[] payload) {
311+
this.payload = payload;
312+
this.hashCode = Arrays.hashCode(payload);
313+
}
314+
315+
@Override
316+
public boolean equals(Object obj) {
317+
return obj instanceof CommitPayloadKey
318+
&& Arrays.equals(payload, ((CommitPayloadKey) obj).payload);
319+
}
320+
321+
@Override
322+
public int hashCode() {
323+
return hashCode;
324+
}
325+
}
326+
}

0 commit comments

Comments
 (0)