Skip to content

Commit 0eb5131

Browse files
authored
[fix](iceberg) Honor standard row-level isolation properties (#69702)
### What problem does this PR solve? Issue Number: N/A Related PR: #69551 Problem Summary: Doris Iceberg DELETE, UPDATE, and MERGE all used the non-standard `delete_isolation_level` property. As a result, the standard per-operation table properties `write.delete.isolation-level`, `write.update.isolation-level`, and `write.merge.isolation-level` were ignored. A table configured for `snapshot` isolation therefore still performed serializable data-file conflict validation and rejected normal concurrent appends. This change selects the Iceberg `TableProperties` key based on `WriteOperation` and parses it with `IsolationLevel.fromName()`. The existing delete-file, base-snapshot, conflict-filter, and referenced-data-file validations remain enabled; only `validateNoConflictingDataFiles` is gated on serializable isolation. Invalid isolation-level values now fail explicitly instead of being treated as snapshot isolation. The three disabled regressions from #69551 are enabled. Unit coverage also verifies per-operation key selection, serializable defaults, and invalid values. ### Release note Iceberg DELETE, UPDATE, and MERGE now honor their standard per-operation isolation-level table properties. ### Check List (For Author) - Test - [ ] Regression test - [x] Unit Test: `IcebergConnectorTransactionTest` and `IcebergWriteKnownBugsTest` pass, including all three concurrent-append snapshot-isolation regressions. - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [x] Yes. Snapshot isolation now permits concurrent data-file appends while retaining the other row-delta validations. - Does this need documentation? - [x] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label
1 parent 56dfbb1 commit 0eb5131

7 files changed

Lines changed: 280 additions & 38 deletions

File tree

‎fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java‎

Lines changed: 39 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import org.apache.iceberg.FileFormat;
3939
import org.apache.iceberg.FileScanTask;
4040
import org.apache.iceberg.HasTableOperations;
41+
import org.apache.iceberg.IsolationLevel;
4142
import org.apache.iceberg.ManifestFile;
4243
import org.apache.iceberg.ManifestFiles;
4344
import org.apache.iceberg.ManifestReader;
@@ -129,8 +130,7 @@
129130
public class IcebergConnectorTransaction implements ConnectorTransaction, RewriteCapableTransaction {
130131

131132
private static final Logger LOG = LogManager.getLogger(IcebergConnectorTransaction.class);
132-
private static final String DELETE_ISOLATION_LEVEL = "delete_isolation_level";
133-
private static final String DELETE_ISOLATION_LEVEL_DEFAULT = "serializable";
133+
private static final String LEGACY_DELETE_ISOLATION_LEVEL = "delete_isolation_level";
134134
// Scope of a bare INSERT OVERWRITE (no PARTITION clause). Key + values are byte-identical to the fe-core
135135
// SessionVariable constants (the connector cannot import fe-core); the value reaches the commit through
136136
// ConnectorSession.getProperty, which reads the session-variable registry by name.
@@ -170,6 +170,7 @@ public class IcebergConnectorTransaction implements ConnectorTransaction, Rewrit
170170
// Op context captured at begin time, consumed by commit() (the volatile transaction write at the end of
171171
// beginWrite publishes these plain writes to the commit thread).
172172
private WriteOperation writeOperation = WriteOperation.INSERT;
173+
private IsolationLevel rowLevelIsolationLevel;
173174
private boolean staticPartitionOverwrite;
174175
// True when iceberg_partition_overwrite_mode = 'static': a bare INSERT OVERWRITE clears the whole table
175176
// (every partition spec) instead of replacing only the partitions the new data lands in. Captured at begin
@@ -292,15 +293,23 @@ public void beginWrite(ConnectorSession session, String db, String tableName, Ic
292293
if (writeSchemaContext != null) {
293294
writeSchemaContext.validateCurrentSchema(loaded, ctx.isOverwrite());
294295
}
296+
// Reject invalid isolation before a sink can dispatch BE file writes. Read the SDK
297+
// transaction's refreshed metadata and keep that selection for this statement's commit.
298+
IsolationLevel isolationLevel = null;
299+
if (writeOperation == WriteOperation.DELETE || writeOperation == WriteOperation.UPDATE
300+
|| writeOperation == WriteOperation.MERGE) {
301+
isolationLevel = resolveIsolationLevel(opened.table(), writeOperation);
302+
}
295303
synchronized (beginLock) {
296304
if (closed) {
297305
this.table = null;
298306
throw new IllegalStateException(
299307
"Iceberg transaction closed while its write table was loading");
300308
}
301-
this.transaction = opened;
309+
this.rowLevelIsolationLevel = isolationLevel;
302310
this.tableLease = retained;
303311
retained = null;
312+
this.transaction = opened;
304313
writeStarted = true;
305314
}
306315
return null;
@@ -1235,7 +1244,7 @@ private void applyRowDeltaValidations(RowDelta rowDelta, Table icebergTable,
12351244
List<TIcebergCommitData> commitData, List<String> referencedDataFiles) {
12361245
applyBaseSnapshotValidation(rowDelta);
12371246
applyConflictDetectionFilter(rowDelta, icebergTable, commitData);
1238-
if (isSerializableIsolationLevel(icebergTable)) {
1247+
if (rowLevelIsolationLevel == IsolationLevel.SERIALIZABLE) {
12391248
rowDelta.validateNoConflictingDataFiles();
12401249
}
12411250
rowDelta.validateDeletedFiles();
@@ -1390,13 +1399,32 @@ private List<String> extractPartitionValues(TIcebergCommitData commitData) {
13901399
return Collections.emptyList();
13911400
}
13921401

1393-
boolean isSerializableIsolationLevel(Table icebergTable) {
1394-
if (icebergTable == null) {
1395-
return true;
1396-
}
1397-
String level = icebergTable.properties()
1398-
.getOrDefault(DELETE_ISOLATION_LEVEL, DELETE_ISOLATION_LEVEL_DEFAULT);
1399-
return "serializable".equalsIgnoreCase(level);
1402+
IsolationLevel resolveIsolationLevel(Table icebergTable, WriteOperation operation) {
1403+
String property;
1404+
String defaultLevel;
1405+
switch (operation) {
1406+
case DELETE:
1407+
property = TableProperties.DELETE_ISOLATION_LEVEL;
1408+
defaultLevel = TableProperties.DELETE_ISOLATION_LEVEL_DEFAULT;
1409+
break;
1410+
case UPDATE:
1411+
property = TableProperties.UPDATE_ISOLATION_LEVEL;
1412+
defaultLevel = TableProperties.UPDATE_ISOLATION_LEVEL_DEFAULT;
1413+
break;
1414+
case MERGE:
1415+
property = TableProperties.MERGE_ISOLATION_LEVEL;
1416+
defaultLevel = TableProperties.MERGE_ISOLATION_LEVEL_DEFAULT;
1417+
break;
1418+
default:
1419+
throw new IllegalArgumentException(
1420+
"Unsupported iceberg row-level write operation: " + operation);
1421+
}
1422+
// Standard per-operation settings take precedence; existing legacy-only tables retain their
1423+
// configured isolation until they migrate. Parse only the selected value, including invalid ones.
1424+
Map<String, String> properties = icebergTable.properties();
1425+
String level = properties.getOrDefault(property,
1426+
properties.getOrDefault(LEGACY_DELETE_ISOLATION_LEVEL, defaultLevel));
1427+
return IsolationLevel.fromName(level);
14001428
}
14011429

14021430
// ─────────────────── V3 deletion-vector "rewrite previous delete files" (legacy :786-851) ───────────────────

0 commit comments

Comments
 (0)