Skip to content

Commit bf278a5

Browse files
authored
[feature](iceberg) Support whole-file metadata delete for DELETE (apache#69176)
### What problem does this PR solve? Issue Number: N/A Problem Summary: For an Iceberg table, a row-level DELETE always scans the matching rows and writes delete files (position deletes on v2, deletion vectors on v3), so a data file whose rows are all gone stays in the snapshot and every later scan still reads it. This PR adds the whole-file "metadata delete" fast path that Spark / Trino use for whole-partition and whole-file predicates: the DELETE is committed as one metadata operation that drops the matching data files, so the statement reads no rows, writes no delete files, and later scans no longer read those files. - The engine only tries the fast path when the statement predicate can be extracted exactly (single-table DELETE, leaf is the target table scan, every conjunct representable). Anything else keeps the current row-level path. `PARTITION(...)` scoped deletes, UPDATE and MERGE are unchanged. - The Iceberg connector lowers the predicate with the strictly all-or-nothing rewrite matrix and only commits when every candidate file is matched in full (whole-partition shortcut, strict partition projection or file metrics); otherwise it falls back to the row-level delete, so a partially matched file is never dropped. - Like the engines above, the committed snapshot applies the predicate to the current table state, and the reported row count is iceberg's `deleted-records` minus the rows the removed delete files had already hidden. ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [x] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [x] Yes. - Does this need documentation? - [x] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label
1 parent 883bf91 commit bf278a5

21 files changed

Lines changed: 1629 additions & 46 deletions

File tree

‎fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@
100100
import java.util.OptionalLong;
101101
import java.util.Set;
102102
import java.util.TreeMap;
103+
import java.util.function.BooleanSupplier;
103104
import java.util.function.Function;
104105
import java.util.function.Supplier;
105106
import java.util.function.ToLongBiFunction;
@@ -2149,6 +2150,21 @@ public void validateRowLevelDmlMode(ConnectorSession session, ConnectorTableHand
21492150
// permissive here -- a throw would newly reject legal plain-hive row-level DML.
21502151
}
21512152

2153+
/**
2154+
* A foreign Iceberg handle must use the sibling's whole-file metadata-delete implementation. The Hive gateway
2155+
* itself has no data-file delete contract, so a real Hive handle keeps the SPI default of falling back.
2156+
*/
2157+
@Override
2158+
public OptionalLong deleteUsingMetadata(ConnectorSession session, ConnectorTableHandle handle,
2159+
Optional<ConnectorExpression> predicate, String boundWriteMetadataIdentity,
2160+
BooleanSupplier isCancelled) {
2161+
if (!(handle instanceof HiveTableHandle)) {
2162+
return siblingMetadata(session, handle).deleteUsingMetadata(
2163+
session, handle, predicate, boundWriteMetadataIdentity, isCancelled);
2164+
}
2165+
return OptionalLong.empty();
2166+
}
2167+
21522168
/**
21532169
* Rejects an illegal static-partition column on {@code INSERT [OVERWRITE] ... PARTITION (col=val)}: the
21542170
* column must be a partition key of this (partitioned) table. Port of upstream #65991's

‎fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataSiblingDelegationTest.java‎

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@
7070
import java.util.OptionalLong;
7171
import java.util.Set;
7272
import java.util.TreeSet;
73+
import java.util.function.BooleanSupplier;
7374
import java.util.function.Supplier;
7475
import java.util.regex.Matcher;
7576
import java.util.regex.Pattern;
@@ -356,12 +357,12 @@ public void foreignHandleFailsLoudWhenNoSiblingConfigured() {
356357
}
357358

358359
@Test
359-
public void everyAlterDdlAndValidateMethodForwardsAForeignHandleToTheSibling() {
360+
public void everyForeignWriteOperationForwardsAForeignHandleToTheSibling() {
360361
HiveConnectorMetadata md = withSibling();
361362

362-
// The 14 ALTER-DDL mutators + 2 write validators: a foreign (iceberg-on-HMS) handle must divert, never
363-
// run the hive branch and never be cast. Change objects are null — the guard fires on the handle type
364-
// before any param is touched.
363+
// The 14 ALTER-DDL mutators + 2 write validators + metadata delete: a foreign (iceberg-on-HMS) handle
364+
// must divert, never run the hive branch and never be cast. Change objects are null — the guard fires on
365+
// the handle type before any param is touched.
365366
md.renameTable(session, foreignHandle, "new");
366367
md.setTableProperties(session, foreignHandle, Collections.emptyMap());
367368
md.addColumn(session, foreignHandle, null, null);
@@ -387,9 +388,21 @@ public void everyAlterDdlAndValidateMethodForwardsAForeignHandleToTheSibling() {
387388
// Empty list on purpose: a foreign handle must forward REGARDLESS of emptiness (the empty-early-return is
388389
// hive-only) — this would fail if the empty check were placed before the foreign-handle divert.
389390
md.validateWritePartitionNames(session, foreignHandle, Collections.emptyList());
391+
ConnectorExpression predicate = new ConnectorColumnRef("id", ConnectorType.of("INT"));
392+
BooleanSupplier cancellation = () -> true;
393+
OptionalLong deleted = md.deleteUsingMetadata(
394+
session, foreignHandle, Optional.of(predicate), "bound-generation", cancellation);
390395

391396
Assertions.assertEquals(RecordingSiblingMetadata.EXPECTED_WRITE_METHODS, siblingMetadata.calls,
392397
"every ALTER-DDL mutator + write validator must forward a foreign handle to the sibling");
398+
Assertions.assertEquals(OptionalLong.of(RecordingSiblingMetadata.SENTINEL_DELETED_ROWS), deleted,
399+
"metadata delete must return the sibling's result");
400+
Assertions.assertSame(predicate, siblingMetadata.lastDeletePredicate,
401+
"metadata delete must forward the exact predicate");
402+
Assertions.assertEquals("bound-generation", siblingMetadata.lastBoundWriteMetadataIdentity,
403+
"metadata delete must forward the write-generation fence");
404+
Assertions.assertSame(cancellation, siblingMetadata.lastCancellation,
405+
"metadata delete must forward the cancellation callback");
393406
}
394407

395408
@Test
@@ -451,6 +464,9 @@ public void hiveHandleAlterDdlThrowsAndValidateIsNoopAndNeverConsultsSibling() {
451464
// plain-hive row-level DML / static-partition INSERTs.
452465
md.validateRowLevelDmlMode(session, hive, null);
453466
md.validateStaticPartitionColumns(session, hive, Collections.emptyList());
467+
Assertions.assertEquals(OptionalLong.empty(),
468+
md.deleteUsingMetadata(session, hive, Optional.empty(), null, () -> false),
469+
"a real Hive handle must keep the metadata-delete fallback");
454470

455471
Assertions.assertEquals(0, siblingConnector.getMetadataCount,
456472
"a hive handle must never build/consult the iceberg sibling for ALTER-DDL / validate");
@@ -721,6 +737,7 @@ private static final class RecordingSiblingMetadata implements ConnectorMetadata
721737
static final ConnectorTransaction SIBLING_TXN = new NoOpConnectorTransaction(4243L, "ICEBERG");
722738
static final long SENTINEL_SIZE = 4242L;
723739
static final long SENTINEL_SNAPSHOT_ID = 99L;
740+
static final long SENTINEL_DELETED_ROWS = 31L;
724741
static final List<Long> SENTINEL_FILE_SIZES = Collections.unmodifiableList(Arrays.asList(7L, 11L));
725742
static final List<ConnectorExpression> SIBLING_PREDICATES = Collections.singletonList(
726743
new ConnectorColumnRef("sibling-pred", ConnectorType.of("STRING")));
@@ -747,9 +764,13 @@ private static final class RecordingSiblingMetadata implements ConnectorMetadata
747764
"modifyNestedColumn", "modifyColumnComment",
748765
"createOrReplaceBranch", "createOrReplaceTag", "dropBranch", "dropTag",
749766
"addPartitionField", "dropPartitionField", "replacePartitionField",
750-
"validateRowLevelDmlMode", "validateStaticPartitionColumns", "validateWritePartitionNames"));
767+
"validateRowLevelDmlMode", "validateStaticPartitionColumns", "validateWritePartitionNames",
768+
"deleteUsingMetadata"));
751769

752770
final List<String> calls = new ArrayList<>();
771+
ConnectorExpression lastDeletePredicate;
772+
String lastBoundWriteMetadataIdentity;
773+
BooleanSupplier lastCancellation;
753774
final Optional<FilterApplicationResult<ConnectorTableHandle>> filterResult =
754775
Optional.of(new FilterApplicationResult<>(SIBLING_HANDLE, null, false));
755776

@@ -1069,5 +1090,16 @@ public void validateWritePartitionNames(ConnectorSession session, ConnectorTable
10691090
List<String> partitionNames) {
10701091
calls.add("validateWritePartitionNames");
10711092
}
1093+
1094+
@Override
1095+
public OptionalLong deleteUsingMetadata(ConnectorSession session, ConnectorTableHandle handle,
1096+
Optional<ConnectorExpression> predicate, String boundWriteMetadataIdentity,
1097+
BooleanSupplier isCancelled) {
1098+
calls.add("deleteUsingMetadata");
1099+
lastDeletePredicate = predicate.orElse(null);
1100+
lastBoundWriteMetadataIdentity = boundWriteMetadataIdentity;
1101+
lastCancellation = isCancelled;
1102+
return OptionalLong.of(SENTINEL_DELETED_ROWS);
1103+
}
10721104
}
10731105
}

0 commit comments

Comments
 (0)