Skip to content

Commit 71f51ed

Browse files
morningmanclaude
andcommitted
[fix](iceberg) Keep the promoted type of a dropped equality-delete key
### What problem does this PR solve? Issue Number: None Related PR: #66538 Problem Summary: Review follow-up. The previous commit resolves an equality-delete key that the scan schema no longer has from the scan snapshot's parent chain first. With Iceberg 1.12 a pending RowDelta is stamped with the schema that is current when it commits, and nothing checks its equality field IDs against that schema. So for a key k promoted from INT to LONG by a schema-only change (no snapshot), a writer that wrote LONG tombstones, and k dropped before that delete commits, the delete snapshot records the schema without k and its parent the INT one: no snapshot on the lineage records LONG. The carrier then said INT, FileScannerV2 compares equality keys in the carrier's type, and a tombstone such as 2147483648 was cast to NULL in non-strict mode and deleted every row whose k is NULL (strict mode failed the query). The previous metadata-list lookup found LONG. A dropped key now keeps its lineage definition (name, optionality, initial default) but takes the widest type its field ID has in any retained schema. Field IDs are never reused and promotion is monotonic and lossless, so that type covers every file written for the field, also when it comes from a schema after a time-travel target. ### Release note None ### Check List (For Author) - Test: Unit Test - IcebergScanPlanProviderTest#equalityCarrierKeepsPromotedTypeOfKeyDroppedBeforeItsDeleteCommits replays the sequence with the SDK; it fails without the widening - fe-connector-iceberg module tests - Behavior changed: No - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent dc21748 commit 71f51ed

2 files changed

Lines changed: 74 additions & 9 deletions

File tree

‎fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java‎

Lines changed: 44 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2284,9 +2284,11 @@ private Set<Integer> cachedApplicableEqualityDeleteFieldIds(Table table, TableSc
22842284
* schema, so a schema's position in the metadata list does not tell when it was last active, and the list may
22852285
* also hold schemas committed after a time-travel or branch target. The scan snapshot's parent chain is
22862286
* therefore searched first, so the definition active on that lineage wins; the whole metadata list, newest
2287-
* first, is the fallback for schema-only changes and expired ancestors. Current fields remain first, allowing
2288-
* a dropped/re-added name to resolve the projected field by name while a live historical equality key
2289-
* resolves by its stable field ID.</p>
2287+
* first, is the fallback for schema-only changes and expired ancestors. A dropped key nevertheless takes the
2288+
* widest type it has in any retained schema: a delete committed after a schema-only promotion holds values of
2289+
* the promoted type even when no snapshot records it. Current fields remain first, allowing a dropped/re-added
2290+
* name to resolve the projected field by name while a live historical equality key resolves by its stable
2291+
* field ID.</p>
22902292
*/
22912293
@VisibleForTesting
22922294
static List<NestedField> schemaForPotentialEqualityDeletes(
@@ -2305,6 +2307,8 @@ static List<NestedField> schemaForPotentialEqualityDeletes(
23052307
}
23062308

23072309
List<NestedField> fields = new ArrayList<>(scanSchema.columns());
2310+
List<Schema> metadataSchemas = metadataSchemaHistory(table);
2311+
Map<Integer, Type> widestTypes = widestPromotedTypes(missing, metadataSchemas);
23082312
// Snapshots on one lineage mostly share a schema, so each schema is indexed at most once.
23092313
Set<Integer> lineageSchemaIds = new HashSet<>();
23102314
for (Snapshot snapshot = scanSnapshot; snapshot != null && !missing.isEmpty();
@@ -2318,11 +2322,10 @@ static List<NestedField> schemaForPotentialEqualityDeletes(
23182322
throw new IllegalStateException(
23192323
"Iceberg snapshot schema " + schemaId + " is absent from table metadata");
23202324
}
2321-
addHistoricalEqualityFields(fields, missing, lineageSchema);
2325+
addHistoricalEqualityFields(fields, missing, lineageSchema, widestTypes);
23222326
}
2323-
List<Schema> metadataSchemas = metadataSchemaHistory(table);
23242327
for (int index = metadataSchemas.size() - 1; index >= 0 && !missing.isEmpty(); index--) {
2325-
addHistoricalEqualityFields(fields, missing, metadataSchemas.get(index));
2328+
addHistoricalEqualityFields(fields, missing, metadataSchemas.get(index), widestTypes);
23262329
}
23272330
if (!missing.isEmpty()) {
23282331
throw new IllegalStateException(
@@ -2338,8 +2341,40 @@ private static List<Schema> metadataSchemaHistory(Table table) {
23382341
return new ArrayList<>(table.schemas().values());
23392342
}
23402343

2341-
private static void addHistoricalEqualityFields(
2342-
List<NestedField> fields, Set<Integer> missingFieldIds, Schema historicalSchema) {
2344+
/**
2345+
* Field IDs are never reused and a field's type only changes by promotion, which is monotonic, so the widest
2346+
* type a field has in any retained schema covers every file written for it.
2347+
*/
2348+
private static Map<Integer, Type> widestPromotedTypes(Set<Integer> fieldIds, List<Schema> schemas) {
2349+
Map<Integer, Type> widestTypes = new HashMap<>();
2350+
for (Schema schema : schemas) {
2351+
for (Integer fieldId : fieldIds) {
2352+
Type type = schema.findType(fieldId);
2353+
if (type == null) {
2354+
continue;
2355+
}
2356+
Type widest = widestTypes.get(fieldId);
2357+
if (widest == null || TypeUtil.isPromotionAllowed(widest, type.asPrimitiveType())) {
2358+
widestTypes.put(fieldId, type);
2359+
}
2360+
}
2361+
}
2362+
return widestTypes;
2363+
}
2364+
2365+
private static List<NestedField> withWidestTypes(List<NestedField> fields, Map<Integer, Type> widestTypes) {
2366+
List<NestedField> widened = new ArrayList<>(fields.size());
2367+
for (NestedField field : fields) {
2368+
Type type = field.type().isStructType()
2369+
? Types.StructType.of(withWidestTypes(field.type().asStructType().fields(), widestTypes))
2370+
: widestTypes.getOrDefault(field.fieldId(), field.type());
2371+
widened.add(type.equals(field.type()) ? field : Types.NestedField.from(field).ofType(type).build());
2372+
}
2373+
return widened;
2374+
}
2375+
2376+
private static void addHistoricalEqualityFields(List<NestedField> fields, Set<Integer> missingFieldIds,
2377+
Schema historicalSchema, Map<Integer, Type> widestTypes) {
23432378
Map<Integer, NestedField> historicalFields = TypeUtil.indexById(historicalSchema.asStruct());
23442379
Set<Integer> selectedFieldIds = new HashSet<>();
23452380
for (Integer fieldId : missingFieldIds) {
@@ -2356,7 +2391,7 @@ private static void addHistoricalEqualityFields(
23562391
return;
23572392
}
23582393
Schema selectedSchema = TypeUtil.select(historicalSchema, selectedFieldIds);
2359-
mergeHistoricalEqualityFields(fields, selectedSchema.columns());
2394+
mergeHistoricalEqualityFields(fields, withWidestTypes(selectedSchema.columns(), widestTypes));
23602395
missingFieldIds.removeAll(selectedFieldIds);
23612396
}
23622397

‎fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
import org.apache.iceberg.Metrics;
6363
import org.apache.iceberg.PartitionData;
6464
import org.apache.iceberg.PartitionSpec;
65+
import org.apache.iceberg.RowDelta;
6566
import org.apache.iceberg.Schema;
6667
import org.apache.iceberg.Snapshot;
6768
import org.apache.iceberg.SnapshotSummary;
@@ -2113,6 +2114,35 @@ null, new IcebergTableHandle("db1", "reused_schema_id_eq_key"),
21132114
"the scan dictionary must carry the dropped equality key after the schema ID is reused");
21142115
}
21152116

2117+
@Test
2118+
public void equalityCarrierKeepsPromotedTypeOfKeyDroppedBeforeItsDeleteCommits() throws Exception {
2119+
// A writer started after a schema-only promotion of k to LONG writes LONG tombstones, and k is dropped before
2120+
// its delete commits. The delete snapshot records the schema without k and its parent the INT one, so only
2121+
// the metadata list knows LONG; an INT carrier would narrow 2147483648 to NULL and match NULL keys.
2122+
Schema schema = new Schema(
2123+
Types.NestedField.required(1, "id", Types.IntegerType.get()),
2124+
Types.NestedField.optional(2, "k", Types.IntegerType.get()));
2125+
Table table = createTable("promoted_dropped_eq_key", schema, PartitionSpec.unpartitioned(),
2126+
Collections.singletonMap(TableProperties.FORMAT_VERSION, "2"));
2127+
table.newAppend().appendFile(dataFile(table.spec(),
2128+
"s3://b/db/promoted_dropped_eq_key/f1.parquet", 1024, null, null)).commit();
2129+
int intSchemaId = table.currentSnapshot().schemaId();
2130+
table.updateSchema().updateColumn("k", Types.LongType.get()).commit();
2131+
RowDelta pendingDelete = table.newRowDelta().addDeletes(equalityDeleteFile(
2132+
"s3://b/db/promoted_dropped_eq_key/eq.parquet", FileFormat.PARQUET, 2));
2133+
table.updateSchema().deleteColumn("k").commit();
2134+
pendingDelete.commit();
2135+
Assertions.assertNull(table.schemas().get(table.currentSnapshot().schemaId()).findField(2));
2136+
Assertions.assertEquals(intSchemaId, table.snapshot(table.currentSnapshot().parentId()).schemaId());
2137+
2138+
List<NestedField> carrier = IcebergScanPlanProvider.schemaForPotentialEqualityDeletes(
2139+
table, table.newScan(), table.schema());
2140+
Assertions.assertEquals(2, carrier.size(), "the dropped equality key must stay resolvable: " + carrier);
2141+
Assertions.assertEquals(2, carrier.get(1).fieldId());
2142+
Assertions.assertEquals("k", carrier.get(1).name());
2143+
Assertions.assertEquals(Types.LongType.get(), carrier.get(1).type());
2144+
}
2145+
21162146
@Test
21172147
public void equalityCarrierResolvesReusedSchemaIdAfterParentSnapshotsExpire() throws Exception {
21182148
Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get()));

0 commit comments

Comments
 (0)