Skip to content

Commit 36bac83

Browse files
morningmanclaude
andcommitted
[fix](iceberg) Convert DATE defaults before rebuilding a promoted equality-delete key
### What problem does this PR solve? Issue Number: None Related PR: #69536 Problem Summary: The previous commit lets a dropped Iceberg equality-delete key take, from any retained schema, the v3 promotion of DATE to an unzoned TIMESTAMP or TIMESTAMP_NS, and rebuilds the key's field with Types.NestedField.from(field).ofType(type). The builder copies the field's initial and write defaults unchanged, and NestedField's constructor casts both to the new type with Literal.to. In Iceberg 1.12.0 a DATE literal converts only to DATE, so the rebuild threw "Cannot cast default value to timestamp". This happened on a v3 table whose DATE key has a default, was promoted without a snapshot and was dropped before its equality delete committed: the lineage selected the DATE definition, the retained promotion widened it, and planning failed for every scan the delete applies to. The SDK promotions (INT to LONG, FLOAT to DOUBLE, decimal precision) convert their literals already, and a change of optionality alone keeps the type, so only DATE defaults were affected. The rebuild now converts a DATE default to midnight of its day in the promoted TIMESTAMP or TIMESTAMP_NS before building the field. A date outside the promoted type's range still fails, as the format requires for this promotion. ### Release note None ### Check List (For Author) - Test: Unit Test - IcebergScanPlanProviderTest: a v3 DATE key with initial and write defaults, promoted to TIMESTAMP or TIMESTAMP_NS without a snapshot and dropped before its delete commits, keeps both defaults as midnight in the promoted type, and the scan dictionary carries the converted initial default - fe-connector-iceberg module tests - Behavior changed: No (fixes a regression of this PR's previous commit) - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent cbf21fb commit 36bac83

2 files changed

Lines changed: 73 additions & 2 deletions

File tree

‎fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@
7979
import org.apache.iceberg.expressions.Expression;
8080
import org.apache.iceberg.expressions.Expressions;
8181
import org.apache.iceberg.expressions.InclusiveMetricsEvaluator;
82+
import org.apache.iceberg.expressions.Literal;
8283
import org.apache.iceberg.expressions.ManifestEvaluator;
8384
import org.apache.iceberg.expressions.Projections;
8485
import org.apache.iceberg.expressions.ResidualEvaluator;
@@ -92,6 +93,7 @@
9293
import org.apache.iceberg.types.TypeUtil;
9394
import org.apache.iceberg.types.Types;
9495
import org.apache.iceberg.types.Types.NestedField;
96+
import org.apache.iceberg.util.DateTimeUtil;
9597
import org.apache.iceberg.util.ScanTaskUtil;
9698
import org.apache.iceberg.util.TableScanUtil;
9799
import org.apache.logging.log4j.LogManager;
@@ -2346,7 +2348,8 @@ private static List<Schema> metadataSchemaHistory(Table table) {
23462348
* the schema-only changes made before the drop from every snapshot. Field IDs are never reused and a field
23472349
* only relaxes compatibly, by promotion or from required to optional, so each field of the selected definition
23482350
* takes the widest promotion of its type and the optionality it has in any retained schema; neither depends on
2349-
* the order of the schemas. A field required in every retained schema stays required.
2351+
* the order of the schemas. A field required in every retained schema stays required, and a promoted field
2352+
* keeps its defaults in the promoted type ({@link #promotedDefault}).
23502353
*/
23512354
private static List<NestedField> withRetainedRelaxations(List<NestedField> fields, List<Schema> schemas) {
23522355
List<NestedField> relaxed = new ArrayList<>(fields.size());
@@ -2368,7 +2371,10 @@ && isWideningPromotion(type.asPrimitiveType(), retained.type().asPrimitiveType()
23682371
type = Types.StructType.of(withRetainedRelaxations(type.asStructType().fields(), schemas));
23692372
}
23702373
relaxed.add(type.equals(field.type()) && optional == field.isOptional()
2371-
? field : Types.NestedField.from(field).ofType(type).isOptional(optional).build());
2374+
? field : Types.NestedField.from(field).ofType(type).isOptional(optional)
2375+
.withInitialDefault(promotedDefault(field.initialDefaultLiteral(), field.type(), type))
2376+
.withWriteDefault(promotedDefault(field.writeDefaultLiteral(), field.type(), type))
2377+
.build());
23722378
}
23732379
return relaxed;
23742380
}
@@ -2388,6 +2394,20 @@ private static boolean isWideningPromotion(Type.PrimitiveType from, Type.Primiti
23882394
|| to.typeId() == Type.TypeID.TIMESTAMP_NANO && !((Types.TimestampNanoType) to).shouldAdjustToUTC();
23892395
}
23902396

2397+
/**
2398+
* {@code value}, a default of a field of type {@code from}, in the type {@code to} the field is rebuilt with.
2399+
* NestedField casts both defaults to its type, which converts the literals of the SDK promotions, but in Iceberg
2400+
* 1.12.0 a DATE literal converts only to DATE: a DATE default moves to midnight of its day in the promoted
2401+
* TIMESTAMP or TIMESTAMP_NS, and a date outside that type's range throws, as the format requires.
2402+
*/
2403+
private static Literal<?> promotedDefault(Literal<?> value, Type from, Type to) {
2404+
if (value == null || from.typeId() != Type.TypeID.DATE || to.equals(from)) {
2405+
return value;
2406+
}
2407+
return Literal.of(DateTimeUtil.microsFromTimestamp(
2408+
DateTimeUtil.dateFromDays((Integer) value.value()).atStartOfDay())).to(to);
2409+
}
2410+
23912411
private static void addHistoricalEqualityFields(List<NestedField> fields, Set<Integer> missingFieldIds,
23922412
Schema historicalSchema, List<Schema> retainedSchemas) {
23932413
Map<Integer, NestedField> historicalFields = TypeUtil.indexById(historicalSchema.asStruct());

‎fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@
8989
import org.apache.iceberg.mapping.NameMappingParser;
9090
import org.apache.iceberg.metrics.ScanReport;
9191
import org.apache.iceberg.types.Conversions;
92+
import org.apache.iceberg.types.Type;
9293
import org.apache.iceberg.types.Types;
9394
import org.apache.iceberg.types.Types.NestedField;
9495
import org.junit.jupiter.api.Assertions;
@@ -2193,6 +2194,56 @@ public void equalityCarrierWidensDateKeyPromotedWithoutASnapshot() throws Except
21932194
Assertions.assertEquals(Types.TimestampNanoType.withoutZone(), carrier.get(1).type());
21942195
}
21952196

2197+
@Test
2198+
public void equalityCarrierConvertsDefaultsOfDateKeyPromotedToTimestamp() throws Exception {
2199+
assertPromotedDateKeyDefaults("promoted_date_default_eq_key", Types.TimestampType.withoutZone());
2200+
}
2201+
2202+
@Test
2203+
public void equalityCarrierConvertsDefaultsOfDateKeyPromotedToTimestampNs() throws Exception {
2204+
assertPromotedDateKeyDefaults("promoted_date_nanos_default_eq_key", Types.TimestampNanoType.withoutZone());
2205+
}
2206+
2207+
/**
2208+
* Gives a v3 DATE key both defaults, promotes it to {@code promotedType} without a snapshot and drops it while an
2209+
* equality delete on it is pending. The lineage selects the DATE definition and the carrier rebuilds it in the
2210+
* promoted type, which casts both defaults; a DATE literal converts only to DATE, so an unconverted default
2211+
* fails planning with "Cannot cast default value".
2212+
*/
2213+
private static void assertPromotedDateKeyDefaults(String name, Type promotedType) {
2214+
Literal<Integer> date = Literal.of("2024-01-02").to(Types.DateType.get());
2215+
Literal<Long> midnight = Literal.of("2024-01-02T00:00:00").to(promotedType);
2216+
Table table = createTable(name, new Schema(
2217+
Types.NestedField.required(1, "id", Types.IntegerType.get()),
2218+
Types.NestedField.optional("k").withId(2).ofType(Types.DateType.get())
2219+
.withInitialDefault(date).withWriteDefault(date).build()), PartitionSpec.unpartitioned(),
2220+
Collections.singletonMap(TableProperties.FORMAT_VERSION, "3"));
2221+
table.newAppend().appendFile(dataFile(table.spec(), "s3://b/db/" + name + "/f1.parquet", 1024, null, null))
2222+
.commit();
2223+
publishSchema(table, new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get()),
2224+
Types.NestedField.optional("k").withId(2).ofType(promotedType)
2225+
.withInitialDefault(midnight).withWriteDefault(midnight).build()), 2);
2226+
RowDelta pendingDelete = table.newRowDelta().addDeletes(equalityDeleteFile(
2227+
"s3://b/db/" + name + "/eq.parquet", FileFormat.PARQUET, 2));
2228+
table.updateSchema().deleteColumn("k").commit();
2229+
pendingDelete.commit();
2230+
Assertions.assertEquals(Types.DateType.get(), table.schemas()
2231+
.get(table.snapshot(table.currentSnapshot().parentId()).schemaId()).findType(2));
2232+
2233+
List<NestedField> carrier = IcebergScanPlanProvider.schemaForPotentialEqualityDeletes(
2234+
table, table.newScan(), table.schema());
2235+
Assertions.assertEquals(2, carrier.size(), "the dropped equality key must stay resolvable: " + carrier);
2236+
Assertions.assertEquals(promotedType, carrier.get(1).type());
2237+
Assertions.assertEquals(midnight, carrier.get(1).initialDefaultLiteral());
2238+
Assertions.assertEquals(midnight, carrier.get(1).writeDefaultLiteral());
2239+
2240+
IcebergScanPlanProvider provider = providerOver(table);
2241+
Map<String, String> props = provider.getScanNodeProperties(null, new IcebergTableHandle("db1", name),
2242+
Collections.singletonList(new IcebergColumnHandle("id", 1)), Optional.empty());
2243+
Assertions.assertEquals("2024-01-02 00:00:00",
2244+
dictFieldById(dictRootFields(provider, props), 2).getInitialDefaultValue());
2245+
}
2246+
21962247
@Test
21972248
public void equalityCarrierKeepsSchemaOnlyOptionalityOfDroppedKey() throws Exception {
21982249
// k becomes optional without a snapshot, and both a pending delete and an append without k commit after k

0 commit comments

Comments
 (0)