Skip to content

Commit 2937f34

Browse files
morningmanclaude
andcommitted
[fix](fe) Bind Iceberg branch writes to the table's current schema
### What problem does this PR solve? Issue Number: None Related PR: #66538, #65851 Problem Summary: A write to an Iceberg branch (INSERT [OVERWRITE] INTO t@branch(b)) pins the schema of the branch head snapshot as its writer schema, while a read of the branch uses the table-current schema. Iceberg validates a branch commit against the table-current schema and stamps that schema on the new snapshot, Spark binds branch writes to the current schema, and the Iceberg docs ("Schema selection with branches and tags") say writes to a branch use the table schema. So once the table schema evolves, a write to an older branch rejects the current column names and accepts the old ones, writes the branch head's write-default instead of the current one, and fails outright when the current partition spec has a source column the branch head schema does not have. Master also needs an extra check (validateBranchWriterSchema) because the files it writes carry a different schema than the one Iceberg records for the new snapshot. branch-4.1 and branch-4.2 (#66538) bind branch writes to table.schema(). This change pins table.schema() for branch writes, keeps the check that the target is an existing branch and not a tag, and fences the commit on the table-current schema id for branch writes as for writes to main. The branch head checks are removed. ### Release note Fix: Writes to an Iceberg branch now use the table's current schema (column names, write-defaults and partition spec), as Spark does, instead of the schema of the branch's head snapshot. ### Check List (For Author) - Test: Regression test / Unit Test - test_iceberg_initial_defaults (a branch write gets the current write-default 36), write/test_iceberg_write_evolution_refs, branch_tag/iceberg_branch_tag_schema_change_extended and test_iceberg_schema_ref_actions_matrix (an older branch takes the current column names and rejects the old ones) now expect what branch-4.2 expects. They pass locally; without this change each of them fails at the new expectation - IcebergWritePlanProviderTest: a branch created before a rename writes the new name; the branch write uses the current write-defaults, binds a current required column without a default, and binds the current partition spec whose source column the branch head lacks; a schema change after pinning fails the commit; a tag or a missing ref is rejected - Behavior changed: Yes. See the release note. - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 12f61ba commit 2937f34

9 files changed

Lines changed: 123 additions & 135 deletions

File tree

‎fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWritePlanProvider.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -303,9 +303,8 @@ private void validateBoundWriteColumns(Table table, IcebergWriteSchemaContext sc
303303
if (writeOperation == WriteOperation.DELETE || boundColumns.isEmpty()) {
304304
return;
305305
}
306-
// A branch write is deliberately bound to the branch-head schema, which can lag the current main
307-
// schema. The schema context separately fences branch evolution, so comparing with table.schema()
308-
// rejects valid branch writes without adding concurrency protection.
306+
// Compare with the schema the statement pinned, which is the table-current schema at pin time for main
307+
// and branch writes alike. The schema context fences any later schema change before the commit.
309308
Schema writeSchema = schemaContext == null ? table.schema() : schemaContext.getSchema();
310309
List<NestedField> currentColumns = writeSchema.columns();
311310
boolean rowLevelWrite = writeOperation == WriteOperation.UPDATE || writeOperation == WriteOperation.MERGE;

‎fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWriteSchemaContext.java‎

Lines changed: 9 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,7 @@
4343
import org.apache.iceberg.TableMetadata;
4444
import org.apache.iceberg.transforms.Transforms;
4545
import org.apache.iceberg.types.Type;
46-
import org.apache.iceberg.types.TypeUtil;
4746
import org.apache.iceberg.types.Types;
48-
import org.apache.iceberg.util.SnapshotUtil;
4947

5048
import java.math.BigDecimal;
5149
import java.nio.ByteBuffer;
@@ -96,11 +94,12 @@ static IcebergWriteSchemaContext create(Table table, String tableName,
9694
Objects.requireNonNull(table, "table should not be null");
9795
Objects.requireNonNull(tableName, "tableName should not be null");
9896
Objects.requireNonNull(branchName, "branchName should not be null");
99-
Schema schema = branchName.isPresent()
100-
? resolveBranchSchema(table, branchName.get(), tableName) : table.schema();
10197
if (branchName.isPresent()) {
102-
validateBranchWriterSchema(schema, table.schema(), branchName.get(), tableName);
98+
validateTargetBranch(table, branchName.get(), tableName);
10399
}
100+
// A branch selects only the parent snapshot / ref. Iceberg validates a branch write against the
101+
// table-current schema and stamps that schema on the new snapshot, and Spark binds it the same way.
102+
Schema schema = table.schema();
104103
int formatVersion = IcebergWriterHelper.getFormatVersion(table);
105104
TableIdentity identity = pinTableIdentity(table, formatVersion);
106105
return new IcebergWriteSchemaContext(
@@ -273,7 +272,7 @@ private static void validateWriterMetadataSources(
273272
}
274273
}
275274

276-
private static Schema resolveBranchSchema(Table table, String branchName, String tableName) {
275+
private static void validateTargetBranch(Table table, String branchName, String tableName) {
277276
SnapshotRef ref = table.refs().get(branchName);
278277
if (ref == null) {
279278
throw new DorisConnectorException(branchName + " is not founded in " + tableName);
@@ -282,51 +281,13 @@ private static Schema resolveBranchSchema(Table table, String branchName, String
282281
throw new DorisConnectorException(branchName
283282
+ " is a tag, not a branch. Tags cannot be targets for producing snapshots");
284283
}
285-
return SnapshotUtil.schemaFor(table, ref.snapshotId());
286-
}
287-
288-
private static void validateBranchWriterSchema(
289-
Schema branchSchema, Schema currentSchema, String branchName, String tableName) {
290-
Map<Integer, Types.NestedField> branchFields = TypeUtil.indexById(branchSchema.asStruct());
291-
Map<Integer, Types.NestedField> currentFields = TypeUtil.indexById(currentSchema.asStruct());
292-
Map<Integer, Integer> currentParents = TypeUtil.indexParents(currentSchema.asStruct());
293-
for (Types.NestedField currentField : currentFields.values()) {
294-
Types.NestedField branchField = branchFields.get(currentField.fieldId());
295-
if (branchField != null) {
296-
if (currentField.isRequired() && branchField.isOptional()) {
297-
throw incompatibleBranchSchema(branchSchema, currentSchema, branchName, tableName,
298-
currentField, "is optional in the pinned branch schema and can contain explicit nulls");
299-
}
300-
continue;
301-
}
302-
Types.NestedField highestMissingField = currentField;
303-
Integer parentId = currentParents.get(currentField.fieldId());
304-
while (parentId != null && !branchFields.containsKey(parentId)) {
305-
highestMissingField = Preconditions.checkNotNull(currentFields.get(parentId),
306-
"Iceberg parent field %s is absent from current schema", parentId);
307-
parentId = currentParents.get(parentId);
308-
}
309-
if (highestMissingField.isRequired() && highestMissingField.initialDefault() == null) {
310-
throw incompatibleBranchSchema(branchSchema, currentSchema, branchName, tableName,
311-
highestMissingField,
312-
"is absent from the pinned branch schema and has no initial default");
313-
}
314-
}
315-
}
316-
317-
private static DorisConnectorException incompatibleBranchSchema(
318-
Schema branchSchema, Schema currentSchema, String branchName, String tableName,
319-
Types.NestedField field, String incompatibility) {
320-
return new DorisConnectorException("Iceberg table current schema " + currentSchema.schemaId()
321-
+ " cannot label files written with pinned branch " + branchName + " schema "
322-
+ branchSchema.schemaId() + " for table " + tableName + ": required field "
323-
+ field.name() + " (id " + field.fieldId() + ") " + incompatibility
324-
+ "; retry after updating the branch schema");
325284
}
326285

327286
void validateCurrentSchema(Table table, boolean requireCurrentPartitionSpec) {
328-
Schema currentSchema = branchName.isPresent()
329-
? resolveBranchSchema(table, branchName.get(), tableName) : table.schema();
287+
if (branchName.isPresent()) {
288+
validateTargetBranch(table, branchName.get(), tableName);
289+
}
290+
Schema currentSchema = table.schema();
330291
int currentFormatVersion = IcebergWriterHelper.getFormatVersion(table);
331292
validateTableIdentity(table, currentFormatVersion);
332293
if (currentSchema.schemaId() != schema.schemaId() || currentFormatVersion != formatVersion) {
@@ -335,9 +296,6 @@ void validateCurrentSchema(Table table, boolean requireCurrentPartitionSpec) {
335296
+ ", current schema " + currentSchema.schemaId() + "/format " + currentFormatVersion
336297
+ "; retry the statement");
337298
}
338-
if (branchName.isPresent()) {
339-
validateBranchWriterSchema(schema, table.schema(), branchName.get(), tableName);
340-
}
341299
PartitionSpec retainedSpec = table.specs().get(partitionSpec.specId());
342300
if (retainedSpec == null
343301
|| !partitionSpecJson.equals(PartitionSpecParser.toJson(retainedSpec))) {

‎fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergWritePlanProviderTest.java‎

Lines changed: 87 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@
101101
import java.util.Optional;
102102
import java.util.Set;
103103
import java.util.UUID;
104+
import java.util.stream.Collectors;
104105

105106
/**
106107
* Pins {@link IcebergWritePlanProvider#planWrite} for INSERT/OVERWRITE against legacy
@@ -559,7 +560,9 @@ public void variantWriteDefaultsMustBeNull() {
559560
}
560561

561562
@Test
562-
public void branchWritePinsBranchHeadSchema() {
563+
public void branchWriteUsesTableCurrentSchema() {
564+
// A branch only selects the parent snapshot. Iceberg stamps a branch commit with the table-current
565+
// schema and Spark binds branch writes to it, so a branch created before a rename writes the new name.
563566
InMemoryCatalog catalog = freshCatalog();
564567
Table table = unpartitionedUnsortedTable(catalog);
565568
table.newAppend().commit();
@@ -569,14 +572,14 @@ public void branchWritePinsBranchHeadSchema() {
569572
IcebergWriteSchemaContext writeContext = IcebergWriteSchemaContext.create(
570573
table, "db1.t2", Optional.of("old_schema"), false, false);
571574

572-
Assertions.assertNotEquals(table.schema().schemaId(), writeContext.getSchema().schemaId());
573-
Assertions.assertNotNull(writeContext.getSchema().findField("name"));
574-
Assertions.assertNull(writeContext.getSchema().findField("renamed_name"));
575+
Assertions.assertEquals(table.schema().schemaId(), writeContext.getSchema().schemaId());
576+
Assertions.assertNotNull(writeContext.getSchema().findField("renamed_name"));
577+
Assertions.assertNull(writeContext.getSchema().findField("name"));
575578
Assertions.assertDoesNotThrow(() -> writeContext.validateCurrentSchema(table, false));
576579
}
577580

578581
@Test
579-
public void planWriteValidatesBoundColumnsAgainstPinnedBranchSchema() {
582+
public void planWriteBindsBranchColumnsToTableCurrentSchema() {
580583
InMemoryCatalog catalog = freshCatalog();
581584
Table table = unpartitionedUnsortedTable(catalog);
582585
table.newAppend().commit();
@@ -589,11 +592,13 @@ public void planWriteValidatesBoundColumnsAgainstPinnedBranchSchema() {
589592
List<ConnectorColumn> branchColumns = provider.getWriteColumns(
590593
session, tableHandle, Optional.of("old_schema")).orElseThrow(AssertionError::new);
591594

595+
Assertions.assertEquals(Arrays.asList("id", "renamed_name"),
596+
branchColumns.stream().map(ConnectorColumn::getName).collect(Collectors.toList()));
592597
ConnectorSinkPlan plan = Assertions.assertDoesNotThrow(() -> provider.planWrite(session,
593598
new WriteHandle(tableHandle).branch("old_schema").boundTargetColumns(branchColumns)));
594599

595-
Assertions.assertTrue(plan.getDataSink().getIcebergTableSink().getSchemaJson().contains("name"));
596-
Assertions.assertFalse(plan.getDataSink().getIcebergTableSink().getSchemaJson().contains("renamed_name"));
600+
Assertions.assertEquals(SchemaParser.toJson(table.schema()),
601+
plan.getDataSink().getIcebergTableSink().getSchemaJson());
597602
}
598603

599604
@Test
@@ -673,55 +678,106 @@ public void writeRoutingSkipsV1VoidPartitionFields() {
673678
}
674679

675680
@Test
676-
public void branchWriteRejectsCurrentRequiredFieldAbsentWithoutDefault() {
681+
public void branchWriteRejectsTagOrMissingRef() {
682+
InMemoryCatalog catalog = freshCatalog();
683+
Table table = unpartitionedUnsortedTable(catalog);
684+
table.newAppend().commit();
685+
table.manageSnapshots().createTag("old_tag", table.currentSnapshot().snapshotId()).commit();
686+
687+
DorisConnectorException tagException = Assertions.assertThrows(DorisConnectorException.class,
688+
() -> IcebergWriteSchemaContext.create(table, "db1.t2", Optional.of("old_tag"), false, false));
689+
Assertions.assertTrue(tagException.getMessage().contains(
690+
"old_tag is a tag, not a branch. Tags cannot be targets for producing snapshots"),
691+
tagException.getMessage());
692+
DorisConnectorException missingException = Assertions.assertThrows(DorisConnectorException.class,
693+
() -> IcebergWriteSchemaContext.create(table, "db1.t2", Optional.of("no_such_ref"), false, false));
694+
Assertions.assertTrue(missingException.getMessage().contains("no_such_ref is not founded in db1.t2"),
695+
missingException.getMessage());
696+
}
697+
698+
@Test
699+
public void branchWriteBindsCurrentRequiredFieldWithoutDefault() {
700+
// A required column added on main after the branch is part of the branch write. It has no default, so
701+
// omitting it fails at analysis exactly as it does for a write to main.
677702
InMemoryCatalog catalog = freshCatalog();
678703
Table table = unpartitionedUnsortedTable(catalog);
679704
table.newAppend().commit();
680705
table.manageSnapshots().createBranch("old_schema", table.currentSnapshot().snapshotId()).commit();
681706
table.updateSchema().allowIncompatibleChanges()
682707
.addRequiredColumn("required_value", Types.IntegerType.get()).commit();
683708

684-
DorisConnectorException exception = Assertions.assertThrows(DorisConnectorException.class,
685-
() -> IcebergWriteSchemaContext.create(
686-
table, "db1.t2", Optional.of("old_schema"), false, false));
709+
IcebergWriteSchemaContext writeContext = IcebergWriteSchemaContext.create(
710+
table, "db1.t2", Optional.of("old_schema"), false, false);
687711

688-
Assertions.assertTrue(exception.getMessage().contains("required field required_value"));
689-
Assertions.assertTrue(exception.getMessage().contains(
690-
"is absent from the pinned branch schema and has no initial default"));
712+
ConnectorColumn requiredValue = writeColumn(writeContext, "required_value");
713+
Assertions.assertFalse(requiredValue.isNullable());
714+
Assertions.assertNull(requiredValue.getDefaultValueSql());
715+
Assertions.assertDoesNotThrow(() -> writeContext.validateCurrentSchema(table, false));
716+
}
717+
718+
@Test
719+
public void branchWriteUsesTableCurrentWriteDefaults() {
720+
InMemoryCatalog catalog = freshCatalog();
721+
Table table = formatVersionThreeTable(catalog);
722+
table.updateSchema().addColumn("default_int", Types.IntegerType.get(), Literal.of(35)).commit();
723+
table.newAppend().commit();
724+
table.manageSnapshots().createBranch("old_defaults", table.currentSnapshot().snapshotId()).commit();
725+
table.updateSchema().updateColumnDefault("default_int", Literal.of(36))
726+
.addRequiredColumn("required_value", Types.IntegerType.get(), Literal.of(7)).commit();
727+
728+
IcebergWriteSchemaContext writeContext = IcebergWriteSchemaContext.create(
729+
table, "db1.tv3", Optional.of("old_defaults"), false, false);
730+
731+
Assertions.assertEquals("36", writeColumn(writeContext, "default_int").getDefaultValueSql());
732+
Assertions.assertEquals("7", writeColumn(writeContext, "required_value").getDefaultValueSql());
733+
Assertions.assertDoesNotThrow(() -> writeContext.validateCurrentSchema(table, false));
691734
}
692735

693736
@Test
694-
public void branchWriteRevalidatesFieldMadeRequiredAfterPlanning() {
737+
public void branchWriteBindsPartitionSpecToTableCurrentSchema() {
738+
// The current spec partitions by a column the branch head schema never had.
695739
InMemoryCatalog catalog = freshCatalog();
696740
Table table = unpartitionedUnsortedTable(catalog);
697741
table.newAppend().commit();
698742
table.manageSnapshots().createBranch("old_schema", table.currentSnapshot().snapshotId()).commit();
699-
IcebergWriteSchemaContext writeContext = IcebergWriteSchemaContext.create(
700-
table, "db1.t2", Optional.of("old_schema"), false, false);
701-
table.updateSchema().allowIncompatibleChanges().requireColumn("name").commit();
743+
table.updateSchema().addColumn("region", Types.StringType.get()).commit();
744+
table.updateSpec().addField("region").commit();
702745

703-
DorisConnectorException exception = Assertions.assertThrows(DorisConnectorException.class,
704-
() -> writeContext.validateCurrentSchema(table, false));
746+
IcebergWriteSchemaContext writeContext = Assertions.assertDoesNotThrow(() -> IcebergWriteSchemaContext.create(
747+
table, "db1.t2", Optional.of("old_schema"), false, false));
705748

706-
Assertions.assertTrue(exception.getMessage().contains("required field name"));
707-
Assertions.assertTrue(exception.getMessage().contains(
708-
"is optional in the pinned branch schema and can contain explicit nulls"));
749+
Assertions.assertEquals(table.spec().specId(), writeContext.getPartitionSpec().specId());
750+
Assertions.assertEquals(table.schema().findField("region").fieldId(),
751+
writeContext.getPartitionSpec().fields().get(0).sourceId());
709752
}
710753

711754
@Test
712-
public void branchWriteAllowsCurrentRequiredFieldWithInitialDefault() {
755+
public void branchWriteRejectsTableSchemaChangeBeforeCommit() {
756+
// The branch head schema stays the same, but the files were written with the pinned current schema, so
757+
// any later change to the table-current schema must fail the commit, as it does for a write to main.
713758
InMemoryCatalog catalog = freshCatalog();
714-
Table table = formatVersionThreeTable(catalog);
759+
Table table = unpartitionedUnsortedTable(catalog);
715760
table.newAppend().commit();
716761
table.manageSnapshots().createBranch("old_schema", table.currentSnapshot().snapshotId()).commit();
717-
table.updateSchema().addRequiredColumn(
718-
"required_value", Types.IntegerType.get(), Literal.of(7)).commit();
719-
720762
IcebergWriteSchemaContext writeContext = IcebergWriteSchemaContext.create(
721-
table, "db1.tv3", Optional.of("old_schema"), false, false);
763+
table, "db1.t2", Optional.of("old_schema"), false, false);
764+
int pinnedSchemaId = table.schema().schemaId();
765+
table.updateSchema().addColumn("late_value", Types.IntegerType.get()).commit();
722766

723-
Assertions.assertNull(writeContext.getSchema().findField("required_value"));
724-
Assertions.assertDoesNotThrow(() -> writeContext.validateCurrentSchema(table, false));
767+
DorisConnectorException exception = Assertions.assertThrows(DorisConnectorException.class,
768+
() -> writeContext.validateCurrentSchema(table, false));
769+
770+
Assertions.assertTrue(exception.getMessage().contains("pinned schema " + pinnedSchemaId + "/format"),
771+
exception.getMessage());
772+
Assertions.assertTrue(exception.getMessage().contains(
773+
"current schema " + table.schema().schemaId() + "/format"), exception.getMessage());
774+
}
775+
776+
private static ConnectorColumn writeColumn(IcebergWriteSchemaContext writeContext, String name) {
777+
return writeContext.getColumns().stream()
778+
.filter(column -> column.getName().equals(name))
779+
.findFirst()
780+
.orElseThrow(() -> new AssertionError("missing write column " + name));
725781
}
726782

727783
@Test

‎regression-test/data/external_table_p0/iceberg/test_iceberg_initial_defaults.out‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -388,20 +388,20 @@ missing_map_col map<text,struct<missing_map_default_int:int>> Yes true \N
388388
4 123e4567-e89b-12d3-a456-426614174000 0A0B0C0D 2A2B2C 2024-12-17 23:59:59.123456+00:00
389389

390390
-- !parquet_branch_overwrite_default --
391-
14 35
391+
14 36
392392

393393
-- !parquet_main_overwrite_default --
394394
15 36
395395

396396
-- !parquet_branch_after_main_overwrite --
397-
14 35
397+
14 36
398398

399399
-- !orc_branch_overwrite_default --
400-
14 35
400+
14 36
401401

402402
-- !orc_main_overwrite_default --
403403
15 36
404404

405405
-- !orc_branch_after_main_overwrite --
406-
14 35
406+
14 36
407407

‎regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_evolution_refs.out‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,13 @@
3838
4 CN-east new-spec
3939
5 DE-west \N
4040
6 \N null-zone
41+
4142
-- !branch_after_insert --
4243
1 CN base-cn \N
4344
2 US base-us \N
4445
3 \N \N \N
45-
7 JP-east branch-insert \N
46-
8 FR-west branch-overwrite-seed \N
46+
7 JP-east branch branch-insert
47+
8 FR-west branch-seed branch-overwrite-seed
4748

4849
-- !main_unchanged_after_branch_insert --
4950
1
@@ -57,7 +58,7 @@
5758
1 CN base-cn \N
5859
2 US base-us \N
5960
3 \N \N \N
60-
7 JP-east branch-insert \N
61+
7 JP-east branch branch-insert
6162
8 FR-west branch-overwrite branch-overwrite
6263

6364
-- !base_tag_after_branch_overwrite --
@@ -72,3 +73,4 @@
7273
4 CN-east new-spec
7374
5 DE-west \N
7475
6 \N null-zone
76+

‎regression-test/suites/external_table_p0/iceberg/branch_tag/iceberg_branch_tag_schema_change_extended.groovy‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -70,14 +70,9 @@ suite("iceberg_branch_tag_schema_change_extended", "p0,external") {
7070
sql """ alter table ${table_name} modify column id bigint """
7171
qt_b3_new_type """ select * from ${table_name}@branch(b3_schema) where id = 1 """ // Should use new type
7272

73-
// Test 3.1.4: Branch writes bind to the branch-head schema
73+
// Test 3.1.4: Branch writes use the shared latest table schema
7474
sql """ alter table ${table_name} add column new_col string """
75-
test {
76-
sql """ insert into ${table_name}@branch(b3_schema)(id, value, new_col) values (3, 30, 'test') """
77-
exception "Unknown column 'new_col'"
78-
}
79-
// A successful old-schema write advances the branch to a snapshot carrying the current table schema.
80-
sql """ insert into ${table_name}@branch(b3_schema)(id, value) values (4, 40) """
75+
// Iceberg branch commits advance the branch to a snapshot written with current table metadata.
8176
sql """ insert into ${table_name}@branch(b3_schema)(id, value, new_col) values (3, 30, 'test') """
8277
qt_b3_with_new_col """ select * from ${table_name}@branch(b3_schema) where id = 3 """
8378

0 commit comments

Comments
 (0)