Skip to content

Commit f1104a6

Browse files
morningmanclaude
andcommitted
[fix](catalog) fe-connector-iceberg: reject case-insensitive column-name collisions on the top-level ALTER ops
Port gap from apache#65329. Upstream `IcebergMetadataOps` called `validateNoCaseInsensitiveSiblingCollision` from five sites: flat ADD, nested ADD, the ADD COLUMNS batch, flat RENAME and nested RENAME. The connector port carried only the two NESTED sites into `IcebergNestedColumnEvolution`; the top-level ops in `CatalogBackedIcebergCatalogOps` went straight to `UpdateSchema`. Iceberg matches field names case-sensitively (`SchemaUpdate` sets `caseSensitive = true`) while Doris column names are case-insensitive, so nothing caught the mismatch: against a Spark-created table with a field `Id`, `ADD COLUMN id` silently committed a second column, and `RENAME COLUMN label ...` died with iceberg's raw "Cannot rename missing column: label" because the real field is `Label`. Wire the three top-level ops to the validators the nested arm already uses: - addColumn - reject a name colliding case-insensitively with an existing field. - addColumns - same per column, plus reject two names of one request that collide with each other; the whole batch is validated before anything is staged, so a partly-valid request commits nothing. - renameColumn - resolve the SOURCE name case-insensitively (a pure re-casing rename is let through by the field-identity escape) and restate identifier-field paths via the existing `applyRenameColumn`. `validateNoCaseInsensitiveSiblingCollision` becomes package-private; its `parentPath.isEmpty()` top-level arm was already written but unreachable. Fixes external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop line 273 (TeamCity Doris_External_Regression #1005291), and the four assertions after it that the suite never reached. Tests: 7 cases in CatalogBackedIcebergCatalogOpsColumnEvolutionTest against a real InMemoryCatalog with a mixed-case fixture (Id / Label / Info.Metric); they fail on the pre-fix code with the exact CI symptoms. Module suite 1145/1145 green (cache-free run), full-reactor test-compile green. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01VK7NGsHDmPGvpHFFK5ojej
1 parent 7ff51a1 commit f1104a6

3 files changed

Lines changed: 164 additions & 6 deletions

File tree

‎fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogOps.java‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -434,7 +434,10 @@ public Optional<String> loadNamespaceLocation(String dbName) {
434434
@Override
435435
public void addColumn(String dbName, String tableName, IcebergColumnChange column,
436436
ConnectorColumnPosition position) {
437-
UpdateSchema updateSchema = loadTable(dbName, tableName).updateSchema();
437+
Table table = loadTable(dbName, tableName);
438+
IcebergNestedColumnEvolution.validateNoCaseInsensitiveSiblingCollision(
439+
table.schema().asStruct(), "", column.getName(), null, "add");
440+
UpdateSchema updateSchema = table.updateSchema();
438441
updateSchema.addColumn(column.getName(), column.getType(), column.getComment(),
439442
column.getDefaultValue());
440443
applyPosition(updateSchema, position, column.getName());
@@ -443,7 +446,9 @@ public void addColumn(String dbName, String tableName, IcebergColumnChange colum
443446

444447
@Override
445448
public void addColumns(String dbName, String tableName, List<IcebergColumnChange> columns) {
446-
UpdateSchema updateSchema = loadTable(dbName, tableName).updateSchema();
449+
Table table = loadTable(dbName, tableName);
450+
IcebergNestedColumnEvolution.validateNoCaseInsensitiveTopLevelCollisions(table.schema(), columns);
451+
UpdateSchema updateSchema = table.updateSchema();
447452
for (IcebergColumnChange column : columns) {
448453
updateSchema.addColumn(column.getName(), column.getType(), column.getComment(),
449454
column.getDefaultValue());
@@ -460,9 +465,7 @@ public void dropColumn(String dbName, String tableName, String columnName) {
460465

461466
@Override
462467
public void renameColumn(String dbName, String tableName, String oldName, String newName) {
463-
UpdateSchema updateSchema = loadTable(dbName, tableName).updateSchema();
464-
updateSchema.renameColumn(oldName, newName);
465-
updateSchema.commit();
468+
IcebergNestedColumnEvolution.renameTopLevelColumn(loadTable(dbName, tableName), oldName, newName);
466469
}
467470

468471
@Override

‎fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergNestedColumnEvolution.java‎

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,11 @@
4949
*
5050
* <p><b>No partial commit:</b> every {@code UpdateSchema.commit()} is the final statement of each entry point, so
5151
* any validation/resolution throw aborts the whole change before anything is committed (legacy parity).</p>
52+
*
53+
* <p>The TOP-LEVEL (flat) column ops stay on {@link IcebergCatalogOps}, but they share this class's name
54+
* resolution and validators — iceberg matches field names case-SENSITIVELY while Doris column names are
55+
* case-insensitive, so both arms must agree. See {@link #validateNoCaseInsensitiveSiblingCollision},
56+
* {@link #validateNoCaseInsensitiveTopLevelCollisions} and {@link #renameTopLevelColumn}.</p>
5257
*/
5358
public final class IcebergNestedColumnEvolution {
5459

@@ -277,7 +282,45 @@ private static ConnectorColumnPath childPath(ConnectorColumnPath parentPath, Str
277282
return ConnectorColumnPath.of(parts);
278283
}
279284

280-
private static void validateNoCaseInsensitiveSiblingCollision(Types.StructType parentType, String parentPath,
285+
/**
286+
* Renames the TOP-LEVEL column {@code oldName} to {@code newName}. {@code oldName} is resolved
287+
* case-insensitively (iceberg's own {@code renameColumn} is case-sensitive and would report the column as
288+
* missing when the user types a different case than the iceberg field), then {@code newName} is checked
289+
* against the other top-level fields. Mirrors the legacy flat {@code IcebergMetadataOps.renameColumn}.
290+
*/
291+
static void renameTopLevelColumn(Table table, String oldName, String newName) {
292+
Schema schema = table.schema();
293+
ResolvedColumnPath oldPath = resolveColumnPath(schema, ConnectorColumnPath.of(oldName), "rename");
294+
validateNoCaseInsensitiveSiblingCollision(schema.asStruct(), "", newName, oldPath.getField(), "rename");
295+
296+
UpdateSchema updateSchema = table.updateSchema();
297+
applyRenameColumn(schema, updateSchema, oldPath, newName);
298+
updateSchema.commit();
299+
}
300+
301+
/**
302+
* Rejects a batch of TOP-LEVEL columns that collides case-insensitively with an existing field or with
303+
* another column of the same request. Validates the whole batch before the caller stages anything, so a
304+
* partly-valid request commits nothing. Mirrors the legacy
305+
* {@code IcebergMetadataOps.validateNoCaseInsensitiveTopLevelCollisions}.
306+
*/
307+
static void validateNoCaseInsensitiveTopLevelCollisions(Schema schema, List<IcebergColumnChange> columns) {
308+
Set<String> requestedNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
309+
for (IcebergColumnChange column : columns) {
310+
validateNoCaseInsensitiveSiblingCollision(schema.asStruct(), "", column.getName(), null, "add");
311+
if (!requestedNames.add(column.getName())) {
312+
throw new DorisConnectorException("Cannot add column '" + column.getName()
313+
+ "': conflicts with another requested column (case-insensitive)");
314+
}
315+
}
316+
}
317+
318+
/**
319+
* Rejects {@code targetName} when it case-insensitively collides with a sibling of {@code parentType} that
320+
* is not {@code sourceField} itself (the identity escape lets a pure re-casing rename through). An empty
321+
* {@code parentPath} addresses the table's top-level columns.
322+
*/
323+
static void validateNoCaseInsensitiveSiblingCollision(Types.StructType parentType, String parentPath,
281324
String targetName, NestedField sourceField, String operation) {
282325
NestedField conflictingField = parentType.caseInsensitiveField(targetName);
283326
if (conflictingField != null

‎fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/CatalogBackedIcebergCatalogOpsColumnEvolutionTest.java‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,118 @@ public void testRenameColumn() {
131131
Assertions.assertNotNull(reload().findField("full_name"));
132132
}
133133

134+
// ---------- case-insensitive name collision on the flat ops (#65329 flat arm) ----------
135+
136+
/**
137+
* Creates db1.mixed with Spark-style mixed-case names ({@code Id}, {@code Label},
138+
* {@code Info STRUCT<Metric:INT>}) — the fixture of the regression suite
139+
* {@code test_iceberg_nested_schema_evolution_spark_doris_interop}. Iceberg itself matches names
140+
* case-SENSITIVELY, so without the connector guards a Doris user could add a second {@code id} next to
141+
* {@code Id} and make the table unreadable through Doris (whose column names are case-insensitive).
142+
*/
143+
private void createMixedCaseTable() {
144+
Schema schema = new Schema(
145+
Types.NestedField.optional(1, "Id", Types.IntegerType.get()),
146+
Types.NestedField.optional(2, "Label", Types.StringType.get()),
147+
Types.NestedField.optional(3, "Info", Types.StructType.of(
148+
Types.NestedField.optional(4, "Metric", Types.IntegerType.get()))));
149+
ops.createTable("db1", "mixed", schema, PartitionSpec.unpartitioned(), null,
150+
IcebergSchemaBuilder.buildTableProperties(Collections.emptyMap()));
151+
}
152+
153+
@Test
154+
public void testAddColumnCaseInsensitiveCollisionFailsLoud() {
155+
createMixedCaseTable();
156+
DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class,
157+
() -> ops.addColumn("db1", "mixed", change("id", Types.StringType.get(), "", true), null));
158+
Assertions.assertTrue(ex.getMessage()
159+
.contains("conflicts with existing Iceberg field 'Id' (case-insensitive)"),
160+
ex.getMessage());
161+
Assertions.assertEquals(3, reload("mixed").columns().size());
162+
}
163+
164+
@Test
165+
public void testAddColumnsCaseInsensitiveCollisionFailsLoud() {
166+
createMixedCaseTable();
167+
DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class,
168+
() -> ops.addColumns("db1", "mixed", Arrays.asList(
169+
change("ok_col", Types.StringType.get(), "", true),
170+
change("id", Types.StringType.get(), "", true))));
171+
Assertions.assertTrue(ex.getMessage()
172+
.contains("conflicts with existing Iceberg field 'Id' (case-insensitive)"),
173+
ex.getMessage());
174+
// Whole batch validated before anything is staged: the valid sibling must not be committed either.
175+
Assertions.assertEquals(3, reload("mixed").columns().size());
176+
}
177+
178+
@Test
179+
public void testAddColumnsDuplicateRequestedNamesFailLoud() {
180+
createMixedCaseTable();
181+
DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class,
182+
() -> ops.addColumns("db1", "mixed", Arrays.asList(
183+
change("new_field", Types.StringType.get(), "", true),
184+
change("NEW_FIELD", Types.StringType.get(), "", true))));
185+
Assertions.assertTrue(ex.getMessage()
186+
.contains("conflicts with another requested column (case-insensitive)"),
187+
ex.getMessage());
188+
Assertions.assertEquals(3, reload("mixed").columns().size());
189+
}
190+
191+
@Test
192+
public void testRenameColumnCaseInsensitiveCollisionFailsLoud() {
193+
createMixedCaseTable();
194+
DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class,
195+
() -> ops.renameColumn("db1", "mixed", "label", "id"));
196+
Assertions.assertTrue(ex.getMessage()
197+
.contains("conflicts with existing Iceberg field 'Id' (case-insensitive)"),
198+
ex.getMessage());
199+
Assertions.assertNotNull(reload("mixed").findField("Label"));
200+
}
201+
202+
@Test
203+
public void testRenameColumnResolvesSourceNameCaseInsensitively() {
204+
createMixedCaseTable();
205+
// The user types the Doris-side (case-insensitive) name; it must resolve to the iceberg field 'Label'
206+
// rather than hitting iceberg's case-sensitive "Cannot rename missing column".
207+
ops.renameColumn("db1", "mixed", "label", "full_label");
208+
Assertions.assertNull(reload("mixed").findField("Label"));
209+
Assertions.assertNotNull(reload("mixed").findField("full_label"));
210+
}
211+
212+
@Test
213+
public void testRenameColumnCaseOnlySelfRenameSucceeds() {
214+
createMixedCaseTable();
215+
// 'Label' -> 'label' collides with itself; the field-identity escape in the guard must let it through,
216+
// otherwise a pure re-casing of a column would be impossible.
217+
ops.renameColumn("db1", "mixed", "label", "label");
218+
Assertions.assertNotNull(reload("mixed").findField("label"));
219+
Assertions.assertEquals(3, reload("mixed").columns().size());
220+
}
221+
222+
@Test
223+
public void testRenameColumnPreservesIdentifierFieldPath() {
224+
// Iceberg drops an identifier-field path when the field is renamed; the flat rename must restate it
225+
// (legacy IcebergMetadataOps.applyRenameColumn parity, already honoured by the nested arm).
226+
Schema schema = new Schema(
227+
Arrays.asList(
228+
Types.NestedField.required(1, "Key", Types.IntegerType.get()),
229+
Types.NestedField.optional(2, "v", Types.StringType.get())),
230+
Collections.singleton(1));
231+
ops.createTable("db1", "ident", schema, PartitionSpec.unpartitioned(), null,
232+
IcebergSchemaBuilder.buildTableProperties(Collections.emptyMap()));
233+
ops.renameColumn("db1", "ident", "key", "renamed_key");
234+
Schema after = reload("ident");
235+
Assertions.assertEquals(Collections.singleton(after.findField("renamed_key").fieldId()),
236+
after.identifierFieldIds());
237+
}
238+
239+
@Test
240+
public void testAddColumnKeepsWorkingWhenNoCollision() {
241+
createMixedCaseTable();
242+
ops.addColumn("db1", "mixed", change("brand_new", Types.StringType.get(), "", true), null);
243+
Assertions.assertNotNull(reload("mixed").findField("brand_new"));
244+
}
245+
134246
// ---------- modifyColumn ----------
135247

136248
@Test

0 commit comments

Comments
 (0)