Skip to content

Commit 25f2916

Browse files
authored
[fix](fe) Reject Iceberg v3 row lineage columns (apache#63825)
Problem Summary: Creating an Iceberg table with format-version 3 could accept user columns named _row_id or _last_updated_sequence_number. Doris later appends Iceberg row lineage hidden columns with the same names, which can produce duplicate hidden-column metadata and confusing insert/query behavior. This change validates Iceberg create table definitions, including CTAS, and rejects those reserved row lineage column names when the requested Iceberg format version is 3 or newer.
1 parent 7b8f62c commit 25f2916

5 files changed

Lines changed: 190 additions & 2 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -373,10 +373,16 @@ public boolean performCreateTable(CreateTableInfo createTableInfo) throws UserEx
373373
Schema schema = new Schema(visit.asNestedType().asStructType().fields());
374374
Map<String, String> properties = createTableInfo.getProperties();
375375
properties.put(ExternalCatalog.DORIS_VERSION, ExternalCatalog.DORIS_VERSION_VALUE);
376-
properties.putIfAbsent(TableProperties.FORMAT_VERSION, "2");
376+
Map<String, String> catalogProperties = dorisCatalog.getProperties();
377+
if (!properties.containsKey(TableProperties.FORMAT_VERSION)
378+
&& !IcebergUtils.hasIcebergCatalogFormatVersion(catalogProperties)) {
379+
properties.put(TableProperties.FORMAT_VERSION, "2");
380+
}
377381
properties.putIfAbsent(TableProperties.DELETE_MODE, RowLevelOperationMode.MERGE_ON_READ.modeName());
378382
properties.putIfAbsent(TableProperties.UPDATE_MODE, RowLevelOperationMode.MERGE_ON_READ.modeName());
379383
properties.putIfAbsent(TableProperties.MERGE_MODE, RowLevelOperationMode.MERGE_ON_READ.modeName());
384+
createTableInfo.validateIcebergRowLineageColumns(
385+
IcebergUtils.getEffectiveIcebergFormatVersion(properties, catalogProperties));
380386
PartitionSpec partitionSpec = IcebergUtils.solveIcebergPartitionSpec(createTableInfo.getPartitionDesc(),
381387
schema);
382388
// Build and create table with optional sort order

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,33 @@ public Integer initialValue() {
189189

190190
private static final Pattern SNAPSHOT_ID = Pattern.compile("\\d+");
191191

192+
public static boolean hasIcebergCatalogFormatVersion(Map<String, String> catalogProperties) {
193+
return catalogProperties.containsKey(CatalogProperties.TABLE_OVERRIDE_PREFIX + TableProperties.FORMAT_VERSION)
194+
|| catalogProperties.containsKey(CatalogProperties.TABLE_DEFAULT_PREFIX
195+
+ TableProperties.FORMAT_VERSION);
196+
}
197+
198+
public static int getEffectiveIcebergFormatVersion(Map<String, String> tableProperties,
199+
Map<String, String> catalogProperties) {
200+
String formatVersion = catalogProperties.get(CatalogProperties.TABLE_OVERRIDE_PREFIX
201+
+ TableProperties.FORMAT_VERSION);
202+
if (formatVersion == null) {
203+
formatVersion = tableProperties.get(TableProperties.FORMAT_VERSION);
204+
if (formatVersion == null) {
205+
formatVersion = catalogProperties.get(CatalogProperties.TABLE_DEFAULT_PREFIX
206+
+ TableProperties.FORMAT_VERSION);
207+
}
208+
}
209+
if (formatVersion == null) {
210+
return 2;
211+
}
212+
try {
213+
return Integer.parseInt(formatVersion);
214+
} catch (NumberFormatException ignored) {
215+
return 2;
216+
}
217+
}
218+
192219
public static Expression convertToIcebergExpr(Expr expr, Schema schema) {
193220
if (expr == null) {
194221
return null;

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import org.apache.doris.datasource.InternalCatalog;
5050
import org.apache.doris.datasource.hive.HMSExternalCatalog;
5151
import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
52+
import org.apache.doris.datasource.iceberg.IcebergUtils;
5253
import org.apache.doris.datasource.maxcompute.MaxComputeExternalCatalog;
5354
import org.apache.doris.datasource.paimon.PaimonExternalCatalog;
5455
import org.apache.doris.mysql.privilege.PrivPredicate;
@@ -380,6 +381,9 @@ private void checkEngineWithCatalog() {
380381
+ " Make sure 'engine' type is specified when use the catalog: " + ctlName);
381382
}
382383
}
384+
if (Strings.isNullOrEmpty(ctlName)) {
385+
return;
386+
}
383387
CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(ctlName);
384388
if (catalog instanceof HMSExternalCatalog && !engineName.equals(ENGINE_HIVE)) {
385389
throw new AnalysisException("Hms type catalog can only use `hive` engine.");
@@ -791,6 +795,10 @@ public void validate(ConnectContext ctx) {
791795
+ "and you can use 'bucket(num, column)' in 'PARTITIONED BY'.");
792796
}
793797

798+
if (engineName.equalsIgnoreCase(ENGINE_ICEBERG)) {
799+
validateIcebergRowLineageColumns();
800+
}
801+
794802
// Validate Iceberg sort order columns
795803
if (sortOrderFields != null && !sortOrderFields.isEmpty()) {
796804
if (!engineName.equalsIgnoreCase(ENGINE_ICEBERG)) {
@@ -1105,6 +1113,34 @@ private void validateKeyColumns() {
11051113
}
11061114
}
11071115

1116+
/**
1117+
* Validate that Iceberg v3 tables do not define row lineage reserved columns.
1118+
*/
1119+
public void validateIcebergRowLineageColumns(int formatVersion) {
1120+
if (formatVersion < IcebergUtils.ICEBERG_ROW_LINEAGE_MIN_VERSION) {
1121+
return;
1122+
}
1123+
for (ColumnDefinition columnDef : columns) {
1124+
if (IcebergUtils.isIcebergRowLineageColumn(columnDef.getName())) {
1125+
throw new AnalysisException("Cannot create Iceberg v" + formatVersion
1126+
+ " table with reserved row lineage column: " + columnDef.getName());
1127+
}
1128+
}
1129+
}
1130+
1131+
private void validateIcebergRowLineageColumns() {
1132+
validateIcebergRowLineageColumns(getEffectiveIcebergFormatVersion());
1133+
}
1134+
1135+
private int getEffectiveIcebergFormatVersion() {
1136+
CatalogIf catalog = Strings.isNullOrEmpty(ctlName) ? null
1137+
: Env.getCurrentEnv().getCatalogMgr().getCatalog(ctlName);
1138+
if (catalog instanceof IcebergExternalCatalog) {
1139+
return IcebergUtils.getEffectiveIcebergFormatVersion(properties, catalog.getProperties());
1140+
}
1141+
return IcebergUtils.getEffectiveIcebergFormatVersion(properties, Collections.emptyMap());
1142+
}
1143+
11081144
/**
11091145
* analyzeEngine
11101146
*/

‎fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.apache.doris.nereids.trees.expressions.NamedExpression;
3737
import org.apache.doris.nereids.trees.expressions.Slot;
3838
import org.apache.doris.nereids.trees.plans.Plan;
39+
import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand;
3940
import org.apache.doris.nereids.trees.plans.commands.DeleteFromCommand;
4041
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
4142
import org.apache.doris.nereids.trees.plans.commands.UpdateCommand;
@@ -273,6 +274,76 @@ public void testIcebergDeletePlanAddsRowIdProject() throws Exception {
273274
assertContainsPhysicalSink(physicalPlan, PhysicalIcebergDeleteSink.class);
274275
}
275276

277+
@Test
278+
public void testCreateIcebergV3TableRejectsRowLineageReservedColumn() throws Exception {
279+
useIceberg();
280+
String rowIdTable = "row_lineage_reserved_" + UUID.randomUUID().toString().replace("-", "");
281+
String rowIdSql = "create table " + rowIdTable
282+
+ " (_row_id bigint) properties('format-version'='3')";
283+
LogicalPlan rowIdPlan = parseStmt(rowIdSql);
284+
Assertions.assertTrue(rowIdPlan instanceof CreateTableCommand);
285+
Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class,
286+
() -> ((CreateTableCommand) rowIdPlan).getCreateTableInfo().validate(connectContext));
287+
288+
String lastUpdatedSequenceNumberTable =
289+
"row_lineage_reserved_" + UUID.randomUUID().toString().replace("-", "");
290+
String lastUpdatedSequenceNumberSql = "create table " + lastUpdatedSequenceNumberTable
291+
+ " (_last_updated_sequence_number bigint) properties('format-version'='3')";
292+
LogicalPlan lastUpdatedSequenceNumberPlan = parseStmt(lastUpdatedSequenceNumberSql);
293+
Assertions.assertTrue(lastUpdatedSequenceNumberPlan instanceof CreateTableCommand);
294+
Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class,
295+
() -> ((CreateTableCommand) lastUpdatedSequenceNumberPlan).getCreateTableInfo()
296+
.validate(connectContext));
297+
298+
String formatV2Table = "row_lineage_reserved_" + UUID.randomUUID().toString().replace("-", "");
299+
String formatV2Sql = "create table " + formatV2Table
300+
+ " (_row_id bigint) properties('format-version'='2')";
301+
LogicalPlan formatV2Plan = parseStmt(formatV2Sql);
302+
Assertions.assertTrue(formatV2Plan instanceof CreateTableCommand);
303+
Assertions.assertDoesNotThrow(
304+
() -> ((CreateTableCommand) formatV2Plan).getCreateTableInfo().validate(connectContext));
305+
}
306+
307+
@Test
308+
public void testCreateIcebergDefaultV3TableRejectsRowLineageReservedColumn() throws Exception {
309+
useIceberg();
310+
IcebergExternalCatalog catalog = (IcebergExternalCatalog) Env.getCurrentEnv()
311+
.getCatalogMgr().getCatalog(catalogName);
312+
catalog.getCatalogProperty().addProperty("table-default.format-version", "3");
313+
try {
314+
String rowIdTable = "row_lineage_reserved_" + UUID.randomUUID().toString().replace("-", "");
315+
String rowIdSql = "create table " + rowIdTable + " (_row_id bigint)";
316+
LogicalPlan rowIdPlan = parseStmt(rowIdSql);
317+
Assertions.assertTrue(rowIdPlan instanceof CreateTableCommand);
318+
Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class,
319+
() -> ((CreateTableCommand) rowIdPlan).getCreateTableInfo().validate(connectContext));
320+
Assertions.assertFalse(catalog.getCatalog().tableExists(TableIdentifier.of(dbName, rowIdTable)));
321+
322+
String formatV2Table = "row_lineage_reserved_" + UUID.randomUUID().toString().replace("-", "");
323+
String formatV2Sql = "create table " + formatV2Table
324+
+ " (_row_id bigint) properties('format-version'='2')";
325+
LogicalPlan formatV2Plan = parseStmt(formatV2Sql);
326+
Assertions.assertTrue(formatV2Plan instanceof CreateTableCommand);
327+
Assertions.assertDoesNotThrow(
328+
() -> ((CreateTableCommand) formatV2Plan).getCreateTableInfo().validate(connectContext));
329+
} finally {
330+
catalog.getCatalogProperty().deleteProperty("table-default.format-version");
331+
}
332+
}
333+
334+
@Test
335+
public void testIcebergV3CtasRejectsRowLineageReservedColumn() throws Exception {
336+
useIceberg();
337+
String ctasTable = "row_lineage_reserved_" + UUID.randomUUID().toString().replace("-", "");
338+
String ctasSql = "create table " + ctasTable
339+
+ " properties('format-version'='3') as select 1 as _row_id";
340+
LogicalPlan ctasPlan = parseStmt(ctasSql);
341+
Assertions.assertTrue(ctasPlan instanceof CreateTableCommand);
342+
Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class,
343+
() -> ((CreateTableCommand) ctasPlan).validateCreateTableAsSelect(
344+
connectContext, ((CreateTableCommand) ctasPlan).getCtasQuery().get()));
345+
}
346+
276347
@Test
277348
public void testIcebergUpdatePlans() throws Exception {
278349
useIceberg();

‎fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
package org.apache.doris.datasource.iceberg;
1919

20+
import org.apache.doris.catalog.Column;
21+
import org.apache.doris.catalog.Type;
2022
import org.apache.doris.common.security.authentication.ExecutionAuthenticator;
2123
import org.apache.doris.datasource.CatalogProperty;
2224
import org.apache.doris.datasource.ExternalDatabase;
@@ -28,14 +30,20 @@
2830
import org.apache.doris.filesystem.FileSystem;
2931
import org.apache.doris.filesystem.Location;
3032
import org.apache.doris.fs.MemoryFileSystem;
33+
import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo;
3134

35+
import org.apache.iceberg.CatalogProperties;
36+
import org.apache.iceberg.PartitionSpec;
37+
import org.apache.iceberg.Schema;
38+
import org.apache.iceberg.TableProperties;
3239
import org.apache.iceberg.catalog.Catalog;
3340
import org.apache.iceberg.catalog.Namespace;
3441
import org.apache.iceberg.catalog.SupportsNamespaces;
3542
import org.apache.iceberg.catalog.TableIdentifier;
3643
import org.apache.iceberg.catalog.ViewCatalog;
3744
import org.junit.Assert;
3845
import org.junit.jupiter.api.Test;
46+
import org.mockito.ArgumentCaptor;
3947
import org.mockito.Mockito;
4048

4149
import java.io.IOException;
@@ -129,6 +137,42 @@ public void testListTableNamesFiltersViewsWhenRestViewEnabled() {
129137
Assert.assertEquals(Collections.singletonList("DORIS_HORIZON_T"), tableNames);
130138
}
131139

140+
@Test
141+
public void testPerformCreateTableRespectsCatalogDefaultFormatVersion() throws Exception {
142+
Map<String, String> catalogProps = new HashMap<>();
143+
catalogProps.put(CatalogProperties.TABLE_DEFAULT_PREFIX + TableProperties.FORMAT_VERSION, "3");
144+
IcebergExternalCatalog dorisCatalog = mockHmsCatalog(catalogProps);
145+
Catalog icebergCatalog = Mockito.mock(Catalog.class,
146+
Mockito.withSettings().extraInterfaces(SupportsNamespaces.class));
147+
IcebergMetadataOps ops = new IcebergMetadataOps(dorisCatalog, icebergCatalog);
148+
149+
ExternalDatabase<?> dorisDb = Mockito.mock(ExternalDatabase.class);
150+
Mockito.when(dorisDb.getRemoteName()).thenReturn("db");
151+
Mockito.when(dorisDb.getTableNullable("tbl")).thenReturn(null);
152+
Mockito.doReturn(dorisDb).when(dorisCatalog).getDbNullable("db");
153+
Mockito.when(dorisCatalog.getName()).thenReturn("iceberg_catalog");
154+
Mockito.when(icebergCatalog.tableExists(TableIdentifier.of("db", "tbl"))).thenReturn(false);
155+
156+
CreateTableInfo createTableInfo = Mockito.mock(CreateTableInfo.class);
157+
Map<String, String> tableProps = new HashMap<>();
158+
Mockito.when(createTableInfo.getDbName()).thenReturn("db");
159+
Mockito.when(createTableInfo.getTableName()).thenReturn("tbl");
160+
Mockito.when(createTableInfo.isIfNotExists()).thenReturn(false);
161+
Mockito.when(createTableInfo.getColumns()).thenReturn(Collections.singletonList(
162+
new Column("id", Type.INT, true)));
163+
Mockito.when(createTableInfo.getProperties()).thenReturn(tableProps);
164+
165+
ops.performCreateTable(createTableInfo);
166+
167+
Mockito.verify(createTableInfo).validateIcebergRowLineageColumns(3);
168+
ArgumentCaptor<Map<String, String>> propsCaptor = ArgumentCaptor.forClass(Map.class);
169+
Mockito.verify(icebergCatalog).createTable(Mockito.eq(TableIdentifier.of("db", "tbl")),
170+
Mockito.any(Schema.class), Mockito.any(PartitionSpec.class), propsCaptor.capture());
171+
Assert.assertFalse(propsCaptor.getValue().containsKey(TableProperties.FORMAT_VERSION));
172+
Assert.assertEquals(3, IcebergUtils.getEffectiveIcebergFormatVersion(
173+
propsCaptor.getValue(), catalogProps));
174+
}
175+
132176
@Test
133177
public void testDropTableCleansEmptyTableLocation() throws Exception {
134178
MemoryFileSystem fs = new MemoryFileSystem();
@@ -214,10 +258,14 @@ public void testDeleteEmptyTableLocationCleansFlatObjectStoreMarkers() throws Ex
214258
}
215259

216260
private IcebergExternalCatalog mockHmsCatalog() {
261+
return mockHmsCatalog(Collections.emptyMap());
262+
}
263+
264+
private IcebergExternalCatalog mockHmsCatalog(Map<String, String> catalogProperties) {
217265
IcebergExternalCatalog dorisCatalog = Mockito.mock(IcebergExternalCatalog.class);
218266
Mockito.when(dorisCatalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() {
219267
});
220-
Mockito.when(dorisCatalog.getProperties()).thenReturn(Collections.emptyMap());
268+
Mockito.when(dorisCatalog.getProperties()).thenReturn(catalogProperties);
221269
Mockito.when(dorisCatalog.getIcebergCatalogType()).thenReturn(IcebergExternalCatalog.ICEBERG_HMS);
222270
Mockito.when(dorisCatalog.getCatalogProperty()).thenReturn(new CatalogProperty(null, Collections.emptyMap()));
223271
return dorisCatalog;

0 commit comments

Comments
 (0)