Skip to content

Commit c7ffbda

Browse files
hubgeterhubgeter
authored
[fix](iceberg) Preserve nullability in schema metadata (#68640)
### What problem does this PR solve? Related PR: #67856 Problem Summary: Iceberg columns declared `NOT NULL` are shown as nullable by `DESC`, `SHOW CREATE TABLE`, `SHOW [FULL] COLUMNS` and `information_schema.columns`. The connector reads every top-level column as nullable on purpose, since old data files can miss a column, and the display paths reuse that read schema. `ConnectorColumn` now also carries the nullability the source declares. The Iceberg connector sets it from the field's required flag, and these metadata commands use it. The columns used for reads and writes stay nullable. ### Release note Show the declared nullability of Iceberg columns in `DESC`, `SHOW CREATE TABLE`, `SHOW COLUMNS` and `information_schema.columns`. ### Check List (For Author) - Test - [x] Regression test - [x] Unit Test - Behavior changed: - [x] Yes. Required Iceberg columns are now shown as `NOT NULL`. - Does this need documentation? - [x] No.
1 parent 3763203 commit c7ffbda

26 files changed

Lines changed: 527 additions & 64 deletions

File tree

‎fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2487,6 +2487,8 @@ private List<ConnectorColumn> parseSchema(Schema schema) {
24872487
// isKey is always true (external-table semantics: DESC shows Key=true),
24882488
// and isAllowNull is always true regardless of the Iceberg required/optional flag (rows can
24892489
// still read NULL under schema-evolution default-fill; do NOT propagate the NOT NULL constraint).
2490+
// The required/optional flag travels separately as the SOURCE nullability, which is what DESC /
2491+
// SHOW CREATE TABLE / SHOW COLUMNS / information_schema display instead of the read nullability.
24902492
// Iceberg write defaults are statement-scoped writer metadata, not catalog display metadata.
24912493
// IcebergWritePlanProvider pins them before write analysis; keeping the cached default null prevents
24922494
// DESCRIBE/SHOW CREATE from exposing them and avoids using a stale default after schema evolution.
@@ -2497,7 +2499,7 @@ private List<ConnectorColumn> parseSchema(Schema schema) {
24972499
field.doc() != null ? field.doc() : "",
24982500
true,
24992501
null,
2500-
true);
2502+
true).withSourceNullable(field.isOptional());
25012503
// Carry the stable iceberg field-id as the column's uniqueId (legacy
25022504
// IcebergUtils.updateIcebergColumnUniqueId set the top-level Column.uniqueId = field.fieldId()).
25032505
// fe-core's ConnectorColumnConverter re-applies it (>= 0); the BE field-id scan path keys the

‎fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataTest.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -888,6 +888,25 @@ public void getTableSchemaCarriesFieldDocAsComment() {
888888
"a doc-less Iceberg field must yield an empty (not null) comment");
889889
}
890890

891+
@Test
892+
public void getTableSchemaDeclaresRequiredFieldsNotNull() {
893+
RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps();
894+
ops.table = new FakeIcebergTable(
895+
"t1", idNameSchema(), PartitionSpec.unpartitioned(),
896+
"s3://bucket/db1/t1", Collections.emptyMap());
897+
898+
ConnectorTableSchema schema =
899+
metadataWith(ops).getTableSchema(null, new IcebergTableHandle("db1", "t1"));
900+
901+
// Required fields are read as nullable but displayed as NOT NULL.
902+
ConnectorColumn id = schema.getColumns().get(0);
903+
Assertions.assertTrue(id.isNullable());
904+
Assertions.assertFalse(id.isSourceNullable());
905+
ConnectorColumn name = schema.getColumns().get(1);
906+
Assertions.assertTrue(name.isNullable());
907+
Assertions.assertTrue(name.isSourceNullable());
908+
}
909+
891910
@Test
892911
public void getTableSchemaCopiesTablePropertiesAndLocation() {
893912
RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps();

‎fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorColumn.java‎

Lines changed: 44 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,22 @@ public final class ConnectorColumn {
7474
private final String defaultValueSql;
7575
// Optional semantic type for textual write input, separate from the exposed binary column type.
7676
private final ConnectorType stringWriteType;
77+
// The nullability the SOURCE schema declares (its required/optional flag), which is what the engine's
78+
// metadata statements show: DESC, SHOW CREATE TABLE, SHOW [FULL] COLUMNS, information_schema.columns and
79+
// the field list a MySQL client asks for. It is deliberately separate from the read nullability above:
80+
// a connector may read a source-required column as nullable (iceberg/paimon do, because a row written
81+
// before the column existed reads back NULL under schema-evolution default-fill) while the source
82+
// contract still declares it NOT NULL. Execution — scans, the optimizer, write validation, the BE
83+
// descriptor — always follows `nullable`, never this field.
84+
//
85+
// Defaults to `nullable`, so every connector that does not opt in keeps display == read and this field
86+
// changes nothing. Set via withSourceNullable().
87+
//
88+
// Excluded from equals()/hashCode() like the "specified" markers above: it is a presentation fact that
89+
// never round-trips (a column converted back from a Doris Column carries no source contract), so it must
90+
// not become part of column identity — otherwise a MODIFY COLUMN would look like a change just because
91+
// the request-side column lacks the display flag.
92+
private final boolean sourceNullable;
7793

7894
public ConnectorColumn(String name, ConnectorType type, String comment,
7995
boolean nullable, String defaultValue) {
@@ -94,14 +110,14 @@ public ConnectorColumn(String name, ConnectorType type, String comment,
94110
boolean nullable, String defaultValue, boolean isKey, boolean isAutoInc,
95111
boolean isAggregated) {
96112
this(name, type, comment, nullable, defaultValue, isKey, isAutoInc, isAggregated, false, true,
97-
UNSET_UNIQUE_ID, false, false, false, null, null);
113+
UNSET_UNIQUE_ID, false, false, false, null, null, nullable);
98114
}
99115

100116
private ConnectorColumn(String name, ConnectorType type, String comment,
101117
boolean nullable, String defaultValue, boolean isKey, boolean isAutoInc,
102118
boolean isAggregated, boolean withTimeZone, boolean visible, int uniqueId,
103119
boolean reservedPassthrough, boolean nullableSpecified, boolean commentSpecified,
104-
String defaultValueSql, ConnectorType stringWriteType) {
120+
String defaultValueSql, ConnectorType stringWriteType, boolean sourceNullable) {
105121
this.name = Objects.requireNonNull(name, "name");
106122
this.type = Objects.requireNonNull(type, "type");
107123
this.comment = comment;
@@ -118,6 +134,7 @@ private ConnectorColumn(String name, ConnectorType type, String comment,
118134
this.commentSpecified = commentSpecified;
119135
this.defaultValueSql = defaultValueSql;
120136
this.stringWriteType = stringWriteType;
137+
this.sourceNullable = sourceNullable;
121138
}
122139

123140
/**
@@ -128,7 +145,7 @@ private ConnectorColumn(String name, ConnectorType type, String comment,
128145
public ConnectorColumn withTimeZone() {
129146
return new ConnectorColumn(name, type, comment, nullable, defaultValue,
130147
isKey, isAutoInc, isAggregated, true, visible, uniqueId, reservedPassthrough,
131-
nullableSpecified, commentSpecified, defaultValueSql, stringWriteType);
148+
nullableSpecified, commentSpecified, defaultValueSql, stringWriteType, sourceNullable);
132149
}
133150

134151
/**
@@ -138,7 +155,7 @@ public ConnectorColumn withTimeZone() {
138155
public ConnectorColumn invisible() {
139156
return new ConnectorColumn(name, type, comment, nullable, defaultValue,
140157
isKey, isAutoInc, isAggregated, withTimeZone, false, uniqueId, reservedPassthrough,
141-
nullableSpecified, commentSpecified, defaultValueSql, stringWriteType);
158+
nullableSpecified, commentSpecified, defaultValueSql, stringWriteType, sourceNullable);
142159
}
143160

144161
/**
@@ -150,7 +167,7 @@ public ConnectorColumn invisible() {
150167
public ConnectorColumn reservedPassthrough() {
151168
return new ConnectorColumn(name, type, comment, nullable, defaultValue,
152169
isKey, isAutoInc, isAggregated, withTimeZone, visible, uniqueId, true,
153-
nullableSpecified, commentSpecified, defaultValueSql, stringWriteType);
170+
nullableSpecified, commentSpecified, defaultValueSql, stringWriteType, sourceNullable);
154171
}
155172

156173
/**
@@ -161,7 +178,7 @@ public ConnectorColumn reservedPassthrough() {
161178
public ConnectorColumn withSpecified(boolean nullableSpecified, boolean commentSpecified) {
162179
return new ConnectorColumn(name, type, comment, nullable, defaultValue,
163180
isKey, isAutoInc, isAggregated, withTimeZone, visible, uniqueId, reservedPassthrough,
164-
nullableSpecified, commentSpecified, defaultValueSql, stringWriteType);
181+
nullableSpecified, commentSpecified, defaultValueSql, stringWriteType, sourceNullable);
165182
}
166183

167184
/**
@@ -173,7 +190,7 @@ public ConnectorColumn withSpecified(boolean nullableSpecified, boolean commentS
173190
public ConnectorColumn withUniqueId(int uniqueId) {
174191
return new ConnectorColumn(name, type, comment, nullable, defaultValue,
175192
isKey, isAutoInc, isAggregated, withTimeZone, visible, uniqueId, reservedPassthrough,
176-
nullableSpecified, commentSpecified, defaultValueSql, stringWriteType);
193+
nullableSpecified, commentSpecified, defaultValueSql, stringWriteType, sourceNullable);
177194
}
178195

179196
/**
@@ -182,7 +199,7 @@ public ConnectorColumn withUniqueId(int uniqueId) {
182199
public ConnectorColumn withDefaultValueSql(String defaultValueSql) {
183200
return new ConnectorColumn(name, type, comment, nullable, defaultValue,
184201
isKey, isAutoInc, isAggregated, withTimeZone, visible, uniqueId, reservedPassthrough,
185-
nullableSpecified, commentSpecified, defaultValueSql, stringWriteType);
202+
nullableSpecified, commentSpecified, defaultValueSql, stringWriteType, sourceNullable);
186203
}
187204

188205
/**
@@ -193,13 +210,23 @@ public ConnectorColumn withDefaultValueSql(String defaultValueSql) {
193210
public ConnectorColumn withStringWriteType(ConnectorType stringWriteType) {
194211
return new ConnectorColumn(name, type, comment, nullable, defaultValue,
195212
isKey, isAutoInc, isAggregated, withTimeZone, visible, uniqueId, reservedPassthrough,
196-
nullableSpecified, commentSpecified, defaultValueSql, stringWriteType);
213+
nullableSpecified, commentSpecified, defaultValueSql, stringWriteType, sourceNullable);
197214
}
198215

199216
public ConnectorType getStringWriteType() {
200217
return stringWriteType;
201218
}
202219

220+
/**
221+
* Returns a copy carrying the nullability the source schema declares, for metadata statements only.
222+
* See {@link #isSourceNullable()}; execution keeps following {@link #isNullable()}.
223+
*/
224+
public ConnectorColumn withSourceNullable(boolean sourceNullable) {
225+
return new ConnectorColumn(name, type, comment, nullable, defaultValue,
226+
isKey, isAutoInc, isAggregated, withTimeZone, visible, uniqueId, reservedPassthrough,
227+
nullableSpecified, commentSpecified, defaultValueSql, stringWriteType, sourceNullable);
228+
}
229+
203230
public String getName() {
204231
return name;
205232
}
@@ -262,6 +289,14 @@ public String getDefaultValueSql() {
262289
return defaultValueSql;
263290
}
264291

292+
/**
293+
* The nullability the source schema declares, used by metadata statements only; see the field comment.
294+
* Defaults to {@link #isNullable()}. Execution always follows {@link #isNullable()}.
295+
*/
296+
public boolean isSourceNullable() {
297+
return sourceNullable;
298+
}
299+
265300
@Override
266301
public boolean equals(Object o) {
267302
if (this == o) {

‎fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,8 +88,9 @@ public void connectorApiMajorTracksTheRecordedSurfaceChange() throws IOException
8888
Assertions.assertNotNull(in, "missing connector plugin API version resource");
8989
version.load(in);
9090
}
91-
// Major 18 adds DiagnosticException to the major 17 scoped streaming surface.
92-
Assertions.assertEquals("18.0", version.getProperty("api.version"));
91+
// Major 19 adds ConnectorColumn#isSourceNullable()/withSourceNullable(boolean) - the nullability the
92+
// source declares, kept separate from the nullability the engine reads the column with.
93+
Assertions.assertEquals("19.0", version.getProperty("api.version"));
9394
}
9495

9596
/** Root entry points plus provider/handle types returned to connector plugins. */

‎fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,11 +52,13 @@ org.apache.doris.connector.spi.ConnectorColumn#isKey():boolean
5252
org.apache.doris.connector.spi.ConnectorColumn#isNullable():boolean
5353
org.apache.doris.connector.spi.ConnectorColumn#isNullableSpecified():boolean
5454
org.apache.doris.connector.spi.ConnectorColumn#isReservedPassthrough():boolean
55+
org.apache.doris.connector.spi.ConnectorColumn#isSourceNullable():boolean
5556
org.apache.doris.connector.spi.ConnectorColumn#isVisible():boolean
5657
org.apache.doris.connector.spi.ConnectorColumn#isWithTimeZone():boolean
5758
org.apache.doris.connector.spi.ConnectorColumn#reservedPassthrough():org.apache.doris.connector.spi.ConnectorColumn
5859
org.apache.doris.connector.spi.ConnectorColumn#toString():java.lang.String
5960
org.apache.doris.connector.spi.ConnectorColumn#withDefaultValueSql(java.lang.String):org.apache.doris.connector.spi.ConnectorColumn
61+
org.apache.doris.connector.spi.ConnectorColumn#withSourceNullable(boolean):org.apache.doris.connector.spi.ConnectorColumn
6062
org.apache.doris.connector.spi.ConnectorColumn#withSpecified(boolean,boolean):org.apache.doris.connector.spi.ConnectorColumn
6163
org.apache.doris.connector.spi.ConnectorColumn#withStringWriteType(org.apache.doris.connector.spi.ConnectorType):org.apache.doris.connector.spi.ConnectorColumn
6264
org.apache.doris.connector.spi.ConnectorColumn#withTimeZone():org.apache.doris.connector.spi.ConnectorColumn

‎fe/fe-connector/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ under the License.
5555
of the latter two means bumping this property as well (and fe-extension-spi means bumping
5656
all five families).
5757
-->
58-
<connector.plugin.api.version>18.0</connector.plugin.api.version>
58+
<connector.plugin.api.version>19.0</connector.plugin.api.version>
5959
</properties>
6060

6161
<modules>

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4735,7 +4735,7 @@ public static void getDdlStmt(Command command, String dbName, TableIf table, Lis
47354735

47364736
sb.append(" (\n");
47374737
int idx = 0;
4738-
List<Column> columns = table.getBaseSchema(false);
4738+
List<Column> columns = table.getBaseSchemaForDisplay(false);
47394739
for (Column column : columns) {
47404740
if (idx++ != 0) {
47414741
sb.append(",\n");

‎fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,8 +173,33 @@ default List<Column> getBaseSchemaOrEmpty() {
173173
}
174174
}
175175

176+
/** {@link #getBaseSchemaOrEmpty()} over the metadata view; see {@link #getBaseSchemaForDisplay()}. */
177+
default List<Column> getBaseSchemaForDisplayOrEmpty() {
178+
try {
179+
return getBaseSchemaForDisplay();
180+
} catch (Exception e) {
181+
LOG.warn("failed to get base schema for table {}", getName(), e);
182+
return Lists.newArrayList();
183+
}
184+
}
185+
176186
List<Column> getBaseSchema(boolean full);
177187

188+
/**
189+
* The base schema shown by metadata statements (DESC, SHOW CREATE TABLE, SHOW COLUMNS and
190+
* information_schema.columns). It differs from {@link #getBaseSchema()} only for a table whose source
191+
* declares columns NOT NULL that are still read as nullable. Execution (scans, the optimizer, write
192+
* validation, the BE descriptor) must keep using {@link #getBaseSchema()}.
193+
*/
194+
default List<Column> getBaseSchemaForDisplay() {
195+
return getBaseSchema();
196+
}
197+
198+
/** Same as {@link #getBaseSchemaForDisplay()}, with the hidden-column choice of {@link #getBaseSchema(boolean)}. */
199+
default List<Column> getBaseSchemaForDisplay(boolean full) {
200+
return getBaseSchema(full);
201+
}
202+
178203
void setNewFullSchema(List<Column> newSchema);
179204

180205
Column getColumn(String name);

‎fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ public ProcNodeInterface lookup(String idxIdStr) throws AnalysisException {
144144
return new RemoteIndexSchemaProcDir(table, schema, bfColumns);
145145
}
146146
} else {
147-
schema = table.getBaseSchema();
147+
schema = table.getBaseSchemaForDisplay();
148148
}
149149
return new IndexSchemaProcNode(schema, bfColumns);
150150
} finally {

0 commit comments

Comments
 (0)