Skip to content

Commit a2a6f69

Browse files
morningmanclaude
andcommitted
[fix](catalog) port apache#65782 collect_column_stats sink flag + write-metrics tests to the iceberg connector
Upstream apache#65782 (Honor disabled write metrics) modified fe-core write-path files that this branch has migrated to fe-connector, so its conflicts were resolved during the rebase: the manifest metrics-filtering logic and the BaseTable->HasTableOperations format-version read now live in the connector IcebergWriterHelper. This commit completes the port with the pieces that are not part of a conflicted file. - IcebergWritePlanProvider: set the TIcebergTableSink/TIcebergMergeSink collect_column_stats flag via IcebergWriterHelper.shouldCollectColumnStats, in buildSink (table schema), buildMergeSink and buildRewriteSink (the v3 row-lineage-appended schema, matching each sink's schema_json). A table whose columns are all metrics=none lets the BE skip stats collection. - IcebergWriterHelperTest: 6 write-side tests over a real InMemoryCatalog table (none / counts / truncate string+binary / orc-preserves-max-without- successor / logical-metrics-suppressed-below-repeated-fields, and a transaction-table v3 case that guards the HasTableOperations change). Field ids are resolved from the created table schema because createTable reassigns them. - StatisticsUtilTest: 2 read-side tests for getIcebergColumnStats returning unknown (empty) when required file metrics are absent or a scan close fails. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
1 parent b955544 commit a2a6f69

3 files changed

Lines changed: 228 additions & 3 deletions

File tree

‎fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWritePlanProvider.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -384,6 +384,9 @@ private TIcebergTableSink buildSink(Table table, IcebergTableHandle tableHandle,
384384

385385
// Schema (no v3 row-lineage append — that is REWRITE/procedures, P6.4).
386386
tSink.setSchemaJson(SchemaParser.toJson(table.schema()));
387+
// #65782: gate BE-side column-stats collection on the table's iceberg metrics policy; a fully-disabled
388+
// table (all columns metrics=none) skips BE collection entirely. Same schema the sink advertises.
389+
tSink.setCollectColumnStats(IcebergWriterHelper.shouldCollectColumnStats(table, table.schema()));
387390

388391
// Partition spec (only for a partitioned table, mirroring legacy spec().isPartitioned()).
389392
if (table.spec().isPartitioned()) {
@@ -442,8 +445,10 @@ private TIcebergTableSink buildRewriteSink(Table table, IcebergTableHandle table
442445
tSink.setWriteType(TIcebergWriteType.REWRITE);
443446
if (IcebergWriterHelper.getFormatVersion(table) >= 3) {
444447
// iceberg v3 format requires the row-lineage fields when rewriting data files.
445-
tSink.setSchemaJson(SchemaParser.toJson(
446-
IcebergWriterHelper.appendRowLineageFieldsForV3(table.schema())));
448+
Schema rewriteSchema = IcebergWriterHelper.appendRowLineageFieldsForV3(table.schema());
449+
tSink.setSchemaJson(SchemaParser.toJson(rewriteSchema));
450+
// #65782: the collect flag must reflect the same (v3-appended) schema the sink advertises.
451+
tSink.setCollectColumnStats(IcebergWriterHelper.shouldCollectColumnStats(table, rewriteSchema));
447452
}
448453
return tSink;
449454
}
@@ -508,6 +513,8 @@ private TIcebergMergeSink buildMergeSink(Table table, IcebergTableHandle tableHa
508513
Schema schema = formatVersion >= 3
509514
? IcebergWriterHelper.appendRowLineageFieldsForV3(table.schema()) : table.schema();
510515
tSink.setSchemaJson(SchemaParser.toJson(schema));
516+
// #65782: gate BE-side column-stats collection on the table's iceberg metrics policy (v3-appended schema).
517+
tSink.setCollectColumnStats(IcebergWriterHelper.shouldCollectColumnStats(table, schema));
511518

512519
if (table.spec().isPartitioned()) {
513520
tSink.setPartitionSpecsJson(Maps.transformValues(table.specs(), PartitionSpecParser::toJson));

‎fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergWriterHelperTest.java‎

Lines changed: 165 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,15 @@
2626
import org.apache.iceberg.DataFiles;
2727
import org.apache.iceberg.DeleteFile;
2828
import org.apache.iceberg.FileFormat;
29+
import org.apache.iceberg.MetadataColumns;
2930
import org.apache.iceberg.PartitionSpec;
3031
import org.apache.iceberg.Schema;
3132
import org.apache.iceberg.Table;
3233
import org.apache.iceberg.catalog.Namespace;
3334
import org.apache.iceberg.catalog.TableIdentifier;
3435
import org.apache.iceberg.inmemory.InMemoryCatalog;
3536
import org.apache.iceberg.io.WriteResult;
37+
import org.apache.iceberg.types.Conversions;
3638
import org.apache.iceberg.types.Types;
3739
import org.junit.jupiter.api.Assertions;
3840
import org.junit.jupiter.api.Test;
@@ -43,6 +45,7 @@
4345
import java.util.Arrays;
4446
import java.util.Collections;
4547
import java.util.List;
48+
import java.util.Map;
4649

4750
/**
4851
* Parity oracle for the connector-resident {@link IcebergWriterHelper} — the self-contained port of legacy
@@ -65,14 +68,31 @@ public class IcebergWriterHelperTest {
6568
private final FileFormat format = FileFormat.PARQUET;
6669

6770
private Table tableWith(String... props) {
71+
return tableWith(schema, props);
72+
}
73+
74+
private Table tableWith(Schema tableSchema, String... props) {
6875
InMemoryCatalog catalog = new InMemoryCatalog();
6976
catalog.initialize("test", Collections.emptyMap());
7077
catalog.createNamespace(Namespace.of("db1"));
7178
java.util.Map<String, String> properties = new java.util.HashMap<>();
7279
for (int i = 0; i + 1 < props.length; i += 2) {
7380
properties.put(props[i], props[i + 1]);
7481
}
75-
return catalog.createTable(TableIdentifier.of("db1", "t"), schema, unpartitionedSpec, properties);
82+
return catalog.createTable(TableIdentifier.of("db1", "t"), tableSchema, unpartitionedSpec, properties);
83+
}
84+
85+
// Builds one DATA-content commit fragment and returns the single converted data file. Row count / file size
86+
// are immaterial to the metrics assertions, so they are fixed; the column metrics come from {@code stats}.
87+
private DataFile writeSingle(Table table, TIcebergColumnStats stats, String path) {
88+
TIcebergCommitData d = new TIcebergCommitData();
89+
d.setFilePath(path);
90+
d.setRowCount(10L);
91+
d.setFileSize(1024L);
92+
d.setFileContent(TFileContent.DATA);
93+
d.setColumnStats(stats);
94+
return IcebergWriterHelper.convertToWriterResult(
95+
table, Collections.singletonList(d), ZoneOffset.UTC).dataFiles()[0];
7696
}
7797

7898
// ─────────────────── convertToDeleteFiles: ported from fe-core IcebergWriterHelperTest ───────────────────
@@ -218,6 +238,150 @@ public void convertToWriterResultCarriesColumnMetrics() {
218238
Assertions.assertEquals(Long.valueOf(2L), df.nullValueCounts().get(1));
219239
}
220240

241+
// ──────────── convertToWriterResult: #65782 honor iceberg metrics policy (ported from fe-core) ────────────
242+
243+
@Test
244+
public void convertToWriterResultRespectsNoneMetricsMode() {
245+
Table table = tableWith("write.format.default", "parquet", "write.metadata.metrics.default", "none");
246+
TIcebergColumnStats stats = new TIcebergColumnStats();
247+
stats.setColumnSizes(Map.of(2, 128L));
248+
stats.setValueCounts(Map.of(2, 10L));
249+
stats.setNullValueCounts(Map.of(2, 0L));
250+
stats.setLowerBounds(Map.of(2, ByteBuffer.wrap(new byte[] {0x01})));
251+
stats.setUpperBounds(Map.of(2, ByteBuffer.wrap(new byte[] {0x02})));
252+
253+
DataFile df = writeSingle(table, stats, "s3://b/db1/t/f.parquet");
254+
255+
Assertions.assertTrue(df.columnSizes() == null || df.columnSizes().isEmpty());
256+
Assertions.assertTrue(df.valueCounts() == null || df.valueCounts().isEmpty());
257+
Assertions.assertTrue(df.nullValueCounts() == null || df.nullValueCounts().isEmpty());
258+
Assertions.assertTrue(df.lowerBounds() == null || df.lowerBounds().isEmpty());
259+
Assertions.assertTrue(df.upperBounds() == null || df.upperBounds().isEmpty());
260+
}
261+
262+
@Test
263+
public void convertToWriterResultCountsModeOmitsBounds() {
264+
Table table = tableWith("write.format.default", "parquet", "write.metadata.metrics.default", "counts");
265+
TIcebergColumnStats stats = new TIcebergColumnStats();
266+
stats.setColumnSizes(Map.of(2, 128L));
267+
stats.setValueCounts(Map.of(2, 10L));
268+
stats.setNullValueCounts(Map.of(2, 0L));
269+
stats.setLowerBounds(Map.of(2, Conversions.toByteBuffer(Types.StringType.get(), "abcdefgh")));
270+
stats.setUpperBounds(Map.of(2, Conversions.toByteBuffer(Types.StringType.get(), "ijklmnop")));
271+
272+
DataFile df = writeSingle(table, stats, "s3://b/db1/t/f.parquet");
273+
274+
Assertions.assertEquals(Long.valueOf(128L), df.columnSizes().get(2));
275+
Assertions.assertEquals(Long.valueOf(10L), df.valueCounts().get(2));
276+
Assertions.assertEquals(Long.valueOf(0L), df.nullValueCounts().get(2));
277+
Assertions.assertTrue(df.lowerBounds() == null || df.lowerBounds().isEmpty());
278+
Assertions.assertTrue(df.upperBounds() == null || df.upperBounds().isEmpty());
279+
}
280+
281+
@Test
282+
public void convertToWriterResultTruncatesStringAndBinaryBounds() {
283+
Schema boundsSchema = new Schema(
284+
Types.NestedField.optional(1, "text", Types.StringType.get()),
285+
Types.NestedField.optional(2, "payload", Types.BinaryType.get()));
286+
Table table = tableWith(boundsSchema, "write.format.default", "parquet",
287+
"write.metadata.metrics.default", "truncate(3)");
288+
TIcebergColumnStats stats = new TIcebergColumnStats();
289+
stats.setLowerBounds(Map.of(
290+
1, Conversions.toByteBuffer(Types.StringType.get(), "abcdef"),
291+
2, ByteBuffer.wrap(new byte[] {1, 2, 3, 4})));
292+
stats.setUpperBounds(Map.of(
293+
1, Conversions.toByteBuffer(Types.StringType.get(), "uvwxyz"),
294+
2, ByteBuffer.wrap(new byte[] {1, 2, 3, 4})));
295+
296+
DataFile df = writeSingle(table, stats, "s3://b/db1/t/f.parquet");
297+
298+
Assertions.assertEquals("abc",
299+
Conversions.fromByteBuffer(Types.StringType.get(), df.lowerBounds().get(1)).toString());
300+
Assertions.assertEquals("uvx",
301+
Conversions.fromByteBuffer(Types.StringType.get(), df.upperBounds().get(1)).toString());
302+
Assertions.assertEquals(ByteBuffer.wrap(new byte[] {1, 2, 3}), df.lowerBounds().get(2));
303+
Assertions.assertEquals(ByteBuffer.wrap(new byte[] {1, 2, 4}), df.upperBounds().get(2));
304+
}
305+
306+
@Test
307+
public void convertToWriterResultPreservesOrcUpperBoundWithoutTruncatedSuccessor() {
308+
Schema boundsSchema = new Schema(Types.NestedField.optional(1, "text", Types.StringType.get()));
309+
Table table = tableWith(boundsSchema, "write.format.default", "orc",
310+
"write.metadata.metrics.default", "truncate(1)");
311+
String maxWithoutSuccessor = new String(Character.toChars(Character.MAX_CODE_POINT)) + "tail";
312+
TIcebergColumnStats stats = new TIcebergColumnStats();
313+
stats.setUpperBounds(Map.of(1, Conversions.toByteBuffer(Types.StringType.get(), maxWithoutSuccessor)));
314+
315+
DataFile df = writeSingle(table, stats, "s3://b/db1/t/f.orc");
316+
317+
Assertions.assertNotNull(df.upperBounds());
318+
Assertions.assertEquals(maxWithoutSuccessor,
319+
Conversions.fromByteBuffer(Types.StringType.get(), df.upperBounds().get(1)).toString());
320+
}
321+
322+
@Test
323+
public void convertToWriterResultHandlesV3TransactionTableLineageMetrics() {
324+
// A commit runs against transaction.table(), which is a HasTableOperations but NOT a BaseTable.
325+
// getFormatVersion must read the real v3 through operations (the reserved format-version property is
326+
// stripped at creation), else the row-lineage columns are not appended and their metric field ids fail
327+
// schema resolution — this test is the regression guard for the BaseTable -> HasTableOperations change.
328+
Table base = tableWith("format-version", "3", "write.format.default", "parquet",
329+
"write.metadata.metrics.default", "truncate(16)");
330+
Table transactionTable = base.newTransaction().table();
331+
332+
int rowId = MetadataColumns.ROW_ID.fieldId();
333+
int seqId = MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId();
334+
ByteBuffer rowIdBound = Conversions.toByteBuffer(MetadataColumns.ROW_ID.type(), 7L);
335+
ByteBuffer seqBound = Conversions.toByteBuffer(MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.type(), 3L);
336+
TIcebergColumnStats stats = new TIcebergColumnStats();
337+
stats.setLowerBounds(Map.of(rowId, rowIdBound, seqId, seqBound));
338+
stats.setUpperBounds(Map.of(rowId, rowIdBound, seqId, seqBound));
339+
340+
DataFile df = writeSingle(transactionTable, stats, "s3://b/db1/t/v3.parquet");
341+
342+
Assertions.assertEquals(rowIdBound, df.lowerBounds().get(rowId));
343+
Assertions.assertEquals(rowIdBound, df.upperBounds().get(rowId));
344+
Assertions.assertEquals(seqBound, df.lowerBounds().get(seqId));
345+
Assertions.assertEquals(seqBound, df.upperBounds().get(seqId));
346+
}
347+
348+
@Test
349+
public void convertToWriterResultSuppressesLogicalMetricsBelowRepeatedFields() {
350+
Schema repeatedSchema = new Schema(
351+
Types.NestedField.optional(1, "items", Types.ListType.ofOptional(2, Types.IntegerType.get())),
352+
Types.NestedField.optional(3, "attributes",
353+
Types.MapType.ofOptional(4, 5, Types.StringType.get(), Types.StringType.get())),
354+
Types.NestedField.optional(6, "top_level", Types.IntegerType.get()));
355+
Table table = tableWith(repeatedSchema, "write.format.default", "parquet",
356+
"write.metadata.metrics.default", "full");
357+
// InMemoryCatalog.createTable reassigns fresh field ids, so resolve the real ids from the stored schema
358+
// (the input ids 2/4/5/6 no longer map to the same fields). The BE keys its stats by these same ids.
359+
Schema stored = table.schema();
360+
int elementId = ((Types.ListType) stored.findField("items").type()).elementId();
361+
int keyId = ((Types.MapType) stored.findField("attributes").type()).keyId();
362+
int valueId = ((Types.MapType) stored.findField("attributes").type()).valueId();
363+
int topId = stored.findField("top_level").fieldId();
364+
TIcebergColumnStats stats = new TIcebergColumnStats();
365+
stats.setColumnSizes(Map.of(elementId, 20L, keyId, 40L, valueId, 50L, topId, 60L));
366+
stats.setValueCounts(Map.of(elementId, 2L, keyId, 4L, valueId, 5L, topId, 6L));
367+
stats.setNullValueCounts(Map.of(elementId, 0L, keyId, 0L, valueId, 0L, topId, 0L));
368+
stats.setLowerBounds(Map.of(
369+
elementId, Conversions.toByteBuffer(Types.IntegerType.get(), 2),
370+
keyId, Conversions.toByteBuffer(Types.StringType.get(), "key"),
371+
valueId, Conversions.toByteBuffer(Types.StringType.get(), "value"),
372+
topId, Conversions.toByteBuffer(Types.IntegerType.get(), 6)));
373+
stats.setUpperBounds(stats.getLowerBounds());
374+
375+
DataFile df = writeSingle(table, stats, "s3://b/db1/t/repeated.parquet");
376+
377+
// columnSizes ignore the repeated-field rule; only the logical count/bound metrics below list/map drop.
378+
Assertions.assertEquals(Map.of(elementId, 20L, keyId, 40L, valueId, 50L, topId, 60L), df.columnSizes());
379+
Assertions.assertEquals(Map.of(topId, 6L), df.valueCounts());
380+
Assertions.assertEquals(Map.of(topId, 0L), df.nullValueCounts());
381+
Assertions.assertEquals(Map.of(topId, stats.getLowerBounds().get(topId)), df.lowerBounds());
382+
Assertions.assertEquals(Map.of(topId, stats.getUpperBounds().get(topId)), df.upperBounds());
383+
}
384+
221385
// ─────────────────── getFileFormat: 3-tier resolution (T04) ───────────────────
222386

223387
@Test

‎fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,11 +44,19 @@
4444
import org.apache.doris.statistics.TableStatsMeta;
4545
import org.apache.doris.thrift.TStorageType;
4646

47+
import org.apache.iceberg.DataFile;
48+
import org.apache.iceberg.FileScanTask;
49+
import org.apache.iceberg.PartitionSpec;
50+
import org.apache.iceberg.Schema;
51+
import org.apache.iceberg.TableScan;
52+
import org.apache.iceberg.io.CloseableIterable;
53+
import org.apache.iceberg.types.Types;
4754
import org.junit.jupiter.api.Assertions;
4855
import org.junit.jupiter.api.Test;
4956
import org.mockito.MockedStatic;
5057
import org.mockito.Mockito;
5158

59+
import java.io.IOException;
5260
import java.time.LocalTime;
5361
import java.time.format.DateTimeFormatter;
5462
import java.util.ArrayList;
@@ -57,6 +65,52 @@
5765
import java.util.Map;
5866

5967
class StatisticsUtilTest {
68+
@Test
69+
void getIcebergColumnStatsReturnsEmptyForDisabledMetrics() {
70+
// Port of #65782 (read side): a metrics-mode=none table omits the column from the data-file stats maps,
71+
// so the reader must return unknown (empty) rather than fabricating zero-valued statistics.
72+
Schema schema = new Schema(Types.NestedField.optional(1, "id", Types.IntegerType.get()));
73+
PartitionSpec spec = PartitionSpec.builderFor(schema).build();
74+
org.apache.iceberg.Table table = Mockito.mock(org.apache.iceberg.Table.class);
75+
TableScan tableScan = Mockito.mock(TableScan.class);
76+
FileScanTask fileScanTask = Mockito.mock(FileScanTask.class);
77+
DataFile dataFile = Mockito.mock(DataFile.class);
78+
Mockito.when(table.newScan()).thenReturn(tableScan);
79+
Mockito.when(tableScan.includeColumnStats()).thenReturn(tableScan);
80+
Mockito.when(tableScan.planFiles())
81+
.thenReturn(CloseableIterable.withNoopClose(List.of(fileScanTask)));
82+
Mockito.when(fileScanTask.spec()).thenReturn(spec);
83+
Mockito.when(fileScanTask.file()).thenReturn(dataFile);
84+
Mockito.when(dataFile.columnSizes()).thenReturn(Map.of());
85+
Mockito.when(dataFile.nullValueCounts()).thenReturn(Map.of());
86+
87+
Assertions.assertTrue(StatisticsUtil.getIcebergColumnStats("id", table).isEmpty());
88+
}
89+
90+
@Test
91+
void getIcebergColumnStatsReturnsEmptyWhenCloseFails() {
92+
// Port of #65782 (read side): a failed scan close must not resurrect partial accumulators into
93+
// fabricated statistics; the reader returns unknown (empty) instead.
94+
Schema schema = new Schema(Types.NestedField.optional(1, "id", Types.IntegerType.get()));
95+
PartitionSpec spec = PartitionSpec.builderFor(schema).build();
96+
org.apache.iceberg.Table table = Mockito.mock(org.apache.iceberg.Table.class);
97+
TableScan tableScan = Mockito.mock(TableScan.class);
98+
FileScanTask fileScanTask = Mockito.mock(FileScanTask.class);
99+
DataFile dataFile = Mockito.mock(DataFile.class);
100+
Mockito.when(table.newScan()).thenReturn(tableScan);
101+
Mockito.when(tableScan.includeColumnStats()).thenReturn(tableScan);
102+
Mockito.when(tableScan.planFiles()).thenReturn(CloseableIterable.combine(
103+
List.of(fileScanTask), () -> {
104+
throw new IOException("close failed");
105+
}));
106+
Mockito.when(fileScanTask.spec()).thenReturn(spec);
107+
Mockito.when(fileScanTask.file()).thenReturn(dataFile);
108+
Mockito.when(dataFile.columnSizes()).thenReturn(Map.of());
109+
Mockito.when(dataFile.nullValueCounts()).thenReturn(Map.of());
110+
111+
Assertions.assertTrue(StatisticsUtil.getIcebergColumnStats("id", table).isEmpty());
112+
}
113+
60114
@Test
61115
void testConvertToDouble() {
62116
try {

0 commit comments

Comments
 (0)