Skip to content

Commit 7224a2d

Browse files
authored
[test](iceberg) Expand capability, transaction and snapshot coverage (apache#69551)
### What problem does this PR solve? Iceberg has broad regression coverage, but feature names alone miss state transitions and weak oracles. For example, expiring fresh snapshots with a cutoff in 2020 never exercises ref retention. Add 45 connector unit tests, two external regression suites, and an inventory mapping the published 84 operations plus read, version, model, lineage, type, scanner and storage dimensions to existing tests and remaining gaps. The tests cover snapshot/schema/ref isolation, atomic writes and failure paths, nested type metadata, exact predicate semantics, partition serialization, actual ref retention, and file positions across deletes and history. Nine individually disabled reproductions preserve correct assertions for five confirmed issue families: DECIMAL-to-DOUBLE pruning, lineage-name collisions during a V2-to-V3 upgrade, ignored per-operation isolation properties, mixed MERGE affected-row counts, and expiration counters built from stale metadata. These were run and observed failing before disabling. Production behavior is unchanged. ### Release note None ### Validation - Full connector reactor: `mvn -f fe/pom.xml -pl :fe-connector-iceberg -am test -DfailIfNoTests=false -Dmaven.build.cache.enabled=false` with the matching Thrift 0.24.0 compiler: BUILD SUCCESS, including FE Checkstyle. - Iceberg connector: 1,734 discovered, 1,720 passed, 14 skipped. New tests: 45 discovered, 36 passed, 9 intentionally disabled defect reproductions. - Both Groovy suites compile with Groovy 4.0.19. They use existing REST/MinIO/Spark fixtures and dynamic comparisons, with no handwritten golden results. - SQL suites were not executed: the managed QA/Jenkins endpoints are unreachable from the validation host. BE unit tests and the full backend/deployment matrix were not run. ### Check List (For Author) - Test - [x] Regression test (added; compilation checked; SQL execution pending) - [x] Unit Test - Behavior changed: - [x] No. - Does this need documentation? - [x] No. Test coverage inventory is included.
1 parent 7d16729 commit 7224a2d

12 files changed

Lines changed: 1965 additions & 0 deletions
Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.connector.iceberg;
19+
20+
import org.apache.doris.connector.iceberg.IcebergCatalogOps.CatalogBackedIcebergCatalogOps;
21+
import org.apache.doris.connector.spi.ConnectorColumn;
22+
import org.apache.doris.connector.spi.ConnectorType;
23+
import org.apache.doris.connector.spi.DorisConnectorException;
24+
25+
import org.apache.iceberg.HasTableOperations;
26+
import org.apache.iceberg.PartitionSpec;
27+
import org.apache.iceberg.Schema;
28+
import org.apache.iceberg.Table;
29+
import org.apache.iceberg.catalog.Namespace;
30+
import org.apache.iceberg.catalog.TableIdentifier;
31+
import org.apache.iceberg.inmemory.InMemoryCatalog;
32+
import org.apache.iceberg.types.Types;
33+
import org.junit.jupiter.api.Assertions;
34+
import org.junit.jupiter.api.Test;
35+
36+
import java.util.Collections;
37+
38+
public class IcebergLineageDdlCoverageTest {
39+
private static final TableIdentifier IDENTIFIER = TableIdentifier.of("db", "events");
40+
private static final IcebergTableHandle HANDLE = new IcebergTableHandle("db", "events");
41+
42+
@Test
43+
public void addDoesNotExposeDuplicateRowIdColumns() {
44+
assertAddPreservesUniqueMetadataNames("_row_id");
45+
}
46+
47+
@Test
48+
public void addDoesNotExposeDuplicateSequenceColumns() {
49+
assertAddPreservesUniqueMetadataNames("_last_updated_sequence_number");
50+
}
51+
52+
@Test
53+
public void renameDoesNotExposeDuplicateRowIdColumns() {
54+
assertRenamePreservesUniqueMetadataNames("_row_id");
55+
}
56+
57+
@Test
58+
public void renameDoesNotExposeDuplicateSequenceColumns() {
59+
assertRenamePreservesUniqueMetadataNames("_last_updated_sequence_number");
60+
}
61+
62+
private void assertAddPreservesUniqueMetadataNames(String reservedName) {
63+
InMemoryCatalog catalog = catalog();
64+
createV3(catalog);
65+
IcebergConnectorMetadata metadata = metadata(catalog);
66+
String previousMetadata = metadataLocation(catalog);
67+
DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class,
68+
() -> metadata.addColumn(null, HANDLE,
69+
new ConnectorColumn(reservedName, ConnectorType.of("BIGINT"), "", true, null, false), null));
70+
Assertions.assertTrue(failure.getMessage().contains("reserved row lineage column"));
71+
Assertions.assertEquals(previousMetadata, metadataLocation(catalog));
72+
Assertions.assertNull(catalog.loadTable(IDENTIFIER).schema().findField(reservedName));
73+
assertUniqueColumnName(metadata, reservedName);
74+
}
75+
76+
private void assertRenamePreservesUniqueMetadataNames(String reservedName) {
77+
InMemoryCatalog catalog = catalog();
78+
createV3(catalog);
79+
IcebergConnectorMetadata metadata = metadata(catalog);
80+
String previousMetadata = metadataLocation(catalog);
81+
DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class,
82+
() -> metadata.renameColumn(null, HANDLE, "id", reservedName));
83+
Assertions.assertTrue(failure.getMessage().contains("reserved row lineage column"));
84+
Assertions.assertEquals(previousMetadata, metadataLocation(catalog));
85+
Assertions.assertNotNull(catalog.loadTable(IDENTIFIER).schema().findField("id"));
86+
Assertions.assertNull(catalog.loadTable(IDENTIFIER).schema().findField(reservedName));
87+
assertUniqueColumnName(metadata, reservedName);
88+
}
89+
90+
private static void assertUniqueColumnName(IcebergConnectorMetadata metadata, String reservedName) {
91+
// A public schema must not contain both a physical user field and hidden metadata under one name.
92+
long matchingColumns = metadata.getTableSchema(null, HANDLE).getColumns().stream()
93+
.filter(column -> reservedName.equalsIgnoreCase(column.getName())).count();
94+
Assertions.assertEquals(1L, matchingColumns,
95+
"The connector exposed duplicate physical and hidden columns named " + reservedName);
96+
}
97+
98+
private static String metadataLocation(InMemoryCatalog catalog) {
99+
Table table = catalog.loadTable(IDENTIFIER);
100+
return ((HasTableOperations) table).operations().current().metadataFileLocation();
101+
}
102+
103+
private static InMemoryCatalog catalog() {
104+
InMemoryCatalog catalog = new InMemoryCatalog();
105+
catalog.initialize("repro", Collections.emptyMap());
106+
catalog.createNamespace(Namespace.of("db"));
107+
return catalog;
108+
}
109+
110+
private static void createV3(InMemoryCatalog catalog) {
111+
catalog.createTable(IDENTIFIER, new Schema(
112+
Types.NestedField.optional(1, "id", Types.LongType.get())),
113+
PartitionSpec.unpartitioned(), Collections.singletonMap("format-version", "3"));
114+
}
115+
116+
private static IcebergConnectorMetadata metadata(InMemoryCatalog catalog) {
117+
return new IcebergConnectorMetadata(new CatalogBackedIcebergCatalogOps(catalog),
118+
IcebergCatalogProperties.of(Collections.singletonMap(
119+
IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_REST)),
120+
new RecordingConnectorContext());
121+
}
122+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.connector.iceberg;
19+
20+
import org.apache.doris.connector.iceberg.IcebergCatalogOps.CatalogBackedIcebergCatalogOps;
21+
22+
import org.apache.iceberg.PartitionSpec;
23+
import org.apache.iceberg.Schema;
24+
import org.apache.iceberg.catalog.Namespace;
25+
import org.apache.iceberg.catalog.TableIdentifier;
26+
import org.apache.iceberg.inmemory.InMemoryCatalog;
27+
import org.apache.iceberg.types.Types;
28+
import org.junit.jupiter.api.Assertions;
29+
import org.junit.jupiter.api.Disabled;
30+
import org.junit.jupiter.api.Test;
31+
32+
import java.util.Collections;
33+
34+
public class IcebergLineageDdlKnownBugsTest {
35+
private static final TableIdentifier IDENTIFIER = TableIdentifier.of("db", "events");
36+
private static final IcebergTableHandle HANDLE = new IcebergTableHandle("db", "events");
37+
38+
@Test
39+
@Disabled("Known issue: a V2-to-V3 upgrade permits a user column to collide with row lineage.")
40+
public void upgradeDoesNotExposeDuplicateRowIdColumns() {
41+
assertUpgradePreservesUniqueMetadataNames("_row_id");
42+
}
43+
44+
@Test
45+
@Disabled("Known issue: a V2-to-V3 upgrade permits a user column to collide with row lineage.")
46+
public void upgradeDoesNotExposeDuplicateSequenceColumns() {
47+
assertUpgradePreservesUniqueMetadataNames("_last_updated_sequence_number");
48+
}
49+
50+
private void assertUpgradePreservesUniqueMetadataNames(String reservedName) {
51+
InMemoryCatalog catalog = catalog();
52+
catalog.createTable(IDENTIFIER, new Schema(
53+
Types.NestedField.optional(1, reservedName, Types.LongType.get())),
54+
PartitionSpec.unpartitioned(), Collections.singletonMap("format-version", "2"));
55+
IcebergConnectorMetadata metadata = metadata(catalog);
56+
metadata.setTableProperties(null, HANDLE, Collections.singletonMap("format-version", "3"));
57+
assertUniqueColumnName(metadata, reservedName);
58+
}
59+
60+
private static void assertUniqueColumnName(IcebergConnectorMetadata metadata, String reservedName) {
61+
// A public schema must not contain both a physical user field and hidden metadata under one name.
62+
long matchingColumns = metadata.getTableSchema(null, HANDLE).getColumns().stream()
63+
.filter(column -> reservedName.equalsIgnoreCase(column.getName())).count();
64+
Assertions.assertEquals(1L, matchingColumns,
65+
"The connector exposed duplicate physical and hidden columns named " + reservedName);
66+
}
67+
68+
private static InMemoryCatalog catalog() {
69+
InMemoryCatalog catalog = new InMemoryCatalog();
70+
catalog.initialize("repro", Collections.emptyMap());
71+
catalog.createNamespace(Namespace.of("db"));
72+
return catalog;
73+
}
74+
75+
private static IcebergConnectorMetadata metadata(InMemoryCatalog catalog) {
76+
return new IcebergConnectorMetadata(new CatalogBackedIcebergCatalogOps(catalog),
77+
IcebergCatalogProperties.of(Collections.singletonMap(
78+
IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_REST)),
79+
new RecordingConnectorContext());
80+
}
81+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.connector.iceberg;
19+
20+
import org.apache.iceberg.PartitionData;
21+
import org.apache.iceberg.PartitionSpec;
22+
import org.apache.iceberg.Schema;
23+
import org.apache.iceberg.transforms.Transform;
24+
import org.apache.iceberg.transforms.Transforms;
25+
import org.apache.iceberg.types.Type;
26+
import org.apache.iceberg.types.Types;
27+
import org.junit.jupiter.api.Assertions;
28+
import org.junit.jupiter.api.Test;
29+
30+
import java.math.BigDecimal;
31+
import java.nio.ByteBuffer;
32+
import java.time.LocalDate;
33+
import java.time.LocalDateTime;
34+
import java.time.ZoneId;
35+
import java.time.ZoneOffset;
36+
import java.util.Arrays;
37+
import java.util.List;
38+
39+
public class IcebergPartitionUtilsCoverageTest {
40+
@Test
41+
public void compoundPartitionWireFormatPreservesEscapingAndExactValues() {
42+
Schema schema = new Schema(
43+
Types.NestedField.optional(1, "text", Types.StringType.get()),
44+
Types.NestedField.optional(2, "nullable", Types.StringType.get()),
45+
Types.NestedField.optional(3, "amount", Types.DecimalType.of(38, 18)),
46+
Types.NestedField.optional(4, "date", Types.DateType.get()),
47+
Types.NestedField.optional(5, "timestamp", Types.TimestampNanoType.withoutZone()),
48+
Types.NestedField.optional(6, "binary", Types.BinaryType.get()));
49+
PartitionSpec spec = PartitionSpec.builderFor(schema).identity("text").identity("nullable")
50+
.identity("amount").identity("date").identity("timestamp").identity("binary").build();
51+
PartitionData original = new PartitionData(spec.partitionType());
52+
List<Object> values = Arrays.asList("null/\"quoted\"\\line\n\u0000雪", null,
53+
new BigDecimal("-99999999999999999999.000000000000000001"), -1,
54+
-1L, ByteBuffer.wrap(new byte[] {0, (byte) 0xff, (byte) 0x80, 34, 92}));
55+
for (int i = 0; i < values.size(); i++) {
56+
original.set(i, values.get(i));
57+
}
58+
59+
for (ZoneId zone : Arrays.asList(ZoneOffset.UTC, ZoneId.of("America/New_York"))) {
60+
List<String> encoded = IcebergPartitionUtils.parsePartitionValuesFromJson(
61+
IcebergPartitionUtils.getPartitionDataJson(original, spec, zone));
62+
Assertions.assertEquals(values.size(), encoded.size());
63+
Assertions.assertEquals(values.get(0), encoded.get(0));
64+
Assertions.assertNull(encoded.get(1));
65+
PartitionData decoded = new PartitionData(spec.partitionType());
66+
for (int i = 0; i < encoded.size(); i++) {
67+
decoded.set(i, IcebergPartitionUtils.parsePartitionValueFromString(
68+
encoded.get(i), spec.partitionType().fields().get(i).type(), zone));
69+
}
70+
// PartitionData stores binary as byte[], whose container equality compares array identities.
71+
for (int i = 0; i < original.size(); i++) {
72+
Assertions.assertEquals(original.get(i), decoded.get(i), "partition field " + i);
73+
}
74+
Assertions.assertEquals(18, ((BigDecimal) decoded.get(2)).scale());
75+
}
76+
}
77+
78+
@Test
79+
public void calendarRangesAgreeWithSdkTransformsAcrossEpochAndLeapBoundaries() {
80+
List<LocalDateTime> timestamps = Arrays.asList(
81+
LocalDateTime.of(1600, 2, 29, 23, 59, 59, 999999000),
82+
LocalDateTime.of(1900, 3, 1, 0, 0),
83+
LocalDateTime.of(1969, 12, 31, 23, 59, 59, 999999000),
84+
LocalDateTime.of(1970, 1, 1, 0, 0),
85+
LocalDateTime.of(2000, 2, 29, 23, 59, 59, 999999000),
86+
LocalDateTime.of(2100, 3, 1, 0, 0));
87+
for (Type type : Arrays.asList(Types.DateType.get(),
88+
Types.TimestampType.withoutZone(), Types.TimestampType.withZone())) {
89+
for (String name : Arrays.asList("year", "month", "day", "hour")) {
90+
if (type.typeId() == Type.TypeID.DATE && name.equals("hour")) {
91+
continue;
92+
}
93+
Transform<Object, Integer> transform = calendarTransform(name);
94+
for (LocalDateTime timestamp : timestamps) {
95+
int ordinal = transform.bind(type).apply(internalValue(type, timestamp));
96+
IcebergPartitionUtils.RangeBuild range = IcebergPartitionUtils.buildRange(
97+
name + "=" + ordinal, Integer.toString(ordinal), name, type, 100L, 200L);
98+
LocalDateTime lower = parseBound(range.getLowerBound().get(0), type);
99+
LocalDateTime upper = parseBound(range.getUpperBound().get(0), type);
100+
LocalDateTime row = type.typeId() == Type.TypeID.DATE
101+
? timestamp.toLocalDate().atStartOfDay() : timestamp;
102+
Assertions.assertFalse(row.isBefore(lower), name + " " + timestamp);
103+
Assertions.assertTrue(row.isBefore(upper), name + " " + timestamp);
104+
// Both ends must describe exactly one SDK transform ordinal, including negative ordinals.
105+
Assertions.assertEquals(ordinal, transform.bind(type).apply(internalValue(type, lower)));
106+
LocalDateTime last = type.typeId() == Type.TypeID.DATE
107+
? upper.minusDays(1) : upper.minusNanos(1000);
108+
Assertions.assertEquals(ordinal, transform.bind(type).apply(internalValue(type, last)));
109+
Assertions.assertEquals(ordinal + 1, transform.bind(type).apply(internalValue(type, upper)));
110+
}
111+
}
112+
}
113+
}
114+
115+
@Test
116+
public void binarySerializationDoesNotConsumeDirectOrReadOnlyBuffers() {
117+
ByteBuffer direct = ByteBuffer.allocateDirect(6);
118+
direct.put(new byte[] {99, 0, (byte) 0xfe, (byte) 0xff, 1, 99});
119+
direct.position(1);
120+
direct.limit(5);
121+
for (ByteBuffer buffer : Arrays.asList(direct, direct.asReadOnlyBuffer(), direct.slice())) {
122+
int position = buffer.position();
123+
int limit = buffer.limit();
124+
for (Type type : Arrays.asList(Types.BinaryType.get(), Types.FixedType.ofLength(4))) {
125+
String encoded = IcebergPartitionUtils.serializePartitionValue(type, buffer, ZoneOffset.UTC);
126+
Assertions.assertEquals("0x00feff01", encoded);
127+
Assertions.assertEquals(buffer, IcebergPartitionUtils.parsePartitionValueFromString(
128+
encoded, type, ZoneOffset.UTC));
129+
Assertions.assertEquals(position, buffer.position());
130+
Assertions.assertEquals(limit, buffer.limit());
131+
}
132+
}
133+
}
134+
135+
private static Transform<Object, Integer> calendarTransform(String name) {
136+
switch (name) {
137+
case "year":
138+
return Transforms.year();
139+
case "month":
140+
return Transforms.month();
141+
case "day":
142+
return Transforms.day();
143+
case "hour":
144+
return Transforms.hour();
145+
default:
146+
throw new IllegalArgumentException(name);
147+
}
148+
}
149+
150+
private static Object internalValue(Type type, LocalDateTime timestamp) {
151+
return type.typeId() == Type.TypeID.DATE ? (Object) (int) timestamp.toLocalDate().toEpochDay()
152+
: timestamp.toEpochSecond(ZoneOffset.UTC) * 1_000_000L + timestamp.getNano() / 1000;
153+
}
154+
155+
private static LocalDateTime parseBound(String bound, Type type) {
156+
return type.typeId() == Type.TypeID.DATE ? LocalDate.parse(bound).atStartOfDay()
157+
: LocalDateTime.parse(bound.replace("+00:00", "").replace(' ', 'T'));
158+
}
159+
}

0 commit comments

Comments
 (0)